├── .github └── workflows │ └── tests.yml ├── LICENSE ├── MANIFEST.in ├── README.md ├── dist ├── zipfly-1.1.3.tar.gz ├── zipfly-1.1.4.tar.gz ├── zipfly-1.1.5.tar.gz ├── zipfly-1.1.6.tar.gz ├── zipfly-1.1.7.tar.gz ├── zipfly-1.1.8.tar.gz ├── zipfly-1.1.9.tar.gz ├── zipfly-1.2.0.tar.gz ├── zipfly-1.2.1.tar.gz ├── zipfly-1.2.2.tar.gz ├── zipfly-2.1.0.tar.gz ├── zipfly-2.1.1.tar.gz ├── zipfly-2.1.2.tar.gz ├── zipfly-3.0.1.tar.gz ├── zipfly-3.0.2.tar.gz ├── zipfly-3.0.5.tar.gz ├── zipfly-3.0.6.tar.gz ├── zipfly-4.0.1.tar.gz ├── zipfly-4.0.2.tar.gz ├── zipfly-4.0.3.tar.gz ├── zipfly-5.0.1.tar.gz ├── zipfly-5.0.2.tar.gz ├── zipfly-5.0.3.tar.gz ├── zipfly-5.0.4.tar.gz ├── zipfly-5.0.5.tar.gz ├── zipfly-6.0.1.tar.gz ├── zipfly-6.0.3.tar.gz └── zipfly-6.0.4.tar.gz ├── examples ├── create_paths.py ├── file_response.py ├── set_comment.py ├── streaming_django.py ├── streaming_flask.py └── zip_size_before_creating_it.py ├── setup.cfg ├── setup.py ├── zipfly.egg-info ├── PKG-INFO ├── SOURCES.txt ├── dependency_links.txt └── top_level.txt └── zipfly ├── __init__.py ├── __pycache__ ├── __init__.cpython-37.pyc ├── api.cpython-37.pyc ├── response.cpython-37.pyc └── zipfly.cpython-37.pyc ├── api.py ├── compat.py ├── response.py ├── test.py ├── version.py └── zipfly.py /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | pull_request: 5 | paths-ignore: 6 | - 'dist/**' 7 | push: 8 | paths-ignore: 9 | - 'dist/**' 10 | 11 | jobs: 12 | tests: 13 | name: Python ${{ matrix.python-version }} 14 | runs-on: ubuntu-latest 15 | 16 | strategy: 17 | matrix: 18 | python-version: 19 | - '3.10' 20 | 21 | steps: 22 | - uses: actions/checkout@v2 23 | 24 | - uses: actions/setup-python@v2 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | 28 | 29 | - name: Upgrade packaging tools 30 | run: python -m pip install --upgrade pip setuptools virtualenv wheel 31 | 32 | - name: Run tests for ${{ matrix.python-version }} 33 | run: python -m unittest 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Cardallot, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/sandes/zipfly/tests.yml) 2 | ![GitHub release (latest by date)](https://img.shields.io/github/v/release/sandes/zipfly) 3 | [![Downloads](https://static.pepy.tech/badge/zipfly)](https://pepy.tech/project/zipfly) 4 | 5 | 6 | # ZipFly 7 | 8 | ZipFly is a zip archive generator based on zipfile.py. 9 | It was created to generate very large ZIP archives for immediate sending out to clients, or for writing large ZIP archives without memory inflation. 10 | 11 | # Requirements 12 | Python 3.6+ Added support for writing to unseekable streams. 13 | 14 | # Install 15 | pip3 install zipfly 16 | 17 | # Basic usage, compress on-the-fly during writes 18 | Using this library will save you from having to write the Zip to disk. Some data will be buffered by the zipfile deflater, but memory inflation is going to be very constrained. Data will be written to destination by default at regular 32KB intervals. 19 | 20 | 21 | `ZipFly` defaults attributes:
22 | - paths: [ ]
23 | - mode: (write) w
24 | - chunksize: (bytes) 32768
25 | - compression: Stored
26 | - allowZip64: True
27 | - compresslevel: None
28 | - storesize: (bytes) 0
29 | - encode: utf-8
30 | 31 |
32 | 33 | 34 | 35 | `paths` list of dictionaries: 36 | 37 | | |. 38 | |---------------- |------------------------------- 39 | |**fs** |Should be the path to a file on the filesystem 40 | |**n** *(Optional)* |Is the name which it will have within the archive
(by default, this will be the same as **fs**) 41 | 42 |
43 | 44 | ```python 45 | 46 | import zipfly 47 | 48 | paths = [ 49 | { 50 | 'fs': '/path/to/large/file' 51 | }, 52 | ] 53 | 54 | zfly = zipfly.ZipFly(paths = paths) 55 | 56 | generator = zfly.generator() 57 | print (generator) 58 | # 59 | 60 | 61 | with open("large.zip", "wb") as f: 62 | for i in generator: 63 | f.write(i) 64 | 65 | ``` 66 | 67 | 68 | # Examples 69 | 70 | > Streaming multiple files in a zip with Django or Flask 71 | Send forth large files to clients with the most popular frameworks 72 | 73 | > Create paths 74 | Easy way to create the array `paths` from a parent folder. 75 | 76 | > Predict the size of the zip file before creating it 77 | Use the `BufferPredictionSize` to compute the correct size of the resulting archive before creating it. 78 | 79 | > Streaming a large file 80 | Efficient way to read a single very large binary file in python 81 | 82 | > Set a comment 83 | Your own comment in the zip file 84 | 85 | 86 | -------------------------------------------------------------------------------- /dist/zipfly-1.1.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.1.3.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-1.1.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.1.4.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-1.1.5.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.1.5.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-1.1.6.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.1.6.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-1.1.7.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.1.7.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-1.1.8.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.1.8.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-1.1.9.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.1.9.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-1.2.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.2.0.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-1.2.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.2.1.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-1.2.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-1.2.2.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-2.1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-2.1.0.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-2.1.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-2.1.1.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-2.1.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-2.1.2.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-3.0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-3.0.1.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-3.0.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-3.0.2.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-3.0.5.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-3.0.5.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-3.0.6.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-3.0.6.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-4.0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-4.0.1.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-4.0.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-4.0.2.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-4.0.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-4.0.3.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-5.0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-5.0.1.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-5.0.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-5.0.2.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-5.0.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-5.0.3.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-5.0.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-5.0.4.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-5.0.5.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-5.0.5.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-6.0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-6.0.1.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-6.0.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-6.0.3.tar.gz -------------------------------------------------------------------------------- /dist/zipfly-6.0.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/dist/zipfly-6.0.4.tar.gz -------------------------------------------------------------------------------- /examples/create_paths.py: -------------------------------------------------------------------------------- 1 | # create paths 2 | 3 | import zipfly 4 | import os 5 | 6 | parent = "/home/user/Documents/folder/" 7 | paths = [] 8 | 9 | # Create paths 10 | for dirpath, dnames, fnames in os.walk(parent): 11 | for f in fnames: 12 | paths.append( 13 | { 14 | 'fs': f'{parent}{f}', 15 | 'n': f'large_folder/{f}', 16 | } 17 | ) 18 | 19 | # ZipFly 20 | zfly = zipfly.ZipFly( paths = paths ) 21 | 22 | with open("large.zip", "wb") as f: 23 | for i in zfly.generator(): 24 | f.write(i) -------------------------------------------------------------------------------- /examples/file_response.py: -------------------------------------------------------------------------------- 1 | import zipfly 2 | 3 | file_location = '/home/user/Documents/file-100-GB.csv' 4 | 5 | # Efficient way to read a single very large binary file in python 6 | go_to_streaming = zipfly.FileResponse( file_location ) 7 | 8 | print ( go_to_streaming ) 9 | # 10 | -------------------------------------------------------------------------------- /examples/set_comment.py: -------------------------------------------------------------------------------- 1 | import zipfly 2 | import os 3 | 4 | 5 | paths = [ 6 | { 7 | 'fs': 'home/user/Videos/jupiter.mp4', 8 | 'n': 'movies/jupiter.mp4', 9 | }, 10 | ] 11 | 12 | 13 | zfly = zipfly.ZipFly( paths=paths ) 14 | 15 | # set a comment 16 | # IMPORTANT: set a comment before than buffer_prediction_size() 17 | zfly.set_comment("Isaac Newton 1234") 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /examples/streaming_django.py: -------------------------------------------------------------------------------- 1 | # django 2 | 3 | from django.http import StreamingHttpResponse 4 | import zipfly 5 | 6 | 7 | paths = [ 8 | { 9 | 'fs': 'home/user/Videos/jupiter.mp4', 10 | 'n': 'movies/jupiter.mp4', 11 | }, 12 | ] 13 | 14 | # constructor 15 | zfly = zipfly.ZipFly( paths=paths ) 16 | 17 | # z is a new generator 18 | z = zfly.generator() 19 | 20 | # django streaming 21 | response = StreamingHttpResponse(z, content_type='application/octet-stream') 22 | response['Content-Disposition'] = 'attachment; filename=file.zip' 23 | 24 | return response 25 | -------------------------------------------------------------------------------- /examples/streaming_flask.py: -------------------------------------------------------------------------------- 1 | # flask 2 | 3 | from flask import Response 4 | import zipfly 5 | 6 | paths = [ 7 | { 8 | 'fs': 'home/user/Videos/jupiter.mp4', 9 | 'n': 'movies/jupiter.mp4', 10 | }, 11 | ] 12 | 13 | 14 | # constructor 15 | zfly = zipfly.ZipFly( paths=paths ) 16 | 17 | # z is a new generator 18 | z = zfly.generator() 19 | 20 | # flask streaming 21 | response = Response(z, mimetype='application/zip',) 22 | response.headers['Content-Disposition'] = 'attachment; filename=file.zip' 23 | 24 | return response 25 | -------------------------------------------------------------------------------- /examples/zip_size_before_creating_it.py: -------------------------------------------------------------------------------- 1 | import zipfly 2 | import os 3 | 4 | 5 | # IMPORTANT: 6 | # BufferPredictionSize using Linux 7 | 8 | paths = [ 9 | { 10 | 'fs': 'home/user/Videos/jupiter.mp4', 11 | 'n': 'movies/jupiter.mp4', 12 | }, 13 | { 14 | 'fs': 'home/user/Videos/mercury.mp4', 15 | 'n': 'movies/mercury.mp4', 16 | } 17 | ] 18 | 19 | storesize = 0 20 | for path in paths: 21 | 22 | # (jupiter.mp4 + mercury.mp4) size in bytes 23 | 24 | f = open(path['fs'], 'rb') 25 | storesize += os.fstat(f.fileno()).st_size 26 | 27 | 28 | zfly = zipfly.ZipFly( paths=paths, storesize=storesize ) 29 | 30 | 31 | # zip size before creating it in bytes 32 | try: 33 | prediction = zfly.buffer_prediction_size() 34 | except zipfly.LargePredictionSize as e: 35 | print (e) 36 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description_file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | import sys 3 | import os 4 | __version__ = '6.0.5' 5 | 6 | CURRENT_PYTHON = sys.version_info[:2] 7 | REQUIRED_PYTHON = (3,6) 8 | 9 | # This check and everything above must remain compatible with Python 3.6 10 | if CURRENT_PYTHON < REQUIRED_PYTHON: 11 | 12 | sys.stderr.write(""" 13 | ========================== 14 | Unsupported Python version 15 | ========================== 16 | This version of ZipFly requires Python {}.{}, but you're trying to 17 | install it on Python {}.{}. 18 | This may be because you are using a version of pip that doesn't 19 | understand the python_requires classifier""".format(*(REQUIRED_PYTHON + CURRENT_PYTHON))) 20 | sys.exit(1) 21 | 22 | # 3.9 23 | 24 | setup( 25 | name='zipfly', 26 | packages=['zipfly'], 27 | description='ZipFly', 28 | version=__version__, 29 | url='http://github.com/sandes/zipfly', 30 | download_url = f'https://github.com/sandes/zipfly/archive/v{__version__}.tar.gz', 31 | author='sandes', 32 | long_description=open('README.md').read(), 33 | long_description_content_type='text/markdown', 34 | author_email='santidebus@gmail.com', 35 | keywords=['zipfly','Santiago Debus','santiagodebus.com'], 36 | install_requires=[], 37 | classifiers=[ 38 | 'Development Status :: 5 - Production/Stable', 39 | 'Intended Audience :: Developers', 40 | 'Topic :: Software Development :: Build Tools', 41 | 'License :: OSI Approved :: MIT License', 42 | 'Programming Language :: Python :: 3.6', 43 | 'Programming Language :: Python :: 3.7', 44 | 'Programming Language :: Python :: 3.8', 45 | 'Programming Language :: Python :: 3.9', 46 | 'Programming Language :: Python :: 3.10', 47 | ], 48 | ) 49 | -------------------------------------------------------------------------------- /zipfly.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: zipfly 3 | Version: 6.0.4 4 | Summary: ZipFly 5 | Home-page: http://github.com/buzonIO/zipfly 6 | Author: Buzon 7 | Author-email: support@buzon.io 8 | License: UNKNOWN 9 | Download-URL: https://github.com/BuzonIO/zipfly/archive/v6.0.4.tar.gz 10 | Keywords: zipfly,buzon 11 | Platform: UNKNOWN 12 | Classifier: Development Status :: 5 - Production/Stable 13 | Classifier: Intended Audience :: Developers 14 | Classifier: Topic :: Software Development :: Build Tools 15 | Classifier: License :: OSI Approved :: MIT License 16 | Classifier: Programming Language :: Python :: 3.6 17 | Classifier: Programming Language :: Python :: 3.7 18 | Classifier: Programming Language :: Python :: 3.8 19 | Classifier: Programming Language :: Python :: 3.9 20 | Description-Content-Type: text/markdown 21 | License-File: LICENSE 22 | 23 | [![Build Status](https://img.shields.io/circleci/build/github/sandes/zipfly/master)](https://app.circleci.com/pipelines/github/sandes/zipfly) 24 | ![GitHub release (latest by date)](https://img.shields.io/github/v/release/buzonio/zipfly) 25 | [![Downloads](https://pepy.tech/badge/zipfly)](https://pepy.tech/project/zipfly) 26 | 27 | # ZipFly 28 | 29 | ZipFly is a zip archive generator based on zipfile.py. 30 | It was created to generate very large ZIP archives for immediate sending out to clients, or for writing large ZIP archives without memory inflation. 31 | 32 | # Requirements 33 | Python 3.6+ Added support for writing to unseekable streams. 34 | 35 | # Install 36 | pip3 install zipfly 37 | 38 | # Basic usage, compress on-the-fly during writes 39 | Using this library will save you from having to write the Zip to disk. Some data will be buffered by the zipfile deflater, but memory inflation is going to be very constrained. Data will be written to destination by default at regular 32KB intervals. 40 | 41 | 42 | `ZipFly` defaults attributes:
43 | - paths: [ ]
44 | - mode: (write) w
45 | - chunksize: (bytes) 32768
46 | - compression: Stored
47 | - allowZip64: True
48 | - compresslevel: None
49 | - storesize: (bytes) 0
50 | - encode: utf-8
51 | 52 |
53 | 54 | 55 | 56 | `paths` list of dictionaries: 57 | 58 | | |. 59 | |---------------- |------------------------------- 60 | |**fs** |Should be the path to a file on the filesystem 61 | |**n** *(Optional)* |Is the name which it will have within the archive
(by default, this will be the same as **fs**) 62 | 63 |
64 | 65 | ```python 66 | 67 | import zipfly 68 | 69 | paths = [ 70 | { 71 | 'fs': '/path/to/large/file' 72 | }, 73 | ] 74 | 75 | zfly = zipfly.ZipFly(paths = paths) 76 | 77 | generator = zfly.generator() 78 | print (generator) 79 | # 80 | 81 | 82 | with open("large.zip", "wb") as f: 83 | for i in generator: 84 | f.write(i) 85 | 86 | ``` 87 | 88 | 89 | # Examples 90 | 91 | > Streaming multiple files in a zip with Django or Flask 92 | Send forth large files to clients with the most popular frameworks 93 | 94 | > Create paths 95 | Easy way to create the array `paths` from a parent folder. 96 | 97 | > Predict the size of the zip file before creating it 98 | Use the `BufferPredictionSize` to compute the correct size of the resulting archive before creating it. 99 | 100 | > Streaming a large file 101 | Efficient way to read a single very large binary file in python 102 | 103 | > Set a comment 104 | Your own comment in the zip file 105 | 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /zipfly.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | LICENSE 2 | MANIFEST.in 3 | README.md 4 | setup.cfg 5 | setup.py 6 | zipfly/__init__.py 7 | zipfly/api.py 8 | zipfly/compat.py 9 | zipfly/response.py 10 | zipfly/test.py 11 | zipfly/version.py 12 | zipfly/zipfly.py 13 | zipfly.egg-info/PKG-INFO 14 | zipfly.egg-info/SOURCES.txt 15 | zipfly.egg-info/dependency_links.txt 16 | zipfly.egg-info/top_level.txt -------------------------------------------------------------------------------- /zipfly.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /zipfly.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | zipfly 2 | -------------------------------------------------------------------------------- /zipfly/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import unicode_literals, print_function, with_statement 4 | 5 | __version__ = '6.0.4' 6 | __author__ = 'Santiago Debus - Buzon, Inc.' 7 | __license__ = 'MIT' 8 | # v 9 | 10 | from .zipfly import ZipFly 11 | from .zipfly import LargePredictionSize 12 | from .response import FileResponse 13 | from .api import Buffer 14 | -------------------------------------------------------------------------------- /zipfly/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/zipfly/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /zipfly/__pycache__/api.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/zipfly/__pycache__/api.cpython-37.pyc -------------------------------------------------------------------------------- /zipfly/__pycache__/response.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/zipfly/__pycache__/response.cpython-37.pyc -------------------------------------------------------------------------------- /zipfly/__pycache__/zipfly.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandes/zipfly/17fd24de45d464c44ee2f5691908b38d7e553db8/zipfly/__pycache__/zipfly.cpython-37.pyc -------------------------------------------------------------------------------- /zipfly/api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | 4 | class Buffer: 5 | 6 | def __init__(self, 7 | paths = [], 8 | storesize = 0, 9 | comment = b''): 10 | 11 | self.paths = paths 12 | self.pfbs = 0 13 | self.storesize = int(storesize) 14 | self.comment = comment 15 | -------------------------------------------------------------------------------- /zipfly/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | pythoncompat 5 | 6 | Copied from requests 7 | """ 8 | 9 | import sys 10 | 11 | PY3 = sys.version_info[0] == 3 12 | 13 | builtin_str = str 14 | str = str 15 | bytes = bytes 16 | basestring = (str, bytes) 17 | numeric_types = (int, float) 18 | 19 | from zipfile import ZIP64_VERSION 20 | from zipfile import BZIP2_VERSION 21 | from zipfile import ZIP_BZIP2 22 | from zipfile import LZMA_VERSION 23 | from zipfile import ZIP_LZMA 24 | from zipfile import ZIP_MAX_COMMENT 25 | 26 | # Copy from io 27 | SEEK_SET = 0 # start of the stream (the default); offset should be zero or positive 28 | SEEK_CUR = 1 # current stream position; offset may be negative 29 | SEEK_END = 2 # end of the stream; offset is usually negative 30 | -------------------------------------------------------------------------------- /zipfly/response.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | class FileResponse: 4 | 5 | """read a file piece by piece. 6 | Default chunk size 0x4000 bytes""" 7 | 8 | def __init__(self, 9 | file_location = '/', 10 | chunksize = int(0x4000)): 11 | 12 | self.file_location = file_location 13 | self.chunksize = chunksize 14 | 15 | 16 | def __iter__(self): 17 | 18 | with open(self.file_location, 'rb') as entry: 19 | 20 | for chunk in iter(lambda: entry.read(self.chunksize), b''): 21 | 22 | yield chunk -------------------------------------------------------------------------------- /zipfly/test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import zipfly 3 | import os 4 | import string 5 | import random 6 | 7 | def rs(N): 8 | return ''.join(random.SystemRandom().choice( 9 | string.ascii_uppercase + \ 10 | string.ascii_lowercase + \ 11 | string.digits) for _ in range(N)) 12 | 13 | def pick_n(): 14 | return int(''.join(random.SystemRandom().choice(string.digits) for _ in range(3))) 15 | 16 | p1 = os.getcwd() 17 | p2 = p1 18 | p3 = p1 19 | 20 | """ 21 | idx1 = p1.rfind("/zipfly") 22 | p1 = p1[0:idx1] 23 | p1 = p1+"/dist/" 24 | 25 | idx2 = p2.rfind("/zipfly") 26 | p2 = p2[0:idx2] 27 | p2 = p2+"/zipfly.egg-info/" 28 | 29 | idx3 = p3.rfind("/zipfly") 30 | p3 = p3[0:idx3] 31 | p3 = p3+"/examples/" 32 | 33 | """ 34 | p1 = p1+"/dist/" 35 | p2 = p2+"/zipfly.egg-info/" 36 | p3 = p3+"/examples/" 37 | 38 | paths1 = [] 39 | 40 | for dirpath, dnames, fnames in os.walk(p1): 41 | for f in fnames: 42 | paths1.append( 43 | { 44 | 'fs':'{}{}'.format(p1,f), 45 | 'n':'{}{}'.format(p1,f), 46 | } 47 | ) 48 | 49 | for dirpath, dnames, fnames in os.walk(p2): 50 | for f in fnames: 51 | paths1.append( 52 | { 53 | 'fs':'{}{}'.format(p2,f), 54 | 'n':'{}{}'.format(p2,f), 55 | } 56 | ) 57 | 58 | for dirpath, dnames, fnames in os.walk(p3): 59 | for f in fnames: 60 | paths1.append( 61 | { 62 | 'fs':'{}{}'.format(p3,f), 63 | 'n':'{}{}'.format(p3,f), 64 | } 65 | ) 66 | 67 | class TestBufferPredictionSize(unittest.TestCase): 68 | 69 | def test_buffer_prediction_size(self): 70 | 71 | print ( 72 | """ 73 | TEST IF REAL ZIP SIZE IS EQUAL TO PREDICTION SIZE 74 | # # # # # # # # # # # # # # # # # # # # # # # # # 75 | """ 76 | ) 77 | 78 | for test_n in range(1, 50): 79 | 80 | with self.subTest(i=test_n): 81 | 82 | storesize = 0 83 | for path in paths1: 84 | f = open(path['fs'], 'rb') 85 | storesize += os.fstat(f.fileno()).st_size 86 | f.close() 87 | 88 | zfly = zipfly.ZipFly( paths = paths1, storesize = storesize ) 89 | 90 | zfly.set_comment(rs(pick_n())) 91 | 92 | # zip size before creating it in bytes 93 | ps = zfly.buffer_prediction_size() 94 | 95 | with open("test{}.zip".format(test_n), "wb") as f: 96 | for i in zfly.generator(): 97 | f.write(i) 98 | 99 | f = open("test{}.zip".format(test_n), 'rb') 100 | zs = os.fstat(f.fileno()).st_size 101 | f.close() 102 | 103 | # fetch 104 | 105 | print ( 106 | "test-{}.zip ->".format(test_n), 107 | "{} KB".format(round(zs/1024,2)), 108 | "({} bytes)".format(zs), 109 | (" ---- OK" if zs==ps else " ---- FAIL") 110 | ) 111 | 112 | self.assertEqual(zs,ps) 113 | 114 | 115 | if __name__ == '__main__': 116 | 117 | unittest.main() 118 | -------------------------------------------------------------------------------- /zipfly/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '6.0.4' 2 | # v 3 | -------------------------------------------------------------------------------- /zipfly/zipfly.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __version__ = '6.0.5' 3 | # v 4 | 5 | import io 6 | import stat 7 | import zipfile 8 | 9 | ZIP64_LIMIT = (1 << 31) + 1 10 | 11 | class LargePredictionSize(Exception): 12 | """ 13 | Raised when Buffer is larger than ZIP64 14 | """ 15 | 16 | class ZipflyStream(io.RawIOBase): 17 | 18 | """ 19 | The RawIOBase ABC extends IOBase. It deals with 20 | the reading and writing of bytes to a stream. FileIO subclasses 21 | RawIOBase to provide an interface to files in the machine’s file system. 22 | """ 23 | 24 | def __init__(self): 25 | self._buffer = b'' 26 | self._size = 0 27 | 28 | def writable(self): 29 | return True 30 | 31 | def write(self, b): 32 | if self.closed: 33 | raise RuntimeError("ZipFly stream was closed!") 34 | self._buffer += b 35 | return len(b) 36 | 37 | def get(self): 38 | chunk = self._buffer 39 | self._buffer = b'' 40 | self._size += len(chunk) 41 | return chunk 42 | 43 | def size(self): 44 | return self._size 45 | 46 | 47 | class ZipFly: 48 | 49 | def __init__(self, 50 | mode = 'w', 51 | paths = [], 52 | chunksize = 0x8000, 53 | compression = zipfile.ZIP_STORED, 54 | allowZip64 = True, 55 | compresslevel = None, 56 | storesize = 0, 57 | filesystem = 'fs', 58 | arcname = 'n', 59 | encode = 'utf-8',): 60 | 61 | """ 62 | @param store size : int : size of all files 63 | in paths without compression 64 | """ 65 | 66 | if mode not in ('w',): 67 | raise RuntimeError("ZipFly requires 'w' mode") 68 | 69 | if compression not in ( zipfile.ZIP_STORED,): 70 | raise RuntimeError("Not compression supported") 71 | 72 | if compresslevel not in (None, ): 73 | raise RuntimeError("Not compression level supported") 74 | 75 | if isinstance(chunksize, str): 76 | chunksize = int(chunksize, 16) 77 | 78 | 79 | 80 | self.comment = f'Written using Zipfly v{__version__}' 81 | self.mode = mode 82 | self.paths = paths 83 | self.filesystem = filesystem 84 | self.arcname = arcname 85 | self.compression = compression 86 | self.chunksize = chunksize 87 | self.allowZip64 = allowZip64 88 | self.compresslevel = compresslevel 89 | self.storesize = storesize 90 | self.encode = encode 91 | self.ezs = int('0x8e', 16) # empty zip size in bytes 92 | 93 | def set_comment(self, comment): 94 | 95 | if not isinstance(comment, bytes): 96 | comment = str.encode(comment) 97 | 98 | if len(comment) >= zipfile.ZIP_MAX_COMMENT: 99 | 100 | # trunk comment 101 | comment = comment[:zipfile.ZIP_MAX_COMMENT] 102 | 103 | self.comment = comment 104 | 105 | 106 | def reader(self, entry): 107 | 108 | def get_chunk(): 109 | return entry.read( self.chunksize ) 110 | 111 | return get_chunk() 112 | 113 | 114 | def buffer_size(self): 115 | 116 | ''' 117 | FOR UNIT TESTING (not used) 118 | using to get the buffer size 119 | this size is different from the size of each file added 120 | ''' 121 | 122 | for i in self.generator(): pass 123 | return self._buffer_size 124 | 125 | 126 | def buffer_prediction_size(self): 127 | 128 | if not self.allowZip64: 129 | raise RuntimeError("ZIP64 extensions required") 130 | 131 | 132 | # End of Central Directory Record 133 | EOCD = int('0x16', 16) 134 | FILE_OFFSET = int('0x5e', 16) * len(self.paths) 135 | 136 | tmp_comment = self.comment 137 | if isinstance(self.comment, bytes): 138 | tmp_comment = ( self.comment ).decode() 139 | 140 | size_comment = len(tmp_comment.encode( self.encode )) 141 | 142 | # path-name 143 | 144 | size_paths = 0 145 | #for path in self.paths: 146 | for idx in range(len(self.paths)): 147 | 148 | ''' 149 | getting bytes from character in UTF-8 format 150 | example: 151 | '传' has 3 bytes in utf-8 format ( b'\xe4\xbc\xa0' ) 152 | ''' 153 | 154 | #path = paths[idx] 155 | name = self.arcname 156 | if not self.arcname in self.paths[idx]: 157 | name = self.filesystem 158 | 159 | tmp_name = self.paths[idx][name] 160 | if (tmp_name)[0] in ('/', ): 161 | 162 | # is dir then trunk 163 | tmp_name = (tmp_name)[ 1 : len( tmp_name ) ] 164 | 165 | size_paths += (len(tmp_name.encode( self.encode )) - int( '0x1', 16)) * int('0x2', 16) 166 | 167 | # zipsize 168 | zs = sum([EOCD,FILE_OFFSET,size_comment,size_paths,self.storesize,]) 169 | 170 | if zs > ZIP64_LIMIT: 171 | raise LargePredictionSize( 172 | "Prediction size for zip file greater than 2 GB not supported" 173 | ) 174 | 175 | return zs 176 | 177 | 178 | def generator(self): 179 | 180 | # stream 181 | stream = ZipflyStream() 182 | 183 | with zipfile.ZipFile( 184 | stream, 185 | mode = self.mode, 186 | compression = self.compression, 187 | allowZip64 = self.allowZip64,) as zf: 188 | 189 | for path in self.paths: 190 | 191 | if not self.filesystem in path: 192 | 193 | raise RuntimeError(f"'{self.filesystem}' key is required") 194 | 195 | """ 196 | filesystem should be the path to a file or directory on the filesystem. 197 | arcname is the name which it will have within the archive (by default, 198 | this will be the same as filename 199 | """ 200 | 201 | if not self.arcname in path: 202 | 203 | # arcname will be default path 204 | path[self.arcname] = path[self.filesystem] 205 | 206 | z_info = zipfile.ZipInfo.from_file( 207 | path[self.filesystem], 208 | path[self.arcname] 209 | ) 210 | 211 | with open( path[self.filesystem], 'rb' ) as e: 212 | # Read from filesystem: 213 | with zf.open( z_info, mode=self.mode ) as d: 214 | 215 | for chunk in iter( lambda: e.read(self.chunksize), b'' ): 216 | 217 | d.write(chunk) 218 | yield stream.get() 219 | 220 | 221 | self.set_comment(self.comment) 222 | zf.comment = self.comment 223 | 224 | yield stream.get() 225 | self._buffer_size = stream.size() 226 | 227 | # Flush and close this stream. 228 | stream.close() 229 | 230 | 231 | def get_size(self): 232 | 233 | return self._buffer_size 234 | --------------------------------------------------------------------------------