├── .dockerignore ├── .flake8 ├── .github └── workflows │ └── docker-publish.yml ├── .gitignore ├── .travis.yml ├── .vscode ├── .ropeproject │ └── config.py ├── launch.json └── tasks.json ├── Dockerfile ├── LICENSE ├── README.md ├── docker-compose.yaml ├── renovate.json ├── requirements.txt ├── test ├── data │ ├── folder │ │ ├── emptyfile │ │ └── subfolder │ │ │ └── file.txt │ ├── script.sh │ ├── test.zip │ └── text.txt └── runtest.sh └── ziprofs.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .idea 3 | test 4 | objectdb 5 | venv/ 6 | log.txt 7 | benchmark 8 | renovate.json 9 | README.md 10 | .travis.yml 11 | .github 12 | .flake8 13 | .gitignore 14 | .git -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 100 -------------------------------------------------------------------------------- /.github/workflows/docker-publish.yml: -------------------------------------------------------------------------------- 1 | name: Docker build 2 | 3 | on: 4 | push: 5 | branches: 6 | - dev 7 | workflow_dispatch: 8 | 9 | env: 10 | # Use docker.io for Docker Hub if empty 11 | REGISTRY: ghcr.io 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | permissions: 17 | contents: read 18 | packages: write 19 | # This is used to complete the identity challenge 20 | # with sigstore/fulcio when running outside of PRs. 21 | id-token: write 22 | 23 | steps: 24 | - name: Checkout repository 25 | uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 26 | 27 | # Workaround: https://github.com/docker/build-push-action/issues/461 28 | - name: Setup Docker buildx 29 | uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1 30 | 31 | - name: Log into registry ${{ env.REGISTRY }} 32 | uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 33 | with: 34 | registry: ${{ env.REGISTRY }} 35 | username: ${{ github.actor }} 36 | password: ${{ secrets.GITHUB_TOKEN }} 37 | 38 | - name: Extract Docker metadata 39 | id: meta 40 | uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 41 | with: 42 | images: ${{ env.REGISTRY }}/${{ github.repository }} 43 | tags: | 44 | type=schedule 45 | type=ref,event=branch 46 | type=ref,event=tag 47 | type=raw,value=latest,enable={{is_default_branch}} 48 | 49 | - name: Setup buildx mount cache 50 | uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 51 | with: 52 | path: | 53 | home-cache-386 54 | home-cache-amd64 55 | home-cache-armv6 56 | home-cache-armv7 57 | home-cache-arm64 58 | key: buildx-mount-cache-${{ github.sha }} 59 | restore-keys: | 60 | buildx-mount-cache- 61 | 62 | - name: Inject buildx mount cache into docker 63 | uses: reproducible-containers/buildkit-cache-dance@5b6db76d1da5c8b307d5d2e0706d266521b710de # v3.1.2 64 | with: 65 | cache-map: | 66 | { 67 | "home-cache-386": { 68 | "target": "/root/.cache", 69 | "id": "home-cache-linux/386" 70 | }, 71 | "home-cache-amd64": { 72 | "target": "/root/.cache", 73 | "id": "home-cache-linux/amd64" 74 | }, 75 | "home-cache-armv6": { 76 | "target": "/root/.cache", 77 | "id": "home-cache-linux/arm/v6" 78 | }, 79 | "home-cache-armv7": { 80 | "target": "/root/.cache", 81 | "id": "home-cache-linux/arm/v7" 82 | }, 83 | "home-cache-arm64": { 84 | "target": "/root/.cache", 85 | "id": "home-cache-linux/arm64" 86 | } 87 | } 88 | 89 | - name: Build and push Docker image 90 | id: build-and-push 91 | uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 92 | with: 93 | context: . 94 | push: true 95 | platforms: linux/amd64,linux/386,linux/arm/v6,linux/arm/v7,linux/arm64 96 | tags: ${{ steps.meta.outputs.tags }} 97 | labels: ${{ steps.meta.outputs.labels }} 98 | cache-from: type=gha 99 | cache-to: type=gha,mode=max 100 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/settings.json 2 | test/test.log 3 | test/mnt/* 4 | objectdb 5 | venv/ 6 | .idea/ 7 | log.txt 8 | benchmark 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: focal 2 | 3 | addons: 4 | apt: 5 | packages: 6 | - fuse 7 | - python3-fusepy 8 | - tree 9 | 10 | script: 11 | - "$TRAVIS_BUILD_DIR/test/runtest.sh -log" 12 | -------------------------------------------------------------------------------- /.vscode/.ropeproject/config.py: -------------------------------------------------------------------------------- 1 | # The default ``config.py`` 2 | # flake8: noqa 3 | 4 | 5 | def set_prefs(prefs): 6 | """This function is called before opening the project""" 7 | 8 | # Specify which files and folders to ignore in the project. 9 | # Changes to ignored resources are not added to the history and 10 | # VCSs. Also they are not returned in `Project.get_files()`. 11 | # Note that ``?`` and ``*`` match all characters but slashes. 12 | # '*.pyc': matches 'test.pyc' and 'pkg/test.pyc' 13 | # 'mod*.pyc': matches 'test/mod1.pyc' but not 'mod/1.pyc' 14 | # '.svn': matches 'pkg/.svn' and all of its children 15 | # 'build/*.o': matches 'build/lib.o' but not 'build/sub/lib.o' 16 | # 'build//*.o': matches 'build/lib.o' and 'build/sub/lib.o' 17 | prefs['ignored_resources'] = ['*.pyc', '*~', '.ropeproject', 18 | '.hg', '.svn', '_svn', '.git', '.tox'] 19 | 20 | # Specifies which files should be considered python files. It is 21 | # useful when you have scripts inside your project. Only files 22 | # ending with ``.py`` are considered to be python files by 23 | # default. 24 | # prefs['python_files'] = ['*.py'] 25 | 26 | # Custom source folders: By default rope searches the project 27 | # for finding source folders (folders that should be searched 28 | # for finding modules). You can add paths to that list. Note 29 | # that rope guesses project source folders correctly most of the 30 | # time; use this if you have any problems. 31 | # The folders should be relative to project root and use '/' for 32 | # separating folders regardless of the platform rope is running on. 33 | # 'src/my_source_folder' for instance. 34 | # prefs.add('source_folders', 'src') 35 | 36 | # You can extend python path for looking up modules 37 | # prefs.add('python_path', '~/python/') 38 | 39 | # Should rope save object information or not. 40 | prefs['save_objectdb'] = True 41 | prefs['compress_objectdb'] = False 42 | 43 | # If `True`, rope analyzes each module when it is being saved. 44 | prefs['automatic_soa'] = True 45 | # The depth of calls to follow in static object analysis 46 | prefs['soa_followed_calls'] = 0 47 | 48 | # If `False` when running modules or unit tests "dynamic object 49 | # analysis" is turned off. This makes them much faster. 50 | prefs['perform_doa'] = True 51 | 52 | # Rope can check the validity of its object DB when running. 53 | prefs['validate_objectdb'] = True 54 | 55 | # How many undos to hold? 56 | prefs['max_history_items'] = 32 57 | 58 | # Shows whether to save history across sessions. 59 | prefs['save_history'] = True 60 | prefs['compress_history'] = False 61 | 62 | # Set the number spaces used for indenting. According to 63 | # :PEP:`8`, it is best to use 4 spaces. Since most of rope's 64 | # unit-tests use 4 spaces it is more reliable, too. 65 | prefs['indent_size'] = 4 66 | 67 | # Builtin and c-extension modules that are allowed to be imported 68 | # and inspected by rope. 69 | prefs['extension_modules'] = [] 70 | 71 | # Add all standard c-extensions to extension_modules list. 72 | prefs['import_dynload_stdmods'] = True 73 | 74 | # If `True` modules with syntax errors are considered to be empty. 75 | # The default value is `False`; When `False` syntax errors raise 76 | # `rope.base.exceptions.ModuleSyntaxError` exception. 77 | prefs['ignore_syntax_errors'] = False 78 | 79 | # If `True`, rope ignores unresolvable imports. Otherwise, they 80 | # appear in the importing namespace. 81 | prefs['ignore_bad_imports'] = False 82 | 83 | # If `True`, rope will insert new module imports as 84 | # `from import ` by default. 85 | prefs['prefer_module_from_imports'] = False 86 | 87 | # If `True`, rope will transform a comma list of imports into 88 | # multiple separate import statements when organizing 89 | # imports. 90 | prefs['split_imports'] = False 91 | 92 | # If `True`, rope will remove all top-level import statements and 93 | # reinsert them at the top of the module when making changes. 94 | prefs['pull_imports_to_top'] = True 95 | 96 | # If `True`, rope will sort imports alphabetically by module name instead 97 | # of alphabetically by import statement, with from imports after normal 98 | # imports. 99 | prefs['sort_imports_alphabetically'] = False 100 | 101 | # Location of implementation of 102 | # rope.base.oi.type_hinting.interfaces.ITypeHintingFactory In general 103 | # case, you don't have to change this value, unless you're an rope expert. 104 | # Change this value to inject you own implementations of interfaces 105 | # listed in module rope.base.oi.type_hinting.providers.interfaces 106 | # For example, you can add you own providers for Django Models, or disable 107 | # the search type-hinting in a class hierarchy, etc. 108 | prefs['type_hinting_factory'] = ( 109 | 'rope.base.oi.type_hinting.factory.default_type_hinting_factory') 110 | 111 | 112 | def project_opened(project): 113 | """This function is called after opening the project""" 114 | # Do whatever you like here! 115 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python: Attach using Process Id", 9 | "type": "python", 10 | "request": "attach", 11 | "processId": "${command:pickProcess}" 12 | } 13 | ] 14 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "runtest.sh", 8 | "type": "process", 9 | "command": "bash", 10 | "args": [ 11 | // "-x", // uncomment for debug 12 | "${workspaceFolder}/test/runtest.sh", 13 | ], 14 | "group": { 15 | "kind": "build", 16 | "isDefault": true 17 | } 18 | }, 19 | { 20 | "label": "run ziprofs", 21 | "type": "shell", 22 | "command": "${workspaceFolder}/ziprofs.py", 23 | "args": [ 24 | "${workspaceFolder}/test/data", 25 | "${workspaceFolder}/test/mnt", 26 | "-o", 27 | "foreground,debug" 28 | ], 29 | } 30 | ] 31 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12.6-alpine@sha256:7130f75b1bb16c7c5d802782131b4024fe3d7a87ce7d936e8948c2d2e0180bc4 2 | 3 | # renovate: datasource=repology depName=alpine_3_20/fuse versioning=loose 4 | ARG FUSE_VERSION="2.9.9-r5" 5 | 6 | ARG TARGETPLATFORM 7 | 8 | WORKDIR /app 9 | 10 | ADD requirements.txt . 11 | 12 | RUN --mount=type=cache,sharing=locked,target=/root/.cache,id=home-cache-$TARGETPLATFORM \ 13 | apk add --no-cache \ 14 | fuse=${FUSE_VERSION} \ 15 | && \ 16 | sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf && \ 17 | pip install -r requirements.txt && \ 18 | chown -R nobody:nogroup /app 19 | 20 | COPY --chown=nobody:nogroup . . 21 | 22 | USER nobody 23 | ENV FUSE_LIBRARY_PATH=/usr/lib/libfuse.so.2 24 | 25 | ENTRYPOINT [ "python", "./ziprofs.py" ] 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Andrew Lutsenko (qu1ck) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ZipROFS 2 | [![Build Status](https://travis-ci.com/openscopeproject/ZipROFS.svg?branch=dev)](https://travis-ci.com/openscopeproject/ZipROFS) 3 | 4 | This is a FUSE filesystem that acts as pass through to another FS except it 5 | expands zip files like folders and allows direct transparent access to the contents. 6 | 7 | ### Dependencies 8 | * FUSE 9 | * fusepy 10 | 11 | ### Limitations 12 | * Read only 13 | * Nested zip files are not expanded, they are still just files 14 | 15 | ### Example usage 16 | To mount run ziprofs.py: 17 | ```shell 18 | $ ./ziprofs.py ~/root ~/mount -o allowother,cachesize=2048 19 | ``` 20 | 21 | Example results: 22 | ```shell 23 | $ tree root 24 | root 25 | ├── folder 26 | ├── test.zip 27 | └── text.txt 28 | 29 | $ tree mount 30 | mount 31 | ├── folder 32 | ├── test.zip 33 | │ ├── folder 34 | │ │ ├── emptyfile 35 | │ │ └── subfolder 36 | │ │ └── file.txt 37 | │ ├── script.sh 38 | │ └── text.txt 39 | └── text.txt 40 | ``` 41 | 42 | You can later unmount it using: 43 | ```shell 44 | $ fusermount -u ~/mount 45 | ``` 46 | 47 | Or: 48 | ```shell 49 | $ umount ~/mount 50 | ``` 51 | 52 | Full help: 53 | ```shell 54 | $ ./ziprofs.py -h 55 | usage: ziprofs.py [-h] [-o options] [root] [mountpoint] 56 | 57 | ZipROFS read only transparent zip filesystem. 58 | 59 | positional arguments: 60 | root filesystem root (default: None) 61 | mountpoint filesystem mount point (default: None) 62 | 63 | optional arguments: 64 | -h, --help show this help message and exit 65 | -o options comma separated list of options: foreground, debug, allowother, async, cachesize=N (default: {}) 66 | ``` 67 | 68 | `foreground` and `allowother` options are passed to FUSE directly. 69 | 70 | `debug` option is used to print all syscall details to stdout. 71 | 72 | By default ZipROFS disables async reads to improve performance since async syscalls can 73 | be reordered in fuse which heavily impacts read speeds. 74 | If async reads are preferable, pass `async` option on mount. 75 | 76 | `cachesize` option determines in memory zipfile cache size, defaults to 1000 77 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | ziprofs: 3 | # build: . 4 | image: ghcr.io/openscopeproject/ziprofs 5 | privileged: true 6 | volumes: 7 | - # Root/source directory 8 | source: ./data/root 9 | target: /app/root 10 | type: bind 11 | - # Mountpoint/target directory 12 | source: ./data/mnt 13 | target: /app/mnt 14 | type: bind 15 | bind: 16 | propagation: rshared 17 | command: 18 | - /app/root 19 | - /app/mnt 20 | - -o 21 | - allowother,foreground 22 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:best-practices", 5 | ":semanticCommits", 6 | ":semanticCommitScopeDisabled", 7 | "docker:enableMajor", 8 | "customManagers:dockerfileVersions", 9 | ":disableRateLimiting", 10 | ":ignoreUnstable", 11 | ":separateMultipleMajorReleases", 12 | ":updateNotScheduled" 13 | ], 14 | "docker-compose": { 15 | "enabled": false 16 | }, 17 | "packageRules": [ 18 | { 19 | "description": "Group OS packages to avoid build errors if more than one package is updated and previous version is not present in repo already", 20 | "matchDatasources": [ 21 | "repology" 22 | ], 23 | "groupName": "OS Packages" 24 | } 25 | ], 26 | "forkProcessing": "enabled" 27 | } 28 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fusepy==3.0.1 2 | -------------------------------------------------------------------------------- /test/data/folder/emptyfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openscopeproject/ZipROFS/22e8ec9caa97f0b48fe9584a2682a3d238d069fc/test/data/folder/emptyfile -------------------------------------------------------------------------------- /test/data/folder/subfolder/file.txt: -------------------------------------------------------------------------------- 1 | Nothing tops a plain pizza. -------------------------------------------------------------------------------- /test/data/script.sh: -------------------------------------------------------------------------------- 1 | echo hello 2 | -------------------------------------------------------------------------------- /test/data/test.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openscopeproject/ZipROFS/22e8ec9caa97f0b48fe9584a2682a3d238d069fc/test/data/test.zip -------------------------------------------------------------------------------- /test/data/text.txt: -------------------------------------------------------------------------------- 1 | hello 2 | -------------------------------------------------------------------------------- /test/runtest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TOTALTESTS=0 4 | PASSEDTESTS=0 5 | PASTELOG=$1 6 | 7 | function runtest { 8 | ((TOTALTESTS+=1)) 9 | TESTDESCR="$1" 10 | EXPECTED="$2" 11 | TESTCOMMAND="$3" 12 | echo "--------------------------" 13 | echo "Running test: $TESTDESCR" 14 | RESULT=$(bash -c "$TESTCOMMAND") 15 | if [[ "$EXPECTED" != "$RESULT" ]]; then 16 | echo "FAIL" 17 | echo -e "Expected:\n$EXPECTED" 18 | echo -e "Got:\n$RESULT" 19 | echo "Diff:" 20 | diff <(echo "$EXPECTED") <(echo "$RESULT") 21 | return 1 22 | else 23 | echo "PASS" 24 | ((PASSEDTESTS+=1)) 25 | return 0 26 | fi 27 | } 28 | 29 | echo "Running ziprofs tests..." 30 | REPODIR="$(dirname $(dirname $(readlink -f "$0")))" 31 | echo "Mounting filesystem" 32 | if [ ! -d "$REPODIR/test/mnt" ]; then 33 | mkdir -p "$REPODIR/test/mnt" 34 | fi 35 | if [ ! -z "$(mount | grep "$REPODIR/test/mnt")" ]; then 36 | fusermount -u "$REPODIR/test/mnt" 37 | fi 38 | "$REPODIR/ziprofs.py" "$REPODIR/test/data" "$REPODIR/test/mnt" -o foreground,debug > "$REPODIR/test/test.log" 2>&1 & 39 | PID=$! 40 | sleep 1 41 | cd "$REPODIR/test/mnt" 42 | 43 | runtest "zip is directory" "./test.zip" 'find ./ -type d -name test.zip' 44 | 45 | TREERESULT=$(tree -a --noreport ../data | tail -n +2 | grep -v test.zip) 46 | runtest "tree" "$TREERESULT" 'tree -a --noreport ./test.zip | tail -n +2' 47 | 48 | runtest "reading file content #1" "$(cat ../data/text.txt)" 'cat test.zip/text.txt' 49 | runtest "reading file content #2" "$(cat ../data/folder/subfolder/file.txt)" 'cat test.zip/folder/subfolder/file.txt' 50 | runtest "checking non-existant file" "does not exist" "[ -e test.zip/nosuchfile ] || echo 'does not exist'" 51 | 52 | runtest "running script" "hello" 'test.zip/script.sh' 53 | 54 | cd .. 55 | echo "Killing ziprofs" 56 | fusermount -u "$REPODIR/test/mnt" 57 | kill $PID 58 | 59 | echo "$PASSEDTESTS/$TOTALTESTS tests passed." 60 | if [[ "$PASTELOG" == "-log" ]]; then 61 | echo "Copying test log in full:" 62 | echo 63 | cat "$REPODIR/test/test.log" 64 | fi 65 | 66 | if [[ $PASSEDTESTS != $TOTALTESTS ]]; then 67 | exit 1 68 | fi 69 | -------------------------------------------------------------------------------- /ziprofs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from __future__ import print_function, absolute_import, division 3 | 4 | from functools import lru_cache 5 | 6 | from os.path import realpath 7 | 8 | import argparse 9 | import ctypes 10 | import errno 11 | import logging 12 | import os 13 | import time 14 | import zipfile 15 | import stat 16 | from threading import RLock 17 | from typing import Optional, Dict 18 | 19 | try: 20 | from fuse import FUSE, FuseOSError, Operations, LoggingMixIn, S_IFDIR, fuse_operations 21 | import fuse as fusepy 22 | except ImportError: 23 | # ubuntu renamed package in repository 24 | from fusepy import FUSE, FuseOSError, Operations, LoggingMixIn, S_IFDIR, fuse_operations 25 | import fusepy 26 | 27 | from collections import OrderedDict 28 | 29 | 30 | @lru_cache(maxsize=2048) 31 | def is_zipfile(path, mtime): 32 | # mtime just to miss cache on changed files 33 | return zipfile.is_zipfile(path) 34 | 35 | 36 | class ZipFile(zipfile.ZipFile): 37 | def __init__(self, *args, **kwargs): 38 | super().__init__(*args, **kwargs) 39 | self.__lock = RLock() 40 | 41 | def lock(self): 42 | return self.__lock 43 | 44 | 45 | class CachedZipFactory(object): 46 | MAX_CACHE_SIZE = 1000 47 | cache = OrderedDict() 48 | log = logging.getLogger('ziprofs.cache') 49 | 50 | def __init__(self): 51 | self.__lock = RLock() 52 | 53 | def _add(self, path: str): 54 | if path in self.cache: 55 | return 56 | while len(self.cache) >= self.MAX_CACHE_SIZE: 57 | oldpath, val = self.cache.popitem(last=False) 58 | self.log.debug('Popping cache entry: %s', oldpath) 59 | val[1].close() 60 | mtime = os.lstat(path).st_mtime 61 | self.log.debug("Caching path (%s:%s)", path, mtime) 62 | self.cache[path] = (mtime, ZipFile(path)) 63 | 64 | def get(self, path: str) -> ZipFile: 65 | with self.__lock: 66 | if path in self.cache: 67 | self.cache.move_to_end(path) 68 | mtime = os.lstat(path).st_mtime 69 | if mtime > self.cache[path][0]: 70 | val = self.cache.pop(path) 71 | val[1].close() 72 | self._add(path) 73 | else: 74 | self._add(path) 75 | return self.cache[path][1] 76 | 77 | 78 | class ZipROFS(Operations): 79 | zip_factory = CachedZipFactory() 80 | 81 | def __init__(self, root, zip_check): 82 | self.root = realpath(root) 83 | self.zip_check = zip_check 84 | # odd file handles are files inside zip, even fhs are system-wide files 85 | self._zip_file_fh: Dict[int, zipfile.ZipExtFile] = {} 86 | self._zip_zfile_fh: Dict[int, ZipFile] = {} 87 | self._fh_locks: Dict[int, RLock] = {} 88 | self._lock = RLock() 89 | 90 | def __call__(self, op, path, *args): 91 | return super().__call__(op, self.root + path, *args) 92 | 93 | def _get_free_zip_fh(self): 94 | i = 5 # avoid confusion with stdin/err/out 95 | while i in self._zip_file_fh: 96 | i += 2 97 | return i 98 | 99 | def get_zip_path(self, path: str) -> Optional[str]: 100 | parts = [] 101 | head, tail = os.path.split(path) 102 | while tail: 103 | parts.append(tail) 104 | head, tail = os.path.split(head) 105 | parts.reverse() 106 | cur_path = '/' 107 | for part in parts: 108 | cur_path = os.path.join(cur_path, part) 109 | if part[-4:] == '.zip' and ( 110 | not self.zip_check or is_zipfile(cur_path, os.lstat(cur_path).st_mtime)): 111 | return cur_path 112 | return None 113 | 114 | def access(self, path, mode): 115 | if self.get_zip_path(path): 116 | if mode & os.W_OK: 117 | raise FuseOSError(errno.EROFS) 118 | else: 119 | if not os.access(path, mode): 120 | raise FuseOSError(errno.EACCES) 121 | 122 | def getattr(self, path, fh=None): 123 | zip_path = self.get_zip_path(path) 124 | st = os.lstat(zip_path) if zip_path else os.lstat(path) 125 | result = {key: getattr(st, key) for key in ( 126 | 'st_atime', 'st_ctime', 'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid' 127 | )} 128 | if zip_path == path: 129 | result['st_mode'] = S_IFDIR | (result['st_mode'] & 0o555) 130 | elif zip_path: 131 | zf = self.zip_factory.get(zip_path) 132 | subpath = path[len(zip_path) + 1:] 133 | info = None 134 | try: 135 | info = zf.getinfo(subpath) 136 | result['st_size'] = info.file_size 137 | result['st_mode'] = stat.S_IFREG | 0o555 138 | except KeyError: 139 | # check if it is a valid subdirectory 140 | try: 141 | info = zf.getinfo(subpath + '/') 142 | except KeyError: 143 | pass 144 | found = False 145 | if not info: 146 | infolist = zf.infolist() 147 | for f in infolist: 148 | if f.filename.find(subpath + '/') == 0: 149 | found = True 150 | break 151 | if found or info: 152 | result['st_mode'] = S_IFDIR | 0o555 153 | else: 154 | raise FuseOSError(errno.ENOENT) 155 | if info: 156 | # update mtime 157 | try: 158 | mtime = time.mktime(info.date_time + (0, 0, -1)) 159 | result['st_mtime'] = mtime 160 | except Exception: 161 | pass 162 | return result 163 | 164 | def open(self, path, flags): 165 | zip_path = self.get_zip_path(path) 166 | if zip_path: 167 | with self._lock: 168 | fh = self._get_free_zip_fh() 169 | zf = self.zip_factory.get(zip_path) 170 | self._zip_zfile_fh[fh] = zf 171 | self._zip_file_fh[fh] = zf.open(path[len(zip_path) + 1:]) 172 | return fh 173 | else: 174 | fh = os.open(path, flags) << 1 175 | self._fh_locks[fh] = RLock() 176 | return fh 177 | 178 | def read(self, path, size, offset, fh): 179 | if fh in self._zip_file_fh: 180 | # should be here (file is first opened, then read) 181 | f = self._zip_file_fh[fh] 182 | with self._zip_zfile_fh[fh].lock(): 183 | if not f.seekable(): 184 | raise FuseOSError(errno.EBADF) 185 | 186 | f.seek(offset) 187 | return f.read(size) 188 | else: 189 | with self._fh_locks[fh]: 190 | os.lseek(fh >> 1, offset, 0) 191 | return os.read(fh >> 1, size) 192 | 193 | def readdir(self, path, fh): 194 | zip_path = self.get_zip_path(path) 195 | if not zip_path: 196 | return ['.', '..'] + os.listdir(path) 197 | subpath = path[len(zip_path) + 1:] 198 | zf = self.zip_factory.get(zip_path) 199 | infolist = zf.infolist() 200 | 201 | result = ['.', '..'] 202 | subdirs = set() 203 | for info in infolist: 204 | if info.filename.find(subpath) == 0 and info.filename > subpath: 205 | suffix = info.filename[len(subpath) + 1 if subpath else 0:] 206 | if not suffix: 207 | continue 208 | if '/' not in suffix: 209 | result.append(suffix) 210 | else: 211 | subdirs.add(suffix[:suffix.find('/')]) 212 | result.extend(subdirs) 213 | return result 214 | 215 | def release(self, path, fh): 216 | if fh in self._zip_file_fh: 217 | with self._lock: 218 | f = self._zip_file_fh[fh] 219 | with self._zip_zfile_fh[fh].lock(): 220 | del self._zip_file_fh[fh] 221 | del self._zip_zfile_fh[fh] 222 | return f.close() 223 | else: 224 | with self._fh_locks[fh]: 225 | del self._fh_locks[fh] 226 | return os.close(fh >> 1) 227 | 228 | def statfs(self, path): 229 | stv = os.statvfs(path) 230 | return dict((key, getattr(stv, key)) for key in ( 231 | 'f_bavail', 'f_bfree', 'f_blocks', 'f_bsize', 'f_favail', 232 | 'f_ffree', 'f_files', 'f_flag', 'f_frsize', 'f_namemax' 233 | )) 234 | 235 | 236 | class ZipROFSDebug(LoggingMixIn, ZipROFS): 237 | def __call__(self, op, path, *args): 238 | return super().__call__(op, self.root + path, *args) 239 | 240 | 241 | class fuse_conn_info(ctypes.Structure): 242 | _fields_ = [ 243 | ('proto_major', ctypes.c_uint), 244 | ('proto_minor', ctypes.c_uint), 245 | ('async_read', ctypes.c_uint), 246 | ('max_write', ctypes.c_uint), 247 | ('max_readahead', ctypes.c_uint), 248 | ('capable', ctypes.c_uint), 249 | ('want', ctypes.c_uint), 250 | ('reserved', ctypes.c_uint, 25)] 251 | 252 | 253 | class ZipROFuse(FUSE): 254 | def __init__(self, operations, mountpoint, **kwargs): 255 | self.support_async = kwargs.get('support_async', False) 256 | del kwargs['support_async'] 257 | if not self.support_async: 258 | # monkeypatch fuse_operations 259 | ops = fuse_operations._fields_ 260 | for i in range(len(ops)): 261 | if ops[i][0] == 'init': 262 | ops[i] = ( 263 | 'init', 264 | ctypes.CFUNCTYPE( 265 | ctypes.c_voidp, ctypes.POINTER(fuse_conn_info)) 266 | ) 267 | fusepy.fuse_operations = type( 268 | 'fuse_operations', (ctypes.Structure,), {'_fields_': ops}) 269 | super().__init__(operations, mountpoint, **kwargs) 270 | 271 | def init(self, conn): 272 | if not self.support_async: 273 | conn[0].async_read = 0 274 | conn[0].want = conn.contents.want & ~1 275 | return self.operations('init', '/') 276 | 277 | 278 | def parse_mount_opts(in_str): 279 | opts = {} 280 | for o in in_str.split(','): 281 | if '=' in o: 282 | name, val = o.split('=', 1) 283 | opts[name] = val 284 | else: 285 | opts[o] = True 286 | return opts 287 | 288 | 289 | if __name__ == '__main__': 290 | parser = argparse.ArgumentParser( 291 | description='ZipROFS read only transparent zip filesystem.', 292 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 293 | parser.add_argument('root', nargs='?', help="filesystem root") 294 | parser.add_argument( 295 | 'mountpoint', 296 | nargs='?', 297 | help="filesystem mount point") 298 | parser.add_argument( 299 | '-o', metavar='options', dest='opts', 300 | help="comma separated list of options: foreground, debug, allowother, " 301 | "nozipcheck, async, cachesize=N", 302 | type=parse_mount_opts, default={}) 303 | arg = parser.parse_args() 304 | 305 | if 'cachesize' in arg.opts: 306 | cache_size = int(arg.opts['cachesize']) 307 | if cache_size < 1: 308 | raise ValueError("Bad cache size") 309 | CachedZipFactory.MAX_CACHE_SIZE = cache_size 310 | 311 | logging.basicConfig( 312 | level=logging.DEBUG if 'debug' in arg.opts else logging.INFO) 313 | 314 | zip_check = 'nozipcheck' not in arg.opts 315 | 316 | if 'debug' in arg.opts: 317 | fs = ZipROFSDebug(arg.root, zip_check) 318 | else: 319 | fs = ZipROFS(arg.root, zip_check) 320 | 321 | fuse = ZipROFuse( 322 | fs, 323 | arg.mountpoint, 324 | foreground=('foreground' in arg.opts), 325 | allow_other=('allowother' in arg.opts), 326 | support_async=('async' in arg.opts) 327 | ) 328 | --------------------------------------------------------------------------------