├── dedupsqlfs ├── db │ ├── migrations │ │ ├── .gitkeep │ │ ├── m20150712001.py │ │ ├── m20170510001.py │ │ ├── m20160122001.py │ │ ├── m20160513001.py │ │ ├── m20160925001.py │ │ ├── m20151113001.py │ │ ├── m20210530001.py │ │ ├── m20220907001.py │ │ ├── m20171103001.py │ │ ├── m20210530004.py │ │ ├── m20180609002.py │ │ └── m20210530002.py │ ├── sqlite │ │ ├── __init__.py │ │ ├── table │ │ │ ├── __init__.py │ │ │ ├── tmp_id_count.py │ │ │ ├── tmp_ids.py │ │ │ ├── _memory.py │ │ │ ├── hash_owner.py │ │ │ └── option.py │ │ └── row.py │ ├── mysql │ │ ├── table │ │ │ ├── __init__.py │ │ │ ├── hash_owner.py │ │ │ ├── tmp_ids.py │ │ │ └── tmp_id_count.py │ │ └── __init__.py │ └── __init__.py ├── app │ ├── __init__.py │ └── actions │ │ └── __init__.py ├── fuse │ ├── __init__.py │ ├── compress │ │ └── __init__.py │ └── helpers │ │ ├── __init__.py │ │ └── repr.py ├── lib │ ├── __init__.py │ ├── cache │ │ ├── __init__.py │ │ └── _base.py │ └── timers_ops.py ├── compression │ ├── __init__.py │ ├── lz4.py │ ├── lzo.py │ ├── snappy.py │ ├── none.py │ ├── lz4h.py │ ├── bz2.py │ ├── zlib.py │ ├── deflate.py │ ├── zstd.py │ ├── brotli.py │ └── lzma.py ├── argp.py ├── fs.py ├── proc.py ├── log.py ├── __init__.py ├── my_formats.py └── get_memory_usage.py ├── lib-dynload ├── _pymysql │ ├── constants │ │ ├── __init__.py │ │ ├── FLAG.py │ │ ├── SERVER_STATUS.py │ │ ├── FIELD_TYPE.py │ │ ├── COMMAND.py │ │ └── CLIENT.py │ ├── util.py │ ├── times.py │ ├── _compat.py │ └── optionfile.py ├── _lz4 │ ├── MANIFEST.in │ ├── _lz4 │ │ ├── block │ │ │ └── __init__.py │ │ ├── version.py │ │ └── __init__.py │ ├── requirements.txt │ ├── setup.cfg │ ├── LICENSE │ ├── __init__.py │ └── lz4libs │ │ └── lz4frame_static.h ├── _llfuse │ ├── Include │ │ ├── libc │ │ │ ├── __init__.py │ │ │ ├── sys │ │ │ │ ├── __init__.py │ │ │ │ └── statvfs.pxd │ │ │ └── dirent.pxd │ │ ├── fuse_opt.pxd │ │ ├── pthread.pxd │ │ └── fuse_common.pxd │ ├── setup.cfg │ ├── MANIFEST.in │ ├── src │ │ ├── lock.h │ │ ├── llfuse.h │ │ ├── gettime.h │ │ ├── darwin_compat.h │ │ └── macros.c │ ├── __init__.py │ └── README.rst ├── _recordclass │ ├── lib │ │ └── recordclass │ │ │ ├── tools │ │ │ ├── __init__.py │ │ │ └── sqlite.py │ │ │ ├── test │ │ │ ├── match │ │ │ │ └── __init__.py │ │ │ ├── typing │ │ │ │ └── __init__.py │ │ │ ├── __init__.py │ │ │ └── test_sqlite.py │ │ │ ├── _litelist.h │ │ │ ├── _dataobject.h │ │ │ ├── __init__.py │ │ │ └── typing │ │ │ └── __init__.py │ ├── setup.cfg │ ├── MANIFEST.in │ ├── LICENSE.txt │ └── __init__.py ├── quicklz │ ├── README │ ├── setup.py │ └── __init__.py ├── brotli │ ├── setup.cfg │ ├── MANIFEST.in │ ├── c │ │ ├── common │ │ │ ├── platform.c │ │ │ ├── constants.c │ │ │ ├── version.h │ │ │ └── dictionary.h │ │ └── enc │ │ │ ├── dictionary_hash.h │ │ │ ├── bit_cost.c │ │ │ ├── literal_cost.h │ │ │ ├── utf8_util.h │ │ │ ├── encoder_dict.c │ │ │ ├── command.c │ │ │ ├── cluster.h │ │ │ ├── block_encoder_inc.h │ │ │ ├── params.h │ │ │ ├── encoder_dict.h │ │ │ ├── static_dict.h │ │ │ ├── backward_references.h │ │ │ ├── histogram_inc.h │ │ │ ├── cluster.c │ │ │ ├── block_splitter.h │ │ │ ├── fast_log.h │ │ │ ├── bit_cost.h │ │ │ ├── histogram.h │ │ │ └── prefix.h │ ├── README │ ├── LICENSE │ ├── __init__.py │ ├── CONTRIBUTING.md │ ├── python │ │ ├── README.md │ │ └── brotli.py │ └── expose.py ├── zstd │ ├── MANIFEST.in │ ├── PKG-INFO │ ├── zstd │ │ └── lib │ │ │ ├── common │ │ │ ├── xxhash.c │ │ │ ├── debug.c │ │ │ ├── allocations.h │ │ │ └── zstd_common.c │ │ │ ├── deprecated │ │ │ └── zbuff_common.c │ │ │ ├── compress │ │ │ ├── zstd_fast.h │ │ │ ├── zstd_compress_superblock.h │ │ │ ├── zstd_preSplit.h │ │ │ ├── zstd_compress_literals.h │ │ │ └── zstd_double_fast.h │ │ │ └── decompress │ │ │ └── zstd_ddict.h │ ├── src │ │ └── debug.h │ └── LICENSE ├── deflate │ └── __init__.py ├── __lzo │ ├── setup.py │ ├── NEWS │ └── __init__.py ├── snappy │ └── __init__.py └── xz │ └── __init__.py ├── contrib └── tests │ ├── copy-tar-qt4.sh │ ├── copy-tar-qt5.sh │ ├── copy-tar-qt6.sh │ ├── rm-qt4.sh │ ├── rm-qt5.sh │ ├── rm-qt6.sh │ ├── rsync-qt4.sh │ ├── rsync-qt5.sh │ ├── rsync-qt6.sh │ ├── unpack-qt4.sh │ ├── unpack-qt5.sh │ ├── unpack-qt6.sh │ ├── vacuum-sqlfs.sh │ ├── defragment-sqlfs.sh │ ├── defragment-clustered-sqlfs.sh │ └── mount-sqlfs.sh ├── .gitignore ├── bin ├── clean-all-modules.sh ├── do.dedupsqlfs ├── fsck.dedupsqlfs ├── mkfs.dedupsqlfs ├── cache_flusher ├── mount.dedupsqlfs └── rebuild-all-modules.sh ├── LICENSE ├── docs └── benchmarks │ ├── 2021-05-31_index_cleanup_speed_bench_1.2.951.ru.md │ ├── 2021-05-31_cython_speed_bench_1.2.951.ru.md │ └── 2021-04-27_simple_speed_bench_1.2.949.ru.md ├── tests ├── cleanupplan │ └── test-dates.py └── compression │ └── minimal-length.py └── TODO.md /dedupsqlfs/db/migrations/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/constants/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dedupsqlfs/app/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sergey' 2 | -------------------------------------------------------------------------------- /dedupsqlfs/fuse/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sergey' 2 | -------------------------------------------------------------------------------- /dedupsqlfs/lib/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sergey' 2 | -------------------------------------------------------------------------------- /lib-dynload/_lz4/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | -------------------------------------------------------------------------------- /dedupsqlfs/fuse/compress/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sergey' 2 | -------------------------------------------------------------------------------- /dedupsqlfs/fuse/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sergey' 2 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/Include/libc/__init__.py: -------------------------------------------------------------------------------- 1 | # Empty file 2 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/Include/libc/sys/__init__.py: -------------------------------------------------------------------------------- 1 | # Empty file 2 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | -------------------------------------------------------------------------------- /dedupsqlfs/app/actions/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Complex actions for do command 3 | """ -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/test/match/__init__.py: -------------------------------------------------------------------------------- 1 | # {{LICENCE}} 2 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/test/typing/__init__.py: -------------------------------------------------------------------------------- 1 | # {{LICENCE}} 2 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build = 3 | tag_date = 0 4 | 5 | -------------------------------------------------------------------------------- /dedupsqlfs/lib/cache/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sergey' 2 | 3 | from ._base import TimedCache 4 | -------------------------------------------------------------------------------- /lib-dynload/_lz4/_lz4/block/__init__.py: -------------------------------------------------------------------------------- 1 | from ._block import compress, decompress, LZ4BlockError # noqa: F401 2 | -------------------------------------------------------------------------------- /lib-dynload/_lz4/requirements.txt: -------------------------------------------------------------------------------- 1 | tox 2 | pytest 3 | pytest-runner 4 | setuptools_scm 5 | pkgconfig 6 | future 7 | -------------------------------------------------------------------------------- /contrib/tests/copy-tar-qt4.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | /usr/bin/time -v \ 4 | cp -vf qt4.tar ~/temp/mount/ 5 | 6 | sudo umount mount -------------------------------------------------------------------------------- /contrib/tests/copy-tar-qt5.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | /usr/bin/time -v \ 4 | cp -vf qt5.tar ~/temp/mount/ 5 | 6 | sudo umount mount -------------------------------------------------------------------------------- /contrib/tests/copy-tar-qt6.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | /usr/bin/time -v \ 4 | cp -vf qt6.tar ~/temp/mount/ 5 | 6 | sudo umount mount -------------------------------------------------------------------------------- /dedupsqlfs/db/sqlite/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # Misc DB functions 4 | # 5 | 6 | __author__ = 'sergey' 7 | -------------------------------------------------------------------------------- /dedupsqlfs/db/mysql/table/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from ._base import Table 6 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from ._base import BaseCompression 6 | -------------------------------------------------------------------------------- /contrib/tests/rm-qt4.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p ~/temp/mount/qt 4 | /usr/bin/time -v \ 5 | rm -rf ~/temp/mount/qt4* 6 | 7 | sudo umount mount -------------------------------------------------------------------------------- /contrib/tests/rm-qt5.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p ~/temp/mount/qt 4 | /usr/bin/time -v \ 5 | rm -rf ~/temp/mount/qt5* 6 | 7 | sudo umount mount -------------------------------------------------------------------------------- /contrib/tests/rm-qt6.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p ~/temp/mount/qt 4 | /usr/bin/time -v \ 5 | rm -rf ~/temp/mount/qt6* 6 | 7 | sudo umount mount -------------------------------------------------------------------------------- /lib-dynload/_llfuse/setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | source-dir = rst 3 | build-dir = doc 4 | 5 | [egg_info] 6 | tag_build = 7 | tag_date = 0 8 | 9 | -------------------------------------------------------------------------------- /lib-dynload/_lz4/setup.cfg: -------------------------------------------------------------------------------- 1 | [build_ext] 2 | inplace = 1 3 | 4 | [aliases] 5 | test=pytest 6 | 7 | [tool:pytest] 8 | addopts = -x --tb=long --showlocals 9 | -------------------------------------------------------------------------------- /lib-dynload/quicklz/README: -------------------------------------------------------------------------------- 1 | A clone of QuickLZ (the FAST compression library) 2 | http://www.quicklz.com/ 3 | 4 | https://github.com/robottwo/quicklz 5 | -------------------------------------------------------------------------------- /contrib/tests/rsync-qt4.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p ~/temp/mount/qt 4 | /usr/bin/time -v \ 5 | rsync -ah ~/temp/mount/qt4 ~/temp/mount/qt 6 | 7 | sudo umount mount -------------------------------------------------------------------------------- /contrib/tests/rsync-qt5.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p ~/temp/mount/qt 4 | /usr/bin/time -v \ 5 | rsync -ah ~/temp/mount/qt5 ~/temp/mount/qt 6 | 7 | sudo umount mount -------------------------------------------------------------------------------- /contrib/tests/rsync-qt6.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p ~/temp/mount/qt 4 | /usr/bin/time -v \ 5 | rsync -ah ~/temp/mount/qt6 ~/temp/mount/qt 6 | 7 | sudo umount mount -------------------------------------------------------------------------------- /dedupsqlfs/db/sqlite/table/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from ._base import Table 6 | from ._memory import MemoryTable 7 | -------------------------------------------------------------------------------- /contrib/tests/unpack-qt4.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p ~/temp/mount/qt4 4 | /usr/bin/time -v \ 5 | tar -xf qt4.tar \ 6 | -C ~/temp/mount/qt4 7 | 8 | sudo umount mount -------------------------------------------------------------------------------- /contrib/tests/unpack-qt5.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p ~/temp/mount/qt5 4 | /usr/bin/time -v \ 5 | tar -xf qt5.tar \ 6 | -C ~/temp/mount/qt5 7 | 8 | sudo umount mount -------------------------------------------------------------------------------- /contrib/tests/unpack-qt6.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p ~/temp/mount/qt6 4 | /usr/bin/time -v \ 5 | tar -xf qt6.tar \ 6 | -C ~/temp/mount/qt6 7 | 8 | sudo umount mount -------------------------------------------------------------------------------- /lib-dynload/brotli/setup.cfg: -------------------------------------------------------------------------------- 1 | [build] 2 | build-base = build 3 | 4 | [yapf] 5 | based_on_style = google 6 | 7 | [egg_info] 8 | tag_build = 9 | tag_date = 0 10 | 11 | -------------------------------------------------------------------------------- /lib-dynload/_lz4/_lz4/version.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # file generated by setuptools_scm 3 | # don't change, don't track in version control 4 | version = '4.0.0' 5 | version_tuple = (4, 0, 0) 6 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/_litelist.h: -------------------------------------------------------------------------------- 1 | typedef struct _PyLiteListObject { 2 | PyObject_HEAD 3 | Py_ssize_t ob_size; 4 | Py_ssize_t allocated; 5 | PyObject **ob_item; 6 | } PyLiteListObject; 7 | 8 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/util.py: -------------------------------------------------------------------------------- 1 | import struct 2 | 3 | 4 | def byte2int(b): 5 | if isinstance(b, int): 6 | return b 7 | else: 8 | return struct.unpack("!B", b)[0] 9 | 10 | 11 | def int2byte(i): 12 | return struct.pack("!B", i) 13 | 14 | -------------------------------------------------------------------------------- /lib-dynload/zstd/MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft src 2 | graft zstd/lib/common 3 | graft zstd/lib/compress 4 | graft zstd/lib/decompress 5 | graft zstd/lib/legacy 6 | graft tests 7 | include LICENSE 8 | include README.md 9 | include zstd/lib/zstd.h 10 | include zstd/lib/zstd_errors.h 11 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/constants/FLAG.py: -------------------------------------------------------------------------------- 1 | NOT_NULL = 1 2 | PRI_KEY = 2 3 | UNIQUE_KEY = 4 4 | MULTIPLE_KEY = 8 5 | BLOB = 16 6 | UNSIGNED = 32 7 | ZEROFILL = 64 8 | BINARY = 128 9 | ENUM = 256 10 | AUTO_INCREMENT = 512 11 | TIMESTAMP = 1024 12 | SET = 2048 13 | PART_KEY = 16384 14 | GROUP = 32767 15 | UNIQUE = 65536 16 | -------------------------------------------------------------------------------- /lib-dynload/deflate/__init__.py: -------------------------------------------------------------------------------- 1 | import zlib 2 | 3 | def compress(data, level=6): 4 | from zlib import compressobj 5 | c = compressobj(level) 6 | return c.compress(data) 7 | 8 | def decompress(cdata): 9 | from zlib import decompressobj 10 | d = decompressobj() 11 | return d.decompress(cdata) -------------------------------------------------------------------------------- /lib-dynload/_llfuse/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include Changes.rst 2 | include LICENSE 3 | graft doc/html 4 | graft Include 5 | graft examples 6 | graft rst 7 | graft util 8 | graft test 9 | prune test/.cache 10 | exclude MANIFEST.in 11 | recursive-include src *.pyx *.pxi *.c *.h 12 | global-exclude *.pyc 13 | global-exclude __pycache__ 14 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/lz4.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for LZ4 compression helper 7 | """ 8 | 9 | from dedupsqlfs.compression import BaseCompression 10 | 11 | class Lz4Compression(BaseCompression): 12 | 13 | _method_name = "_lz4" 14 | 15 | _minimal_size = 15 16 | 17 | _has_comp_level_options = False 18 | 19 | pass 20 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/lzo.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for LZO compression helper 7 | """ 8 | 9 | from dedupsqlfs.compression import BaseCompression 10 | 11 | class LzoCompression(BaseCompression): 12 | 13 | _method_name = "__lzo" 14 | 15 | _minimal_size = 38 16 | 17 | _has_comp_level_options = False 18 | 19 | pass 20 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/snappy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for Snappy compression helper 7 | """ 8 | 9 | from dedupsqlfs.compression import BaseCompression 10 | 11 | class SnappyCompression(BaseCompression): 12 | 13 | _method_name = "snappy" 14 | 15 | _minimal_size = 17 16 | 17 | _has_comp_level_options = False 18 | 19 | pass 20 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/constants/SERVER_STATUS.py: -------------------------------------------------------------------------------- 1 | 2 | SERVER_STATUS_IN_TRANS = 1 3 | SERVER_STATUS_AUTOCOMMIT = 2 4 | SERVER_MORE_RESULTS_EXISTS = 8 5 | SERVER_QUERY_NO_GOOD_INDEX_USED = 16 6 | SERVER_QUERY_NO_INDEX_USED = 32 7 | SERVER_STATUS_CURSOR_EXISTS = 64 8 | SERVER_STATUS_LAST_ROW_SENT = 128 9 | SERVER_STATUS_DB_DROPPED = 256 10 | SERVER_STATUS_NO_BACKSLASH_ESCAPES = 512 11 | SERVER_STATUS_METADATA_CHANGED = 1024 12 | -------------------------------------------------------------------------------- /dedupsqlfs/db/sqlite/row.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # Misc DB functions 4 | # 5 | 6 | __author__ = 'sergey' 7 | 8 | def dict_factory(cursor, row): 9 | d = {} 10 | for idx, col in enumerate(cursor.description): 11 | d[col[0]] = row[idx] 12 | return d 13 | 14 | def tuple_factory(cursor, row): 15 | t = () 16 | for idx, col in enumerate(cursor.description): 17 | t += (row[idx],) 18 | return t 19 | -------------------------------------------------------------------------------- /contrib/tests/vacuum-sqlfs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | dod=~/src/dedupsqlfs/bin/do.dedupsqlfs 4 | 5 | /usr/bin/time -vp python3.13 $dod \ 6 | --data ~/sqlfs \ 7 | --data-clustered ~/clsqlfs \ 8 | --block-partitions 10 \ 9 | --no-sync \ 10 | --minimal-compress-size -1 \ 11 | --compress zstd \ 12 | --journal-mode off \ 13 | --auto-vacuum 2 \ 14 | -v -v -M \ 15 | --log-file vacuum-dedupsqlfs.log \ 16 | --vacuum 17 | -------------------------------------------------------------------------------- /contrib/tests/defragment-sqlfs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | dod=~/src/dedupsqlfs/bin/do.dedupsqlfs 4 | 5 | /usr/bin/time -vp python3.13 $dod \ 6 | --data ~/sqlfs \ 7 | --data-clustered ~/clsqlfs \ 8 | --block-partitions 10 \ 9 | --no-sync \ 10 | --minimal-compress-size -1 \ 11 | --compress zstd \ 12 | --journal-mode off \ 13 | --auto-vacuum 2 \ 14 | -v -v -M \ 15 | --log-file defragment-dedupsqlfs.log \ 16 | --defragment 17 | -------------------------------------------------------------------------------- /lib-dynload/brotli/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CONTRIBUTING.md 2 | include c/common/*.c 3 | include c/common/*.h 4 | include c/dec/*.c 5 | include c/dec/*.h 6 | include c/enc/*.c 7 | include c/enc/*.h 8 | include c/include/brotli/*.h 9 | include LICENSE 10 | include MANIFEST.in 11 | include python/_brotli.cc 12 | include python/bro.py 13 | include python/brotli.py 14 | include python/README.md 15 | include README.md 16 | include setup.py 17 | include c/tools/brotli.c 18 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/times.py: -------------------------------------------------------------------------------- 1 | from time import localtime 2 | from datetime import date, datetime, time, timedelta 3 | 4 | 5 | Date = date 6 | Time = time 7 | TimeDelta = timedelta 8 | Timestamp = datetime 9 | 10 | 11 | def DateFromTicks(ticks): 12 | return date(*localtime(ticks)[:3]) 13 | 14 | 15 | def TimeFromTicks(ticks): 16 | return time(*localtime(ticks)[3:6]) 17 | 18 | 19 | def TimestampFromTicks(ticks): 20 | return datetime(*localtime(ticks)[:6]) 21 | -------------------------------------------------------------------------------- /contrib/tests/defragment-clustered-sqlfs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | dod=~/src/dedupsqlfs/bin/do.dedupsqlfs 4 | 5 | /usr/bin/time -vp python3.13 $dod \ 6 | --data ~/sqlfs \ 7 | --data-clustered ~/clsqlfs \ 8 | --block-partitions 10 \ 9 | --no-sync \ 10 | --minimal-compress-size -1 \ 11 | --compress zstd \ 12 | --journal-mode off \ 13 | --auto-vacuum 2 \ 14 | -v -v -M \ 15 | --log-file defragment-clustered-dedupsqlfs.log \ 16 | --defragment-clustered 17 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/Include/fuse_opt.pxd: -------------------------------------------------------------------------------- 1 | ''' 2 | fuse_opt.pxd 3 | 4 | This file contains Cython definitions for fuse_opt.h 5 | 6 | Copyright © 2010 Nikolaus Rath 7 | 8 | This file is part of Python-LLFUSE. This work may be distributed under 9 | the terms of the GNU LGPL. 10 | ''' 11 | 12 | 13 | # Based on fuse sources, revision tag fuse_2_8_3 14 | cdef extern from "" nogil: 15 | struct fuse_args: 16 | int argc 17 | char **argv 18 | int allocated 19 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/src/lock.h: -------------------------------------------------------------------------------- 1 | /* 2 | lock.h - Header file for lock.c 3 | 4 | Copyright © 2015 Nikolaus Rath 5 | 6 | This file is part of Python-LLFUSE. This work may be distributed under 7 | the terms of the GNU LGPL. 8 | */ 9 | 10 | 11 | #ifndef _LLFUSE_LOCK_H_ 12 | #define _LLFUSE_LOCK_H_ 13 | 14 | #include 15 | 16 | int acquire(double timeout); 17 | int release(void); 18 | int c_yield(int count); 19 | void init_lock(void); 20 | 21 | 22 | #endif /* _LLFUSE_LOCK_H_ */ 23 | -------------------------------------------------------------------------------- /lib-dynload/zstd/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.1 2 | Name: zstd 3 | Version: 1.5.2.4 4 | Summary: Simple python bindings to Yann Collet ZSTD compression library 5 | Home-page: https://github.com/sergey-dryabzhinsky/python-zstd 6 | Author: Sergey Dryabzhinsky 7 | Author-email: sergey.dryabzhinsky@gmail.com 8 | License: BSD 9 | Download-URL: https://github.com/sergey-dryabzhinsky/python-zstd/archive/v1.5.2.4.tar.gz 10 | Description: Simple ZSTandarD bindings for Python 11 | Keywords: zstd,zstandard,compression 12 | Platform: POSIX 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #Python 2 | *.py[cod] 3 | __pycache__ 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Packages 9 | *.egg 10 | *.egg-info 11 | dist 12 | build 13 | eggs 14 | parts 15 | bin 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | lib 21 | lib64 22 | tests 23 | 24 | # Installer logs 25 | pip-log.txt 26 | 27 | # Unit test / coverage reports 28 | .coverage 29 | .tox 30 | nosetests.xml 31 | 32 | # Translations 33 | *.mo 34 | 35 | # Mr Developer 36 | .mr.developer.cfg 37 | .project 38 | .pydevproject 39 | 40 | # PyCharm 41 | .idea -------------------------------------------------------------------------------- /contrib/tests/mount-sqlfs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mnt=~/src/dedupsqlfs/bin/mount.dedupsqlfs 4 | 5 | python3.13 $mnt \ 6 | --data ~/sqlfs \ 7 | --data-clustered ~/clsqlfs \ 8 | --block-partitions 10 \ 9 | --no-cache-flusher --no-sync \ 10 | --minimal-compress-size -1 \ 11 | --flush-interval 5 \ 12 | --block-size 65536 \ 13 | --journal-mode off \ 14 | --auto-vacuum 2 \ 15 | --compress zstd \ 16 | -v -v -M --verbose-stats \ 17 | -o noatime \ 18 | --log-file mount-dedupsqlfs.log \ 19 | ~/temp/mount -------------------------------------------------------------------------------- /dedupsqlfs/argp.py: -------------------------------------------------------------------------------- 1 | # Helpers for ArgParse 2 | 3 | import argparse 4 | 5 | # 6 | class SmartFormatter(argparse.HelpFormatter): 7 | 8 | def _split_lines(self, text, width): 9 | if text.startswith('R|'): 10 | lines = [] 11 | for line in text[2:].splitlines(): 12 | lines.extend(argparse.HelpFormatter._split_lines(self, line, width)) 13 | return lines 14 | # this is the RawTextHelpFormatter._split_lines 15 | return argparse.HelpFormatter._split_lines(self, text, width) 16 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/constants/FIELD_TYPE.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | DECIMAL = 0 4 | TINY = 1 5 | SHORT = 2 6 | LONG = 3 7 | FLOAT = 4 8 | DOUBLE = 5 9 | NULL = 6 10 | TIMESTAMP = 7 11 | LONGLONG = 8 12 | INT24 = 9 13 | DATE = 10 14 | TIME = 11 15 | DATETIME = 12 16 | YEAR = 13 17 | NEWDATE = 14 18 | VARCHAR = 15 19 | BIT = 16 20 | JSON = 245 21 | NEWDECIMAL = 246 22 | ENUM = 247 23 | SET = 248 24 | TINY_BLOB = 249 25 | MEDIUM_BLOB = 250 26 | LONG_BLOB = 251 27 | BLOB = 252 28 | VAR_STRING = 253 29 | STRING = 254 30 | GEOMETRY = 255 31 | 32 | CHAR = TINY 33 | INTERVAL = ENUM 34 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/_compat.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | PY2 = sys.version_info[0] == 2 4 | PYPY = hasattr(sys, 'pypy_translation_info') 5 | JYTHON = sys.platform.startswith('java') 6 | IRONPYTHON = sys.platform == 'cli' 7 | CPYTHON = not PYPY and not JYTHON and not IRONPYTHON 8 | 9 | if PY2: 10 | import __builtin__ 11 | range_type = xrange 12 | text_type = unicode 13 | long_type = long 14 | str_type = basestring 15 | unichr = __builtin__.unichr 16 | else: 17 | range_type = range 18 | text_type = str 19 | long_type = int 20 | str_type = str 21 | unichr = chr 22 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/common/platform.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include 8 | 9 | #include "./platform.h" 10 | #include 11 | 12 | /* Default brotli_alloc_func */ 13 | void* BrotliDefaultAllocFunc(void* opaque, size_t size) { 14 | BROTLI_UNUSED(opaque); 15 | return malloc(size); 16 | } 17 | 18 | /* Default brotli_free_func */ 19 | void BrotliDefaultFreeFunc(void* opaque, void* address) { 20 | BROTLI_UNUSED(opaque); 21 | free(address); 22 | } 23 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/none.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for NONE compression helper 7 | """ 8 | 9 | from dedupsqlfs.compression import BaseCompression 10 | 11 | class NoneCompression(BaseCompression): 12 | 13 | _method_name = "none" 14 | 15 | _minimal_size = 0 16 | 17 | _has_comp_level_options = False 18 | 19 | def _init_module(self): 20 | return None 21 | 22 | def _noop(self, data): 23 | return data 24 | 25 | def _get_comp_func(self): 26 | return self._noop 27 | 28 | def _get_decomp_func(self): 29 | return self._noop 30 | 31 | -------------------------------------------------------------------------------- /bin/clean-all-modules.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #### 3 | # This script will try to clean 4 | # every available compression module 5 | ### 6 | 7 | 8 | THIS_DIR=`dirname $0` 9 | 10 | if [ -z "$PY" ]; then 11 | # generic 12 | PY=python3 13 | fi 14 | 15 | PY=`which $PY` 16 | 17 | cd "${THIS_DIR}/../lib-dynload" 18 | 19 | for mdir in `ls .` 20 | do 21 | if [ -d ${mdir} ]; then 22 | 23 | if [ ! -r ${mdir}/setup.py ]; then 24 | continue 25 | fi 26 | 27 | echo "" 28 | echo "MODULE clean: ${mdir}" 29 | 30 | cd ${mdir} 31 | $PY setup.py clean -a 32 | [ -d ".eggs" ] && rm -rf .eggs 33 | find . -type d -name "__pycache__" -exec rm -rf '{}' \; 34 | cd .. 35 | fi 36 | done 37 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/Include/libc/dirent.pxd: -------------------------------------------------------------------------------- 1 | ''' 2 | dirent.pxd 3 | 4 | This file contains Cython definitions for dirent.h and sys/types.h 5 | 6 | Copyright © 2010 Nikolaus Rath 7 | 8 | This file is part of Python-LLFUSE. This work may be distributed under 9 | the terms of the GNU LGPL. 10 | ''' 11 | 12 | cdef extern from "" nogil: 13 | ctypedef struct DIR: 14 | pass 15 | cdef struct dirent: 16 | char* d_name 17 | 18 | dirent* readdir(DIR* dirp) 19 | int readdir_r(DIR *dirp, dirent *entry, dirent **result) 20 | 21 | cdef extern from "" nogil: 22 | DIR *opendir(char *name) 23 | int closedir(DIR* dirp) 24 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/common/constants.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include "./constants.h" 8 | 9 | const BrotliPrefixCodeRange 10 | _kBrotliPrefixCodeRanges[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = { 11 | {1, 2}, {5, 2}, {9, 2}, {13, 2}, {17, 3}, {25, 3}, 12 | {33, 3}, {41, 3}, {49, 4}, {65, 4}, {81, 4}, {97, 4}, 13 | {113, 5}, {145, 5}, {177, 5}, {209, 5}, {241, 6}, {305, 6}, 14 | {369, 7}, {497, 8}, {753, 9}, {1265, 10}, {2289, 11}, {4337, 12}, 15 | {8433, 13}, {16625, 24}}; 16 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/common/xxhash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xxHash - Extremely Fast Hash algorithm 3 | * Copyright (c) Yann Collet - Meta Platforms, Inc 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* 12 | * xxhash.c instantiates functions defined in xxhash.h 13 | */ 14 | 15 | #define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ 16 | #define XXH_IMPLEMENTATION /* access definitions */ 17 | 18 | #include "xxhash.h" 19 | -------------------------------------------------------------------------------- /lib-dynload/brotli/README: -------------------------------------------------------------------------------- 1 | BROTLI DATA COMPRESSIOM LIBRARY 2 | 3 | Brotli is a generic-purpose lossless compression algorithm that compresses data 4 | using a combination of a modern variant of the LZ77 algorithm, Huffman coding 5 | and 2nd order context modeling, with a compression ratio comparable to the best 6 | currently available general-purpose compression methods. It is similar in speed 7 | with deflate but offers more dense compression. 8 | 9 | The specification of the Brotli Compressed Data Format is defined in RFC 7932 10 | https://tools.ietf.org/html/rfc7932 11 | 12 | Brotli is open-sourced under the MIT License, see the LICENSE file. 13 | 14 | Brotli mailing list: 15 | https://groups.google.com/forum/#!forum/brotli 16 | -------------------------------------------------------------------------------- /lib-dynload/_lz4/_lz4/__init__.py: -------------------------------------------------------------------------------- 1 | # Although the canonical way to get the package version is using pkg_resources 2 | # as below, this turns out to be very slow on systems with lots of packages. 3 | # So, until that is remedied, we'll import the version from a local file 4 | # created by setuptools_scm. 5 | 6 | # from pkg_resources import get_distribution, DistributionNotFound 7 | # try: 8 | # __version__ = get_distribution(__name__).version 9 | # except DistributionNotFound: 10 | # # package is not installed 11 | # pass 12 | 13 | from .version import version as __version__ 14 | from ._version import ( # noqa: F401 15 | library_version_number, 16 | library_version_string, 17 | ) 18 | 19 | VERSION = __version__ 20 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/tools/sqlite.py: -------------------------------------------------------------------------------- 1 | from recordclass import make_dataclass 2 | 3 | def make_row_factory(cls_factory, **kw): 4 | def row_factory(cursor, row, cls=[None]): 5 | rf = cls[0] 6 | if rf is None: 7 | fields = [col[0] for col in cursor.description] 8 | cls[0] = cls_factory("Row", fields, **kw) 9 | print(cls[0]) 10 | return cls[0](*row) 11 | return rf(*row) 12 | return row_factory 13 | 14 | def dataclass_row_factory(cls=None): 15 | if cls is None: 16 | return make_row_factory(make_dataclass, fast_new=True) 17 | else: 18 | def row_factory(cursor, row): 19 | return cls(*row) 20 | return row_factory 21 | -------------------------------------------------------------------------------- /dedupsqlfs/fs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | """ 3 | Module to work with file system 4 | """ 5 | 6 | __author__ = 'sergey' 7 | 8 | import os 9 | 10 | 11 | def which(bin_exe): 12 | """ 13 | Поиск исполняемого файла 14 | 15 | @return: False или полный путь до исполняемого файла 16 | @rtype: str|bool 17 | """ 18 | # support Linux / POSIX here? 19 | paths = ["/usr/local/bin", "/usr/local/sbin", "/bin", "/sbin", "/usr/bin", "/usr/sbin"] 20 | if "PATH" in os.environ: 21 | paths = os.environ["PATH"].split(":") 22 | for p in paths: 23 | full_path = os.path.join(p, bin_exe) 24 | if os.path.isfile(full_path) and os.access(full_path, os.X_OK): 25 | return full_path 26 | return False 27 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/Include/libc/sys/statvfs.pxd: -------------------------------------------------------------------------------- 1 | ''' 2 | statvfs.pxd 3 | 4 | This file contains Cython definitions for sys/statvfs.h 5 | 6 | Copyright © 2010 Nikolaus Rath 7 | 8 | This file is part of Python-LLFUSE. This work may be distributed under 9 | the terms of the GNU LGPL. 10 | ''' 11 | 12 | cdef extern from "" nogil: 13 | ctypedef int fsblkcnt_t 14 | ctypedef int fsfilcnt_t 15 | 16 | struct statvfs: 17 | unsigned long f_bsize 18 | unsigned long f_frsize 19 | fsblkcnt_t f_blocks 20 | fsblkcnt_t f_bfree 21 | fsblkcnt_t f_bavail 22 | fsfilcnt_t f_files 23 | fsfilcnt_t f_ffree 24 | fsfilcnt_t f_favail 25 | unsigned long f_namemax 26 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/optionfile.py: -------------------------------------------------------------------------------- 1 | from ._compat import PY2 2 | 3 | if PY2: 4 | import ConfigParser as configparser 5 | else: 6 | import configparser 7 | 8 | 9 | class Parser(configparser.RawConfigParser): 10 | def __init__(self, **kwargs): 11 | kwargs['allow_no_value'] = True 12 | configparser.RawConfigParser.__init__(self, **kwargs) 13 | 14 | def __remove_quotes(self, value): 15 | quotes = ["'", "\""] 16 | for quote in quotes: 17 | if len(value) >= 2 and value[0] == value[-1] == quote: 18 | return value[1:-1] 19 | return value 20 | 21 | def get(self, section, option): 22 | value = configparser.RawConfigParser.get(self, section, option) 23 | return self.__remove_quotes(value) 24 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/dictionary_hash.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Hash table on the 4-byte prefixes of static dictionary words. */ 8 | 9 | #ifndef BROTLI_ENC_DICTIONARY_HASH_H_ 10 | #define BROTLI_ENC_DICTIONARY_HASH_H_ 11 | 12 | #include 13 | 14 | #if defined(__cplusplus) || defined(c_plusplus) 15 | extern "C" { 16 | #endif 17 | 18 | extern const uint16_t kStaticDictionaryHashWords[32768]; 19 | extern const uint8_t kStaticDictionaryHashLengths[32768]; 20 | 21 | #if defined(__cplusplus) || defined(c_plusplus) 22 | } /* extern "C" */ 23 | #endif 24 | 25 | #endif /* BROTLI_ENC_DICTIONARY_HASH_H_ */ 26 | -------------------------------------------------------------------------------- /dedupsqlfs/db/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # Misc DB functions 4 | # 5 | __author__ = 'sergey' 6 | 7 | def check_engines(): 8 | engines = () 9 | try: 10 | import sqlite3 11 | engines += ('sqlite',) 12 | except: 13 | pass 14 | try: 15 | import pymysql 16 | 17 | from dedupsqlfs.db.mysql import get_table_engines 18 | 19 | if len(get_table_engines()): 20 | engines += ('mysql',) 21 | except: 22 | pass 23 | 24 | msg = "" 25 | if engines: 26 | msg = "Use selected storage engine. One of: "+", ".join(engines)+". Default is "+engines[0]+"." 27 | if 'sqlite' in engines: 28 | msg += " Note: 'sqlite' use less disk space, and may work slowly on large data." 29 | 30 | return engines, msg 31 | -------------------------------------------------------------------------------- /dedupsqlfs/db/mysql/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sergey' 2 | 3 | from dedupsqlfs.fs import which 4 | import subprocess 5 | 6 | def get_table_engines(): 7 | 8 | table_engines = () 9 | 10 | mysqld_bin = which("mysqld") 11 | 12 | if not mysqld_bin: 13 | return table_engines 14 | 15 | try: 16 | output = subprocess.check_output([mysqld_bin, "--verbose", "--help"], stderr=subprocess.DEVNULL) 17 | table_engines = ['MyISAM', 'InnoDB','MEMORY'] 18 | if output.find(b'--aria[='): 19 | table_engines.append('Aria') 20 | if output.find(b'--tokudb[='): 21 | table_engines.append('TokuDB') 22 | if output.find(b'--rocksdb[='): 23 | table_engines.append('RocksDB') 24 | except: 25 | # No MySQL? 26 | table_engines = () 27 | 28 | return table_engines 29 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/constants/COMMAND.py: -------------------------------------------------------------------------------- 1 | 2 | COM_SLEEP = 0x00 3 | COM_QUIT = 0x01 4 | COM_INIT_DB = 0x02 5 | COM_QUERY = 0x03 6 | COM_FIELD_LIST = 0x04 7 | COM_CREATE_DB = 0x05 8 | COM_DROP_DB = 0x06 9 | COM_REFRESH = 0x07 10 | COM_SHUTDOWN = 0x08 11 | COM_STATISTICS = 0x09 12 | COM_PROCESS_INFO = 0x0a 13 | COM_CONNECT = 0x0b 14 | COM_PROCESS_KILL = 0x0c 15 | COM_DEBUG = 0x0d 16 | COM_PING = 0x0e 17 | COM_TIME = 0x0f 18 | COM_DELAYED_INSERT = 0x10 19 | COM_CHANGE_USER = 0x11 20 | COM_BINLOG_DUMP = 0x12 21 | COM_TABLE_DUMP = 0x13 22 | COM_CONNECT_OUT = 0x14 23 | COM_REGISTER_SLAVE = 0x15 24 | COM_STMT_PREPARE = 0x16 25 | COM_STMT_EXECUTE = 0x17 26 | COM_STMT_SEND_LONG_DATA = 0x18 27 | COM_STMT_CLOSE = 0x19 28 | COM_STMT_RESET = 0x1a 29 | COM_SET_OPTION = 0x1b 30 | COM_STMT_FETCH = 0x1c 31 | COM_DAEMON = 0x1d 32 | COM_BINLOG_DUMP_GTID = 0x1e 33 | COM_END = 0x1f 34 | -------------------------------------------------------------------------------- /lib-dynload/zstd/src/debug.h: -------------------------------------------------------------------------------- 1 | #ifndef DEBUG_H 2 | #define DEBUG_H 3 | 4 | #ifndef ZSTD_DEBUG 5 | #define ZSTD_DEBUG 0 6 | #endif 7 | 8 | #ifndef ZSTD_DEBUG_NOTICE 9 | #define ZSTD_DEBUG_NOTICE 0 10 | #endif 11 | 12 | #ifndef ZSTD_DEBUG_INFO 13 | #define ZSTD_DEBUG_INFO 0 14 | #endif 15 | 16 | #ifndef ZSTD_DEBUG_ERROR 17 | #define ZSTD_DEBUG_ERROR 0 18 | #endif 19 | 20 | #define DEBUG_FMT_MAX_LEN 64 21 | 22 | /*prints messages to stderr only if debug defined */ 23 | int printd(const char* fmt, int dv); 24 | int printdn(const char* fmt, int dv); 25 | int printdi(const char* fmt, int dv); 26 | int printde(const char* fmt, int dv); 27 | int printdes(const char* fmt, const char* dv); 28 | int printd2(const char* fmt, int dv, int dv2); 29 | int printd2n(const char* fmt, int dv, int dv2); 30 | int printd2i(const char* fmt, int dv, int dv2); 31 | int printd2e(const char* fmt, int dv, int dv2); 32 | 33 | #endif -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/test/__init__.py: -------------------------------------------------------------------------------- 1 | # {{LICENCE.txt}} 2 | 3 | from recordclass.test.test_recordclass import * 4 | from recordclass.test.test_arrayclass import * 5 | from recordclass.test.test_dataobject import * 6 | from recordclass.test.test_litelist import * 7 | from recordclass.test.test_litetuple import * 8 | 9 | import sys 10 | _PY310 = sys.version_info[:2] >= (3, 10) 11 | 12 | from recordclass.test.typing.test_recordclass import * 13 | from recordclass.test.typing.test_dataobject import * 14 | from recordclass.test.typing.test_datastruct import * 15 | 16 | 17 | if _PY310: 18 | from recordclass.test.match.test_dataobject_match import * 19 | 20 | try: 21 | import sqlite3 as sql 22 | except: 23 | sql = None 24 | 25 | if sql is not None: 26 | from recordclass.test.test_sqlite import * 27 | 28 | def test_all(): 29 | import unittest 30 | unittest.main(verbosity=3) 31 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/lz4h.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for LZ4 compression helper High Level 7 | """ 8 | 9 | import sys 10 | from dedupsqlfs.compression import BaseCompression 11 | 12 | class Lz4hCompression(BaseCompression): 13 | 14 | _method_name = "lz4" 15 | 16 | _minimal_size = 15 17 | 18 | _has_comp_level_options = False 19 | 20 | def _init_module(self): 21 | if not self._module: 22 | if self._method_name in sys.modules: 23 | del sys.modules[ self._method_name ] 24 | self._module = __import__(self._method_name) 25 | 26 | func_comp = getattr(self._module, "compress") 27 | def compress_high(data): 28 | return func_comp(data, "high_compression") 29 | 30 | self._func_comp = compress_high 31 | self._func_decomp = getattr(self._module, "decompress") 32 | return 33 | 34 | pass 35 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/bit_cost.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions to estimate the bit cost of Huffman trees. */ 8 | 9 | #include "./bit_cost.h" 10 | 11 | #include "../common/constants.h" 12 | #include "../common/platform.h" 13 | #include 14 | #include "./fast_log.h" 15 | #include "./histogram.h" 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | #define FN(X) X ## Literal 22 | #include "./bit_cost_inc.h" /* NOLINT(build/include) */ 23 | #undef FN 24 | 25 | #define FN(X) X ## Command 26 | #include "./bit_cost_inc.h" /* NOLINT(build/include) */ 27 | #undef FN 28 | 29 | #define FN(X) X ## Distance 30 | #include "./bit_cost_inc.h" /* NOLINT(build/include) */ 31 | #undef FN 32 | 33 | #if defined(__cplusplus) || defined(c_plusplus) 34 | } /* extern "C" */ 35 | #endif 36 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/common/version.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Version definition. */ 8 | 9 | #ifndef BROTLI_COMMON_VERSION_H_ 10 | #define BROTLI_COMMON_VERSION_H_ 11 | 12 | /* This macro should only be used when library is compiled together with client. 13 | If library is dynamically linked, use BrotliDecoderVersion and 14 | BrotliEncoderVersion methods. */ 15 | 16 | /* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */ 17 | #define BROTLI_VERSION 0x1000009 18 | 19 | /* This macro is used by build system to produce Libtool-friendly soname. See 20 | https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html 21 | */ 22 | 23 | /* ABI version, calculated as (CURRENT << 24) | (REVISION << 12) | AGE */ 24 | #define BROTLI_ABI_VERSION 0x1009000 25 | 26 | #endif /* BROTLI_COMMON_VERSION_H_ */ 27 | -------------------------------------------------------------------------------- /lib-dynload/_pymysql/constants/CLIENT.py: -------------------------------------------------------------------------------- 1 | # https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags 2 | LONG_PASSWORD = 1 3 | FOUND_ROWS = 1 << 1 4 | LONG_FLAG = 1 << 2 5 | CONNECT_WITH_DB = 1 << 3 6 | NO_SCHEMA = 1 << 4 7 | COMPRESS = 1 << 5 8 | ODBC = 1 << 6 9 | LOCAL_FILES = 1 << 7 10 | IGNORE_SPACE = 1 << 8 11 | PROTOCOL_41 = 1 << 9 12 | INTERACTIVE = 1 << 10 13 | SSL = 1 << 11 14 | IGNORE_SIGPIPE = 1 << 12 15 | TRANSACTIONS = 1 << 13 16 | SECURE_CONNECTION = 1 << 15 17 | MULTI_STATEMENTS = 1 << 16 18 | MULTI_RESULTS = 1 << 17 19 | PS_MULTI_RESULTS = 1 << 18 20 | PLUGIN_AUTH = 1 << 19 21 | CONNECT_ATTRS = 1 << 20 22 | PLUGIN_AUTH_LENENC_CLIENT_DATA = 1 << 21 23 | CAPABILITIES = ( 24 | LONG_PASSWORD | LONG_FLAG | PROTOCOL_41 | TRANSACTIONS 25 | | SECURE_CONNECTION | MULTI_RESULTS 26 | | PLUGIN_AUTH | PLUGIN_AUTH_LENENC_CLIENT_DATA | CONNECT_ATTRS) 27 | 28 | # Not done yet 29 | HANDLE_EXPIRED_PASSWORDS = 1 << 22 30 | SESSION_TRACK = 1 << 23 31 | DEPRECATE_EOF = 1 << 24 32 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/literal_cost.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Literal cost model to allow backward reference replacement to be efficient. 8 | */ 9 | 10 | #ifndef BROTLI_ENC_LITERAL_COST_H_ 11 | #define BROTLI_ENC_LITERAL_COST_H_ 12 | 13 | #include "../common/platform.h" 14 | #include 15 | 16 | #if defined(__cplusplus) || defined(c_plusplus) 17 | extern "C" { 18 | #endif 19 | 20 | /* Estimates how many bits the literals in the interval [pos, pos + len) in the 21 | ring-buffer (data, mask) will take entropy coded and writes these estimates 22 | to the cost[0..len) array. */ 23 | BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals( 24 | size_t pos, size_t len, size_t mask, const uint8_t* data, float* cost); 25 | 26 | #if defined(__cplusplus) || defined(c_plusplus) 27 | } /* extern "C" */ 28 | #endif 29 | 30 | #endif /* BROTLI_ENC_LITERAL_COST_H_ */ 31 | -------------------------------------------------------------------------------- /dedupsqlfs/proc.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | 4 | def pid_exists(pid): 5 | """Check whether pid exists in the current process table. 6 | UNIX only. 7 | """ 8 | if pid < 0: 9 | return False 10 | if pid == 0: 11 | # According to "man 2 kill" PID 0 refers to every process 12 | # in the process group of the calling process. 13 | # On certain systems 0 is a valid PID but we have no way 14 | # to know that in a portable fashion. 15 | raise ValueError('invalid PID 0') 16 | try: 17 | os.kill(pid, 0) 18 | except OSError as err: 19 | if err.errno == errno.ESRCH: 20 | # ESRCH == No such process 21 | return False 22 | elif err.errno == errno.EPERM: 23 | # EPERM clearly means there's a process to deny access to 24 | return True 25 | else: 26 | # According to "man 2 kill" possible error values are 27 | # (EINVAL, EPERM, ESRCH) 28 | raise 29 | else: 30 | return True 31 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/utf8_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Heuristics for deciding about the UTF8-ness of strings. */ 8 | 9 | #ifndef BROTLI_ENC_UTF8_UTIL_H_ 10 | #define BROTLI_ENC_UTF8_UTIL_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | static const double kMinUTF8Ratio = 0.75; 20 | 21 | /* Returns 1 if at least min_fraction of the bytes between pos and 22 | pos + length in the (data, mask) ring-buffer is UTF8-encoded, otherwise 23 | returns 0. */ 24 | BROTLI_INTERNAL BROTLI_BOOL BrotliIsMostlyUTF8( 25 | const uint8_t* data, const size_t pos, const size_t mask, 26 | const size_t length, const double min_fraction); 27 | 28 | #if defined(__cplusplus) || defined(c_plusplus) 29 | } /* extern "C" */ 30 | #endif 31 | 32 | #endif /* BROTLI_ENC_UTF8_UTIL_H_ */ 33 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/bz2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for BZ2 compression helper 7 | """ 8 | 9 | from dedupsqlfs.compression import BaseCompression 10 | 11 | class Bz2Compression(BaseCompression): 12 | 13 | _method_name = "bz2" 14 | 15 | _minimal_size = 40 16 | 17 | _has_comp_level_options = True 18 | 19 | def getFastCompressionOptions(self): 20 | return ( 1, ) 21 | 22 | def getNormCompressionOptions(self): 23 | return ( 6, ) 24 | 25 | def getBestCompressionOptions(self): 26 | return ( 9, ) 27 | 28 | def getDefaultCompressionOptions(self): 29 | return ( 2, ) 30 | 31 | def getCustomCompressionOptions(self): 32 | try: 33 | level = int(self._custom_comp_level) 34 | if level < 1: 35 | level = 1 36 | elif level > 9: 37 | level = 9 38 | opts = (level, ) 39 | except: 40 | opts = False 41 | pass 42 | return opts 43 | 44 | pass 45 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/zlib.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for ZLIB compression helper 7 | """ 8 | 9 | from dedupsqlfs.compression import BaseCompression 10 | 11 | class ZlibCompression(BaseCompression): 12 | 13 | _method_name = "zlib" 14 | 15 | _minimal_size = 12 16 | 17 | _has_comp_level_options = True 18 | 19 | def getFastCompressionOptions(self): 20 | return ( 1, ) 21 | 22 | def getNormCompressionOptions(self): 23 | return ( 6, ) 24 | 25 | def getBestCompressionOptions(self): 26 | return ( 9, ) 27 | 28 | def getDefaultCompressionOptions(self): 29 | return ( 3, ) 30 | 31 | def getCustomCompressionOptions(self): 32 | try: 33 | level = int(self._custom_comp_level) 34 | if level < 0: 35 | level = 0 36 | elif level > 9: 37 | level = 9 38 | opts = (level, ) 39 | except: 40 | opts = False 41 | pass 42 | return opts 43 | 44 | pass 45 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/deflate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for ZLIB compression helper 7 | """ 8 | 9 | from dedupsqlfs.compression import BaseCompression 10 | 11 | class DeflateCompression(BaseCompression): 12 | 13 | _method_name = "deflate" 14 | 15 | _minimal_size = 3 16 | 17 | _has_comp_level_options = True 18 | 19 | def getFastCompressionOptions(self): 20 | return ( 1, ) 21 | 22 | def getNormCompressionOptions(self): 23 | return ( 6, ) 24 | 25 | def getBestCompressionOptions(self): 26 | return ( 9, ) 27 | 28 | def getDefaultCompressionOptions(self): 29 | return ( 3, ) 30 | 31 | def getCustomCompressionOptions(self): 32 | try: 33 | level = int(self._custom_comp_level) 34 | if level < 0: 35 | level = 0 36 | elif level > 9: 37 | level = 9 38 | opts = (level, ) 39 | except: 40 | opts = False 41 | pass 42 | return opts 43 | 44 | pass 45 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include setup.py 2 | include README.md 3 | include LICENSE.txt 4 | include test_all.py 5 | include test_performance.py 6 | include lib/recordclass/__init__.py 7 | include lib/recordclass/about.py 8 | include lib/recordclass/recordclass.py 9 | include lib/recordclass/dataclass.py 10 | include lib/recordclass/dictclass.py 11 | include lib/recordclass/arrayclass.py 12 | include lib/recordclass/datatype.py 13 | include lib/recordclass/utils.py 14 | include lib/recordclass/adapter.py 15 | include lib/recordclass/_dataobject.c 16 | include lib/recordclass/_dataobject.h 17 | include lib/recordclass/_litelist.h 18 | include lib/recordclass/_litelist.c 19 | include lib/recordclass/_linkedlist.h 20 | include lib/recordclass/_linkedlist.c 21 | include lib/recordclass/_litetuple.c 22 | include lib/recordclass/tools/*.py 23 | include lib/recordclass/typing/*.py 24 | include lib/recordclass/test/*.py 25 | include lib/recordclass/test/typing/*.py 26 | include lib/recordclass/test/match/*.py 27 | include lib/recordclass/tools/*.py 28 | exclude lib/recordclass/.* 29 | 30 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/encoder_dict.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include "./encoder_dict.h" 8 | 9 | #include "../common/dictionary.h" 10 | #include "../common/transform.h" 11 | #include "./dictionary_hash.h" 12 | #include "./hash.h" 13 | 14 | #if defined(__cplusplus) || defined(c_plusplus) 15 | extern "C" { 16 | #endif 17 | 18 | void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) { 19 | dict->words = BrotliGetDictionary(); 20 | dict->num_transforms = (uint32_t)BrotliGetTransforms()->num_transforms; 21 | 22 | dict->hash_table_words = kStaticDictionaryHashWords; 23 | dict->hash_table_lengths = kStaticDictionaryHashLengths; 24 | dict->buckets = kStaticDictionaryBuckets; 25 | dict->dict_words = kStaticDictionaryWords; 26 | 27 | dict->cutoffTransformsCount = kCutoffTransformsCount; 28 | dict->cutoffTransforms = kCutoffTransforms; 29 | } 30 | 31 | #if defined(__cplusplus) || defined(c_plusplus) 32 | } /* extern "C" */ 33 | #endif 34 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/src/llfuse.h: -------------------------------------------------------------------------------- 1 | /* 2 | llfuse.h 3 | 4 | Copyright © 2013 Nikolaus Rath 5 | 6 | This file is part of Python-LLFUSE. This work may be distributed under 7 | the terms of the GNU LGPL. 8 | */ 9 | 10 | 11 | #define PLATFORM_LINUX 1 12 | #define PLATFORM_BSD 2 13 | #define PLATFORM_DARWIN 3 14 | 15 | #ifdef __linux__ 16 | #define PLATFORM PLATFORM_LINUX 17 | #elif __FreeBSD_kernel__&&__GLIBC__ 18 | #define PLATFORM PLATFORM_LINUX 19 | #elif __FreeBSD__ 20 | #define PLATFORM PLATFORM_BSD 21 | #elif __NetBSD__ 22 | #define PLATFORM PLATFORM_BSD 23 | #elif __APPLE__ && __MACH__ 24 | #define PLATFORM PLATFORM_DARWIN 25 | #else 26 | #error "Unable to determine system (Linux/FreeBSD/NetBSD/Darwin)" 27 | #endif 28 | 29 | #if PLATFORM == PLATFORM_DARWIN 30 | #include "darwin_compat.h" 31 | #else 32 | /* See also: Include/pthreads.pxd */ 33 | #include 34 | #endif 35 | 36 | #include 37 | 38 | #if FUSE_VERSION < 29 39 | #error FUSE version too old, 2.9.0 or newer required 40 | #endif 41 | 42 | #if FUSE_MAJOR_VERSION != 2 43 | #error This version of the FUSE library is not yet supported. 44 | #endif 45 | -------------------------------------------------------------------------------- /bin/do.dedupsqlfs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 -O 2 | # -*- coding: utf8 -*- 3 | 4 | """ 5 | Commands for dedupsqlfs 6 | 7 | @author: sergey 8 | @copyright: 2013 9 | @since: 2013-07-20 10 | """ 11 | 12 | import sys 13 | import os 14 | 15 | dirname = "dedupsqlfs" 16 | 17 | # Figure out the directy which is the prefix 18 | # path-of-current-file/.. 19 | curpath = os.path.abspath( sys.argv[0] ) 20 | if os.path.islink(curpath): 21 | curpath = os.readlink(curpath) 22 | currentdir = os.path.dirname( curpath ) 23 | basedir = os.path.abspath( os.path.join( currentdir, ".." ) ) 24 | 25 | # Add the base directory where the application is installed in to sys.path 26 | if not os.path.exists( os.path.join( basedir, dirname ) ): 27 | raise SystemExit( "ERROR: Could not find required directory: %s" % 28 | os.path.join( basedir, dirname ) ) 29 | 30 | dynloaddir = os.path.abspath( os.path.join( basedir, "lib-dynload" ) ) 31 | 32 | sys.path.insert( 0, dynloaddir ) 33 | sys.path.insert( 0, basedir ) 34 | 35 | from dedupsqlfs.app.do import main 36 | 37 | try: 38 | sys.exit( main( ) ) 39 | except KeyboardInterrupt: 40 | raise SystemExit 41 | -------------------------------------------------------------------------------- /bin/fsck.dedupsqlfs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 -O 2 | # -*- coding: utf8 -*- 3 | 4 | """ 5 | Check dedupsqlfs filesystem 6 | 7 | @author: sergey 8 | @copyright: 2013 9 | @since: 2013-07-28 10 | """ 11 | 12 | import sys 13 | import os 14 | 15 | dirname = "dedupsqlfs" 16 | 17 | # Figure out the directy which is the prefix 18 | # path-of-current-file/.. 19 | curpath = os.path.abspath( sys.argv[0] ) 20 | if os.path.islink(curpath): 21 | curpath = os.readlink(curpath) 22 | currentdir = os.path.dirname( curpath ) 23 | basedir = os.path.abspath( os.path.join( currentdir, ".." ) ) 24 | 25 | # Add the base directory where the application is installed in to sys.path 26 | if not os.path.exists( os.path.join( basedir, dirname ) ): 27 | raise SystemExit( "ERROR: Could not find required directory: %s" % 28 | os.path.join( basedir, dirname ) ) 29 | 30 | dynloaddir = os.path.abspath( os.path.join( basedir, "lib-dynload" ) ) 31 | 32 | sys.path.insert( 0, dynloaddir ) 33 | sys.path.insert( 0, basedir ) 34 | 35 | from dedupsqlfs.app.fsck import main 36 | 37 | try: 38 | sys.exit( main( ) ) 39 | except KeyboardInterrupt: 40 | raise SystemExit 41 | -------------------------------------------------------------------------------- /bin/mkfs.dedupsqlfs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 -O 2 | # -*- coding: utf8 -*- 3 | 4 | """ 5 | Make dedupsqlfs filesystem 6 | 7 | @author: sergey 8 | @copyright: 2013 9 | @since: 2013-07-28 10 | """ 11 | 12 | import sys 13 | import os 14 | 15 | dirname = "dedupsqlfs" 16 | 17 | # Figure out the directy which is the prefix 18 | # path-of-current-file/.. 19 | curpath = os.path.abspath( sys.argv[0] ) 20 | if os.path.islink(curpath): 21 | curpath = os.readlink(curpath) 22 | currentdir = os.path.dirname( curpath ) 23 | basedir = os.path.abspath( os.path.join( currentdir, ".." ) ) 24 | 25 | # Add the base directory where the application is installed in to sys.path 26 | if not os.path.exists( os.path.join( basedir, dirname ) ): 27 | raise SystemExit( "ERROR: Could not find required directory: %s" % 28 | os.path.join( basedir, dirname ) ) 29 | 30 | dynloaddir = os.path.abspath( os.path.join( basedir, "lib-dynload" ) ) 31 | 32 | sys.path.insert( 0, dynloaddir ) 33 | sys.path.insert( 0, basedir ) 34 | 35 | from dedupsqlfs.app.mkfs import main 36 | 37 | try: 38 | sys.exit( main( ) ) 39 | except KeyboardInterrupt: 40 | raise SystemExit 41 | -------------------------------------------------------------------------------- /lib-dynload/quicklz/setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from distutils.core import setup, Extension 3 | from distutils import ccompiler 4 | 5 | EXTRA_OPT=0 6 | if "--extra-optimization" in sys.argv: 7 | # Support legacy output format functions 8 | EXTRA_OPT=1 9 | sys.argv.remove("--extra-optimization") 10 | 11 | if ccompiler.get_default_compiler() == "msvc": 12 | extra_compile_args = ["/Wall"] 13 | if EXTRA_OPT: 14 | extra_compile_args.insert(0, "/O2") 15 | else: 16 | extra_compile_args.insert(0, "/Ot") 17 | else: 18 | extra_compile_args = ["-std=c99", "-Wall", "-D_FORTIFY_SOURCE=2", "-fstack-protector"] 19 | if EXTRA_OPT: 20 | extra_compile_args.insert(0, "-march=native") 21 | extra_compile_args.insert(0, "-O3") 22 | else: 23 | extra_compile_args.insert(0, "-O2") 24 | 25 | setup( 26 | name = "qlz", 27 | version = "1.0", 28 | packages=[], 29 | package_dir={'': 'src'}, 30 | ext_modules = [ 31 | Extension( 32 | "qlz", 33 | ["src/quicklz.c", "src/quicklzpy.c"], 34 | extra_compile_args=extra_compile_args 35 | ) 36 | ] 37 | ) 38 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/deprecated/zbuff_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /*-************************************* 12 | * Dependencies 13 | ***************************************/ 14 | #include "../common/error_private.h" 15 | #include "zbuff.h" 16 | 17 | /*-**************************************** 18 | * ZBUFF Error Management (deprecated) 19 | ******************************************/ 20 | 21 | /*! ZBUFF_isError() : 22 | * tells if a return value is an error code */ 23 | unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } 24 | /*! ZBUFF_getErrorName() : 25 | * provides error code string from function result (useful for debugging) */ 26 | const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } 27 | -------------------------------------------------------------------------------- /bin/cache_flusher: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 -OO 2 | # -*- coding: utf8 -*- 3 | 4 | """ 5 | Flush caches in dedupsqlfs filesystem 6 | 7 | @author: sergey 8 | @copyright: 2017 9 | @since: 2017-05-12 10 | """ 11 | 12 | import sys 13 | import os 14 | 15 | dirname = "dedupsqlfs" 16 | 17 | # Figure out the directy which is the prefix 18 | # path-of-current-file/.. 19 | curpath = os.path.abspath( sys.argv[0] ) 20 | if os.path.islink(curpath): 21 | curpath = os.readlink(curpath) 22 | currentdir = os.path.dirname( curpath ) 23 | basedir = os.path.abspath( os.path.join( currentdir, ".." ) ) 24 | 25 | # Add the base directory where the application is installed in to sys.path 26 | if not os.path.exists( os.path.join( basedir, dirname ) ): 27 | raise SystemExit( "ERROR: Could not find required directory: %s" % 28 | os.path.join( basedir, dirname ) ) 29 | 30 | sys.path.insert( 0, basedir ) 31 | 32 | from dedupsqlfs.app.cache_flusher import main 33 | 34 | try: 35 | sys.exit( main( ) ) 36 | except KeyboardInterrupt: 37 | raise SystemExit 38 | except Exception as e: 39 | print(e) 40 | import traceback 41 | traceback.print_exc() 42 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/command.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include "./command.h" 8 | 9 | #include 10 | 11 | #if defined(__cplusplus) || defined(c_plusplus) 12 | extern "C" { 13 | #endif 14 | 15 | const uint32_t kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES] = { 16 | 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 17 | 34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594}; 18 | const uint32_t kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES] = { 19 | 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24}; 20 | const uint32_t kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES] = { 21 | 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22 | 22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094, 2118}; 23 | const uint32_t kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES] = { 24 | 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24}; 25 | 26 | #if defined(__cplusplus) || defined(c_plusplus) 27 | } /* extern "C" */ 28 | #endif 29 | -------------------------------------------------------------------------------- /lib-dynload/brotli/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /lib-dynload/__lzo/setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from distutils.core import setup, Extension 3 | from distutils import ccompiler 4 | 5 | EXTRA_OPT=0 6 | if "--extra-optimization" in sys.argv: 7 | # Support legacy output format functions 8 | EXTRA_OPT=1 9 | sys.argv.remove("--extra-optimization") 10 | 11 | if ccompiler.get_default_compiler() == "msvc": 12 | extra_compile_args = ["/Wall"] 13 | if EXTRA_OPT: 14 | extra_compile_args.insert(0, "/O2") 15 | else: 16 | extra_compile_args.insert(0, "/Ot") 17 | else: 18 | extra_compile_args = ["-std=c99", "-Wall", "-D_FORTIFY_SOURCE=2", "-fstack-protector"] 19 | if EXTRA_OPT: 20 | extra_compile_args.insert(0, "-march=native") 21 | extra_compile_args.insert(0, "-O3") 22 | else: 23 | extra_compile_args.insert(0, "-O2") 24 | 25 | setup( 26 | name = "_lzo", 27 | version = "1.15", 28 | packages=[], 29 | package_dir={'': 'src'}, 30 | ext_modules = [ 31 | Extension( 32 | "_lzo", 33 | ["src/lzomodule.c"], 34 | libraries=['lzo2'], 35 | extra_compile_args=extra_compile_args 36 | ) 37 | ] 38 | ) 39 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/_dataobject.h: -------------------------------------------------------------------------------- 1 | typedef struct { 2 | PyObject ob_head; 3 | PyObject *ob_items[1]; 4 | } PyDataStruct; 5 | 6 | struct PyFactoryObject { 7 | PyObject_HEAD 8 | PyObject *factory; 9 | }; 10 | 11 | typedef struct { 12 | PyObject_HEAD 13 | Py_ssize_t index; 14 | int readonly; 15 | } dataobjectproperty_object; 16 | 17 | 18 | #define PyDataObject_ITEMS(op) (PyObject**)(((PyDataStruct*)op)->ob_items) 19 | 20 | #define PyDataObject_NUMITEMS(tp) (tp->tp_itemsize) 21 | 22 | #define PyDataObject_LEN(o) (Py_TYPE(o)->tp_itemsize) 23 | #define PyDataObject_GET_ITEM(op, i) (((PyDataStruct*)(op))->ob_items[(i)]) 24 | #define PyDataObject_SET_ITEM(op, i, v) (((PyDataStruct*)(op))->ob_items[(i)]=(v)) 25 | 26 | #define PyDataObject_DICTPTR(type, op) ((PyObject**)((char*)(op) + (type)->tp_dictoffset)) 27 | #define PyDataObject_WEAKLISTPTR(type, op) ((PyObject**)((char*)op + type->tp_weaklistoffset)) 28 | #define PyDataObject_HAS_DICT(type) (type->tp_dictoffset != 0) 29 | #define PyDataObject_HAS_WEAKLIST(type) (type->tp_weaklistoffset != 0) 30 | 31 | #define Py_TP_BASE(o) (Py_TYPE(o)->tp_base) 32 | #define Py_METATYPE(o) Py_TYPE(Py_TYPE(o)) 33 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/zstd.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for Zstd compression helper 7 | New version 1.0+ 8 | 9 | Since libzstd-1.3.4 - support ultra-fast levels: -100..-1 10 | """ 11 | 12 | from dedupsqlfs.compression import BaseCompression 13 | 14 | class ZstdCompression(BaseCompression): 15 | 16 | # new bundled module name 17 | _method_name = "zstd" 18 | 19 | _minimal_size = 18 20 | 21 | _has_comp_level_options = True 22 | 23 | def getFastCompressionOptions(self): 24 | return ( 1, ) 25 | 26 | def getNormCompressionOptions(self): 27 | return ( 9, ) 28 | 29 | def getBestCompressionOptions(self): 30 | return ( 18, ) 31 | 32 | def getDefaultCompressionOptions(self): 33 | return ( 3, ) 34 | 35 | def getCustomCompressionOptions(self): 36 | try: 37 | level = int(self._custom_comp_level) 38 | if level < -100: 39 | level = -100 40 | elif level > 20: 41 | level = 20 42 | opts = (level, ) 43 | except: 44 | opts = False 45 | pass 46 | return opts 47 | 48 | pass 49 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/brotli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for Google Brotli compression helper 7 | """ 8 | 9 | from dedupsqlfs.compression import BaseCompression 10 | 11 | class BrotliCompression(BaseCompression): 12 | 13 | _method_name = "brotli" 14 | 15 | _minimal_size = 19 16 | 17 | _has_comp_level_options = True 18 | 19 | def getFastCompressionOptions(self): 20 | return ( self._module.MODE_GENERIC, 0, ) 21 | 22 | def getNormCompressionOptions(self): 23 | return ( self._module.MODE_GENERIC, 4, ) 24 | 25 | def getBestCompressionOptions(self): 26 | return ( self._module.MODE_GENERIC, 11, ) 27 | 28 | def getDefaultCompressionOptions(self): 29 | return ( self._module.MODE_GENERIC, 2, ) 30 | 31 | def getCustomCompressionOptions(self): 32 | try: 33 | level = int(self._custom_comp_level) 34 | if level < 0: 35 | level = 0 36 | elif level > 11: 37 | level = 11 38 | opts = (self._module.MODE_GENERIC, level, ) 39 | except: 40 | opts = False 41 | pass 42 | return opts 43 | 44 | pass 45 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) «2015-2023» «Shibzukhov Zaur, szport at gmail dot com» 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software - recordclass library - and associated documentation files 7 | (the "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, distribute, 9 | sublicense, and/or sell copies of the Software, and to permit persons to whom 10 | the Software is furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/common/debug.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * debug 3 | * Part of FSE library 4 | * Copyright (c) Meta Platforms, Inc. and affiliates. 5 | * 6 | * You can contact the author at : 7 | * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | 15 | 16 | /* 17 | * This module only hosts one global variable 18 | * which can be used to dynamically influence the verbosity of traces, 19 | * such as DEBUGLOG and RAWLOG 20 | */ 21 | 22 | #include "debug.h" 23 | 24 | #if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) 25 | /* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a 26 | * translation unit is empty. So remove this from Linux kernel builds, but 27 | * otherwise just leave it in. 28 | */ 29 | int g_debuglevel = DEBUGLEVEL; 30 | #endif 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013-2020 Sergey Dryabzhinsky 4 | Copyright 2010 Peter Odding . 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of 7 | this software and associated documentation files (the "Software"), to deal in 8 | the Software without restriction, including without limitation the rights to 9 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 10 | the Software, and to permit persons to whom the Software is furnished to do so, 11 | subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 18 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 19 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/src/gettime.h: -------------------------------------------------------------------------------- 1 | /* 2 | * gettime.h 3 | * 4 | * Platform-independent interface to system clock 5 | * 6 | * Copyright © 2015 Nikolaus Rath 7 | * 8 | * This file is part of Python-LLFUSE. This work may be distributed under the 9 | * terms of the GNU LGPL. 10 | */ 11 | 12 | /* 13 | * Linux 14 | */ 15 | #if PLATFORM == PLATFORM_LINUX 16 | #include 17 | 18 | static int gettime_realtime(struct timespec *tp) { 19 | return clock_gettime(CLOCK_REALTIME, tp); 20 | } 21 | 22 | 23 | /* 24 | * FreeBSD & NetBSD 25 | */ 26 | #elif PLATFORM == PLATFORM_BSD 27 | #include 28 | 29 | static int gettime_realtime(struct timespec *tp) { 30 | return clock_gettime(CLOCK_REALTIME, tp); 31 | } 32 | 33 | /* 34 | * Darwin 35 | */ 36 | #elif PLATFORM == PLATFORM_DARWIN 37 | #include 38 | 39 | static int gettime_realtime(struct timespec *tp) { 40 | struct timeval tv; 41 | int res; 42 | 43 | res = gettimeofday(&tv, NULL); 44 | if(res != 0) 45 | return -1; 46 | 47 | tp->tv_sec = tv.tv_sec; 48 | tp->tv_nsec = tv.tv_usec * 1000; 49 | 50 | return 0; 51 | } 52 | 53 | 54 | /* 55 | * Unknown system 56 | */ 57 | #else 58 | #error This should not happen 59 | #endif 60 | -------------------------------------------------------------------------------- /bin/mount.dedupsqlfs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 -O 2 | # -*- coding: utf8 -*- 3 | 4 | """ 5 | Mount dedupsqlfs filesystem 6 | 7 | @author: sergey 8 | @copyright: 2013 9 | @since: 2013-07-28 10 | """ 11 | 12 | import sys 13 | import os 14 | 15 | dirname = "dedupsqlfs" 16 | 17 | # Figure out the directy which is the prefix 18 | # path-of-current-file/.. 19 | curpath = os.path.abspath( sys.argv[0] ) 20 | if os.path.islink(curpath): 21 | curpath = os.readlink(curpath) 22 | currentdir = os.path.dirname( curpath ) 23 | basedir = os.path.abspath( os.path.join( currentdir, ".." ) ) 24 | 25 | # Add the base directory where the application is installed in to sys.path 26 | if not os.path.exists( os.path.join( basedir, dirname ) ): 27 | raise SystemExit( "ERROR: Could not find required directory: %s" % 28 | os.path.join( basedir, dirname ) ) 29 | 30 | dynloaddir = os.path.abspath( os.path.join( basedir, "lib-dynload" ) ) 31 | 32 | sys.path.insert( 0, dynloaddir ) 33 | sys.path.insert( 0, basedir ) 34 | 35 | from dedupsqlfs.app.mount import main 36 | 37 | try: 38 | sys.exit( main( ) ) 39 | except KeyboardInterrupt: 40 | raise SystemExit 41 | except Exception as e: 42 | print(e) 43 | import traceback 44 | traceback.print_exc() 45 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/cluster.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions for clustering similar histograms together. */ 8 | 9 | #ifndef BROTLI_ENC_CLUSTER_H_ 10 | #define BROTLI_ENC_CLUSTER_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | #include "./histogram.h" 15 | #include "./memory.h" 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | typedef struct HistogramPair { 22 | uint32_t idx1; 23 | uint32_t idx2; 24 | double cost_combo; 25 | double cost_diff; 26 | } HistogramPair; 27 | 28 | #define CODE(X) /* Declaration */; 29 | 30 | #define FN(X) X ## Literal 31 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 32 | #undef FN 33 | 34 | #define FN(X) X ## Command 35 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 36 | #undef FN 37 | 38 | #define FN(X) X ## Distance 39 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 40 | #undef FN 41 | 42 | #undef CODE 43 | 44 | #if defined(__cplusplus) || defined(c_plusplus) 45 | } /* extern "C" */ 46 | #endif 47 | 48 | #endif /* BROTLI_ENC_CLUSTER_H_ */ 49 | -------------------------------------------------------------------------------- /dedupsqlfs/log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | import logging 6 | 7 | # This is less than 'error' but more than warning 8 | IMPORTANT = logging.ERROR - 1 9 | 10 | # This is more than 'info' but less than warning 11 | NOTE = logging.INFO + 1 12 | 13 | # This is for output more than debug, an extra debug 14 | DEBUG_VERBOSE = logging.DEBUG - 1 15 | 16 | logging.addLevelName(DEBUG_VERBOSE, "DEBUGV") 17 | logging.addLevelName(NOTE, "NOTE") 18 | logging.addLevelName(IMPORTANT, "IMPORTANT") 19 | 20 | def debugv(self, message, *args, **kws): 21 | # Yes, logger takes its '*args' as 'args'. 22 | if self.isEnabledFor(DEBUG_VERBOSE): 23 | self._log(DEBUG_VERBOSE, message, args, **kws) 24 | return 25 | 26 | def note(self, message, *args, **kws): 27 | # Yes, logger takes its '*args' as 'args'. 28 | if self.isEnabledFor(NOTE): 29 | self._log(NOTE, message, args, **kws) 30 | return 31 | 32 | def important(self, message, *args, **kws): 33 | # Yes, logger takes its '*args' as 'args'. 34 | if self.isEnabledFor(IMPORTANT): 35 | self._log(IMPORTANT, message, args, **kws) 36 | return 37 | 38 | logging.Logger.debugv = debugv 39 | logging.Logger.note = note 40 | logging.Logger.important = important 41 | -------------------------------------------------------------------------------- /dedupsqlfs/compression/lzma.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | """ 6 | Class for LZMA compression helper 7 | """ 8 | 9 | from dedupsqlfs.compression import BaseCompression 10 | 11 | class LzmaCompression(BaseCompression): 12 | 13 | _method_name = "xz" 14 | 15 | _minimal_size = 69 16 | 17 | _has_comp_level_options = True 18 | 19 | def getFastCompressionOptions(self): 20 | return { 21 | "preset": 0, 22 | } 23 | 24 | def getNormCompressionOptions(self): 25 | return { 26 | "preset": 4, 27 | } 28 | 29 | def getBestCompressionOptions(self): 30 | return { 31 | "preset": 9, 32 | } 33 | 34 | def getDefaultCompressionOptions(self): 35 | return { 36 | "preset": 2, 37 | } 38 | 39 | def getCustomCompressionOptions(self): 40 | try: 41 | level = int(self._custom_comp_level) 42 | if level < 0: 43 | level = 0 44 | elif level > 9: 45 | level = 9 46 | opts = { 47 | "preset": level, 48 | } 49 | except: 50 | opts = False 51 | pass 52 | return opts 53 | 54 | pass 55 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/block_encoder_inc.h: -------------------------------------------------------------------------------- 1 | /* NOLINT(build/header_guard) */ 2 | /* Copyright 2014 Google Inc. All Rights Reserved. 3 | 4 | Distributed under MIT license. 5 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 6 | */ 7 | 8 | /* template parameters: FN */ 9 | 10 | #define HistogramType FN(Histogram) 11 | 12 | /* Creates entropy codes for all block types and stores them to the bit 13 | stream. */ 14 | static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self, 15 | const HistogramType* histograms, const size_t histograms_size, 16 | const size_t alphabet_size, HuffmanTree* tree, 17 | size_t* storage_ix, uint8_t* storage) { 18 | const size_t table_size = histograms_size * self->histogram_length_; 19 | self->depths_ = BROTLI_ALLOC(m, uint8_t, table_size); 20 | self->bits_ = BROTLI_ALLOC(m, uint16_t, table_size); 21 | if (BROTLI_IS_OOM(m)) return; 22 | 23 | { 24 | size_t i; 25 | for (i = 0; i < histograms_size; ++i) { 26 | size_t ix = i * self->histogram_length_; 27 | BuildAndStoreHuffmanTree(&histograms[i].data_[0], self->histogram_length_, 28 | alphabet_size, tree, &self->depths_[ix], &self->bits_[ix], 29 | storage_ix, storage); 30 | } 31 | } 32 | } 33 | 34 | #undef HistogramType 35 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/compress/zstd_fast.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_FAST_H 12 | #define ZSTD_FAST_H 13 | 14 | #include "../common/mem.h" /* U32 */ 15 | #include "zstd_compress_internal.h" 16 | 17 | void ZSTD_fillHashTable(ZSTD_MatchState_t* ms, 18 | void const* end, ZSTD_dictTableLoadMethod_e dtlm, 19 | ZSTD_tableFillPurpose_e tfp); 20 | size_t ZSTD_compressBlock_fast( 21 | ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 22 | void const* src, size_t srcSize); 23 | size_t ZSTD_compressBlock_fast_dictMatchState( 24 | ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 25 | void const* src, size_t srcSize); 26 | size_t ZSTD_compressBlock_fast_extDict( 27 | ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 28 | void const* src, size_t srcSize); 29 | 30 | #endif /* ZSTD_FAST_H */ 31 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/compress/zstd_compress_superblock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_COMPRESS_ADVANCED_H 12 | #define ZSTD_COMPRESS_ADVANCED_H 13 | 14 | /*-************************************* 15 | * Dependencies 16 | ***************************************/ 17 | 18 | #include "../zstd.h" /* ZSTD_CCtx */ 19 | 20 | /*-************************************* 21 | * Target Compressed Block Size 22 | ***************************************/ 23 | 24 | /* ZSTD_compressSuperBlock() : 25 | * Used to compress a super block when targetCBlockSize is being used. 26 | * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ 27 | size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, 28 | void* dst, size_t dstCapacity, 29 | void const* src, size_t srcSize, 30 | unsigned lastBlock); 31 | 32 | #endif /* ZSTD_COMPRESS_ADVANCED_H */ 33 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/compress/zstd_preSplit.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_PRESPLIT_H 12 | #define ZSTD_PRESPLIT_H 13 | 14 | #include /* size_t */ 15 | 16 | #define ZSTD_SLIPBLOCK_WORKSPACESIZE 8208 17 | 18 | /* ZSTD_splitBlock(): 19 | * @level must be a value between 0 and 4. 20 | * higher levels spend more energy to detect block boundaries. 21 | * @workspace must be aligned for size_t. 22 | * @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE 23 | * note: 24 | * For the time being, this function only accepts full 128 KB blocks. 25 | * Therefore, @blockSize must be == 128 KB. 26 | * While this could be extended to smaller sizes in the future, 27 | * it is not yet clear if this would be useful. TBD. 28 | */ 29 | size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize, 30 | int level, 31 | void* workspace, size_t wkspSize); 32 | 33 | #endif /* ZSTD_PRESPLIT_H */ 34 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/params.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Parameters for the Brotli encoder with chosen quality levels. */ 8 | 9 | #ifndef BROTLI_ENC_PARAMS_H_ 10 | #define BROTLI_ENC_PARAMS_H_ 11 | 12 | #include 13 | #include "./encoder_dict.h" 14 | 15 | typedef struct BrotliHasherParams { 16 | int type; 17 | int bucket_bits; 18 | int block_bits; 19 | int hash_len; 20 | int num_last_distances_to_check; 21 | } BrotliHasherParams; 22 | 23 | typedef struct BrotliDistanceParams { 24 | uint32_t distance_postfix_bits; 25 | uint32_t num_direct_distance_codes; 26 | uint32_t alphabet_size_max; 27 | uint32_t alphabet_size_limit; 28 | size_t max_distance; 29 | } BrotliDistanceParams; 30 | 31 | /* Encoding parameters */ 32 | typedef struct BrotliEncoderParams { 33 | BrotliEncoderMode mode; 34 | int quality; 35 | int lgwin; 36 | int lgblock; 37 | size_t stream_offset; 38 | size_t size_hint; 39 | BROTLI_BOOL disable_literal_context_modeling; 40 | BROTLI_BOOL large_window; 41 | BrotliHasherParams hasher; 42 | BrotliDistanceParams dist; 43 | BrotliEncoderDictionary dictionary; 44 | } BrotliEncoderParams; 45 | 46 | #endif /* BROTLI_ENC_PARAMS_H_ */ 47 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/encoder_dict.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #ifndef BROTLI_ENC_ENCODER_DICT_H_ 8 | #define BROTLI_ENC_ENCODER_DICT_H_ 9 | 10 | #include "../common/dictionary.h" 11 | #include "../common/platform.h" 12 | #include 13 | #include "./static_dict_lut.h" 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | /* Dictionary data (words and transforms) for 1 possible context */ 20 | typedef struct BrotliEncoderDictionary { 21 | const BrotliDictionary* words; 22 | uint32_t num_transforms; 23 | 24 | /* cut off for fast encoder */ 25 | uint32_t cutoffTransformsCount; 26 | uint64_t cutoffTransforms; 27 | 28 | /* from dictionary_hash.h, for fast encoder */ 29 | const uint16_t* hash_table_words; 30 | const uint8_t* hash_table_lengths; 31 | 32 | /* from static_dict_lut.h, for slow encoder */ 33 | const uint16_t* buckets; 34 | const DictWord* dict_words; 35 | } BrotliEncoderDictionary; 36 | 37 | BROTLI_INTERNAL void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict); 38 | 39 | #if defined(__cplusplus) || defined(c_plusplus) 40 | } /* extern "C" */ 41 | #endif 42 | 43 | #endif /* BROTLI_ENC_ENCODER_DICT_H_ */ 44 | -------------------------------------------------------------------------------- /docs/benchmarks/2021-05-31_index_cleanup_speed_bench_1.2.951.ru.md: -------------------------------------------------------------------------------- 1 | # Простые тесты скорости работы DedupSQLfs v1.2.951 с несколькиим методами сжатия 2 | 3 | Проверка после оптимизации структуры БД. 4 | 5 | Распаковка LibreOffice 6.4.7 + 7.1.3 6 | 7 | Проверка работы на движке: sqlite 3.34.1 8 | 9 | CPU: i5-6600K @ 3.50GHz 10 | 11 | System: Ubuntu 16.04 amd64 12 | 13 | Kernel: 4.15.0-142-lowlatency 14 | 15 | Python: 3.9.5 16 | 17 | Размер LibreOffice в tar: 18 | 19 | * 6.4.7 - 1133M 20 | * 7.1.3 - 1183M 21 | 22 | Другое: включен autocommit, выключен sync, hash=md5 23 | 24 | Команда: python3.9 ./mount.dedupsqlfs -vv --verbose-stats --data ~/Temp/sqlfs/data/ --compress {method} --no-sync --no-cache-flusher --minimal-compress-size -1 ~/Temp/sqlfs/mount 25 | 26 | Извлечение: tar xf {libre-tar} -C ~/Temp/sqlfs/mount 27 | 28 | ## Тесты 29 | 30 | | method | untar 6.4.7: time, size, speed | untar 7.1.3: time, size, speed | 31 | | ------ |:------------------------------:|:------------------------------:| 32 | | none | 1:37.95, 1101M, 26.94 MiB/s | 1:34.15, 1653M, 37.08 MiB/s | 33 | | *none* | 1:39.50, 1095M, 32.03 MiB/s | 1:33.57, 1641M, 36.40 MiB/s | 34 | | zstd | 1:49.58, 410M, 25.13 MiB/s | 1:42.24, 597M, 30.80 MiB/s | 35 | | *zstd* | 1:37.39, 404M, 31.06 MiB/s | 1:32.48, 585M, 32.67 MiB/s | 36 | 37 | ## Результат 38 | 39 | Увеличение скорости на ~20%, уменьшение размера базы на ~1%. 40 | -------------------------------------------------------------------------------- /bin/rebuild-all-modules.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #### 3 | # This script will try to build 4 | # every available compression module 5 | # If build fails - it will continue to another module 6 | ### 7 | 8 | 9 | THIS_DIR=`dirname $0` 10 | 11 | if [ -z "$PY" ]; then 12 | # generic 13 | PY=python3 14 | fi 15 | 16 | PY=`which $PY` 17 | 18 | EXTRA_OPT=$1 19 | if [ -n "${EXTRA_OPT}" ]; then 20 | EXTRA_OPT="--extra-optimization" 21 | EXTRA_OPT="" 22 | export RC_EXTRAOPT=1 23 | export CFLAGS="-O3 -march=native" 24 | export CXXFLAGS="-O3 -march=native" 25 | else 26 | EXTRA_OPT="" 27 | fi 28 | 29 | ONLY_MODULE=$2 30 | 31 | if [ "${EXTRA_OPT}" = "-h" ]; then 32 | echo "Usage: [env PY=python3.x] $0 [extra-optimizations=/y only-module=/zstd/..]" 33 | exit 0 34 | fi 35 | 36 | if [ -z "$CC" ]; then 37 | export CC=gcc 38 | fi 39 | if [ -z "$CXX" ]; then 40 | export CXX=g++ 41 | fi 42 | 43 | cd "${THIS_DIR}/../lib-dynload" 44 | 45 | for mdir in `ls .` 46 | do 47 | if [ -d ${mdir} ]; then 48 | 49 | if [ -n "$ONLY_MODULE" ] && [ "$ONLY_MODULE" != "$mdir" ]; then 50 | continue 51 | fi 52 | 53 | if [ ! -r ${mdir}/setup.py ]; then 54 | continue 55 | fi 56 | 57 | echo "" 58 | echo "MODULE rebuild: ${mdir}" 59 | 60 | cd ${mdir} 61 | $PY setup.py clean -a 62 | $PY setup.py build_ext ${EXTRA_OPT} clean || (echo "---= !ERROR! =---" && $PY setup.py clean -a) 63 | cd .. 64 | fi 65 | done 66 | -------------------------------------------------------------------------------- /lib-dynload/brotli/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | p1, p2 = sys.version_info[:2] 5 | 6 | curpath = os.path.abspath( sys.argv[0] ) 7 | if os.path.islink(curpath): 8 | curpath = os.readlink(curpath) 9 | currentdir = os.path.dirname( curpath ) 10 | 11 | build_dir = os.path.abspath( os.path.join(currentdir, "lib-dynload", "brotli", "build") ) 12 | if not os.path.isdir(build_dir): 13 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "lib-dynload", "brotli", "build") ) 14 | if not os.path.isdir(build_dir): 15 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "..", "lib-dynload", "brotli", "build") ) 16 | 17 | dirs = os.listdir(build_dir) 18 | for d in dirs: 19 | 20 | found = 0 21 | if d.find("lib.") == 0: 22 | found += 1 23 | if d.find("-%s.%s" % (p1, p2)) != -1: 24 | found += 1 25 | # python 3.10+ 26 | if d.find("-cpython-%s%s" % (p1, p2)) != -1: 27 | found += 1 28 | # pypy 29 | if d.find("-pypy%s%s" % (p1, p2)) != -1: 30 | found += 1 31 | if found <= 1: 32 | continue 33 | 34 | sys.path.insert(0, os.path.join(build_dir, d) ) 35 | 36 | import importlib 37 | _brotli = importlib.import_module("_brotli") 38 | 39 | from .expose import * 40 | 41 | sys.path.pop(0) 42 | 43 | del importlib 44 | 45 | break 46 | 47 | del p1, p2, d, found 48 | del curpath, currentdir, build_dir, dirs 49 | -------------------------------------------------------------------------------- /lib-dynload/zstd/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015-2021, Sergey Dryabzhinsky, Anton Stuk 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | 25 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/static_dict.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Class to model the static dictionary. */ 8 | 9 | #ifndef BROTLI_ENC_STATIC_DICT_H_ 10 | #define BROTLI_ENC_STATIC_DICT_H_ 11 | 12 | #include "../common/dictionary.h" 13 | #include "../common/platform.h" 14 | #include 15 | #include "./encoder_dict.h" 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | #define BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN 37 22 | static const uint32_t kInvalidMatch = 0xFFFFFFF; 23 | 24 | /* Matches data against static dictionary words, and for each length l, 25 | for which a match is found, updates matches[l] to be the minimum possible 26 | (distance << 5) + len_code. 27 | Returns 1 if matches have been found, otherwise 0. 28 | Prerequisites: 29 | matches array is at least BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1 long 30 | all elements are initialized to kInvalidMatch */ 31 | BROTLI_INTERNAL BROTLI_BOOL BrotliFindAllStaticDictionaryMatches( 32 | const BrotliEncoderDictionary* dictionary, 33 | const uint8_t* data, size_t min_length, size_t max_length, 34 | uint32_t* matches); 35 | 36 | #if defined(__cplusplus) || defined(c_plusplus) 37 | } /* extern "C" */ 38 | #endif 39 | 40 | #endif /* BROTLI_ENC_STATIC_DICT_H_ */ 41 | -------------------------------------------------------------------------------- /lib-dynload/snappy/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | p1, p2 = sys.version_info[:2] 5 | 6 | curpath = os.path.abspath( sys.argv[0] ) 7 | if os.path.islink(curpath): 8 | curpath = os.readlink(curpath) 9 | currentdir = os.path.dirname( curpath ) 10 | 11 | build_dir = os.path.abspath( os.path.join(currentdir, "lib-dynload", "snappy", "build") ) 12 | if not os.path.isdir(build_dir): 13 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "lib-dynload", "snappy", "build") ) 14 | if not os.path.isdir(build_dir): 15 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "..", "lib-dynload", "snappy", "build") ) 16 | 17 | dirs = os.listdir(build_dir) 18 | for d in dirs: 19 | 20 | found = 0 21 | if d.find("lib.") == 0: 22 | found += 1 23 | if d.find("-%s.%s" % (p1, p2)) != -1: 24 | found += 1 25 | # python 3.10+ 26 | if d.find("-cpython-%s%s" % (p1, p2)) != -1: 27 | found += 1 28 | # pypy 29 | if d.find("-pypy%s%s" % (p1, p2)) != -1: 30 | found += 1 31 | if found <= 1: 32 | continue 33 | 34 | sys.path.insert(0, os.path.join(build_dir, d) ) 35 | 36 | import importlib 37 | module = importlib.import_module("_snappy") 38 | 39 | compress = module.compress 40 | decompress = module.decompress 41 | 42 | sys.path.pop(0) 43 | 44 | del importlib, module 45 | 46 | break 47 | 48 | del p1, p2, d, found 49 | del curpath, currentdir, build_dir, dirs 50 | -------------------------------------------------------------------------------- /docs/benchmarks/2021-05-31_cython_speed_bench_1.2.951.ru.md: -------------------------------------------------------------------------------- 1 | # Простые тесты скорости работы DedupSQLfs v1.2.951 со сборкой через cython 2 | 3 | Проверка после оптимизации структуры БД. 4 | 5 | Распаковка LibreOffice 6.4.7 + 7.1.3 6 | 7 | Проверка работы на движке: sqlite 3.34.1 8 | 9 | CPU: i5-6600K @ 3.50GHz 10 | 11 | System: Ubuntu 16.04 amd64 12 | 13 | Kernel: 4.15.0-142-lowlatency 14 | 15 | Python: 3.9.5 + cython 0.29.23 16 | 17 | Размер LibreOffice в tar: 18 | 19 | * 6.4.7 - 1133M 20 | * 7.1.3 - 1183M 21 | 22 | Другое: включен autocommit, выключен sync, hash=md5 23 | 24 | Команда: python3.9 ./mount.dedupsqlfs -vv --verbose-stats --data ~/Temp/sqlfs/data/ --compress {method} --no-sync --no-cache-flusher --minimal-compress-size -1 ~/Temp/sqlfs/mount 25 | 26 | Извлечение: tar xf {libre-tar} -C ~/Temp/sqlfs/mount 27 | 28 | ## Тесты 29 | 30 | | method | untar 6.4.7: time, size, speed | untar 7.1.3: time, size, speed | 31 | | ------ |:------------------------------:|:------------------------------:| 32 | | none | 1:39.50, 1095M, 32.03 MiB/s | 1:33.57, 1641M, 36.40 MiB/s | 33 | | *none* | 1:49.55, 1095M, 31.61 MiB/s | 1:44.10, 1641M, 30.81 MiB/s | 34 | | zstd | 1:37.39, 404M, 31.06 MiB/s | 1:32.48, 585M, 32.67 MiB/s | 35 | | *zstd* | 1:53.25, 404M, 23.22 MiB/s | 1:46.52, 585M, 30.91 MiB/s | 36 | 37 | ## Результат 38 | 39 | Похоже, что с python3.9 сборка через cython замедляет работу. 40 | 41 | Возможно cython помогает коду и более старым версиям python работать быстрее. 42 | -------------------------------------------------------------------------------- /dedupsqlfs/fuse/helpers/repr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | def entry_attributes_to_dict(attr): 6 | """ 7 | @param attr: Attributes object 8 | @type attr: llfuse.EntryAttributes 9 | 10 | @return: dict 11 | @rtype: dict 12 | """ 13 | return { 14 | 'attr_timeout': attr.attr_timeout, 15 | 'entry_timeout': attr.entry_timeout, 16 | 'generation': attr.generation, 17 | 'st_atime_ns': attr.st_atime_ns, 18 | 'st_birthtime_ns': attr.st_birthtime_ns, 19 | 'st_blksize': attr.st_blksize, 20 | 'st_blocks': attr.st_blocks, 21 | 'st_ctime_ns': attr.st_ctime_ns, 22 | 'st_gid': attr.st_gid, 23 | 'st_ino': attr.st_ino, 24 | 'st_mode': attr.st_mode, 25 | 'st_mtime_ns': attr.st_mtime_ns, 26 | 'st_nlink': attr.st_nlink, 27 | 'st_rdev': attr.st_rdev, 28 | 'st_size': attr.st_size, 29 | 'st_uid': attr.st_uid 30 | } 31 | 32 | def setattr_fields_to_dict(fields): 33 | """ 34 | @param fields: SetattrFields object 35 | @type fields: llfuse.SetattrFields 36 | 37 | @return: dict 38 | @rtype: dict 39 | """ 40 | return { 41 | 'update_atime': fields.update_atime, 42 | 'update_gid': fields.update_gid, 43 | 'update_mode': fields.update_mode, 44 | 'update_mtime': fields.update_mtime, 45 | 'update_size': fields.update_size, 46 | 'update_uid': fields.update_uid, 47 | } 48 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/backward_references.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Function to find backward reference copies. */ 8 | 9 | #ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_ 10 | #define BROTLI_ENC_BACKWARD_REFERENCES_H_ 11 | 12 | #include "../common/constants.h" 13 | #include "../common/context.h" 14 | #include "../common/dictionary.h" 15 | #include "../common/platform.h" 16 | #include 17 | #include "./command.h" 18 | #include "./hash.h" 19 | #include "./quality.h" 20 | 21 | #if defined(__cplusplus) || defined(c_plusplus) 22 | extern "C" { 23 | #endif 24 | 25 | /* "commands" points to the next output command to write to, "*num_commands" is 26 | initially the total amount of commands output by previous 27 | CreateBackwardReferences calls, and must be incremented by the amount written 28 | by this call. */ 29 | BROTLI_INTERNAL void BrotliCreateBackwardReferences(size_t num_bytes, 30 | size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, 31 | ContextLut literal_context_lut, const BrotliEncoderParams* params, 32 | Hasher* hasher, int* dist_cache, size_t* last_insert_len, 33 | Command* commands, size_t* num_commands, size_t* num_literals); 34 | 35 | #if defined(__cplusplus) || defined(c_plusplus) 36 | } /* extern "C" */ 37 | #endif 38 | 39 | #endif /* BROTLI_ENC_BACKWARD_REFERENCES_H_ */ 40 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20150712001.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2015-07-12 4 | # 5 | # Add two fields to subvol table 6 | # 7 | __author__ = 'sergey' 8 | 9 | __NUMBER__ = 20150712001 10 | 11 | 12 | def run(manager): 13 | """ 14 | :param manager: Database manager 15 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 16 | :return: bool 17 | """ 18 | 19 | try: 20 | table_sv = manager.getTable("subvolume") 21 | """ 22 | :type table_sv: dedupsqlfs.db.sqlite.table.subvolume.TableSubvolume | 23 | dedupsqlfs.db.mysql.table.subvolume.TableSubvolume 24 | """ 25 | 26 | cur = table_sv.getCursor() 27 | 28 | if not table_sv.hasField('stats'): 29 | cur.execute("ALTER TABLE subvolume ADD COLUMN stats TEXT;") 30 | if not table_sv.hasField('stats_at'): 31 | cur.execute("ALTER TABLE subvolume ADD COLUMN stats_at INTEGER;") 32 | 33 | table_sv.commit() 34 | except Exception as e: 35 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 36 | return False 37 | 38 | table_opts = manager.getTable("option") 39 | 40 | table_opts.getCursor() 41 | mignumber = table_opts.get("migration") 42 | if not mignumber: 43 | table_opts.insert("migration", __NUMBER__) 44 | else: 45 | table_opts.update("migration", __NUMBER__) 46 | 47 | table_opts.commit() 48 | 49 | return True 50 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/test/test_sqlite.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sqlite3 as sql 3 | from recordclass import dataobject, make_dataclass 4 | from recordclass.tools.sqlite import dataclass_row_factory 5 | 6 | class DataobjectSqliteTest(unittest.TestCase): 7 | 8 | def test_row_factory_1(self): 9 | class Planet(dataobject): 10 | name:str 11 | radius:int 12 | 13 | con = sql.connect(":memory:") 14 | cur = con.execute("SELECT 'Earth' AS name, 6378 AS radius") 15 | cur.row_factory = dataclass_row_factory(Planet) 16 | row = cur.fetchone() 17 | print(type(row), row) 18 | self.assertEqual(row.name, 'Earth') 19 | self.assertEqual(row.radius, 6378) 20 | self.assertEqual(type(row).__name__, 'Planet') 21 | 22 | def test_row_factory_2(self): 23 | class Planet(dataobject, mapping=True): 24 | name:str 25 | radius:int 26 | 27 | con = sql.connect(":memory:") 28 | cur = con.execute("SELECT 'Earth' AS name, 6378 AS radius") 29 | cur.row_factory = dataclass_row_factory(Planet) 30 | row = cur.fetchone() 31 | print(type(row), row) 32 | self.assertEqual(row['name'], 'Earth') 33 | self.assertEqual(row['radius'], 6378) 34 | self.assertEqual(type(row).__name__, 'Planet') 35 | 36 | def main(): 37 | suite = unittest.TestSuite() 38 | suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(DataobjectSqliteTest)) 39 | return suite 40 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/decompress/zstd_ddict.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | #ifndef ZSTD_DDICT_H 13 | #define ZSTD_DDICT_H 14 | 15 | /*-******************************************************* 16 | * Dependencies 17 | *********************************************************/ 18 | #include "../common/zstd_deps.h" /* size_t */ 19 | #include "../zstd.h" /* ZSTD_DDict, and several public functions */ 20 | 21 | 22 | /*-******************************************************* 23 | * Interface 24 | *********************************************************/ 25 | 26 | /* note: several prototypes are already published in `zstd.h` : 27 | * ZSTD_createDDict() 28 | * ZSTD_createDDict_byReference() 29 | * ZSTD_createDDict_advanced() 30 | * ZSTD_freeDDict() 31 | * ZSTD_initStaticDDict() 32 | * ZSTD_sizeof_DDict() 33 | * ZSTD_estimateDDictSize() 34 | * ZSTD_getDictID_fromDict() 35 | */ 36 | 37 | const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); 38 | size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); 39 | 40 | void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); 41 | 42 | 43 | 44 | #endif /* ZSTD_DDICT_H */ 45 | -------------------------------------------------------------------------------- /tests/cleanupplan/test-dates.py: -------------------------------------------------------------------------------- 1 | #/usr/bin/env python3 2 | 3 | import sys 4 | import os 5 | 6 | dirname = "dedupsqlfs" 7 | 8 | # Figure out the directy which is the prefix 9 | # path-of-current-file/.. 10 | curpath = os.path.abspath( sys.argv[0] ) 11 | if os.path.islink(curpath): 12 | curpath = os.readlink(curpath) 13 | currentdir = os.path.dirname( curpath ) 14 | basedir = os.path.abspath( os.path.join( currentdir, "..", ".." ) ) 15 | 16 | sys.path.insert( 0, basedir ) 17 | os.chdir(basedir) 18 | 19 | from pprint import pprint 20 | from datetime import datetime, timedelta 21 | from dedupsqlfs.dt import CleanUpPlan 22 | 23 | daysToTest = 800 24 | 25 | today = datetime.now() - timedelta(days=daysToTest) 26 | dates = [] 27 | 28 | # Days to test 29 | for x in range(1, daysToTest): 30 | 31 | dt = timedelta(days=1) 32 | 33 | today += dt 34 | 35 | print("\nToday: %s" % today) 36 | 37 | dates.append(today) 38 | 39 | cleanUp = CleanUpPlan(today) 40 | cleanUp.setCleanUpPlanDaily(14) 41 | cleanUp.setCleanUpPlanWeekly(8) 42 | cleanUp.setCleanUpPlanMonthly(6) 43 | cleanUp.setCleanUpPlanYearly(2) 44 | 45 | dates.sort() 46 | 47 | print("Dates to clean:") 48 | pprint(dates) 49 | 50 | cleanUp.setDates(dates) 51 | 52 | cleanedDates = cleanUp.getCleanedUpList() 53 | 54 | print("Cleaned dates:") 55 | pprint(cleanedDates) 56 | 57 | removedDates = cleanUp.getRemovedList() 58 | 59 | print("Removed dates:") 60 | pprint(removedDates) 61 | 62 | dates = cleanedDates 63 | 64 | print("") 65 | 66 | -------------------------------------------------------------------------------- /lib-dynload/__lzo/NEWS: -------------------------------------------------------------------------------- 1 | ======================================================================= 2 | User visible changes for Python-LZO 3 | ======================================================================= 4 | 5 | Changes in 1.15 (XX Jan 2022) 6 | * Remove python 2.x support. 7 | * Update to PEP517 compliance. 8 | * Migrate CI to github actions. 9 | * Provide Windows Wheels. 10 | * Update windows/macos support to lzo 2.10 11 | 12 | Changes in 1.14 (26 Dec 2021) 13 | * 1.13 was accidently not python2.7 compatible in packaging. Try again. 14 | 15 | Changes in 1.13 (26 Dec 2021) 16 | * Add python2 support statement 17 | * Fix 32bit int limitations 18 | 19 | Changes in 1.12 (23 May 2018) 20 | * Update Windows support to lzo 2.0x. 21 | * Start Windows CI testing. 22 | * Adjust setup.py for jhbuild compatibility. 23 | 24 | Changes in 1.11 (25 Jan 2016) 25 | * Fix a versioning problem in the 1.10 release. 26 | * Fiddle with travis testing. 27 | 28 | Changes in 1.10 (20 Jan 2016) 29 | * Make Binary header metadata optional. 30 | 31 | Changes in 1.09 (19 Nov 2015) 32 | * Add CI testing. 33 | * Update with Python 3 support. 34 | 35 | Changes in 1.08 (17 Jul 2002) 36 | * build system updated to use distutils 37 | * all Python versions supported, tested with 1.5.2 up to 2.2.1 38 | * requires LZO library v1.08 or newer 39 | * [info: version jumped from 1.00 to 1.08 to match the LZO version] 40 | 41 | Changes in 1.00 (21 Aug 1998) 42 | * first public release - requires LZO library v1.03 or newer 43 | -------------------------------------------------------------------------------- /lib-dynload/brotli/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Want to contribute? Great! First, read this page (including the small print at 2 | the end). 3 | 4 | ### Before you contribute 5 | Before we can use your code, you must sign the 6 | [Google Individual Contributor License Agreement] 7 | (https://cla.developers.google.com/about/google-individual) 8 | (CLA), which you can do online. The CLA is necessary mainly because you own the 9 | copyright to your changes, even after your contribution becomes part of our 10 | codebase, so we need your permission to use and distribute your code. We also 11 | need to be sure of various other things—for instance that you'll tell us if you 12 | know that your code infringes on other people's patents. You don't have to sign 13 | the CLA until after you've submitted your code for review and a member has 14 | approved it, but you must do it before we can put your code into our codebase. 15 | Before you start working on a larger contribution, you should get in touch with 16 | us first through the issue tracker with your idea so that we can help out and 17 | possibly guide you. Coordinating up front makes it much easier to avoid 18 | frustration later on. 19 | 20 | ### Code reviews 21 | All submissions, including submissions by project members, require review. We 22 | use Github pull requests for this purpose. 23 | 24 | ### The small print 25 | Contributions made by corporations are covered by a different agreement than 26 | the one above, the [Software Grant and Corporate Contributor License Agreement] 27 | (https://cla.developers.google.com/about/google-corporate). 28 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | TODO list 2 | ========== 3 | 4 | Here are some things on to-do list, in no particular order: 5 | 6 | * Try to store blocks in several database files or tables. Something 7 | like partitioning. Hide it in Block storage class. 8 | 9 | * Implement creation of subvolumes, snapshots on-the-fly via commands through 10 | socket interface. And other commands too. 11 | 12 | * Automatically switch to a larger block size to reduce the overhead for files 13 | that rarely change after being created (like >= 100MB video files :-) 14 | Or use defragmentation tool for that. 15 | Block size must be stored for every file if it differs from default. 16 | 17 | * Implement adaptive compression for different file types: 18 | - no compression for audio/video/images/archives - *.mp3/*.mp4/*.jpg/*.zip 19 | - fast compression for some types - *.wav/*.pdf 20 | - best for texts and some other types - *.txt/*.html/*.bmp 21 | There must be variants for pairs - types:compression. Even custom compression 22 | methods: "*.wav,*.pdf:custom:zlib=fast,snappy". 23 | Compression for file types must be redefinable. 24 | Compression can be overrided for every file. 25 | Need recompression tool. And fsck support for old-new compression options. 26 | 27 | * Use configuration file with defined default compression, hash algo, and other options. 28 | 29 | * Implement rename() independently of link()/unlink() to improve performance? 30 | Need benchmarks. 31 | 32 | * Implement `--verify-reads` option that recalculates hashes when reading to 33 | check for data block corruption? 34 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20170510001.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2017-05-10 4 | # 5 | # - Rename current Lz4 compression into 'lz4r07' 6 | # - Add new 'lz4' compression for 1.7+ 7 | # 8 | __author__ = 'sergey' 9 | 10 | __NUMBER__ = 20170510001 11 | 12 | 13 | def run(manager): 14 | """ 15 | :param manager: Database manager 16 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 17 | :return: bool 18 | """ 19 | 20 | try: 21 | table_ct = manager.getTable("compression_type") 22 | """ 23 | :type table_ct: dedupsqlfs.db.sqlite.table.compression_type.TableCompressionType | 24 | dedupsqlfs.db.mysql.table.compression_type.TableCompressionType 25 | """ 26 | 27 | if not table_ct.find('lz4r07'): 28 | cur = table_ct.getCursor() 29 | 30 | cur.execute("UPDATE compression_type SET value='lz4r07' WHERE value='lz4';") 31 | cur.execute("INSERT INTO compression_type (value) VALUES ('lz4');") 32 | 33 | table_ct.commit() 34 | except Exception as e: 35 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 36 | return False 37 | 38 | table_opts = manager.getTable("option") 39 | 40 | table_opts.getCursor() 41 | mignumber = table_opts.get("migration") 42 | if not mignumber: 43 | table_opts.insert("migration", __NUMBER__) 44 | else: 45 | table_opts.update("migration", __NUMBER__) 46 | 47 | table_opts.commit() 48 | 49 | return True 50 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20160122001.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2016-01-22 4 | # 5 | # - Rename current ZSTD compression into 'zstd036' 6 | # - Add new 'zstd' compression for 0.4+ 7 | # 8 | __author__ = 'sergey' 9 | 10 | __NUMBER__ = 20160122001 11 | 12 | 13 | def run(manager): 14 | """ 15 | :param manager: Database manager 16 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 17 | :return: bool 18 | """ 19 | 20 | try: 21 | table_ct = manager.getTable("compression_type") 22 | """ 23 | :type table_ct: dedupsqlfs.db.sqlite.table.compression_type.TableCompressionType | 24 | dedupsqlfs.db.mysql.table.compression_type.TableCompressionType 25 | """ 26 | 27 | if not table_ct.find('zstd036'): 28 | cur = table_ct.getCursor() 29 | 30 | cur.execute("UPDATE compression_type SET value='zstd036' WHERE value='zstd';") 31 | cur.execute("INSERT INTO compression_type (value) VALUES ('zstd');") 32 | 33 | table_ct.commit() 34 | except Exception as e: 35 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 36 | return False 37 | 38 | table_opts = manager.getTable("option") 39 | 40 | table_opts.getCursor() 41 | mignumber = table_opts.get("migration") 42 | if not mignumber: 43 | table_opts.insert("migration", __NUMBER__) 44 | else: 45 | table_opts.update("migration", __NUMBER__) 46 | 47 | table_opts.commit() 48 | 49 | return True 50 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20160513001.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2016-05-13 4 | # 5 | # - Rename current ZSTD compression into 'zstd047' 6 | # - Add new 'zstd' compression for 0.6+ 7 | # 8 | __author__ = 'sergey' 9 | 10 | __NUMBER__ = 20160513001 11 | 12 | 13 | def run(manager): 14 | """ 15 | :param manager: Database manager 16 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 17 | :return: bool 18 | """ 19 | 20 | try: 21 | table_ct = manager.getTable("compression_type") 22 | """ 23 | :type table_ct: dedupsqlfs.db.sqlite.table.compression_type.TableCompressionType | 24 | dedupsqlfs.db.mysql.table.compression_type.TableCompressionType 25 | """ 26 | 27 | if not table_ct.find('zstd047'): 28 | cur = table_ct.getCursor() 29 | 30 | cur.execute("UPDATE compression_type SET value='zstd047' WHERE value='zstd';") 31 | cur.execute("INSERT INTO compression_type (value) VALUES ('zstd');") 32 | 33 | table_ct.commit() 34 | except Exception as e: 35 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 36 | return False 37 | 38 | table_opts = manager.getTable("option") 39 | 40 | table_opts.getCursor() 41 | mignumber = table_opts.get("migration") 42 | if not mignumber: 43 | table_opts.insert("migration", __NUMBER__) 44 | else: 45 | table_opts.update("migration", __NUMBER__) 46 | 47 | table_opts.commit() 48 | 49 | return True 50 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20160925001.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2016-09-25 4 | # 5 | # - Rename current ZSTD compression into 'zstd061' 6 | # - Add new 'zstd' compression for 1.0+ 7 | # 8 | __author__ = 'sergey' 9 | 10 | __NUMBER__ = 20160925001 11 | 12 | 13 | def run(manager): 14 | """ 15 | :param manager: Database manager 16 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 17 | :return: bool 18 | """ 19 | 20 | try: 21 | table_ct = manager.getTable("compression_type") 22 | """ 23 | :type table_ct: dedupsqlfs.db.sqlite.table.compression_type.TableCompressionType | 24 | dedupsqlfs.db.mysql.table.compression_type.TableCompressionType 25 | """ 26 | 27 | if not table_ct.find('zstd061'): 28 | cur = table_ct.getCursor() 29 | 30 | cur.execute("UPDATE compression_type SET value='zstd061' WHERE value='zstd';") 31 | cur.execute("INSERT INTO compression_type (value) VALUES ('zstd');") 32 | 33 | table_ct.commit() 34 | except Exception as e: 35 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 36 | return False 37 | 38 | table_opts = manager.getTable("option") 39 | 40 | table_opts.getCursor() 41 | mignumber = table_opts.get("migration") 42 | if not mignumber: 43 | table_opts.insert("migration", __NUMBER__) 44 | else: 45 | table_opts.update("migration", __NUMBER__) 46 | 47 | table_opts.commit() 48 | 49 | return True 50 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/src/darwin_compat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2006-2008 Amit Singh/Google Inc. 3 | * Copyright (c) 2011-2013 Benjamin Fleischer 4 | */ 5 | 6 | #ifndef _DARWIN_COMPAT_ 7 | #define _DARWIN_COMPAT_ 8 | 9 | #include 10 | 11 | /* Semaphores */ 12 | 13 | typedef struct darwin_sem { 14 | int id; 15 | union { 16 | struct 17 | { 18 | unsigned int count; 19 | pthread_mutex_t count_lock; 20 | pthread_cond_t count_cond; 21 | } local; 22 | } __data; 23 | } darwin_sem_t; 24 | 25 | #define DARWIN_SEM_VALUE_MAX ((int32_t)32767) 26 | 27 | int darwin_sem_init(darwin_sem_t *sem, int pshared, unsigned int value); 28 | int darwin_sem_destroy(darwin_sem_t *sem); 29 | int darwin_sem_getvalue(darwin_sem_t *sem, unsigned int *value); 30 | int darwin_sem_post(darwin_sem_t *sem); 31 | int darwin_sem_timedwait(darwin_sem_t *sem, const struct timespec *abs_timeout); 32 | int darwin_sem_trywait(darwin_sem_t *sem); 33 | int darwin_sem_wait(darwin_sem_t *sem); 34 | 35 | /* Caller must not include */ 36 | 37 | typedef darwin_sem_t sem_t; 38 | 39 | #define sem_init(s, p, v) darwin_sem_init(s, p, v) 40 | #define sem_destroy(s) darwin_sem_destroy(s) 41 | #define sem_getvalue(s, v) darwin_sem_getvalue(s, v) 42 | #define sem_post(s) darwin_sem_post(s) 43 | #define sem_timedwait(s, t) darwin_sem_timedwait(s, t) 44 | #define sem_trywait(s) darwin_sem_trywait(s) 45 | #define sem_wait(s) darwin_sem_wait(s) 46 | 47 | #define SEM_VALUE_MAX DARWIN_SEM_VALUE_MAX 48 | 49 | #endif /* _DARWIN_COMPAT_ */ 50 | -------------------------------------------------------------------------------- /lib-dynload/_lz4/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2013, Steeve Morin 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of Steeve Morin nor the names of its contributors may be 15 | used to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 22 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | p1, p2 = sys.version_info[:2] 5 | 6 | curpath = os.path.abspath( sys.argv[0] ) 7 | if os.path.islink(curpath): 8 | curpath = os.readlink(curpath) 9 | currentdir = os.path.dirname( curpath ) 10 | 11 | build_dir = os.path.abspath( os.path.join(currentdir, "lib-dynload", "_recordclass", "build") ) 12 | if not os.path.isdir(build_dir): 13 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "lib-dynload", "_recordclass", "build") ) 14 | if not os.path.isdir(build_dir): 15 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "..", "lib-dynload", "_recordclass", "build") ) 16 | 17 | module = None 18 | loaded = False 19 | __version__ = "0.0.0" 20 | 21 | dirs = os.listdir(build_dir) 22 | for d in dirs: 23 | 24 | found = 0 25 | if d.find("lib.") == 0: 26 | found += 1 27 | if d.find("-%s.%s" % (p1, p2)) != -1: 28 | found += 1 29 | # python 3.10+ 30 | if d.find("-cpython-%s%s" % (p1, p2)) != -1: 31 | found += 1 32 | # pypy 33 | if d.find("-pypy%s%s" % (p1, p2)) != -1: 34 | found += 1 35 | if found <= 1: 36 | continue 37 | 38 | svp = sys.path.pop(0) 39 | sys.path.insert(0, os.path.join(build_dir, d) ) 40 | 41 | import importlib 42 | module = importlib.import_module("recordclass") 43 | 44 | loaded = True 45 | 46 | __version__ = "0.23.1" 47 | 48 | sys.path.pop(0) 49 | sys.path.insert(0, svp) 50 | 51 | del importlib 52 | 53 | break 54 | 55 | del p1, p2, d, found, svp 56 | del curpath, currentdir, build_dir, dirs 57 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20151113001.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2015-07-12 4 | # 5 | # - Rename current ZSTD compression into 'zstd001' 6 | # - Add new 'zstd' compression for 0.3+ 7 | # 8 | __author__ = 'sergey' 9 | 10 | __NUMBER__ = 20151113001 11 | 12 | 13 | def run(manager): 14 | """ 15 | :param manager: Database manager 16 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager | 17 | dedupsqlfs.db.mysql.manager.DbManager 18 | :return: bool 19 | """ 20 | 21 | try: 22 | table_ct = manager.getTable("compression_type") 23 | """ 24 | :type table_ct: dedupsqlfs.db.sqlite.table.compression_type.TableCompressionType | 25 | dedupsqlfs.db.mysql.table.compression_type.TableCompressionType 26 | """ 27 | 28 | if not table_ct.find('zstd001'): 29 | cur = table_ct.getCursor() 30 | 31 | cur.execute("UPDATE compression_type SET value='zstd001' WHERE value='zstd';") 32 | cur.execute("INSERT INTO compression_type (value) VALUES ('zstd');") 33 | 34 | table_ct.commit() 35 | except Exception as e: 36 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 37 | return False 38 | 39 | table_opts = manager.getTable("option") 40 | 41 | table_opts.getCursor() 42 | mignumber = table_opts.get("migration") 43 | if not mignumber: 44 | table_opts.insert("migration", __NUMBER__) 45 | else: 46 | table_opts.update("migration", __NUMBER__) 47 | 48 | table_opts.commit() 49 | 50 | return True 51 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/histogram_inc.h: -------------------------------------------------------------------------------- 1 | /* NOLINT(build/header_guard) */ 2 | /* Copyright 2013 Google Inc. All Rights Reserved. 3 | 4 | Distributed under MIT license. 5 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 6 | */ 7 | 8 | /* template parameters: Histogram, DATA_SIZE, DataType */ 9 | 10 | /* A simple container for histograms of data in blocks. */ 11 | 12 | typedef struct FN(Histogram) { 13 | uint32_t data_[DATA_SIZE]; 14 | size_t total_count_; 15 | double bit_cost_; 16 | } FN(Histogram); 17 | 18 | static BROTLI_INLINE void FN(HistogramClear)(FN(Histogram)* self) { 19 | memset(self->data_, 0, sizeof(self->data_)); 20 | self->total_count_ = 0; 21 | self->bit_cost_ = HUGE_VAL; 22 | } 23 | 24 | static BROTLI_INLINE void FN(ClearHistograms)( 25 | FN(Histogram)* array, size_t length) { 26 | size_t i; 27 | for (i = 0; i < length; ++i) FN(HistogramClear)(array + i); 28 | } 29 | 30 | static BROTLI_INLINE void FN(HistogramAdd)(FN(Histogram)* self, size_t val) { 31 | ++self->data_[val]; 32 | ++self->total_count_; 33 | } 34 | 35 | static BROTLI_INLINE void FN(HistogramAddVector)(FN(Histogram)* self, 36 | const DataType* p, size_t n) { 37 | self->total_count_ += n; 38 | n += 1; 39 | while (--n) ++self->data_[*p++]; 40 | } 41 | 42 | static BROTLI_INLINE void FN(HistogramAddHistogram)(FN(Histogram)* self, 43 | const FN(Histogram)* v) { 44 | size_t i; 45 | self->total_count_ += v->total_count_; 46 | for (i = 0; i < DATA_SIZE; ++i) { 47 | self->data_[i] += v->data_[i]; 48 | } 49 | } 50 | 51 | static BROTLI_INLINE size_t FN(HistogramDataSize)(void) { return DATA_SIZE; } 52 | -------------------------------------------------------------------------------- /tests/compression/minimal-length.py: -------------------------------------------------------------------------------- 1 | #/usr/bin/env python3 2 | 3 | import sys 4 | import os 5 | 6 | dirname = "dedupsqlfs" 7 | 8 | # Figure out the directy which is the prefix 9 | # path-of-current-file/.. 10 | curpath = os.path.abspath( sys.argv[0] ) 11 | if os.path.islink(curpath): 12 | curpath = os.readlink(curpath) 13 | currentdir = os.path.dirname( curpath ) 14 | basedir = os.path.abspath( os.path.join( currentdir, "..", ".." ) ) 15 | 16 | dynloaddir = os.path.abspath( os.path.join( basedir, "lib-dynload" ) ) 17 | 18 | sys.path.insert( 0, dynloaddir ) 19 | sys.path.insert( 0, basedir ) 20 | os.chdir(basedir) 21 | 22 | COMPRESSION_SUPPORTED=('lzo', 'zlib', 'deflate', 'bz2', 'xz', 'snappy', 'zstd', 'brotli',) 23 | 24 | CLENGTHS={} 25 | 26 | for l in range(1, 256, 1): 27 | print("Length: %d" % l) 28 | 29 | done = True 30 | 31 | for c in COMPRESSION_SUPPORTED: 32 | 33 | if type(c) is tuple: 34 | _c = c 35 | c = _c[1] 36 | m = __import__(_c[0]) 37 | method = getattr(m, _c[2]) 38 | else: 39 | m = __import__(c) 40 | method = getattr(m, "compress") 41 | 42 | if not c in CLENGTHS: 43 | CLENGTHS[ c ] = { 44 | "done" : False 45 | } 46 | 47 | if CLENGTHS[ c ]["done"]: 48 | continue 49 | else: 50 | done = False 51 | 52 | CLENGTHS[ c ]["length"] = l 53 | 54 | s = b'\0' * l 55 | cs = method(s) 56 | if len(s) > len(cs): 57 | CLENGTHS[ c ]["done"] = True 58 | 59 | 60 | if done: 61 | break 62 | 63 | for c in CLENGTHS: 64 | print("method: %r, min length: %r" % (c, CLENGTHS[c]["length"])) 65 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | p1, p2 = sys.version_info[:2] 5 | 6 | curpath = os.path.abspath( sys.argv[0] ) 7 | if os.path.islink(curpath): 8 | curpath = os.readlink(curpath) 9 | currentdir = os.path.dirname( curpath ) 10 | 11 | build_dir = os.path.abspath( os.path.join(currentdir, "lib-dynload", "_llfuse", "build") ) 12 | if not os.path.isdir(build_dir): 13 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "lib-dynload", "_llfuse", "build") ) 14 | if not os.path.isdir(build_dir): 15 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "..", "lib-dynload", "_llfuse", "build") ) 16 | 17 | module = None 18 | loaded = False 19 | 20 | if os.path.isdir(build_dir): 21 | dirs = os.listdir(build_dir) 22 | else: dirs = [] 23 | 24 | found = 0 25 | for d in dirs: 26 | 27 | found = 0 28 | if d.find("lib.") == 0: 29 | found += 1 30 | if d.find("-%s.%s" % (p1, p2)) != -1: 31 | found += 1 32 | # python 3.10+ 33 | if d.find("-cpython-%s%s" % (p1, p2)) != -1: 34 | found += 1 35 | # pypy 36 | if d.find("-pypy%s%s" % (p1, p2)) != -1: 37 | found += 1 38 | if found <= 1: 39 | continue 40 | 41 | break 42 | 43 | if found: 44 | # print(d) 45 | svp = sys.path.pop(0) 46 | sys.path.insert(0, os.path.join(build_dir, d) ) 47 | 48 | import importlib 49 | try: 50 | module = importlib.import_module("llfuse") 51 | loaded = True 52 | except Excepion as e: 53 | print(e) 54 | loaded = False 55 | pass 56 | 57 | sys.path.pop(0) 58 | sys.path.insert(0, svp) 59 | 60 | del importlib 61 | 62 | del p1, p2, d, found 63 | del curpath, currentdir, build_dir, dirs 64 | -------------------------------------------------------------------------------- /dedupsqlfs/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # Documentation. {{{1 3 | 4 | """ 5 | This Python library implements a file system in user space using FUSE. It's 6 | called DedupFS because the file system's primary feature is deduplication, 7 | which enables it to store virtually unlimited copies of files because data 8 | is only stored once. 9 | 10 | In addition to deduplication the file system also supports transparent 11 | compression using any of the compression methods zlib, bz2, lzma 12 | and optionaly lzo, lz4, snappy, zstd. 13 | 14 | These two properties make the file system ideal for backups: I'm currently 15 | storing 250 GB worth of backups using only 8 GB of disk space. 16 | 17 | The latest version is available at https://github.com/sergey-dryabzhinsky/dedupsqlfs 18 | 19 | DedupFS is licensed under the MIT license. 20 | Copyright 2010 Peter Odding . 21 | Copyright 2013-2020 Sergey Dryabzhinsky . 22 | """ 23 | 24 | __name__ = "DedupSQLfs" 25 | # for fuse mount 26 | __fsname__ = "dedupsqlfs" 27 | __fsversion__ = "3.8" 28 | # Future 1.3 29 | __version__ = "1.2.956-dev" 30 | 31 | # Check the Python version, warn the user if untested. 32 | import sys 33 | 34 | if sys.version_info[0] < 3 or \ 35 | (sys.version_info[0] == 3 and sys.version_info[1] < 4): 36 | msg = "Warning: %s(%s, %s) has been tested on Python 3.4+, while you're running Python %d.%d!\n" 37 | sys.stderr.write(msg % (__name__, __fsversion__, __version__, 38 | int(sys.version_info[0]), int(sys.version_info[1]) 39 | )) 40 | 41 | # Do not abuse GC - we generate alot objects 42 | import gc 43 | if hasattr(gc, "set_threshold"): 44 | gc.set_threshold(100000, 2000, 200) 45 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/cluster.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions for clustering similar histograms together. */ 8 | 9 | #include "./cluster.h" 10 | 11 | #include "../common/platform.h" 12 | #include 13 | #include "./bit_cost.h" /* BrotliPopulationCost */ 14 | #include "./fast_log.h" 15 | #include "./histogram.h" 16 | #include "./memory.h" 17 | 18 | #if defined(__cplusplus) || defined(c_plusplus) 19 | extern "C" { 20 | #endif 21 | 22 | static BROTLI_INLINE BROTLI_BOOL HistogramPairIsLess( 23 | const HistogramPair* p1, const HistogramPair* p2) { 24 | if (p1->cost_diff != p2->cost_diff) { 25 | return TO_BROTLI_BOOL(p1->cost_diff > p2->cost_diff); 26 | } 27 | return TO_BROTLI_BOOL((p1->idx2 - p1->idx1) > (p2->idx2 - p2->idx1)); 28 | } 29 | 30 | /* Returns entropy reduction of the context map when we combine two clusters. */ 31 | static BROTLI_INLINE double ClusterCostDiff(size_t size_a, size_t size_b) { 32 | size_t size_c = size_a + size_b; 33 | return (double)size_a * FastLog2(size_a) + 34 | (double)size_b * FastLog2(size_b) - 35 | (double)size_c * FastLog2(size_c); 36 | } 37 | 38 | #define CODE(X) X 39 | 40 | #define FN(X) X ## Literal 41 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 42 | #undef FN 43 | 44 | #define FN(X) X ## Command 45 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 46 | #undef FN 47 | 48 | #define FN(X) X ## Distance 49 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 50 | #undef FN 51 | 52 | #undef CODE 53 | 54 | #if defined(__cplusplus) || defined(c_plusplus) 55 | } /* extern "C" */ 56 | #endif 57 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/README.rst: -------------------------------------------------------------------------------- 1 | .. 2 | NOTE: We cannot use sophisticated ReST syntax (like 3 | e.g. :file:`foo`) here because this isn't rendered correctly 4 | by PyPi. 5 | 6 | The Python-LLFUSE Module 7 | ======================== 8 | 9 | 10 | .. start-intro 11 | 12 | **Warning - no longer developed!** 13 | 14 | Python-LLFUSE is no longer actively developed and just receiving 15 | community-contributed maintenance to keep it alive for some time. 16 | 17 | Python-LLFUSE is a set of Python bindings for the low level FUSE_ 18 | API. It requires at least FUSE 2.8.0 and supports both Python 2.x and 19 | 3.x. Like FUSE itself, Python-LLFUSE is developed for Linux systems, 20 | but it should be compatible with OS-X, FreeBSD and NetBSD as well. 21 | 22 | Python-LLFUSE releases can be downloaded from PyPi_. The documentation 23 | can be `read online`__ and is also included in the ``doc/html`` 24 | directory of the Python-LLFUSE tarball. 25 | 26 | 27 | Getting Help 28 | ------------ 29 | 30 | Please report any bugs on the `issue tracker`_. For discussion and 31 | questions, please use the general `FUSE mailing list`_. A searchable 32 | `mailing list archive`_ is kindly provided by Gmane_. 33 | 34 | 35 | Contributing 36 | ------------ 37 | 38 | The Python-LLFUSE source code is available on GitHub_. 39 | 40 | 41 | .. __: http://www.rath.org/llfuse-docs/ 42 | .. _FUSE: http://github.com/libfuse/libfuse 43 | .. _FUSE mailing list: https://lists.sourceforge.net/lists/listinfo/fuse-devel 44 | .. _issue tracker: https://github.com/python-llfuse/python-llfuse/issues 45 | .. _mailing list archive: http://dir.gmane.org/gmane.comp.file-systems.fuse.devel 46 | .. _Gmane: http://www.gmane.org/ 47 | .. _PyPi: https://pypi.python.org/pypi/llfuse/ 48 | .. _GitHub: https://github.com/python-llfuse/python-llfuse 49 | -------------------------------------------------------------------------------- /lib-dynload/brotli/python/README.md: -------------------------------------------------------------------------------- 1 | This directory contains the code for the Python `brotli` module, 2 | `bro.py` tool, and roundtrip tests. 3 | 4 | Only Python 2.7+ is supported. 5 | 6 | We provide a `Makefile` to simplify common development commands. 7 | 8 | ### Installation 9 | 10 | If you just want to install the latest release of the Python `brotli` 11 | module, we recommend installing from [PyPI][]: 12 | 13 | $ pip install brotli 14 | 15 | Alternatively, you may install directly from source by running the 16 | following command from this directory: 17 | 18 | $ make install 19 | 20 | ### Development 21 | 22 | You may run the following commands from this directory: 23 | 24 | $ make # Build the module in-place 25 | 26 | $ make test # Test the module 27 | 28 | $ make clean # Remove all temporary files and build output 29 | 30 | If you wish to make the module available while still being 31 | able to edit the source files, you can use the `setuptools` 32 | "[development mode][]": 33 | 34 | $ make develop # Install the module in "development mode" 35 | 36 | ### Code Style 37 | 38 | Brotli's code follows the [Google Python Style Guide][]. To 39 | automatically format your code, first install [YAPF][]: 40 | 41 | $ pip install yapf 42 | 43 | Then, to format all files in the project, you can run: 44 | 45 | $ make fix # Automatically format code 46 | 47 | See the [YAPF usage][] documentation for more information. 48 | 49 | 50 | [PyPI]: https://pypi.org/project/Brotli/ 51 | [development mode]: https://setuptools.readthedocs.io/en/latest/setuptools.html#development-mode 52 | [Google Python Style Guide]: https://google.github.io/styleguide/pyguide.html 53 | [YAPF]: https://github.com/google/yapf 54 | [YAPF usage]: https://github.com/google/yapf#usage 55 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/compress/zstd_compress_literals.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_COMPRESS_LITERALS_H 12 | #define ZSTD_COMPRESS_LITERALS_H 13 | 14 | #include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */ 15 | 16 | 17 | size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); 18 | 19 | /* ZSTD_compressRleLiteralsBlock() : 20 | * Conditions : 21 | * - All bytes in @src are identical 22 | * - dstCapacity >= 4 */ 23 | size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); 24 | 25 | /* ZSTD_compressLiterals(): 26 | * @entropyWorkspace: must be aligned on 4-bytes boundaries 27 | * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE 28 | * @suspectUncompressible: sampling checks, to potentially skip huffman coding 29 | */ 30 | size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity, 31 | const void* src, size_t srcSize, 32 | void* entropyWorkspace, size_t entropyWorkspaceSize, 33 | const ZSTD_hufCTables_t* prevHuf, 34 | ZSTD_hufCTables_t* nextHuf, 35 | ZSTD_strategy strategy, int disableLiteralCompression, 36 | int suspectUncompressible, 37 | int bmi2); 38 | 39 | #endif /* ZSTD_COMPRESS_LITERALS_H */ 40 | -------------------------------------------------------------------------------- /dedupsqlfs/db/sqlite/table/tmp_id_count.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from dedupsqlfs.db.sqlite.table import MemoryTable 6 | 7 | class TableTmpIdCount( MemoryTable ): 8 | 9 | _table_name = "tmp_id_count" 10 | 11 | def create( self ): 12 | c = self.getCursor() 13 | 14 | # Create table 15 | c.execute( 16 | "CREATE TABLE IF NOT EXISTS `%s` (" % self.getName()+ 17 | "id INTEGER PRIMARY KEY,"+ 18 | "cnt INTEGER"+ 19 | ");" 20 | ) 21 | return 22 | 23 | def insert( self, some_id): 24 | """ 25 | :return: int 26 | """ 27 | self.startTimer() 28 | cur = self.getCursor() 29 | 30 | cur.execute("INSERT INTO `%s`(id, cnt) VALUES (?,1)" % self.getName(), ( 31 | some_id, 32 | )) 33 | 34 | item = cur.lastrowid 35 | self.stopTimer('insert') 36 | return item 37 | 38 | def find( self, some_id): 39 | """ 40 | :param inode: int 41 | :return: int 42 | """ 43 | self.startTimer() 44 | cur = self.getCursor() 45 | cur.execute("SELECT `id` FROM `%s` WHERE `id`=?" % self.getName(), ( 46 | some_id, 47 | )) 48 | item = cur.fetchone() 49 | self.stopTimer('find') 50 | return item 51 | 52 | def inc( self, some_id): 53 | """ 54 | :param inode: int 55 | :return: int 56 | """ 57 | self.startTimer() 58 | cur = self.getCursor() 59 | cur.execute("UPDATE `%s` SET `cnt`=`cnt`+1 WHERE `id`=?" % self.getName(), ( 60 | some_id, 61 | )) 62 | item = cur.fetchone() 63 | self.stopTimer('inc') 64 | return item 65 | 66 | pass 67 | -------------------------------------------------------------------------------- /dedupsqlfs/db/sqlite/table/tmp_ids.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from dedupsqlfs.db.sqlite.table import MemoryTable 6 | 7 | class TableTmpIds( MemoryTable ): 8 | 9 | _table_name = "tmp_ids" 10 | 11 | def create( self ): 12 | c = self.getCursor() 13 | 14 | # Create table 15 | c.execute( 16 | "CREATE TABLE IF NOT EXISTS `%s` (" % self.getName()+ 17 | "id INTEGER PRIMARY KEY"+ 18 | ");" 19 | ) 20 | return 21 | 22 | def insert( self, some_id): 23 | """ 24 | :param values: dict | None 25 | :return: int 26 | """ 27 | self.startTimer() 28 | cur = self.getCursor() 29 | 30 | cur.execute("INSERT INTO `%s`(id) VALUES (?)" % self.getName(), ( 31 | some_id, 32 | )) 33 | 34 | item = cur.lastrowid 35 | self.stopTimer('insert') 36 | return item 37 | 38 | def find( self, some_id): 39 | """ 40 | :param inode: int 41 | :return: int 42 | """ 43 | self.startTimer() 44 | cur = self.getCursor() 45 | cur.execute("SELECT `id` FROM `%s` WHERE `id`=?" % self.getName(), ( 46 | some_id, 47 | )) 48 | item = cur.fetchone() 49 | self.stopTimer('find') 50 | return item 51 | 52 | def get_ids_by_ids(self, ids): 53 | self.startTimer() 54 | ret_ids = () 55 | id_str = ",".join(ids) 56 | if id_str: 57 | cur = self.getCursor() 58 | cur.execute("SELECT `id` FROM `%s` " % self.getName()+ 59 | " WHERE `id` IN (%s)" % (id_str,)) 60 | ret_ids = set(str(item["id"]) for item in iter(cur.fetchone,None)) 61 | self.stopTimer('get_ids_by_ids') 62 | return ret_ids 63 | 64 | pass 65 | -------------------------------------------------------------------------------- /dedupsqlfs/db/sqlite/table/_memory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | import platform 6 | 7 | from dedupsqlfs.db.sqlite.table._base import Table 8 | from dedupsqlfs.db.sqlite.row import dict_factory 9 | 10 | class MemoryTable( Table ): 11 | 12 | def _decompress(self): 13 | return True 14 | 15 | def _compress(self): 16 | return True 17 | 18 | def connect( self ): 19 | import sqlite3 20 | 21 | pageSize = self.calcFilePageSize() 22 | 23 | cacheSize = 64*1024*1024 / pageSize 24 | 25 | conn = sqlite3.connect(":memory:") 26 | 27 | conn.row_factory = dict_factory 28 | conn.text_factory = bytes 29 | 30 | # Set journal mode early as possible 31 | conn.execute("PRAGMA journal_mode=OFF") 32 | conn.execute("PRAGMA journal_size_limit=0") 33 | 34 | # Dirty hack again for pypy 35 | isPyPy = platform.python_implementation() == 'PyPy' 36 | if isPyPy: 37 | # Something wrong with sqlite3 module in pypy3 38 | # Works correctly only with autocommit 39 | self.getManager().setAutocommit(True) 40 | 41 | conn.execute("PRAGMA read_uncommitted=ON") 42 | conn.isolation_level = "DEFERRED" 43 | 44 | conn.execute("PRAGMA synchronous=OFF") 45 | 46 | conn.execute("PRAGMA page_size=%i" % pageSize) 47 | conn.execute("PRAGMA cache_size=%i" % cacheSize) 48 | 49 | self._conn = conn 50 | return 51 | 52 | def getFileSize(self): 53 | return 0 54 | 55 | def vacuum(self): 56 | self.startTimer() 57 | self.getConnection().execute("VACUUM") 58 | self.stopTimer("vacuum") 59 | return 0 60 | 61 | def drop(self): 62 | self.startTimer() 63 | self.close() 64 | self.stopTimer("drop") 65 | return self 66 | 67 | pass 68 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20210530001.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2021-05-30 4 | # 5 | # Table `hash_compression_type` don't uses index `hct_type` 6 | # 7 | __author__ = 'sergey' 8 | 9 | __NUMBER__ = 20210530001 10 | 11 | def run(manager): 12 | """ 13 | :param manager: Database manager 14 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 15 | :return: bool 16 | """ 17 | 18 | try: 19 | table_hct = manager.getTable("hash_compression_type") 20 | """ 21 | :type table_hct: dedupsqlfs.db.sqlite.table.TableHashCompressionType | 22 | dedupsqlfs.db.mysql.table.TableHashCompressionType 23 | """ 24 | 25 | manager.getLogger().info("Migration #%s" % (__NUMBER__,)) 26 | 27 | manager.getLogger().info("Migrate `hash_compression_type` table") 28 | 29 | manager.getLogger().info("Drop unused index on `type_id`") 30 | if manager.TYPE == "mysql": 31 | table_hct.dropIndex("type") 32 | if manager.TYPE == "sqlite": 33 | c = table_hct.getCursor() 34 | c.execute("DROP INDEX IF EXISTS `hct_type`") 35 | 36 | table_hct.commit() 37 | table_hct.close() 38 | 39 | 40 | except Exception as e: 41 | import traceback 42 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 43 | manager.getLogger().error("Migration #%s trace:\n%s" % (__NUMBER__, traceback.format_exc(),)) 44 | return False 45 | 46 | table_opts = manager.getTable("option") 47 | 48 | table_opts.getCursor() 49 | 50 | mignumber = table_opts.get("migration") 51 | if not mignumber: 52 | table_opts.insert("migration", __NUMBER__) 53 | else: 54 | table_opts.update("migration", __NUMBER__) 55 | 56 | table_opts.commit() 57 | 58 | return True 59 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/common/allocations.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* This file provides custom allocation primitives 12 | */ 13 | 14 | #define ZSTD_DEPS_NEED_MALLOC 15 | #include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ 16 | 17 | #include "compiler.h" /* MEM_STATIC */ 18 | #define ZSTD_STATIC_LINKING_ONLY 19 | #include "../zstd.h" /* ZSTD_customMem */ 20 | 21 | #ifndef ZSTD_ALLOCATIONS_H 22 | #define ZSTD_ALLOCATIONS_H 23 | 24 | /* custom memory allocation functions */ 25 | 26 | MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) 27 | { 28 | if (customMem.customAlloc) 29 | return customMem.customAlloc(customMem.opaque, size); 30 | return ZSTD_malloc(size); 31 | } 32 | 33 | MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) 34 | { 35 | if (customMem.customAlloc) { 36 | /* calloc implemented as malloc+memset; 37 | * not as efficient as calloc, but next best guess for custom malloc */ 38 | void* const ptr = customMem.customAlloc(customMem.opaque, size); 39 | ZSTD_memset(ptr, 0, size); 40 | return ptr; 41 | } 42 | return ZSTD_calloc(1, size); 43 | } 44 | 45 | MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) 46 | { 47 | if (ptr!=NULL) { 48 | if (customMem.customFree) 49 | customMem.customFree(customMem.opaque, ptr); 50 | else 51 | ZSTD_free(ptr); 52 | } 53 | } 54 | 55 | #endif /* ZSTD_ALLOCATIONS_H */ 56 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/common/zstd_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | 13 | /*-************************************* 14 | * Dependencies 15 | ***************************************/ 16 | #define ZSTD_DEPS_NEED_MALLOC 17 | #include "error_private.h" 18 | #include "zstd_internal.h" 19 | 20 | 21 | /*-**************************************** 22 | * Version 23 | ******************************************/ 24 | unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } 25 | 26 | const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } 27 | 28 | 29 | /*-**************************************** 30 | * ZSTD Error Management 31 | ******************************************/ 32 | #undef ZSTD_isError /* defined within zstd_internal.h */ 33 | /*! ZSTD_isError() : 34 | * tells if a return value is an error code 35 | * symbol is required for external callers */ 36 | unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } 37 | 38 | /*! ZSTD_getErrorName() : 39 | * provides error code string from function result (useful for debugging) */ 40 | const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } 41 | 42 | /*! ZSTD_getError() : 43 | * convert a `size_t` function result into a proper ZSTD_errorCode enum */ 44 | ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } 45 | 46 | /*! ZSTD_getErrorString() : 47 | * provides error code string from enum */ 48 | const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } 49 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/block_splitter.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Block split point selection utilities. */ 8 | 9 | #ifndef BROTLI_ENC_BLOCK_SPLITTER_H_ 10 | #define BROTLI_ENC_BLOCK_SPLITTER_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | #include "./command.h" 15 | #include "./memory.h" 16 | #include "./quality.h" 17 | 18 | #if defined(__cplusplus) || defined(c_plusplus) 19 | extern "C" { 20 | #endif 21 | 22 | typedef struct BlockSplit { 23 | size_t num_types; /* Amount of distinct types */ 24 | size_t num_blocks; /* Amount of values in types and length */ 25 | uint8_t* types; 26 | uint32_t* lengths; 27 | 28 | size_t types_alloc_size; 29 | size_t lengths_alloc_size; 30 | } BlockSplit; 31 | 32 | BROTLI_INTERNAL void BrotliInitBlockSplit(BlockSplit* self); 33 | BROTLI_INTERNAL void BrotliDestroyBlockSplit(MemoryManager* m, 34 | BlockSplit* self); 35 | 36 | BROTLI_INTERNAL void BrotliSplitBlock(MemoryManager* m, 37 | const Command* cmds, 38 | const size_t num_commands, 39 | const uint8_t* data, 40 | const size_t offset, 41 | const size_t mask, 42 | const BrotliEncoderParams* params, 43 | BlockSplit* literal_split, 44 | BlockSplit* insert_and_copy_split, 45 | BlockSplit* dist_split); 46 | 47 | #if defined(__cplusplus) || defined(c_plusplus) 48 | } /* extern "C" */ 49 | #endif 50 | 51 | #endif /* BROTLI_ENC_BLOCK_SPLITTER_H_ */ 52 | -------------------------------------------------------------------------------- /dedupsqlfs/my_formats.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | from math import floor 4 | 5 | def format_timespan(seconds): # {{{1 6 | """ 7 | Format a timespan in seconds as a human-readable string. 8 | """ 9 | result = [] 10 | units = [('day', 60 * 60 * 24), ('hour', 60 * 60), ('minute', 60), ('second', 1)] 11 | for name, size in units: 12 | if seconds >= size: 13 | count = seconds / size 14 | seconds %= size 15 | if seconds < 1: 16 | result.append('%.3f %s%s' % (count, name, floor(count) != 1 and 's' or '')) 17 | else: 18 | result.append('%i %s%s' % (count, name, floor(count) != 1 and 's' or '')) 19 | if result == []: 20 | return 'less than a second' 21 | if len(result) == 1: 22 | return result[0] 23 | else: 24 | return ', '.join(result[:-1]) + ' and ' + result[-1] 25 | 26 | def format_size(nbytes, si=False): 27 | """ 28 | Format a byte count as a human-readable file size. 29 | 30 | @param si: In SI format (/1000) or Binary (/1024) 31 | """ 32 | if si: 33 | return nbytes < 1000 and '%i bytes' % nbytes \ 34 | or nbytes < (1000 ** 2) and __round(nbytes, 1000, 'kB') \ 35 | or nbytes < (1000 ** 3) and __round(nbytes, 1000 ** 2, 'MB') \ 36 | or nbytes < (1000 ** 4) and __round(nbytes, 1000 ** 3, 'GB') \ 37 | or __round(nbytes, 1000 ** 4, 'TB') 38 | 39 | return nbytes < 1024 and '%i bytes' % nbytes \ 40 | or nbytes < (1024 ** 2) and __round(nbytes, 1024, 'KiB') \ 41 | or nbytes < (1024 ** 3) and __round(nbytes, 1024 ** 2, 'MiB') \ 42 | or nbytes < (1024 ** 4) and __round(nbytes, 1024 ** 3, 'GiB') \ 43 | or __round(nbytes, 1024 ** 4, 'TiB') 44 | 45 | def __round(nbytes, divisor, suffix): 46 | _nbytes = float(nbytes) / divisor 47 | if floor(_nbytes) == _nbytes: 48 | return str(int(_nbytes)) + ' ' + suffix 49 | else: 50 | return '%.2f %s' % (_nbytes, suffix) 51 | 52 | # vim: sw=2 sw=2 et 53 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/fast_log.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Utilities for fast computation of logarithms. */ 8 | 9 | #ifndef BROTLI_ENC_FAST_LOG_H_ 10 | #define BROTLI_ENC_FAST_LOG_H_ 11 | 12 | #include 13 | 14 | #include "../common/platform.h" 15 | #include 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | static BROTLI_INLINE uint32_t Log2FloorNonZero(size_t n) { 22 | #if defined(BROTLI_BSR32) 23 | return BROTLI_BSR32((uint32_t)n); 24 | #else 25 | uint32_t result = 0; 26 | while (n >>= 1) result++; 27 | return result; 28 | #endif 29 | } 30 | 31 | #define BROTLI_LOG2_TABLE_SIZE 256 32 | 33 | /* A lookup table for small values of log2(int) to be used in entropy 34 | computation. */ 35 | BROTLI_INTERNAL extern const double kBrotliLog2Table[BROTLI_LOG2_TABLE_SIZE]; 36 | 37 | /* Visual Studio 2012 and Android API levels < 18 do not have the log2() 38 | * function defined, so we use log() and a multiplication instead. */ 39 | #if !defined(BROTLI_HAVE_LOG2) 40 | #if ((defined(_MSC_VER) && _MSC_VER <= 1700) || \ 41 | (defined(__ANDROID_API__) && __ANDROID_API__ < 18)) 42 | #define BROTLI_HAVE_LOG2 0 43 | #else 44 | #define BROTLI_HAVE_LOG2 1 45 | #endif 46 | #endif 47 | 48 | #define LOG_2_INV 1.4426950408889634 49 | 50 | /* Faster logarithm for small integers, with the property of log2(0) == 0. */ 51 | static BROTLI_INLINE double FastLog2(size_t v) { 52 | if (v < BROTLI_LOG2_TABLE_SIZE) { 53 | return kBrotliLog2Table[v]; 54 | } 55 | #if !(BROTLI_HAVE_LOG2) 56 | return log((double)v) * LOG_2_INV; 57 | #else 58 | return log2((double)v); 59 | #endif 60 | } 61 | 62 | #if defined(__cplusplus) || defined(c_plusplus) 63 | } /* extern "C" */ 64 | #endif 65 | 66 | #endif /* BROTLI_ENC_FAST_LOG_H_ */ 67 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/bit_cost.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions to estimate the bit cost of Huffman trees. */ 8 | 9 | #ifndef BROTLI_ENC_BIT_COST_H_ 10 | #define BROTLI_ENC_BIT_COST_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | #include "./fast_log.h" 15 | #include "./histogram.h" 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | static BROTLI_INLINE double ShannonEntropy( 22 | const uint32_t* population, size_t size, size_t* total) { 23 | size_t sum = 0; 24 | double retval = 0; 25 | const uint32_t* population_end = population + size; 26 | size_t p; 27 | if (size & 1) { 28 | goto odd_number_of_elements_left; 29 | } 30 | while (population < population_end) { 31 | p = *population++; 32 | sum += p; 33 | retval -= (double)p * FastLog2(p); 34 | odd_number_of_elements_left: 35 | p = *population++; 36 | sum += p; 37 | retval -= (double)p * FastLog2(p); 38 | } 39 | if (sum) retval += (double)sum * FastLog2(sum); 40 | *total = sum; 41 | return retval; 42 | } 43 | 44 | static BROTLI_INLINE double BitsEntropy( 45 | const uint32_t* population, size_t size) { 46 | size_t sum; 47 | double retval = ShannonEntropy(population, size, &sum); 48 | if (retval < sum) { 49 | /* At least one bit per literal is needed. */ 50 | retval = (double)sum; 51 | } 52 | return retval; 53 | } 54 | 55 | BROTLI_INTERNAL double BrotliPopulationCostLiteral(const HistogramLiteral*); 56 | BROTLI_INTERNAL double BrotliPopulationCostCommand(const HistogramCommand*); 57 | BROTLI_INTERNAL double BrotliPopulationCostDistance(const HistogramDistance*); 58 | 59 | #if defined(__cplusplus) || defined(c_plusplus) 60 | } /* extern "C" */ 61 | #endif 62 | 63 | #endif /* BROTLI_ENC_BIT_COST_H_ */ 64 | -------------------------------------------------------------------------------- /lib-dynload/zstd/zstd/lib/compress/zstd_double_fast.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_DOUBLE_FAST_H 12 | #define ZSTD_DOUBLE_FAST_H 13 | 14 | #include "../common/mem.h" /* U32 */ 15 | #include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ 16 | 17 | #ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR 18 | 19 | void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms, 20 | void const* end, ZSTD_dictTableLoadMethod_e dtlm, 21 | ZSTD_tableFillPurpose_e tfp); 22 | 23 | size_t ZSTD_compressBlock_doubleFast( 24 | ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 25 | void const* src, size_t srcSize); 26 | size_t ZSTD_compressBlock_doubleFast_dictMatchState( 27 | ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 28 | void const* src, size_t srcSize); 29 | size_t ZSTD_compressBlock_doubleFast_extDict( 30 | ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 31 | void const* src, size_t srcSize); 32 | 33 | #define ZSTD_COMPRESSBLOCK_DOUBLEFAST ZSTD_compressBlock_doubleFast 34 | #define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE ZSTD_compressBlock_doubleFast_dictMatchState 35 | #define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT ZSTD_compressBlock_doubleFast_extDict 36 | #else 37 | #define ZSTD_COMPRESSBLOCK_DOUBLEFAST NULL 38 | #define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE NULL 39 | #define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT NULL 40 | #endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */ 41 | 42 | #endif /* ZSTD_DOUBLE_FAST_H */ 43 | -------------------------------------------------------------------------------- /dedupsqlfs/db/mysql/table/hash_owner.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from dedupsqlfs.db.mysql.table import Table 6 | 7 | class TableHashOwner( Table ): 8 | 9 | _table_name = "hash_owner" 10 | 11 | def create( self ): 12 | c = self.getCursor() 13 | 14 | # Create table 15 | c.execute( 16 | "CREATE TABLE IF NOT EXISTS `%s` (" % self.getName()+ 17 | "`hash_id` BIGINT UNSIGNED PRIMARY KEY,"+ 18 | "`uuid` char(32)"+ 19 | ")"+ 20 | self._getCreationAppendString() 21 | ) 22 | self.createIndexIfNotExists("hash", ('hash_id',)) 23 | self.createIndexIfNotExists("owner", ('uuid',)) 24 | return 25 | 26 | def addHashOwner( self, hash_id, uuid): 27 | """ 28 | :param hash_id: int 29 | :param uuid: str 30 | :return: None 31 | """ 32 | self.startTimer() 33 | cur = self.getCursor() 34 | 35 | cur.execute( 36 | "INSERT INTO `%s` " % self.getName()+ 37 | " (`hash_id`,`uuid`) VALUES (%(id)s, '%(uuid)s')", 38 | { 39 | "id": hash_id, 40 | "uuid":uuid, 41 | } 42 | ) 43 | 44 | self.stopTimer('addHashOwner') 45 | return None 46 | 47 | def hashHasOwner( self, hash_id, uuid): 48 | """ 49 | :param hash_id: int 50 | :param uuid: str 51 | :return: int|null 52 | """ 53 | self.startTimer() 54 | cur = self.getCursor() 55 | cur.execute( 56 | "SELECT COUNT(1) AS `cnt` FROM `%s` " % self.getName()+ 57 | " WHERE `hash_id`=%(id)s AND `uuid`='%(uuid)s'", 58 | { 59 | "id": some_id, 60 | "uuid":uuid, 61 | } 62 | ) 63 | item = cur.fetchone() 64 | self.stopTimer('hashHasOwner') 65 | return item['cnt'] 66 | 67 | pass 68 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20220907001.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2022-09-07 4 | # 5 | # - Remove old compression methods 6 | # 7 | __author__ = 'sergey' 8 | 9 | __NUMBER__ = 20220907001 10 | 11 | 12 | def run(manager): 13 | """ 14 | :param manager: Database manager 15 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager | 16 | dedupsqlfs.db.mysql.manager.DbManager 17 | :return: bool 18 | """ 19 | 20 | try: 21 | table_ct = manager.getTable("compression_type") 22 | """ 23 | :type table_ct: dedupsqlfs.db.sqlite.table.compression_type.TableCompressionType | 24 | dedupsqlfs.db.mysql.table.compression_type.TableCompressionType 25 | """ 26 | 27 | cur = table_ct.getCursor() 28 | 29 | cur.execute("DELETE compression_type WHERE value='zstd001';") 30 | cur.execute("DELETE compression_type WHERE value='zstd036';") 31 | cur.execute("DELETE compression_type WHERE value='zstd047';") 32 | cur.execute("DELETE compression_type WHERE value='zstd061';") 33 | cur.execute("DELETE compression_type WHERE value='quicklz';") 34 | cur.execute("DELETE compression_type WHERE value='quicklzf';") 35 | cur.execute("DELETE compression_type WHERE value='quicklzm';") 36 | cur.execute("DELETE compression_type WHERE value='quicklzb';") 37 | cur.execute("DELETE compression_type WHERE value='lz4r07';") 38 | 39 | table_ct.commit() 40 | except Exception as e: 41 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 42 | return False 43 | 44 | table_opts = manager.getTable("option") 45 | 46 | table_opts.getCursor() 47 | mignumber = table_opts.get("migration") 48 | if not mignumber: 49 | table_opts.insert("migration", __NUMBER__) 50 | else: 51 | table_opts.update("migration", __NUMBER__) 52 | 53 | table_opts.commit() 54 | 55 | return True 56 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/Include/pthread.pxd: -------------------------------------------------------------------------------- 1 | ''' 2 | pthreads.pxd 3 | 4 | This file contains Cython definitions for pthread.h 5 | 6 | Copyright © 2015 Nikolaus Rath 7 | 8 | This file is part of Python-LLFUSE. This work may be distributed under 9 | the terms of the GNU LGPL. 10 | ''' 11 | 12 | from posix.signal cimport sigset_t 13 | 14 | cdef extern from "" nogil: 15 | # POSIX says this might be a struct, but CPython (and llfuse) 16 | # rely on it being an integer. 17 | ctypedef int pthread_t 18 | 19 | ctypedef struct pthread_attr_t: 20 | pass 21 | ctypedef struct pthread_mutexattr_t: 22 | pass 23 | ctypedef struct pthread_mutex_t: 24 | pass 25 | 26 | enum: 27 | PTHREAD_CANCEL_ENABLE 28 | PTHREAD_CANCEL_DISABLE 29 | 30 | int pthread_cancel(pthread_t thread) 31 | int pthread_setcancelstate(int state, int *oldstate) 32 | pthread_t pthread_self() 33 | int pthread_sigmask(int how, sigset_t *set, sigset_t *oldset) 34 | int pthread_equal(pthread_t t1, pthread_t t2) 35 | int pthread_create(pthread_t *thread, pthread_attr_t *attr, 36 | void *(*start_routine) (void *), void *arg) 37 | int pthread_join(pthread_t thread, void **retval) 38 | int pthread_kill(pthread_t thread, int sig) 39 | 40 | int pthread_mutex_init(pthread_mutex_t *mutex, pthread_mutexattr_t *mutexattr) 41 | int pthread_mutex_lock(pthread_mutex_t *mutex) 42 | int pthread_mutex_unlock(pthread_mutex_t *mutex) 43 | 44 | # The sem_* functions actually need the semaphone.h header file. However, under 45 | # OS-X we use a compatibility layer that breaks if we include the native 46 | # semaphore.h file. Therefore, we pretend that no header file is required and 47 | # conditionally include semaphore.h in llfuse.h. 48 | cdef extern from * nogil: 49 | ctypedef struct sem_t: 50 | pass 51 | 52 | int sem_init(sem_t *sem, int pshared, unsigned int value) 53 | int sem_post(sem_t *sem) 54 | int sem_wait(sem_t *sem) 55 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/__init__.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) <2015-2024> 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | 24 | from recordclass.datatype import datatype, MATCH 25 | from recordclass._dataobject import dataobject, datastruct, astuple, asdict, clone, update, make, Factory 26 | from recordclass._litelist import litelist, litelist_fromargs 27 | from recordclass._litetuple import litetuple, mutabletuple 28 | from recordclass.recordclass import recordclass 29 | from recordclass.typing import RecordClass 30 | from recordclass.dataclass import make_dataclass, make_structclass, make_class, join_dataclasses 31 | from recordclass.dictclass import make_dictclass 32 | from recordclass.arrayclass import make_arrayclass 33 | from recordclass.adapter import as_dataclass, as_record 34 | 35 | structclass = make_structclass 36 | 37 | from recordclass.about import __version__ 38 | -------------------------------------------------------------------------------- /dedupsqlfs/db/mysql/table/tmp_ids.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from dedupsqlfs.db.mysql.table import Table 6 | 7 | class TableTmpIds( Table ): 8 | 9 | _engine = "MEMORY" 10 | 11 | _table_name = "tmp_ids" 12 | 13 | def create( self ): 14 | c = self.getCursor() 15 | 16 | # Create table 17 | c.execute( 18 | "CREATE TABLE IF NOT EXISTS `%s` (" % self.getName()+ 19 | "`id` BIGINT UNSIGNED PRIMARY KEY"+ 20 | ")"+ 21 | self._getCreationAppendString() 22 | ) 23 | return 24 | 25 | def insert( self, some_id): 26 | """ 27 | :return: int 28 | """ 29 | self.startTimer() 30 | cur = self.getCursor() 31 | 32 | cur.execute( 33 | "INSERT INTO `%s` " % self.getName()+ 34 | " (`id`) VALUES (%(id)s)", 35 | { 36 | "id": some_id, 37 | } 38 | ) 39 | 40 | item = cur.lastrowid 41 | self.stopTimer('insert') 42 | return item 43 | 44 | def find( self, some_id): 45 | """ 46 | :param some_id: int 47 | :return: int 48 | """ 49 | self.startTimer() 50 | cur = self.getCursor() 51 | cur.execute( 52 | "SELECT `id` FROM `%s` " % self.getName()+ 53 | " WHERE `id`=%(id)s", 54 | { 55 | "id": some_id 56 | } 57 | ) 58 | item = cur.fetchone() 59 | self.stopTimer('find') 60 | return item 61 | 62 | def get_ids_by_ids(self, ids): 63 | self.startTimer() 64 | ret_ids = () 65 | id_str = ",".join(ids) 66 | if id_str: 67 | cur = self.getCursor() 68 | cur.execute("SELECT `id` FROM `%s` " % self.getName()+ 69 | " WHERE `id` IN (%s)" % (id_str,)) 70 | ret_ids = set(str(item["id"]) for item in cur) 71 | self.stopTimer('get_ids_by_ids') 72 | return ret_ids 73 | 74 | pass 75 | -------------------------------------------------------------------------------- /dedupsqlfs/get_memory_usage.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | """ 4 | The function in this Python module determines the current memory usage of the 5 | current process by reading the VmSize value from /proc/$pid/status. It's based 6 | on the following entry in the Python cookbook: 7 | http://code.activestate.com/recipes/286222/ 8 | """ 9 | 10 | import os 11 | 12 | _units = { 'KB': 1024, 'MB': 1024**2, 'GB': 1024**3 } 13 | _handle = open('/proc/%d/status' % os.getpid()) 14 | 15 | def get_memory_usage(): 16 | global _proc_status, _units, _handle 17 | try: 18 | for line in _handle: 19 | if line.startswith('VmSize:'): 20 | label, count, unit = line.split() 21 | return int(count) * _units[unit.upper()] 22 | except: 23 | return 0 24 | finally: 25 | _handle.seek(0) 26 | 27 | def get_real_memory_usage(): 28 | global _proc_status, _units, _handle 29 | try: 30 | for line in _handle: 31 | if line.startswith('VmRSS:'): 32 | label, count, unit = line.split() 33 | return int(count) * _units[unit.upper()] 34 | except: 35 | return 0 36 | finally: 37 | _handle.seek(0) 38 | 39 | 40 | if __name__ == '__main__': 41 | from dedupsqlfs.my_formats import format_size 42 | megabyte = 1024**2 43 | counter = megabyte 44 | limit = megabyte * 50 45 | memory = [] 46 | old_memory_usage = get_memory_usage() 47 | assert old_memory_usage > 0 48 | while counter < limit: 49 | memory.append('a' * counter) 50 | msg = "I've just allocated %s and get_memory_usage() returns %s (%s more, deviation is %s)" 51 | new_memory_usage = get_memory_usage() 52 | difference = new_memory_usage - old_memory_usage 53 | deviation = max(difference, counter) - min(difference, counter) 54 | assert deviation < 1024*100 55 | print (msg % (format_size(counter), format_size(new_memory_usage), format_size(difference), format_size(deviation))) 56 | old_memory_usage = new_memory_usage 57 | counter += megabyte 58 | print ("Stopped allocating new strings at %s" % format_size(limit)) 59 | 60 | # vim: ts=2 sw=2 et 61 | -------------------------------------------------------------------------------- /lib-dynload/brotli/expose.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Brotli Authors. All rights reserved. 2 | # 3 | # Distributed under MIT license. 4 | # See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | 6 | """Functions to compress and decompress data using the Brotli library.""" 7 | 8 | import _brotli 9 | 10 | 11 | # The library version. 12 | __version__ = _brotli.__version__ 13 | 14 | # The compression mode. 15 | MODE_GENERIC = _brotli.MODE_GENERIC 16 | MODE_TEXT = _brotli.MODE_TEXT 17 | MODE_FONT = _brotli.MODE_FONT 18 | 19 | # The Compressor object. 20 | Compressor = _brotli.Compressor 21 | 22 | # The Decompressor object. 23 | Decompressor = _brotli.Decompressor 24 | 25 | # Compress a byte string. 26 | def compress(string, mode=MODE_GENERIC, quality=1, lgwin=16, lgblock=0): 27 | """Compress a byte string. 28 | 29 | Args: 30 | string (bytes): The input data. 31 | mode (int, optional): The compression mode can be MODE_GENERIC (default), 32 | MODE_TEXT (for UTF-8 format text input) or MODE_FONT (for WOFF 2.0). 33 | quality (int, optional): Controls the compression-speed vs compression- 34 | density tradeoff. The higher the quality, the slower the compression. 35 | Range is 0 to 11. Defaults to 1. 36 | lgwin (int, optional): Base 2 logarithm of the sliding window size. Range 37 | is 10 to 24. Defaults to 16. 38 | lgblock (int, optional): Base 2 logarithm of the maximum input block size. 39 | Range is 16 to 24. If set to 0, the value will be set based on the 40 | quality. Defaults to 0. 41 | 42 | Returns: 43 | The compressed byte string. 44 | 45 | Raises: 46 | brotli.error: If arguments are invalid, or compressor fails. 47 | """ 48 | compressor = Compressor(mode=mode, quality=quality, lgwin=lgwin, 49 | lgblock=lgblock) 50 | return compressor.process(string) + compressor.finish() 51 | 52 | # Decompress a compressed byte string. 53 | decompress = _brotli.decompress 54 | 55 | # Raised if compression or decompression fails. 56 | error = _brotli.error 57 | -------------------------------------------------------------------------------- /dedupsqlfs/lib/cache/_base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | """ 3 | @author Sergey Dryabzhinsky 4 | """ 5 | 6 | from time import time 7 | 8 | class TimedCache(object): 9 | """ 10 | Cache storage with timers 11 | """ 12 | 13 | _enable_timers = True 14 | 15 | def __init__(self): 16 | self._time_spent = {} 17 | self._op_count = {} 18 | pass 19 | 20 | def setEnableTimers(self, flag=True): 21 | self._enable_timers = flag is True 22 | return self 23 | 24 | def getOperationsCount(self): 25 | return self._op_count 26 | 27 | def getAllOperationsCount(self): 28 | s = 0 29 | if not self._enable_timers: 30 | return s 31 | for op, c in self._op_count.items(): 32 | s += c 33 | return s 34 | 35 | def incOperationsCount(self, op): 36 | if not self._enable_timers: 37 | return self 38 | if not (op in self._op_count): 39 | self._op_count[ op ] = 0 40 | self._op_count[ op ] += 1 41 | return self 42 | 43 | def getTimeSpent(self): 44 | return self._time_spent 45 | 46 | def getAllTimeSpent(self): 47 | s = 0 48 | if not self._enable_timers: 49 | return s 50 | for op, t in self._time_spent.items(): 51 | s += t 52 | return s 53 | 54 | def incOperationsTimeSpent(self, op, start_time): 55 | if not self._enable_timers: 56 | return self 57 | if not (op in self._time_spent): 58 | self._time_spent[ op ] = 0 59 | self._time_spent[ op ] += time() - start_time 60 | return self 61 | 62 | def startTimer(self): 63 | if not self._enable_timers: 64 | return self 65 | self._last_time = time() 66 | return self 67 | 68 | def stopTimer(self, op): 69 | if not self._enable_timers: 70 | return self 71 | 72 | self.incOperationsCount(op) 73 | self.incOperationsTimeSpent(op, self._last_time) 74 | 75 | self._last_time = None 76 | return self 77 | -------------------------------------------------------------------------------- /lib-dynload/brotli/python/brotli.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Brotli Authors. All rights reserved. 2 | # 3 | # Distributed under MIT license. 4 | # See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | 6 | """Functions to compress and decompress data using the Brotli library.""" 7 | 8 | import _brotli 9 | 10 | 11 | # The library version. 12 | __version__ = _brotli.__version__ 13 | 14 | # The compression mode. 15 | MODE_GENERIC = _brotli.MODE_GENERIC 16 | MODE_TEXT = _brotli.MODE_TEXT 17 | MODE_FONT = _brotli.MODE_FONT 18 | 19 | # The Compressor object. 20 | Compressor = _brotli.Compressor 21 | 22 | # The Decompressor object. 23 | Decompressor = _brotli.Decompressor 24 | 25 | # Compress a byte string. 26 | def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0): 27 | """Compress a byte string. 28 | 29 | Args: 30 | string (bytes): The input data. 31 | mode (int, optional): The compression mode can be MODE_GENERIC (default), 32 | MODE_TEXT (for UTF-8 format text input) or MODE_FONT (for WOFF 2.0). 33 | quality (int, optional): Controls the compression-speed vs compression- 34 | density tradeoff. The higher the quality, the slower the compression. 35 | Range is 0 to 11. Defaults to 11. 36 | lgwin (int, optional): Base 2 logarithm of the sliding window size. Range 37 | is 10 to 24. Defaults to 22. 38 | lgblock (int, optional): Base 2 logarithm of the maximum input block size. 39 | Range is 16 to 24. If set to 0, the value will be set based on the 40 | quality. Defaults to 0. 41 | 42 | Returns: 43 | The compressed byte string. 44 | 45 | Raises: 46 | brotli.error: If arguments are invalid, or compressor fails. 47 | """ 48 | compressor = Compressor(mode=mode, quality=quality, lgwin=lgwin, 49 | lgblock=lgblock) 50 | return compressor.process(string) + compressor.finish() 51 | 52 | # Decompress a compressed byte string. 53 | decompress = _brotli.decompress 54 | 55 | # Raised if compression or decompression fails. 56 | error = _brotli.error 57 | -------------------------------------------------------------------------------- /docs/benchmarks/2021-04-27_simple_speed_bench_1.2.949.ru.md: -------------------------------------------------------------------------------- 1 | # Простые тесты скорости работы DedupSQLfs v1.2.949 2 | 3 | Распаковка LibreOffice 6.4.7 + 7.1.2 4 | 5 | Включено сжатие: zstd:1 + brotli:0 6 | 7 | Создание одного снимка FS 8 | 9 | Проверка работы движков: 10 | 11 | * sqlite 3.34.1 12 | * mariadb 10.5 + myisam 13 | * mariadb 10.5 + innodb + сжатие страниц zlib:1 14 | * mariadb 10.5 + tokudb + сжатие small 15 | * mariadb 10.5 + rocksdb - сжатие по-умолчанию 16 | 17 | CPU: i5-6600K @ 3.50GHz 18 | 19 | System: Ubuntu 16.04 amd64 20 | 21 | Kernel: 4.15.0-142-lowlatency 22 | 23 | Python: 3.5.10 24 | 25 | Размер LibreOffice в tar: 26 | 27 | * 6.4.7 - 1133M 28 | * 7.1.2 - 1182M 29 | 30 | Другое: включен autocommit 31 | 32 | ## Тесты 33 | 34 | ### Медленно 35 | 36 | Режим работы: +sync 37 | 38 | | engine | untar 6.4.7: time, size | untar 7.1.2: time, size | snapshot: time, size | 39 | | ------ |:-----------------------:|:-----------------------:|:--------------------:| 40 | | sqlite | 3:39.20, 434M | 2:46.10, 625M | 0:00.34, 662M | 41 | | myisam | 1:56:56, 460M | 1:27:33, 610M | 0:05.82, 689M | 42 | | innodb | 28:25.82, 380M | 24:39.39, 553M | 0:18.79, 591M | 43 | | tokudb | 1:40:25, 528M | 1:13:26, 658M | 0:25.55, 727M | 44 | | rocksdb | 1:27:35, 840M | 1:20:10, 1.3G | 0:17.46, 578M | 45 | 46 | ### Быстрее 47 | 48 | Режим работы: -sync 49 | 50 | | engine | untar 6.4.7: time, size | untar 7.1.2: time, size | snapshot: time, size | 51 | | ------ |:-----------------------:|:-----------------------:|:--------------------:| 52 | | sqlite | 2:00.96, 436M | 2:07.95, 626M | 0:00.34, 662M | 53 | | myisam | 7:01.26, 415M | 6:26.50, 609M | 0:05.58, 688M | 54 | | innodb | 8:02.81, 375M | 7:15.16, 545M | 0:14.01, 587M | 55 | | tokudb | 1:37:18, 528M | 1:17:00, 676M | 0:26.85, 688M | 56 | | rocksdb | 1:36:33, 835M | 1:21:51, 1.3G | 0:16.02, 568M | 57 | 58 | ## Результат 59 | 60 | 1. sqlite - самый быстрый. 61 | 62 | 2. innodb + page compression - может занимать меньше места, чем sqlite. 63 | 64 | 3. rocksdb - компактная упаковка только при остановке сервера. 65 | 66 | 4. tokudb - не оправдал надежд на компактное хранение. 67 | -------------------------------------------------------------------------------- /lib-dynload/quicklz/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | p1, p2 = sys.version_info[:2] 5 | 6 | compress = None 7 | decompress = None 8 | 9 | curpath = os.path.abspath( sys.argv[0] ) 10 | if os.path.islink(curpath): 11 | curpath = os.readlink(curpath) 12 | currentdir = os.path.dirname( curpath ) 13 | 14 | build_dir = os.path.abspath( os.path.join(currentdir, "lib-dynload", "quicklz", "build") ) 15 | if not os.path.isdir(build_dir): 16 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "lib-dynload", "quicklz", "build") ) 17 | if not os.path.isdir(build_dir): 18 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "..", "lib-dynload", "quicklz", "build") ) 19 | 20 | dirs = os.listdir(build_dir) 21 | for d in dirs: 22 | 23 | found = 0 24 | if d.find("lib.") == 0: 25 | found += 1 26 | if d.find("-%s.%s" % (p1, p2)) != -1: 27 | found += 1 28 | # python 3.10+ 29 | if d.find("-cpython-%s%s" % (p1, p2)) != -1: 30 | found += 1 31 | # pypy 32 | if d.find("-pypy%s%s" % (p1, p2)) != -1: 33 | found += 1 34 | if found <= 1: 35 | continue 36 | 37 | sys.path.insert(0, os.path.join(build_dir, d) ) 38 | 39 | import importlib 40 | module = importlib.import_module("qlz") 41 | 42 | def compress_data(raw_data): 43 | """ 44 | This function accepts uncompressed data as an argument, 45 | and returns the compressed form of that data. 46 | """ 47 | state = module.QLZStateCompress() 48 | return module.qlz_compress(raw_data, state) 49 | 50 | def decompress_data(compressed_chunk): 51 | """ 52 | This function accepts a self-contained (not streamed) chunk of data 53 | and returns the decompressed contents. 54 | """ 55 | state = module.QLZStateDecompress() 56 | return module.qlz_decompress(compressed_chunk, state) 57 | 58 | compress = compress_data 59 | decompress = decompress_data 60 | 61 | sys.path.pop(0) 62 | 63 | del importlib, module 64 | 65 | break 66 | 67 | del p1, p2, d, found 68 | del curpath, currentdir, build_dir, dirs 69 | -------------------------------------------------------------------------------- /dedupsqlfs/lib/timers_ops.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from time import time 6 | 7 | class TimersOps( object ): 8 | 9 | _last_time = None 10 | _time_spent = None 11 | _op_count = None 12 | 13 | _log = None 14 | 15 | _enable_timers = False 16 | 17 | def __init__(self, manager): 18 | self._time_spent = {} 19 | self._op_count = {} 20 | pass 21 | 22 | def setEnableTimers(self, flag=True): 23 | self._enable_timers = flag is True 24 | return self 25 | 26 | def getOperationsCount(self): 27 | return self._op_count 28 | 29 | def getAllOperationsCount(self): 30 | s = 0 31 | if not self._enable_timers: 32 | return s 33 | for op, c in self._op_count.items(): 34 | s += c 35 | return s 36 | 37 | def incOperationsCount(self, op): 38 | if not self._enable_timers: 39 | return self 40 | if not (op in self._op_count): 41 | self._op_count[ op ] = 0 42 | self._op_count[ op ] += 1 43 | return self 44 | 45 | def getTimeSpent(self): 46 | return self._time_spent 47 | 48 | def getAllTimeSpent(self): 49 | s = 0 50 | if not self._enable_timers: 51 | return s 52 | for op, t in self._time_spent.items(): 53 | s += t 54 | return s 55 | 56 | def incOperationsTimeSpent(self, op, start_time): 57 | if not self._enable_timers: 58 | return self 59 | if not (op in self._time_spent): 60 | self._time_spent[ op ] = 0 61 | self._time_spent[ op ] += time() - start_time 62 | return self 63 | 64 | def startTimer(self): 65 | if not self._enable_timers: 66 | return self 67 | self._last_time = time() 68 | return self 69 | 70 | def stopTimer(self, op): 71 | if not self._enable_timers: 72 | return self 73 | 74 | self.incOperationsCount(op) 75 | self.incOperationsTimeSpent(op, self._last_time) 76 | 77 | self._last_time = None 78 | return self 79 | 80 | -------------------------------------------------------------------------------- /dedupsqlfs/db/sqlite/table/hash_owner.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | """ 3 | Таблица для дефрагментации 4 | Счетчик используемых hash_id в индексах блоков 5 | Увеличивается при сохранении нового 6 | Уменьшается при удалении 7 | """ 8 | 9 | __author__ = 'sergey' 10 | 11 | from dedupsqlfs.db.sqlite.table import Table 12 | 13 | class TableHashOwner( Table ): 14 | 15 | _table_name = "hash_owner" 16 | 17 | def create( self ): 18 | c = self.getCursor() 19 | 20 | # Create table 21 | c.execute( 22 | "CREATE TABLE IF NOT EXISTS `%s` (" % self.getName()+ 23 | "hash_id INTEGER PRIMARY KEY,"+ 24 | "uuid TEXT"+ 25 | ");" 26 | ) 27 | self.createIndexIfNotExists('owner', ("uuid",)) 28 | return 29 | 30 | def hashHasOwner( self, hash_id, uuid): 31 | """ 32 | :param hash_id: int 33 | :param uuid: str 34 | :return: int 35 | """ 36 | self.startTimer() 37 | cur = self.getCursor() 38 | cur.execute("SELECT COUNT(1) AS `cnt` FROM `%s` WHERE `hash_id`=? AND `uuid`=?" % self.getName(), ( 39 | hash_id, uuid, 40 | )) 41 | item = cur.fetchone() 42 | self.stopTimer('hashHasOwner') 43 | return item['cnt'] 44 | 45 | def addHashOwner( self, hash_id, uuid): 46 | """ 47 | :param hash_id: int 48 | :param uuid: str 49 | :return: None 50 | """ 51 | self.startTimer() 52 | cur = self.getCursor() 53 | cur.execute("INSERT INTO `%s` (`hash_id`,`uuid`) VALUES (?,?)" % self.getName(), ( 54 | hash_id, uuid, 55 | )) 56 | self.stopTimer('addHashOwner') 57 | return None 58 | 59 | def remove_by_ids(self, id_str): 60 | self.startTimer() 61 | count = 0 62 | if id_str: 63 | cur = self.getCursor() 64 | cur.execute("DELETE FROM `%s` " % self.getName()+ 65 | " WHERE `hash_id` IN (%s)" % (id_str,)) 66 | count = cur.rowcount 67 | self.stopTimer('remove_by_ids') 68 | return count 69 | 70 | pass 71 | -------------------------------------------------------------------------------- /lib-dynload/__lzo/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | p1, p2 = sys.version_info[:2] 5 | 6 | curpath = os.path.abspath( sys.argv[0] ) 7 | if os.path.islink(curpath): 8 | curpath = os.readlink(curpath) 9 | currentdir = os.path.dirname( curpath ) 10 | 11 | build_dir = os.path.abspath( os.path.join(currentdir, "lib-dynload", "__lzo", "build") ) 12 | if not os.path.isdir(build_dir): 13 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "lib-dynload", "__lzo", "build") ) 14 | if not os.path.isdir(build_dir): 15 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "..", "lib-dynload", "__lzo", "build") ) 16 | 17 | version = False 18 | if os.path.isdir(build_dir): 19 | dirs = os.listdir(build_dir) 20 | else: dirs = [] 21 | found = 0 22 | for d in dirs: 23 | 24 | found = 0 25 | if d.find("lib.") == 0: 26 | found += 1 27 | if d.find("-%s.%s" % (p1, p2)) != -1: 28 | found += 1 29 | # python 3.10+ 30 | if d.find("-cpython-%s%s" % (p1, p2)) != -1: 31 | found += 1 32 | # pypy 33 | if d.find("-pypy%s%s" % (p1, p2)) != -1: 34 | found += 1 35 | if found <= 1: 36 | continue 37 | 38 | break 39 | 40 | import importlib 41 | svp = sys.path.pop(0) 42 | if found: 43 | sys.path.insert(0, os.path.join(build_dir, d) ) 44 | 45 | try: 46 | module = importlib.import_module("_lzo") 47 | # print(dir(module)) 48 | version = module.LZO_VERSION_STRING 49 | except: 50 | found = False 51 | version = "1.15/exc" 52 | pass 53 | else: 54 | # try system or pypi module 55 | try: 56 | sys.path.insert(0, svp) 57 | module = importlib.import_module("lzo") 58 | 59 | # print(dir(module)) 60 | version = module.LZO_VERSION_STRING 61 | except: 62 | found = False 63 | pass 64 | 65 | if version: 66 | # print(version) 67 | compress = module.compress 68 | decompress = module.decompress 69 | 70 | sys.path.pop(0) 71 | 72 | del importlib 73 | 74 | sys.path.insert(0, svp) 75 | 76 | del p1, p2, svp, found 77 | del curpath, currentdir, build_dir, dirs 78 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/histogram.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Models the histograms of literals, commands and distance codes. */ 8 | 9 | #ifndef BROTLI_ENC_HISTOGRAM_H_ 10 | #define BROTLI_ENC_HISTOGRAM_H_ 11 | 12 | #include /* memset */ 13 | 14 | #include "../common/constants.h" 15 | #include "../common/context.h" 16 | #include "../common/platform.h" 17 | #include 18 | #include "./block_splitter.h" 19 | #include "./command.h" 20 | 21 | #if defined(__cplusplus) || defined(c_plusplus) 22 | extern "C" { 23 | #endif 24 | 25 | /* The distance symbols effectively used by "Large Window Brotli" (32-bit). */ 26 | #define BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS 544 27 | 28 | #define FN(X) X ## Literal 29 | #define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS 30 | #define DataType uint8_t 31 | #include "./histogram_inc.h" /* NOLINT(build/include) */ 32 | #undef DataType 33 | #undef DATA_SIZE 34 | #undef FN 35 | 36 | #define FN(X) X ## Command 37 | #define DataType uint16_t 38 | #define DATA_SIZE BROTLI_NUM_COMMAND_SYMBOLS 39 | #include "./histogram_inc.h" /* NOLINT(build/include) */ 40 | #undef DATA_SIZE 41 | #undef FN 42 | 43 | #define FN(X) X ## Distance 44 | #define DATA_SIZE BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS 45 | #include "./histogram_inc.h" /* NOLINT(build/include) */ 46 | #undef DataType 47 | #undef DATA_SIZE 48 | #undef FN 49 | 50 | BROTLI_INTERNAL void BrotliBuildHistogramsWithContext( 51 | const Command* cmds, const size_t num_commands, 52 | const BlockSplit* literal_split, const BlockSplit* insert_and_copy_split, 53 | const BlockSplit* dist_split, const uint8_t* ringbuffer, size_t pos, 54 | size_t mask, uint8_t prev_byte, uint8_t prev_byte2, 55 | const ContextType* context_modes, HistogramLiteral* literal_histograms, 56 | HistogramCommand* insert_and_copy_histograms, 57 | HistogramDistance* copy_dist_histograms); 58 | 59 | #if defined(__cplusplus) || defined(c_plusplus) 60 | } /* extern "C" */ 61 | #endif 62 | 63 | #endif /* BROTLI_ENC_HISTOGRAM_H_ */ 64 | -------------------------------------------------------------------------------- /dedupsqlfs/db/mysql/table/tmp_id_count.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from dedupsqlfs.db.mysql.table import Table 6 | 7 | class TableTmpIdCount( Table ): 8 | 9 | _engine = "MEMORY" 10 | 11 | _table_name = "tmp_id_count" 12 | 13 | def create( self ): 14 | c = self.getCursor() 15 | 16 | # Create table 17 | c.execute( 18 | "CREATE TABLE IF NOT EXISTS `%s` (" % self.getName()+ 19 | "`id` BIGINT UNSIGNED PRIMARY KEY,"+ 20 | "`cnt` BIGINT UNSIGNED"+ 21 | ")"+ 22 | self._getCreationAppendString() 23 | ) 24 | return 25 | 26 | def insert( self, some_id): 27 | """ 28 | :return: int 29 | """ 30 | self.startTimer() 31 | cur = self.getCursor() 32 | 33 | cur.execute( 34 | "INSERT INTO `%s` " % self.getName()+ 35 | " (`id`,`cnt`) VALUES (%(id)s, 1)", 36 | { 37 | "id": some_id, 38 | } 39 | ) 40 | 41 | item = cur.lastrowid 42 | self.stopTimer('insert') 43 | return item 44 | 45 | def find( self, some_id): 46 | """ 47 | :param some_id: int 48 | :return: int 49 | """ 50 | self.startTimer() 51 | cur = self.getCursor() 52 | cur.execute( 53 | "SELECT `id` FROM `%s` " % self.getName()+ 54 | " WHERE `id`=%(id)s", 55 | { 56 | "id": some_id 57 | } 58 | ) 59 | item = cur.fetchone() 60 | self.stopTimer('find') 61 | return item 62 | 63 | def inc( self, some_id): 64 | """ 65 | :param some_id: int 66 | :return: int 67 | """ 68 | self.startTimer() 69 | cur = self.getCursor() 70 | cur.execute( 71 | "UPDATE FROM `%s` " % self.getName()+ 72 | " SET `cnt`=`cnt`+1 WHERE `id`=%(id)s", 73 | { 74 | "id": some_id 75 | } 76 | ) 77 | item = cur.fetchone() 78 | self.stopTimer('inc') 79 | return item 80 | 81 | pass 82 | -------------------------------------------------------------------------------- /dedupsqlfs/db/sqlite/table/option.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __author__ = 'sergey' 4 | 5 | from dedupsqlfs.db.sqlite.table import Table 6 | 7 | class TableOption( Table ): 8 | 9 | _table_name = "option" 10 | 11 | def create( self ): 12 | c = self.getCursor() 13 | 14 | # Create table 15 | c.execute( 16 | "CREATE TABLE IF NOT EXISTS `%s` (" % self._table_name+ 17 | "name TEXT NOT NULL PRIMARY KEY, "+ 18 | "value TEXT NULL"+ 19 | ")" 20 | ) 21 | return 22 | 23 | def insert( self, name, value ): 24 | self.startTimer() 25 | cur = self.getCursor() 26 | cur.execute("INSERT INTO `%s`(name, value) VALUES (?, ?)" % self._table_name, (name, value)) 27 | item = cur.lastrowid 28 | self.stopTimer('insert') 29 | return item 30 | 31 | def update( self, name, value ): 32 | """ 33 | @return: count updated rows 34 | @rtype: int 35 | """ 36 | self.startTimer() 37 | cur = self.getCursor() 38 | cur.execute("UPDATE `%s` SET value=? WHERE name=?" % self._table_name, (value, name)) 39 | count = cur.rowcount 40 | self.stopTimer('update') 41 | return count 42 | 43 | def get( self, name, raw=False ): 44 | self.startTimer() 45 | cur = self.getCursor() 46 | cur.execute("SELECT value FROM `%s` WHERE name=:name" % self._table_name, 47 | {"name": name} 48 | ) 49 | item = cur.fetchone() 50 | if item: 51 | if raw: 52 | item = item["value"] 53 | else: 54 | item = item["value"].decode() 55 | self.stopTimer('get') 56 | return item 57 | 58 | def getAll( self ): 59 | self.startTimer() 60 | cur = self.getCursor() 61 | cur.execute("SELECT * FROM `%s`" % self._table_name) 62 | items = cur.fetchall() 63 | opts = {} 64 | for item in items: 65 | opts[ item["name"].decode() ] = item["value"].decode() 66 | self.stopTimer('getAll') 67 | return opts 68 | 69 | pass 70 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/Include/fuse_common.pxd: -------------------------------------------------------------------------------- 1 | ''' 2 | fuse_common.pxd 3 | 4 | This file contains Cython definitions for fuse_common.h 5 | 6 | Copyright © 2010 Nikolaus Rath 7 | 8 | This file is part of Python-LLFUSE. This work may be distributed under 9 | the terms of the GNU LGPL. 10 | ''' 11 | 12 | from fuse_opt cimport * 13 | from posix.types cimport off_t 14 | from libc.stdint cimport uint64_t 15 | 16 | # Based on fuse sources, revision tag fuse_2_9_4 17 | cdef extern from * nogil: # fuse_common.h should not be included 18 | 19 | struct fuse_file_info: 20 | int flags 21 | unsigned int direct_io 22 | unsigned int keep_cache 23 | unsigned int nonseekable 24 | uint64_t fh 25 | uint64_t lock_owner 26 | 27 | struct fuse_conn_info: 28 | pass 29 | 30 | struct fuse_session: 31 | pass 32 | 33 | struct fuse_chan: 34 | pass 35 | 36 | fuse_chan *fuse_mount(char *mountpoint, fuse_args *args) 37 | void fuse_unmount(char *mountpoint, fuse_chan *ch) 38 | int fuse_set_signal_handlers(fuse_session *se) 39 | void fuse_remove_signal_handlers(fuse_session *se) 40 | 41 | # fuse_common.h declares these as enums, but they are 42 | # actually flags (i.e., FUSE_BUF_IS_FD|FUSE_BUF_FD_SEEK) 43 | # is a valid variable. Therefore, we declare the type 44 | # as integer instead. 45 | ctypedef int fuse_buf_flags 46 | enum: 47 | FUSE_BUF_IS_FD 48 | FUSE_BUF_FD_SEEK 49 | FUSE_BUF_FD_RETRY 50 | 51 | ctypedef int fuse_buf_copy_flags 52 | enum: 53 | FUSE_BUF_NO_SPLICE 54 | FUSE_BUF_FORCE_SPLICE 55 | FUSE_BUF_SPLICE_MOVE 56 | FUSE_BUF_SPLICE_NONBLOCK 57 | 58 | struct fuse_buf: 59 | size_t size 60 | fuse_buf_flags flags 61 | void *mem 62 | int fd 63 | off_t pos 64 | 65 | struct fuse_bufvec: 66 | size_t count 67 | size_t idx 68 | size_t off 69 | fuse_buf buf[1] 70 | 71 | size_t fuse_buf_size(fuse_bufvec *bufv) 72 | ssize_t fuse_buf_copy(fuse_bufvec *dst, fuse_bufvec *src, 73 | fuse_buf_copy_flags flags) 74 | -------------------------------------------------------------------------------- /lib-dynload/_llfuse/src/macros.c: -------------------------------------------------------------------------------- 1 | /* 2 | macros.c - Pre-processor macros 3 | 4 | Copyright © 2013 Nikolaus Rath 5 | 6 | This file is part of Python-LLFUSE. This work may be distributed under 7 | the terms of the GNU LGPL. 8 | */ 9 | 10 | 11 | /* 12 | * Macros to access the nanosecond attributes in struct stat in a 13 | * platform independent way. Stolen from fuse_misc.h. 14 | */ 15 | 16 | #if PLATFORM == PLATFORM_LINUX 17 | #define GET_ATIME_NS(stbuf) ((stbuf)->st_atim.tv_nsec) 18 | #define GET_CTIME_NS(stbuf) ((stbuf)->st_ctim.tv_nsec) 19 | #define GET_MTIME_NS(stbuf) ((stbuf)->st_mtim.tv_nsec) 20 | #define SET_ATIME_NS(stbuf, val) (stbuf)->st_atim.tv_nsec = (val) 21 | #define SET_CTIME_NS(stbuf, val) (stbuf)->st_ctim.tv_nsec = (val) 22 | #define SET_MTIME_NS(stbuf, val) (stbuf)->st_mtim.tv_nsec = (val) 23 | 24 | #define GET_BIRTHTIME_NS(stbuf) (0) 25 | #define GET_BIRTHTIME(stbuf) (0) 26 | #define SET_BIRTHTIME_NS(stbuf, val) do {} while (0) 27 | #define SET_BIRTHTIME(stbuf, val) do {} while (0) 28 | 29 | /* BSD and OS-X */ 30 | #else 31 | #define GET_BIRTHTIME(stbuf) ((stbuf)->st_birthtime) 32 | #define SET_BIRTHTIME(stbuf, val) ((stbuf)->st_birthtime = (val)) 33 | 34 | #define GET_ATIME_NS(stbuf) ((stbuf)->st_atimespec.tv_nsec) 35 | #define GET_CTIME_NS(stbuf) ((stbuf)->st_ctimespec.tv_nsec) 36 | #define GET_MTIME_NS(stbuf) ((stbuf)->st_mtimespec.tv_nsec) 37 | #define GET_BIRTHTIME_NS(stbuf) ((stbuf)->st_birthtimespec.tv_nsec) 38 | #define SET_ATIME_NS(stbuf, val) ((stbuf)->st_atimespec.tv_nsec = (val)) 39 | #define SET_CTIME_NS(stbuf, val) ((stbuf)->st_ctimespec.tv_nsec = (val)) 40 | #define SET_MTIME_NS(stbuf, val) ((stbuf)->st_mtimespec.tv_nsec = (val)) 41 | #define SET_BIRTHTIME_NS(stbuf, val) ((stbuf)->st_birthtimespec.tv_nsec = (val)) 42 | #endif 43 | 44 | 45 | #if PLATFORM == PLATFORM_LINUX || PLATFORM == PLATFORM_BSD 46 | #define ASSIGN_DARWIN(x,y) 47 | #define ASSIGN_NOT_DARWIN(x,y) ((x) = (y)) 48 | #elif PLATFORM == PLATFORM_DARWIN 49 | #define ASSIGN_DARWIN(x,y) ((x) = (y)) 50 | #define ASSIGN_NOT_DARWIN(x,y) 51 | #else 52 | #error This should not happen 53 | #endif 54 | 55 | /* 56 | * Constants 57 | */ 58 | #define NOTIFY_INVAL_INODE 1 59 | #define NOTIFY_INVAL_ENTRY 2 60 | -------------------------------------------------------------------------------- /lib-dynload/_lz4/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | p1, p2 = sys.version_info[:2] 5 | 6 | curpath = os.path.abspath( sys.argv[0] ) 7 | if os.path.islink(curpath): 8 | curpath = os.readlink(curpath) 9 | currentdir = os.path.dirname( curpath ) 10 | 11 | build_dir = os.path.abspath( os.path.join(currentdir, "lib-dynload", "_lz4", "build") ) 12 | if not os.path.isdir(build_dir): 13 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "lib-dynload", "_lz4", "build") ) 14 | if not os.path.isdir(build_dir): 15 | build_dir = os.path.abspath( os.path.join(currentdir, "..", "..", "lib-dynload", "_lz4", "build") ) 16 | 17 | dirs = [] 18 | if os.path.isdir(build_dir): 19 | dirs = os.listdir(build_dir) 20 | 21 | svp=sys.path[0] 22 | found = 0 23 | for d in dirs: 24 | 25 | found = 0 26 | if d.find("lib.") == 0: 27 | found += 1 28 | if d.find("-%s.%s" % (p1, p2)) != -1: 29 | found += 1 30 | # python 3.10+ 31 | if d.find("-cpython-%s%s" % (p1, p2)) != -1: 32 | found += 1 33 | # pypy 34 | if d.find("-pypy%s%s" % (p1, p2)) != -1: 35 | found += 1 36 | if found <= 1: 37 | continue 38 | 39 | svp = sys.path,pop(0) 40 | sys.path.insert(0, os.path.join(build_dir, d, "_lz4") ) 41 | break 42 | 43 | import importlib 44 | if found: 45 | try: 46 | module = importlib.import_module("_block") 47 | # from _lz4.block import _block as module 48 | # print("bundled") 49 | except Exception as e: 50 | print(e) 51 | module = None 52 | pass 53 | 54 | # try system or pypi 55 | else: # not found 56 | try: 57 | # print(sys.path) 58 | from lz4.block import _block as module 59 | # print("system") 60 | except Exception as e: 61 | print(e) 62 | pass 63 | 64 | # print(dir(module)) 65 | compress = module.compress 66 | decompress = module.decompress 67 | 68 | 69 | def compressHC(data): 70 | return compress(data, mode='high_compression') 71 | 72 | sys.path.pop(0) 73 | 74 | del module, importlib 75 | 76 | sys.path.insert(0, svp) 77 | 78 | del p1, p2, svp, found 79 | del curpath, build_dir, dirs, currentdir 80 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/enc/prefix.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions for encoding of integers into prefix codes the amount of extra 8 | bits, and the actual values of the extra bits. */ 9 | 10 | #ifndef BROTLI_ENC_PREFIX_H_ 11 | #define BROTLI_ENC_PREFIX_H_ 12 | 13 | #include "../common/constants.h" 14 | #include "../common/platform.h" 15 | #include 16 | #include "./fast_log.h" 17 | 18 | #if defined(__cplusplus) || defined(c_plusplus) 19 | extern "C" { 20 | #endif 21 | 22 | /* Here distance_code is an intermediate code, i.e. one of the special codes or 23 | the actual distance increased by BROTLI_NUM_DISTANCE_SHORT_CODES - 1. */ 24 | static BROTLI_INLINE void PrefixEncodeCopyDistance(size_t distance_code, 25 | size_t num_direct_codes, 26 | size_t postfix_bits, 27 | uint16_t* code, 28 | uint32_t* extra_bits) { 29 | if (distance_code < BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_codes) { 30 | *code = (uint16_t)distance_code; 31 | *extra_bits = 0; 32 | return; 33 | } else { 34 | size_t dist = ((size_t)1 << (postfix_bits + 2u)) + 35 | (distance_code - BROTLI_NUM_DISTANCE_SHORT_CODES - num_direct_codes); 36 | size_t bucket = Log2FloorNonZero(dist) - 1; 37 | size_t postfix_mask = (1u << postfix_bits) - 1; 38 | size_t postfix = dist & postfix_mask; 39 | size_t prefix = (dist >> bucket) & 1; 40 | size_t offset = (2 + prefix) << bucket; 41 | size_t nbits = bucket - postfix_bits; 42 | *code = (uint16_t)((nbits << 10) | 43 | (BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_codes + 44 | ((2 * (nbits - 1) + prefix) << postfix_bits) + postfix)); 45 | *extra_bits = (uint32_t)((dist - offset) >> postfix_bits); 46 | } 47 | } 48 | 49 | #if defined(__cplusplus) || defined(c_plusplus) 50 | } /* extern "C" */ 51 | #endif 52 | 53 | #endif /* BROTLI_ENC_PREFIX_H_ */ 54 | -------------------------------------------------------------------------------- /lib-dynload/_recordclass/lib/recordclass/typing/__init__.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | 3 | # Copyright (c) «2015-2022» «Shibzukhov Zaur, szport at gmail dot com» 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software - recordclass library - and associated documentation files 7 | # (the "Software"), to deal in the Software without restriction, including 8 | # without limitation the rights to use, copy, modify, merge, publish, distribute, 9 | # sublicense, and/or sell copies of the Software, and to permit persons to whom 10 | # the Software is furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | from recordclass.datatype import datatype 24 | from recordclass._dataobject import dataobject 25 | from recordclass.recordclass import _add_namedtuple_api 26 | 27 | class recordclassmeta(datatype): 28 | 29 | def __new__(metatype, typename, bases, ns, *, 30 | gc=False, fast_new=True, readonly=False, 31 | use_dict=False, use_weakref=False, hashable=False): 32 | 33 | ns.update(_add_namedtuple_api(typename, readonly)) 34 | 35 | if readonly: 36 | hashable = True 37 | 38 | return datatype.__new__(metatype, typename, bases, ns, 39 | gc=gc, fast_new=fast_new, readonly=readonly, iterable=True, 40 | sequence=True, use_dict=use_dict, use_weakref=use_weakref, hashable=hashable) 41 | 42 | class RecordClass(dataobject, metaclass=recordclassmeta): 43 | pass 44 | 45 | __all__ = 'RecordClass', 46 | 47 | -------------------------------------------------------------------------------- /lib-dynload/brotli/c/common/dictionary.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Collection of static dictionary words. */ 8 | 9 | #ifndef BROTLI_COMMON_DICTIONARY_H_ 10 | #define BROTLI_COMMON_DICTIONARY_H_ 11 | 12 | #include 13 | #include 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | typedef struct BrotliDictionary { 20 | /** 21 | * Number of bits to encode index of dictionary word in a bucket. 22 | * 23 | * Specification: Appendix A. Static Dictionary Data 24 | * 25 | * Words in a dictionary are bucketed by length. 26 | * @c 0 means that there are no words of a given length. 27 | * Dictionary consists of words with length of [4..24] bytes. 28 | * Values at [0..3] and [25..31] indices should not be addressed. 29 | */ 30 | uint8_t size_bits_by_length[32]; 31 | 32 | /* assert(offset[i + 1] == offset[i] + (bits[i] ? (i << bits[i]) : 0)) */ 33 | uint32_t offsets_by_length[32]; 34 | 35 | /* assert(data_size == offsets_by_length[31]) */ 36 | size_t data_size; 37 | 38 | /* Data array is not bound, and should obey to size_bits_by_length values. 39 | Specified size matches default (RFC 7932) dictionary. Its size is 40 | defined by data_size */ 41 | const uint8_t* data; 42 | } BrotliDictionary; 43 | 44 | BROTLI_COMMON_API const BrotliDictionary* BrotliGetDictionary(void); 45 | 46 | /** 47 | * Sets dictionary data. 48 | * 49 | * When dictionary data is already set / present, this method is no-op. 50 | * 51 | * Dictionary data MUST be provided before BrotliGetDictionary is invoked. 52 | * This method is used ONLY in multi-client environment (e.g. C + Java), 53 | * to reduce storage by sharing single dictionary between implementations. 54 | */ 55 | BROTLI_COMMON_API void BrotliSetDictionaryData(const uint8_t* data); 56 | 57 | #define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4 58 | #define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24 59 | 60 | #if defined(__cplusplus) || defined(c_plusplus) 61 | } /* extern "C" */ 62 | #endif 63 | 64 | #endif /* BROTLI_COMMON_DICTIONARY_H_ */ 65 | -------------------------------------------------------------------------------- /lib-dynload/xz/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | search_paths = [] 5 | 6 | curpath = os.path.abspath( sys.argv[0] ) 7 | search_paths.append(curpath) 8 | 9 | if os.path.islink(curpath): 10 | curpath = os.readlink(curpath) 11 | currentdir = os.path.dirname( curpath ) 12 | search_paths.append(currentdir) 13 | 14 | # Evil hack for import not local module 15 | imported = False 16 | 17 | if not imported: 18 | p1, p2 = sys.version_info[:2] 19 | 20 | build_dir = None 21 | for d in search_paths: 22 | build_dir = os.path.abspath( os.path.join(d, "lib-dynload", "xz", "build") ) 23 | if not os.path.isdir(build_dir): 24 | build_dir = os.path.abspath( os.path.join(d, "..", "lib-dynload", "xz", "build") ) 25 | if not os.path.isdir(build_dir): 26 | build_dir = os.path.abspath( os.path.join(d, "..", "..", "lib-dynload", "xz", "build") ) 27 | if os.path.isdir(build_dir): 28 | break 29 | 30 | dirs = [] 31 | if not build_dir or not os.path.isdir(build_dir): 32 | pass 33 | else: 34 | dirs = os.listdir(build_dir) 35 | for d in dirs: 36 | found = 0 37 | if d.find("lib.") == 0: 38 | found += 1 39 | if d.find("-%s.%s" % (p1, p2)) != -1: 40 | found += 1 41 | # python 3.10+ 42 | if d.find("-cpython-%s%s" % (p1, p2)) != -1: 43 | found += 1 44 | # pypy 45 | if d.find("-pypy%s%s" % (p1, p2)) != -1: 46 | found += 1 47 | if found <= 1: 48 | continue 49 | 50 | sys.path.insert(0, os.path.join(build_dir, d) ) 51 | 52 | from .module import compress, decompress 53 | imported = True 54 | 55 | sys.path.pop(0) 56 | 57 | break 58 | 59 | del d 60 | 61 | if not imported: 62 | path = sys.path.pop(0) 63 | try: 64 | import importlib 65 | module = importlib.import_module("lzma") 66 | 67 | compress = module.compress 68 | decompress = module.decompress 69 | 70 | imported = True 71 | except: 72 | pass 73 | 74 | sys.path.insert(0, path) 75 | del path, importlib, module 76 | 77 | del search_paths, p1, p2, dirs, build_dir, curpath, currentdir 78 | -------------------------------------------------------------------------------- /lib-dynload/_lz4/lz4libs/lz4frame_static.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 auto-framing library 3 | Header File for static linking only 4 | Copyright (C) 2011-2020, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 source repository : https://github.com/lz4/lz4 33 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 34 | */ 35 | 36 | #ifndef LZ4FRAME_STATIC_H_0398209384 37 | #define LZ4FRAME_STATIC_H_0398209384 38 | 39 | /* The declarations that formerly were made here have been merged into 40 | * lz4frame.h, protected by the LZ4F_STATIC_LINKING_ONLY macro. Going forward, 41 | * it is recommended to simply include that header directly. 42 | */ 43 | 44 | #define LZ4F_STATIC_LINKING_ONLY 45 | #include "lz4frame.h" 46 | 47 | #endif /* LZ4FRAME_STATIC_H_0398209384 */ 48 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20171103001.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2017-11-03 4 | # 5 | # New statistics for subvolume - root diff in blocks / bytes 6 | # 7 | __author__ = 'sergey' 8 | 9 | __NUMBER__ = 20171103001 10 | 11 | def run(manager): 12 | """ 13 | :param manager: Database manager 14 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 15 | :return: bool 16 | """ 17 | 18 | try: 19 | table_sv = manager.getTable("subvolume") 20 | """ 21 | :type table_sv: dedupsqlfs.db.sqlite.table.subvolume.TableSubvolume | 22 | dedupsqlfs.db.mysql.table.subvolume.TableSubvolume 23 | """ 24 | 25 | from dedupsqlfs.lib.constants import ROOT_SUBVOLUME_NAME 26 | 27 | cur = table_sv.getCursor() 28 | 29 | manager.getLogger().info("Migration #%s" % (__NUMBER__,)) 30 | 31 | if not table_sv.hasField('root_diff'): 32 | if manager.TYPE == "sqlite": 33 | cur.execute("ALTER TABLE `subvolume` ADD COLUMN `root_diff` TEXT;") 34 | if manager.TYPE == "mysql": 35 | cur.execute("ALTER TABLE `subvolume` ADD COLUMN `root_diff` TEXT;") 36 | 37 | if not table_sv.hasField('root_diff_at'): 38 | if manager.TYPE == "sqlite": 39 | cur.execute("ALTER TABLE `subvolume` ADD COLUMN `root_diff_at` INTEGER;") 40 | if manager.TYPE == "mysql": 41 | cur.execute("ALTER TABLE `subvolume` ADD COLUMN `root_diff_at` INT UNSIGNED;") 42 | 43 | table_sv.commit() 44 | table_sv.close() 45 | 46 | 47 | except Exception as e: 48 | import traceback 49 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 50 | manager.getLogger().error("Migration #%s trace:\n%s" % (__NUMBER__, traceback.format_exc(),)) 51 | return False 52 | 53 | table_opts = manager.getTable("option") 54 | 55 | table_opts.getCursor() 56 | 57 | mignumber = table_opts.get("migration") 58 | if not mignumber: 59 | table_opts.insert("migration", __NUMBER__) 60 | else: 61 | table_opts.update("migration", __NUMBER__) 62 | 63 | table_opts.commit() 64 | 65 | return True 66 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20210530004.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2021-05-30 4 | # 5 | # Table `inode_hash_block` : remove indexes `inode` + `hash_inode` 6 | # 7 | __author__ = 'sergey' 8 | 9 | __NUMBER__ = 20210530004 10 | 11 | def run(manager): 12 | """ 13 | :param manager: Database manager 14 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 15 | :return: bool 16 | """ 17 | 18 | try: 19 | table_sv = manager.getTable("subvolume") 20 | """ 21 | :type table_sv: dedupsqlfs.db.sqlite.table.TableSubvolume | 22 | dedupsqlfs.db.mysql.table.TableSubvolume 23 | """ 24 | 25 | manager.getLogger().info("Migration #%s" % (__NUMBER__,)) 26 | 27 | manager.getLogger().info("Migrate `tree` table on all subvolumes") 28 | 29 | cur = table_sv.getCursor(True) 30 | 31 | cur.execute("SELECT id FROM `%s`" % table_sv.getName()) 32 | 33 | for subvol in iter(cur.fetchone, None): 34 | 35 | tname = "inode_hash_block_%d" % (subvol['id'],) 36 | table_ihb = manager.getTable(tname, True) 37 | """ 38 | :type table_t: dedupsqlfs.db.sqlite.table.TableInodeHashBlock | 39 | dedupsqlfs.db.mysql.table.TableInodeHashBlock 40 | """ 41 | 42 | manager.getLogger().info("Drop indexes `inode` and `hash_inode` on `%s`" % tname) 43 | table_ihb.dropIndex('inode') 44 | table_ihb.dropIndex('hash_inode') 45 | 46 | table_ihb.commit() 47 | table_ihb.close() 48 | 49 | except Exception as e: 50 | import traceback 51 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 52 | manager.getLogger().error("Migration #%s trace:\n%s" % (__NUMBER__, traceback.format_exc(),)) 53 | return False 54 | 55 | table_opts = manager.getTable("option") 56 | 57 | table_opts.getCursor() 58 | 59 | mignumber = table_opts.get("migration") 60 | if not mignumber: 61 | table_opts.insert("migration", __NUMBER__) 62 | else: 63 | table_opts.update("migration", __NUMBER__) 64 | 65 | table_opts.commit() 66 | 67 | return True 68 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20180609002.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 002 by 2018-06-09 4 | # 5 | # - Add new 'xz' compression 6 | # - Remove old 'lzma' compression 7 | # - change all old lzma to new xz compression 8 | # 9 | __author__ = 'sergey' 10 | 11 | __NUMBER__ = 20180609002 12 | 13 | 14 | def run(manager): 15 | """ 16 | :param manager: Database manager 17 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 18 | :return: bool 19 | """ 20 | 21 | try: 22 | table_ct = manager.getTable("compression_type") 23 | """ 24 | :type table_ct: dedupsqlfs.db.sqlite.table.compression_type.TableCompressionType | 25 | dedupsqlfs.db.mysql.table.compression_type.TableCompressionType 26 | """ 27 | table_hct = manager.getTable("hash_compression_type") 28 | """ 29 | :type table_ct: dedupsqlfs.db.sqlite.table.compression_type.TableHashCompressionType | 30 | dedupsqlfs.db.mysql.table.compression_type.TableHashCompressionType 31 | """ 32 | 33 | newHashTypeId = table_ct.find('xz') 34 | if not newHashTypeId: 35 | cur = table_ct.getCursor() 36 | cur.execute("INSERT INTO compression_type (value) VALUES ('xz');") 37 | newHashTypeId = cur.lastrowid 38 | table_ct.commit() 39 | 40 | hashTypeId = table_ct.find('lzma') 41 | if hashTypeId: 42 | cur = table_ct.getCursor() 43 | cur.execute("DELETE FROM compression_type WHERE id=?;", (hashTypeId,)) 44 | table_ct.commit() 45 | 46 | cur2 = table_hct.getCursor() 47 | cur2.execute("UPDATE hash_compression_type SET type_id=? WHERE type_id=?", (hashTypeId, newHashTypeId,)) 48 | table_hct.commit() 49 | 50 | except Exception as e: 51 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 52 | return False 53 | 54 | table_opts = manager.getTable("option") 55 | 56 | table_opts.getCursor() 57 | mignumber = table_opts.get("migration") 58 | if not mignumber: 59 | table_opts.insert("migration", __NUMBER__) 60 | else: 61 | table_opts.update("migration", __NUMBER__) 62 | 63 | table_opts.commit() 64 | 65 | return True 66 | -------------------------------------------------------------------------------- /dedupsqlfs/db/migrations/m20210530002.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | # 3 | # DB migration 001 by 2021-05-30 4 | # 5 | # Table `tree` don't uses index `tree_name` 6 | # 7 | __author__ = 'sergey' 8 | 9 | __NUMBER__ = 20210530002 10 | 11 | def run(manager): 12 | """ 13 | :param manager: Database manager 14 | :type manager: dedupsqlfs.db.sqlite.manager.DbManager|dedupsqlfs.db.mysql.manager.DbManager 15 | :return: bool 16 | """ 17 | 18 | try: 19 | table_sv = manager.getTable("subvolume") 20 | """ 21 | :type table_sv: dedupsqlfs.db.sqlite.table.TableSubvolume | 22 | dedupsqlfs.db.mysql.table.TableSubvolume 23 | """ 24 | 25 | manager.getLogger().info("Migration #%s" % (__NUMBER__,)) 26 | 27 | manager.getLogger().info("Migrate `tree` table on all subvolumes") 28 | 29 | cur = table_sv.getCursor(True) 30 | 31 | cur.execute("SELECT id FROM `%s`" % table_sv.getName()) 32 | 33 | for subvol in iter(cur.fetchone, None): 34 | 35 | tname = "tree_%d" % (subvol['id'],) 36 | table_t = manager.getTable(tname, True) 37 | """ 38 | :type table_t: dedupsqlfs.db.sqlite.table.TableTree | 39 | dedupsqlfs.db.mysql.table.TableTree 40 | """ 41 | 42 | manager.getLogger().info("Drop unused index `name` on `%s`" % tname) 43 | 44 | if manager.TYPE == "mysql": 45 | table_t.dropIndex("name") 46 | if manager.TYPE == "sqlite": 47 | c = table_t.getCursor() 48 | c.execute("DROP INDEX IF EXISTS `tree_name`") 49 | 50 | table_t.commit() 51 | table_t.close() 52 | 53 | except Exception as e: 54 | import traceback 55 | manager.getLogger().error("Migration #%s error: %s" % (__NUMBER__, e,)) 56 | manager.getLogger().error("Migration #%s trace:\n%s" % (__NUMBER__, traceback.format_exc(),)) 57 | return False 58 | 59 | table_opts = manager.getTable("option") 60 | 61 | table_opts.getCursor() 62 | 63 | mignumber = table_opts.get("migration") 64 | if not mignumber: 65 | table_opts.insert("migration", __NUMBER__) 66 | else: 67 | table_opts.update("migration", __NUMBER__) 68 | 69 | table_opts.commit() 70 | 71 | return True 72 | --------------------------------------------------------------------------------