├── .dir-locals.el ├── .editorconfig ├── .gitignore ├── .vimrc ├── LICENSE.LGPL2.1 ├── NEWS ├── README.md ├── TODO ├── coccinelle └── fprintf.cocci ├── doc ├── casync.rst ├── conf.py ├── index.rst └── meson.build ├── meson.build ├── meson_options.txt ├── mkosi.build ├── mkosi.default ├── shell-completion └── bash │ ├── casync │ └── meson.build ├── src ├── 75-casync.rules.in ├── cacache.c ├── cacache.h ├── cachunk.c ├── cachunk.h ├── cachunker.c ├── cachunker.h ├── cachunkid.c ├── cachunkid.h ├── cacommon.h ├── cacompression.c ├── cacompression.h ├── cadecoder.c ├── cadecoder.h ├── cadigest.c ├── cadigest.h ├── caencoder.c ├── caencoder.h ├── cafileroot.c ├── cafileroot.h ├── caformat-util.c ├── caformat-util.h ├── caformat.h ├── cafuse.c ├── cafuse.h ├── caindex.c ├── caindex.h ├── calocation.c ├── calocation.h ├── camakebst.c ├── camakebst.h ├── camatch.c ├── camatch.h ├── canametable.c ├── canametable.h ├── canbd.c ├── canbd.h ├── caorigin.c ├── caorigin.h ├── caprotocol-util.c ├── caprotocol-util.h ├── caprotocol.h ├── caremote.c ├── caremote.h ├── caseed.c ├── caseed.h ├── castore.c ├── castore.h ├── casync-http.c ├── casync-tool.c ├── casync.c ├── casync.h ├── cautil.c ├── cautil.h ├── chattr.c ├── chattr.h ├── compressor.c ├── compressor.h ├── copy.c ├── def.h ├── dirent-util.c ├── dirent-util.h ├── fssize.c ├── fssize.h ├── gc.c ├── gc.h ├── gcc-macro.h ├── hash-funcs.c ├── hash-funcs.h ├── hashmap.c ├── hashmap.h ├── log.c ├── log.h ├── mempool.c ├── mempool.h ├── meson.build ├── notify.c ├── notify.h ├── parse-util.c ├── parse-util.h ├── quota-projid.c ├── quota-projid.h ├── realloc-buffer.c ├── realloc-buffer.h ├── reflink.c ├── reflink.h ├── rm-rf.c ├── rm-rf.h ├── set.h ├── signal-handler.c ├── signal-handler.h ├── siphash24.c ├── siphash24.h ├── time-util.c ├── time-util.h ├── udev-util.h ├── util.c └── util.h ├── test-files ├── .gitignore ├── a symlink with umläüts ├── bar ├── file ├── gęślą │ ├── 000000 │ │ └── file │ ├── a │ │ └── file │ ├── b │ │ └── file │ └── file ├── jaźn │ ├── 000000 │ │ └── file │ ├── a │ │ └── file │ ├── b │ │ └── file │ └── file ├── large ├── test-files.sh ├── äöüß └── żażółcić │ ├── 000000 │ └── file │ ├── a │ └── file │ ├── b │ └── file │ └── file ├── test ├── fuzz │ ├── fuzz-compress.c │ ├── fuzz-main.c │ ├── fuzz.h │ └── meson.build ├── http-server.py ├── meson-check-help.sh ├── meson.build ├── notify-wait.c ├── pseudo-ssh ├── report-holes.py ├── semaphore-run ├── test-cache.sh.in ├── test-cachunk.c ├── test-cachunker-histogram.c ├── test-cachunker.c ├── test-cadigest.c ├── test-caencoder.c ├── test-caformat.c ├── test-caindex.c ├── test-calc-digest.c ├── test-calocation.c ├── test-camakebst.c ├── test-camatch.c ├── test-caorigin.c ├── test-casync.c ├── test-cautil.c ├── test-feature-flags.c ├── test-fuse.sh.in ├── test-nbd.sh.in ├── test-script-gzip.sh.in ├── test-script-sha256.sh.in ├── test-script-xz.sh.in ├── test-script.sh.in └── test-util.c └── tools └── oss-fuzz.sh /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ; Sets emacs variables based on mode. 2 | ; A list of (major-mode . ((var1 . value1) (var2 . value2))) 3 | ; Mode can be nil, which gives default values. 4 | 5 | ; NOTE: If you update this file make sure to update .vimrc and .editorconfig, 6 | ; too. 7 | 8 | ((c-mode . ((fill-column . 109) 9 | (c-basic-offset . 8) 10 | (eval . (c-set-offset 'substatement-open 0)) 11 | (eval . (c-set-offset 'statement-case-open 0)) 12 | (eval . (c-set-offset 'case-label 0)) 13 | (eval . (c-set-offset 'arglist-intro '++)) 14 | (eval . (c-set-offset 'arglist-close 0)))) 15 | (nxml-mode . ((nxml-child-indent . 2) 16 | (fill-column . 109))) 17 | (meson-mode . ((meson-indent-basic . 8))) 18 | (sh-mode . ((sh-basic-offset . 8) 19 | (sh-indentation . 8))) 20 | (awk-mode . ((c-basic-offset . 8))) 21 | (nil . ((indent-tabs-mode . nil) 22 | (tab-width . 8) 23 | (fill-column . 79))) 24 | ) 25 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig configuration for systemd 2 | # http://EditorConfig.org 3 | 4 | # NOTE: If you update this file make sure to update .dir-locals.el and .vimrc, 5 | # too. 6 | 7 | # Top-most EditorConfig file 8 | root = true 9 | 10 | # Unix-style newlines with a newline ending every file, utf-8 charset 11 | [*] 12 | end_of_line = lf 13 | insert_final_newline = true 14 | trim_trailing_whitespace = true 15 | charset = utf-8 16 | 17 | # Match config files, set indent to spaces with width of eight 18 | [*.{c,h}] 19 | indent_style = space 20 | indent_size = 8 21 | 22 | [meson.build] 23 | indent_style = space 24 | indent_size = 8 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.cache 2 | *.cache-pre-dev 3 | *.cache-pre-inst 4 | *.plist 5 | *.pyc 6 | *.stamp 7 | *.swp 8 | .gdb_history 9 | *~ 10 | /*.tar.bz2 11 | /*.tar.gz 12 | /*.tar.xz 13 | /GPATH 14 | /GRTAGS 15 | /GSYMS 16 | /GTAGS 17 | /TAGS 18 | /tags 19 | /build*/ 20 | -------------------------------------------------------------------------------- /.vimrc: -------------------------------------------------------------------------------- 1 | " 'set exrc' in ~/.vimrc will read .vimrc from the current directory 2 | " Warning: Enabling exrc is dangerous! You can do nearly everything from a 3 | " vimrc configuration file, including write operations and shell execution. 4 | " You should consider setting 'set secure' as well, which is highly 5 | " recommended! 6 | 7 | " NOTE: If you update this file make sure to update .dir-locals.el and 8 | " .editorconfig, too. 9 | 10 | set tabstop=8 11 | set shiftwidth=8 12 | set expandtab 13 | set makeprg=GCC_COLORS=\ make 14 | set tw=79 15 | au BufRead,BufNewFile *.xml set tw=109 shiftwidth=2 smarttab 16 | au FileType c set tw=109 17 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- 1 | ~~~ casync 2 ~~~ 2 | 3 | - casync now supports retrieving index and chunk data from sftp:// URLs. (In 4 | addition to the existing ftp://, http:// and https:// support). 5 | 6 | - casync will now honour $TMP if it is set, for placing temporary files and 7 | directories. 8 | 9 | - casync now saves/restores basic btrfs subvolume information. (Specifically it 10 | will store whether a directory is a subvolume, and whether it has the 11 | read-only bit set.) Control this metadata option with the new 12 | --with=subvolume/--without=subvolume and 13 | --with=subvolume-ro/--without=subvolume-ro switches. 14 | 15 | - casync now saves/restores SELinux label information. Control this metadata 16 | option with the new --with=selinux/--without=selinux switches. 17 | 18 | - The libgcrypt dependency has been replaced with an OpenSSL dependency, as 19 | that appears to be better supported today, and may be used to generate 20 | SHA512/256 hashes (see below). 21 | 22 | - casync now permits selecting the hash function to use with the new --digest= 23 | option. SHA512/256 is now supported in addition to the old SHA256 algorithm, 24 | which continues to be supported. The new default however is SHA512/256, as it 25 | is substantially faster at otherwise equal properties on today's 64bit 26 | processors. In specific environments SHA256 might perform better, hence both 27 | algorithms remain supported. Index files contain information about the hash 28 | algorithm used, hence automatic compatibility is retained. 29 | 30 | - casync now permits selecting the compression format to use with the new 31 | option --compression=. In addition to the originally reported xz compression, 32 | gzip and zstd compression are now supported, the latter being the new default 33 | as it provides excellent compression at very high speeds. It's OK to mix 34 | chunks compressed with different algorithms in the same store, but of course 35 | clients downloading them need to be new enough to read chunks in non-xz 36 | formats. Note that the file suffix for compressed chunks changed ".xz" → 37 | ".cacnk", as they now may contain either compression, and continuing to use 38 | the ".xz" suffix would be misleading. To retain compatibility with older 39 | casync, the environment variable $CASYNC_COMPRESSED_CHUNK_SUFFIX may be set 40 | to ".xz", to force usage of the old suffix. 41 | 42 | - When extracting archives or archive indexes a subset of the metadata stored 43 | in the archive may now be selected to be replayed, using the usual --with= 44 | and --without= options. For example, if an archive containing full metadata 45 | is extracted with --without=privileged only the unprivileged metadata fields 46 | are extracted (i.e. no file ownership, ACLs, SELinux labels, ...). 47 | 48 | - After completing an operation statistics about downloaded chunks are now 49 | shown. 50 | 51 | - When invoking "casync mkdev" the third parameter may now be an arbitrarily 52 | selected path below /dev which is then created as a symlink to the block 53 | device used, and registered with udev. This means the usual device 54 | enumeration will find the block device under the name picked. Example: 55 | 56 | # casync mkdev /somepath/tomy/index-file.caibx /dev/quux 57 | 58 | This will expose the block image /somepath/tomy/index-file.caibx as /dev/quux. 59 | 60 | Contributions from: David Guibert, enkore, Felipe Sateler, Igor Gnatenko, Jesus 61 | Rodriguez, John Paul Adrian Glaubitz, Lennart Poettering, Martin Pitt, Silvio 62 | Fricke, Zbigniew Jędrzejewski-Szmek 63 | 64 | ~~~ casync 1 ~~~ 65 | 66 | - Initial release 67 | 68 | Contributions from: Daniel Mack, Djalal Harouni, Lennart Poettering, Martin 69 | Pitt, Nikita Puzyryov, Thomas Hindoe Paaboel Andersen, Zbigniew 70 | Jędrzejewski-Szmek 71 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | SHORT-TERM: 2 | * use btrfs reflink with contents checking 3 | * rework how default --with= and --without= are calculated, so that --except stuff doesn't leak into blob index 4 | * whenever we set the timestamp on a file, reread the value set and check if 5 | the required granularity has been available. 6 | 7 | TO MAKE IT USEFUL FOR BACKUPS: 8 | * encryption: aes256 of rotating hash function + HMAC for identifying chunks + individually encrypted chunks 9 | 10 | LATER: 11 | * verify 12 | * diff 13 | * save/restore hardlinks? 14 | * check fs features when restoring 15 | * exclude patterns 16 | * build seed while extracting 17 | * optionally make seeds persistent 18 | * acquire gpg signature along with caidx/caibx/catar 19 | * coalesce index frames sent over protocol 20 | * rework uploading via ssh to use seed instead of cache store for providing chunks to server 21 | * casync-http: parallel http GETs 22 | * casync-http: try all configured stores one after the other before sending MISSING 23 | * add support for compressed index files and archive files 24 | * define mime types for our files 25 | * define http-based url protocol prefix for caibx+caidx 26 | * support accessing base trees through native protocol 27 | * implicitly generate index + chunks when accessing base trees or archives through native protocol 28 | * rework caindex to read multiple chunks per read (use reallocbuffer like elsewhere) 29 | * permit 511 (or 4095?) redundant NUL bytes at the end of archive and index files, so that they could in theory stored on block devices 30 | * optionally import from/export to classic tar ball (and zip?) 31 | * optionally interpret aufs/union mount whiteout files? 32 | * fuse: expose acls and fcaps, selinux, quota projid 33 | * fuse: possibly translate user names on access 34 | * fuse: provide "mount.casync" compat symlink so that people can list casync mounts in /etc/fstab 35 | * encoder: change seeking to be more like decoder's seeking (i.e. delay returned events until the next ca_encoder_step() call) 36 | * rename offset accessor functions (drop the "archive") 37 | * send progress information via sd_notify(), so that people can wrap casync nicely in UIs 38 | * maybe turn "recursive" mode into a numeric value specifying how far to descend? 39 | * make "casync stat" work on a directory with a subpath 40 | * tweak chunker: shift cut to last "marker". 41 | * define sane errors we can show user messages about 42 | * introduce a --best-effort mode when replaying, which means we'll ignore what we can't apply 43 | * when building the cache, also build a seed 44 | * recognize NULL chunks and handle them specially 45 | * make archive digest generation optional 46 | * add "index" digest 47 | * maybe add a concept of "packfiles", to contain multiple chunks in one? 48 | * add libsmbclient backend (so that Lennart can backup to his synology NAS in the easiest way) 49 | * make sure "casync list /etc/fstab" does something useful 50 | * rework CaSeed logic to use CaCache as backend, and then add a new command "casync cache" or so, to explicitly generate a cache/seed 51 | * support blake2 as hashes 52 | * parallelize image generation: when storing chunks in the store do so in a thread 53 | * in "casync stat" output show which flags enable what 54 | * save/restore xfs/ext4 projid 55 | -------------------------------------------------------------------------------- /coccinelle/fprintf.cocci: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | @@ 4 | expression e; 5 | local idexpression r; 6 | expression list s; 7 | @@ 8 | - if (e) { 9 | - fprintf(stderr, s); 10 | - return r; 11 | - } 12 | + if (e) 13 | + return log_error_errno(r, s); 14 | @@ 15 | expression list s; 16 | @@ 17 | - fprintf(stderr, s); 18 | + log_error(s); 19 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # SPDX-License-Identifier: LGPL-2.1+ 4 | # 5 | # casync documentation build configuration file, created by 6 | # sphinx-quickstart on Tue Jun 20 16:46:39 2017. 7 | # 8 | # This file is execfile()d with the current directory set to its 9 | # containing dir. 10 | # 11 | # Note that not all possible configuration values are present in this 12 | # autogenerated file. 13 | # 14 | # All configuration values have a default; values that are commented out 15 | # serve to show the default. 16 | 17 | # If extensions (or modules to document with autodoc) are in another directory, 18 | # add these directories to sys.path here. If the directory is relative to the 19 | # documentation root, use os.path.abspath to make it absolute, like shown here. 20 | # 21 | # import os 22 | # import sys 23 | # sys.path.insert(0, os.path.abspath('.')) 24 | 25 | 26 | # -- General configuration ------------------------------------------------ 27 | 28 | # If your documentation needs a minimal Sphinx version, state it here. 29 | # 30 | # needs_sphinx = '1.0' 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 34 | # ones. 35 | extensions = ['sphinx.ext.todo'] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix(es) of source filenames. 41 | # You can specify multiple suffix as a list of string: 42 | # 43 | # source_suffix = ['.rst', '.md'] 44 | source_suffix = '.rst' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = 'casync' 51 | author = '' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = '1' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '1' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This patterns also effect to html_static_path and html_extra_path 72 | exclude_patterns = [] 73 | 74 | # The name of the Pygments (syntax highlighting) style to use. 75 | pygments_style = 'sphinx' 76 | 77 | # If true, `todo` and `todoList` produce output, else they produce nothing. 78 | todo_include_todos = True 79 | 80 | 81 | # -- Options for HTML output ---------------------------------------------- 82 | 83 | # The theme to use for HTML and HTML Help pages. See the documentation for 84 | # a list of builtin themes. 85 | # 86 | html_theme = 'alabaster' 87 | 88 | # Theme options are theme-specific and customize the look and feel of a theme 89 | # further. For a list of options available for each theme, see the 90 | # documentation. 91 | # 92 | # html_theme_options = {} 93 | 94 | # Add any paths that contain custom static files (such as style sheets) here, 95 | # relative to this directory. They are copied after the builtin static files, 96 | # so a file named "default.css" will overwrite the builtin "default.css". 97 | html_static_path = ['_static'] 98 | 99 | 100 | # -- Options for HTMLHelp output ------------------------------------------ 101 | 102 | # Output file base name for HTML help builder. 103 | htmlhelp_basename = 'casyncdoc' 104 | 105 | # -- Options for manual page output --------------------------------------- 106 | 107 | # One entry per manual page. List of tuples 108 | # (source start file, name, description, authors, manual section). 109 | man_pages = [ 110 | ('casync', 'casync', 'casync Documentation', 111 | [], 1) 112 | ] 113 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. SPDX-License-Identifier: LGPL-2.1+ 2 | .. casync documentation master file, created by 3 | sphinx-quickstart on Tue Jun 20 16:46:39 2017. 4 | 5 | Welcome to casync's documentation! 6 | ================================== 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | :caption: Contents: 11 | 12 | Indices and tables 13 | ================== 14 | 15 | * :ref:`genindex` 16 | * :ref:`modindex` 17 | * :ref:`search` 18 | -------------------------------------------------------------------------------- /doc/meson.build: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: LGPL-2.1+ 2 | 3 | sphinx_sources = [ 4 | 'conf.py', 5 | 'casync.rst', 6 | 'index.rst'] 7 | 8 | man_pages = [ 9 | 'casync.1'] 10 | 11 | mandir1 = join_paths(get_option('mandir'), 'man1') 12 | 13 | if get_option('man') 14 | sphinx_build = find_program('sphinx-build-3', 'sphinx-build') 15 | 16 | custom_target( 17 | 'man', 18 | command : [sphinx_build, 19 | '-b', 'man', 20 | meson.current_source_dir(), 21 | meson.current_build_dir()], 22 | input : sphinx_sources, 23 | output : man_pages, 24 | install : true, 25 | install_dir : mandir1) 26 | endif 27 | -------------------------------------------------------------------------------- /meson_options.txt: -------------------------------------------------------------------------------- 1 | # -*- mode: meson -*- 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | option('fuse', type : 'boolean', value : true, 5 | description : 'build the FUSE integration (requires fuse-devel)') 6 | option('selinux', type : 'boolean', value : true, 7 | description : 'build the SELinux backend (requires libselinux-devel)') 8 | option('udev', type : 'boolean', value : true, 9 | description : 'build the libudev integration (requires libudev-devel)') 10 | option('udevrulesdir', type : 'string', value: '', 11 | description: 'Path where udev rules are installed to (Defaults to udevdir specified in udev.pc)') 12 | option('man', type : 'boolean', value : true, 13 | description : 'build and install man pages (requires sphinx-build') 14 | option('libzstd', type : 'feature', 15 | description : 'link to libzstd') 16 | option('liblzma', type : 'feature', 17 | description : 'link to liblzma (for XZ compression)') 18 | option('libz', type : 'feature', 19 | description : 'link to zlib') 20 | 21 | option('oss-fuzz', type : 'boolean', value : 'false', 22 | description : 'build against oss-fuzz') 23 | option('llvm-fuzz', type : 'boolean', value : 'false', 24 | description : 'build against LLVM libFuzzer') 25 | 26 | option('bashcompletiondir', type : 'string', 27 | description : 'directory for bash completion scripts ["no" disables]') 28 | -------------------------------------------------------------------------------- /mkosi.build: -------------------------------------------------------------------------------- 1 | #!/bin/sh -ex 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | export LC_CTYPE=C.UTF-8 5 | meson build 6 | ninja -C build all 7 | ninja -C build test 8 | ninja -C build install 9 | -------------------------------------------------------------------------------- /mkosi.default: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: LGPL-2.1+ 2 | 3 | [Distribution] 4 | Distribution=fedora 5 | Release=31 6 | 7 | [Output] 8 | Format=raw_gpt 9 | Bootable=yes 10 | 11 | [Partitions] 12 | RootSize=2G 13 | 14 | [Packages] 15 | BuildPackages= 16 | diffutils 17 | fuse-devel 18 | gcc 19 | libacl-devel 20 | libcurl-devel 21 | libzstd-devel 22 | openssl-devel 23 | meson 24 | /usr/bin/sphinx-build 25 | pkgconfig 26 | rsync 27 | xz-devel 28 | Packages= 29 | fuse 30 | -------------------------------------------------------------------------------- /shell-completion/bash/meson.build: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: LGPL-2.1+ 2 | 3 | bashcompletiondir = get_option('bashcompletiondir') 4 | if bashcompletiondir == '' 5 | bash_completion = dependency('bash-completion', required : false) 6 | if bash_completion.found() 7 | bashcompletiondir = bash_completion.get_pkgconfig_variable('completionsdir') 8 | else 9 | bashcompletiondir = join_paths(datadir, 'bash-completion/completions') 10 | endif 11 | endif 12 | 13 | if bashcompletiondir != 'no' 14 | install_data('casync', install_dir : bashcompletiondir) 15 | endif 16 | -------------------------------------------------------------------------------- /src/75-casync.rules.in: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: LGPL-2.1+ 2 | 3 | ACTION=="remove", GOTO="casync_end" 4 | 5 | SUBSYSTEM=="block", KERNEL=="nbd*", IMPORT{program}="@bindir_unquoted@/casync udev %N" 6 | SUBSYSTEM=="block", KERNEL=="nbd*", ENV{CASYNC_NAME}=="?*", SYMLINK+="$env{CASYNC_NAME}" 7 | 8 | LABEL="casync_end" 9 | -------------------------------------------------------------------------------- /src/cacache.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocacachehfoo 4 | #define foocacachehfoo 5 | 6 | #include "cachunkid.h" 7 | #include "calocation.h" 8 | #include "caorigin.h" 9 | #include "util.h" 10 | 11 | /* Implements a cache, that given a location returns a chunk ID plus a matching origin object encoding where the chunk 12 | * is sourced from. This is used to speed up generation of chunks: when we looked at a specific chunk in an earlier 13 | * run, then there's no need to regenerate its hash again as long as the origin inodes haven't changed */ 14 | 15 | typedef struct CaCache CaCache; 16 | 17 | CaCache *ca_cache_new(void); 18 | CaCache *ca_cache_unref(CaCache *c); 19 | CaCache *ca_cache_ref(CaCache *c); 20 | 21 | int ca_cache_set_digest_type(CaCache *c, CaDigestType type); 22 | 23 | int ca_cache_set_fd(CaCache *c, int fd); 24 | int ca_cache_set_path(CaCache *c, const char *path); 25 | 26 | int ca_cache_get(CaCache *c, CaLocation *location, CaChunkID *ret_chunk_id, CaOrigin **ret_origin); 27 | int ca_cache_put(CaCache *c, CaOrigin *origin, const CaChunkID *chunk_id); 28 | int ca_cache_remove(CaCache *c, CaLocation *location); 29 | 30 | DEFINE_TRIVIAL_CLEANUP_FUNC(CaCache*, ca_cache_unref); 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /src/cachunk.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocachunkhfoo 4 | #define foocachunkhfoo 5 | 6 | #include "cachunkid.h" 7 | #include "cacompression.h" 8 | #include "realloc-buffer.h" 9 | 10 | /* The hardcoded, maximum chunk size, after which we refuse operation */ 11 | #define CA_CHUNK_SIZE_LIMIT_MAX ((size_t) (128U*1024U*1024U)) 12 | #define CA_CHUNK_SIZE_LIMIT_MIN ((size_t) 1) 13 | 14 | typedef enum CaChunkCompression { 15 | CA_CHUNK_UNCOMPRESSED, 16 | CA_CHUNK_COMPRESSED, 17 | CA_CHUNK_AS_IS, 18 | _CA_CHUNK_COMPRESSION_MAX, 19 | } CaChunkCompression; 20 | 21 | int ca_load_fd(int fd, ReallocBuffer *buffer); 22 | int ca_load_and_decompress_fd(int fd, ReallocBuffer *buffer); 23 | int ca_load_and_compress_fd(int fd, CaCompressionType compression_type, ReallocBuffer *buffer); 24 | 25 | int ca_save_fd(int fd, const void *data, size_t size); 26 | int ca_save_and_decompress_fd(int fd, const void *data, size_t size); 27 | int ca_save_and_compress_fd(int fd, CaCompressionType compression_type, const void *data, size_t size); 28 | 29 | int ca_decompress(const void *data, size_t size, ReallocBuffer *buffer); 30 | int ca_compress(CaCompressionType compression_type, const void *data, size_t size, ReallocBuffer *buffer); 31 | 32 | int ca_chunk_file_open(int cache_fd, const char *prefix, const CaChunkID *chunkid, const char *suffix, int flags); 33 | 34 | int ca_chunk_file_test(int cache_fd, const char *prefix, const CaChunkID *chunkid); 35 | int ca_chunk_file_load(int cache_fd, const char *prefix, const CaChunkID *chunkid, CaChunkCompression desired_compression, CaCompressionType compression_type, ReallocBuffer *buffer, CaChunkCompression *ret_effective_compression); 36 | int ca_chunk_file_save(int cache_fd, const char *prefix, const CaChunkID *chunkid, CaChunkCompression effective_compression, CaChunkCompression desired_compression, CaCompressionType compression_type, const void *p, uint64_t l); 37 | int ca_chunk_file_mark_missing(int cache_fd, const char *prefix, const CaChunkID *chunkid); 38 | int ca_chunk_file_remove(int chunk_fd, const char *prefix, const CaChunkID *chunkid); 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /src/cachunker.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foochunkerhfoo 4 | #define foochunkerhfoo 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | /* The default average chunk size */ 11 | #define CA_CHUNK_SIZE_AVG_DEFAULT ((size_t) (64U*1024U)) 12 | 13 | /* Our checksum window size */ 14 | #define CA_CHUNKER_WINDOW_SIZE 48 15 | 16 | /* The chunk cut discriminator. In order to get an average chunk size of avg, we cut whenever for a hash value "h" at 17 | * byte "i" given the descriminator "d(avg)": h(i) mod d(avg) == d(avg) - 1. Note that the discriminator 18 | * calculated like this only yields correct results as long as the minimal chunk size is picked as avg/4, and the 19 | * maximum chunk size as avg*4. If they are picked differently the result might be skewed into either direction. */ 20 | #define CA_CHUNKER_DISCRIMINATOR_FROM_AVG(avg) ((size_t) (avg / (-1.42888852e-7 * avg + 1.33237515))) 21 | 22 | typedef struct CaChunker { 23 | uint32_t h; 24 | 25 | size_t window_size; 26 | size_t chunk_size; 27 | 28 | size_t chunk_size_min; 29 | size_t chunk_size_max; 30 | size_t chunk_size_avg; 31 | 32 | size_t discriminator; 33 | 34 | uint8_t window[CA_CHUNKER_WINDOW_SIZE]; 35 | } CaChunker; 36 | 37 | /* The default initializer for the chunker. We pick an average chunk size equivalent to 64K */ 38 | #define CA_CHUNKER_INIT \ 39 | { \ 40 | .chunk_size_min = CA_CHUNK_SIZE_AVG_DEFAULT/4, \ 41 | .chunk_size_avg = CA_CHUNK_SIZE_AVG_DEFAULT, \ 42 | .chunk_size_max = CA_CHUNK_SIZE_AVG_DEFAULT*4, \ 43 | .discriminator = CA_CHUNKER_DISCRIMINATOR_FROM_AVG(CA_CHUNK_SIZE_AVG_DEFAULT), \ 44 | } 45 | 46 | /* Set the min/avg/max chunk size. Each parameter may be 0, in which case a default is used. */ 47 | int ca_chunker_set_size(CaChunker *c, size_t min_size, size_t avg_size, size_t max_size); 48 | 49 | /* Scans the specified data for a chunk border. Returns (size_t) -1 if none was found (and the function should be 50 | * called with more data later on), or another value indicating the position of a border. */ 51 | size_t ca_chunker_scan(CaChunker *c, const void* p, size_t n); 52 | 53 | /* Low-level buzhash functions. Only exported for testing purposes. */ 54 | uint32_t ca_chunker_start(CaChunker *c, const void *p, size_t n); 55 | uint32_t ca_chunker_roll(CaChunker *c, uint8_t pop_byte, uint8_t push_byte); 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /src/cachunkid.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | 6 | #include "cachunk.h" 7 | #include "cachunkid.h" 8 | #include "log.h" 9 | 10 | static char encode_char(uint8_t x) { 11 | x &= 0xF; 12 | return (x < 10 ? '0' : 'a' - 10) + x; 13 | } 14 | 15 | static int decode_char(char x) { 16 | if (x >= '0' && x <= '9') 17 | return x - '0'; 18 | if (x >= 'a' && x <= 'f') 19 | return x - 'a' + 10; 20 | 21 | return -EINVAL; 22 | } 23 | 24 | CaChunkID* ca_chunk_id_parse(const char *v, CaChunkID *ret) { 25 | CaChunkID id; 26 | size_t i; 27 | 28 | assert(v); 29 | assert(ret); 30 | 31 | for (i = 0; i < sizeof(CaChunkID); i++) { 32 | int x, y; 33 | 34 | x = decode_char(v[i*2]); 35 | if (x < 0) 36 | return NULL; 37 | y = decode_char(v[i*2+1]); 38 | if (y < 0) 39 | return NULL; 40 | 41 | id.bytes[i] = (uint8_t) x << 4 | (uint8_t) y; 42 | } 43 | 44 | if (v[sizeof(CaChunkID)*2] != 0) 45 | return NULL; 46 | 47 | *ret = id; 48 | return ret; 49 | } 50 | 51 | char* ca_chunk_id_format(const CaChunkID *id, char v[CA_CHUNK_ID_FORMAT_MAX]) { 52 | size_t i; 53 | 54 | assert(id); 55 | assert(v); 56 | 57 | for (i = 0; i < sizeof(CaChunkID); i++) { 58 | v[i*2] = encode_char(id->bytes[i] >> 4); 59 | v[i*2+1] = encode_char(id->bytes[i] & 0xF); 60 | } 61 | 62 | v[sizeof(CaChunkID) * 2] = 0; 63 | return v; 64 | } 65 | 66 | int ca_chunk_id_make(CaDigest *digest, const void *p, size_t l, CaChunkID *ret) { 67 | if (!digest) 68 | return -EINVAL; 69 | if (!p) 70 | return -EINVAL; 71 | if (l < CA_CHUNK_SIZE_LIMIT_MIN) 72 | return -EINVAL; 73 | if (l > CA_CHUNK_SIZE_LIMIT_MAX) 74 | return -EINVAL; 75 | if (!ret) 76 | return -EINVAL; 77 | 78 | if (ca_digest_get_size(digest) != sizeof(CaChunkID)) 79 | return -EINVAL; 80 | 81 | ca_digest_reset(digest); 82 | ca_digest_write(digest, p, l); 83 | 84 | memcpy(ret, ca_digest_read(digest), sizeof(CaChunkID)); 85 | return 0; 86 | } 87 | 88 | char* ca_chunk_id_format_path( 89 | const char *prefix, 90 | const CaChunkID *chunkid, 91 | const char *suffix, 92 | char buffer[]) { 93 | 94 | size_t n; 95 | 96 | assert(chunkid); 97 | assert(buffer); 98 | 99 | if (prefix) { 100 | n = strlen(prefix); 101 | memcpy(buffer, prefix, n); 102 | } else 103 | n = 0; 104 | 105 | ca_chunk_id_format(chunkid, buffer + n + 4 + 1); 106 | memcpy(buffer + n, buffer + n + 4 + 1, 4); 107 | buffer[n + 4] = '/'; 108 | 109 | if (suffix) 110 | strcpy(buffer + n + 4 + 1 + CA_CHUNK_ID_FORMAT_MAX - 1, suffix); 111 | 112 | return buffer; 113 | } 114 | -------------------------------------------------------------------------------- /src/cachunkid.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocaobjectidhfoo 4 | #define foocaobjectidhfoo 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "cadigest.h" 12 | 13 | #define CA_CHUNK_ID_SIZE 32 14 | #define CA_CHUNK_ID_FORMAT_MAX (CA_CHUNK_ID_SIZE*2+1) 15 | 16 | typedef union CaChunkID { 17 | /* Depending on context either a SHA256 or SHA512/256 sum */ 18 | uint8_t bytes[CA_CHUNK_ID_SIZE]; 19 | uint64_t u64[CA_CHUNK_ID_SIZE / sizeof(uint64_t)]; 20 | } CaChunkID; 21 | 22 | CaChunkID* ca_chunk_id_parse(const char *v, CaChunkID *ret); 23 | char *ca_chunk_id_format(const CaChunkID *id, char v[CA_CHUNK_ID_FORMAT_MAX]); 24 | 25 | static inline bool ca_chunk_id_equal(const CaChunkID *a, const CaChunkID *b) { 26 | 27 | if (a == b) 28 | return true; 29 | 30 | if (!a || !b) 31 | return false; 32 | 33 | return memcmp(a, b, sizeof(CaChunkID)) == 0; 34 | } 35 | 36 | static inline bool ca_chunk_id_is_null(const CaChunkID *a) { 37 | size_t i; 38 | 39 | if (!a) 40 | return true; 41 | 42 | for (i = 0; i < CA_CHUNK_ID_SIZE / sizeof(uint64_t); i++) 43 | if (a->u64[0] != 0) 44 | return false; 45 | 46 | return true; 47 | } 48 | 49 | int ca_chunk_id_make(CaDigest *digest, const void *p, size_t l, CaChunkID *ret); 50 | 51 | #define CA_CHUNK_ID_PATH_SIZE(prefix, suffix) \ 52 | (strlen_null(prefix) + 4 + 1 + CA_CHUNK_ID_FORMAT_MAX + strlen_null(suffix)) 53 | 54 | char* ca_chunk_id_format_path(const char *prefix, const CaChunkID *chunkid, const char *suffix, char buffer[]); 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /src/cacommon.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef cacommonhfoo 4 | #define cacommonhfoo 5 | 6 | typedef enum CaIterate { 7 | CA_ITERATE_CURRENT, 8 | CA_ITERATE_FIRST, 9 | CA_ITERATE_LAST, 10 | CA_ITERATE_NEXT, 11 | CA_ITERATE_PREVIOUS, 12 | _CA_ITERATE_MAX, 13 | _CA_ITERATE_INVALID = -1, 14 | } CaIterate; 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /src/cacompression.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "cacompression.h" 4 | #include "util.h" 5 | 6 | static const char* const table[_CA_COMPRESSION_TYPE_MAX] = { 7 | [CA_COMPRESSION_XZ] = "xz", 8 | [CA_COMPRESSION_GZIP] = "gzip", 9 | [CA_COMPRESSION_ZSTD] = "zstd", 10 | }; 11 | 12 | const char* ca_compression_type_to_string(CaCompressionType c) { 13 | if (c < 0) 14 | return NULL; 15 | if (c >= _CA_COMPRESSION_TYPE_MAX) 16 | return NULL; 17 | 18 | return table[c]; 19 | } 20 | 21 | CaCompressionType ca_compression_type_from_string(const char *s) { 22 | CaCompressionType i; 23 | 24 | if (isempty(s)) 25 | return _CA_COMPRESSION_TYPE_INVALID; 26 | 27 | if (streq(s, "default")) 28 | return CA_COMPRESSION_DEFAULT; 29 | 30 | for (i = 0; i < _CA_COMPRESSION_TYPE_MAX; i++) { 31 | if (streq(table[i], s)) 32 | return i; 33 | } 34 | 35 | return _CA_COMPRESSION_TYPE_INVALID; 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/cacompression.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocacompressorhfoo 4 | #define foocacompressorhfoo 5 | 6 | typedef enum CaCompressionType { 7 | CA_COMPRESSION_XZ, 8 | CA_COMPRESSION_GZIP, 9 | CA_COMPRESSION_ZSTD, 10 | _CA_COMPRESSION_TYPE_MAX, 11 | _CA_COMPRESSION_TYPE_INVALID = -1, 12 | 13 | CA_COMPRESSION_DEFAULT = 14 | #if HAVE_LIBZSTD 15 | CA_COMPRESSION_ZSTD, 16 | #elif HAVE_LIBZ 17 | CA_COMPRESSION_GZIP, 18 | #elif HAVE_LIBLZMA 19 | CA_COMPRESSION_XZ, 20 | #else 21 | _CA_COMPRESSION_TYPE_INVALID 22 | #endif 23 | } CaCompressionType; 24 | 25 | const char* ca_compression_type_to_string(CaCompressionType c); 26 | CaCompressionType ca_compression_type_from_string(const char *s); 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/cadecoder.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocadecoderhfoo 4 | #define foocadecoderhfoo 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "cachunkid.h" 11 | #include "cacommon.h" 12 | #include "calocation.h" 13 | #include "caorigin.h" 14 | 15 | typedef struct CaDecoder CaDecoder; 16 | 17 | enum { 18 | /* Stream events */ 19 | CA_DECODER_FINISHED, /* The end of the stream has been reached */ 20 | CA_DECODER_STEP, /* We processed a bit, call us again */ 21 | CA_DECODER_NEXT_FILE, /* We started processing a new file */ 22 | CA_DECODER_DONE_FILE, /* We finished processing a file */ 23 | CA_DECODER_PAYLOAD, /* We have some payload data for you */ 24 | 25 | /* Requests to the caller */ 26 | CA_DECODER_REQUEST, /* We need more data */ 27 | CA_DECODER_SEEK, /* Please seek */ 28 | CA_DECODER_SKIP, /* Please skip some bytes */ 29 | 30 | /* Response to seeks */ 31 | CA_DECODER_FOUND, /* Seek completed successfully */ 32 | CA_DECODER_NOT_FOUND, /* Seek to file was requested, but file didn't exist */ 33 | }; 34 | 35 | CaDecoder *ca_decoder_new(void); 36 | CaDecoder *ca_decoder_unref(CaDecoder *d); 37 | 38 | /* The actual feature flags in effect if known */ 39 | int ca_decoder_get_feature_flags(CaDecoder *d, uint64_t *ret); 40 | 41 | /* The feature flags to expect. When the actual feature flags don't match this, this is considered an error */ 42 | int ca_decoder_set_expected_feature_flags(CaDecoder *d, uint64_t flags); 43 | 44 | /* A mask configuring which feature flags to actually honour when recreating files */ 45 | int ca_decoder_set_feature_flags_mask(CaDecoder *d, uint64_t flags); 46 | 47 | /* Various booleans to configure the mode of operation */ 48 | int ca_decoder_set_punch_holes(CaDecoder *d, bool enabled); 49 | int ca_decoder_set_reflink(CaDecoder *d, bool enabled); 50 | int ca_decoder_set_hardlink(CaDecoder *d, bool enabled); 51 | int ca_decoder_set_delete(CaDecoder *d, bool enabled); 52 | int ca_decoder_set_payload(CaDecoder *d, bool enabled); 53 | int ca_decoder_set_undo_immutable(CaDecoder *d, bool enabled); 54 | 55 | /* Apply UID shifting */ 56 | int ca_decoder_set_uid_shift(CaDecoder *e, uid_t u); 57 | int ca_decoder_set_uid_range(CaDecoder *e, uid_t u); 58 | 59 | /* Output: a file descriptor to a directory tree, block device node, or regular file */ 60 | int ca_decoder_set_base_fd(CaDecoder *d, int fd); 61 | int ca_decoder_set_boundary_fd(CaDecoder *d, int fd); 62 | 63 | /* Output: if no output to the file system is desired: specify instead what kind of object is to be read */ 64 | int ca_decoder_set_base_mode(CaDecoder *d, mode_t mode); 65 | 66 | /* Input: set the archive size, to make this seekable */ 67 | int ca_decoder_set_archive_size(CaDecoder *d, uint64_t size); 68 | 69 | /* The core of loop, returns one of the CA_DECODER_XYZ events defined above */ 70 | int ca_decoder_step(CaDecoder *d); 71 | 72 | /* If ca_decoder_step() returned CA_DECODER_REQUEST, which offset we are at now */ 73 | int ca_decoder_get_request_offset(CaDecoder *d, uint64_t *offset); 74 | 75 | /* If ca_decoder_step() returned CA_DECODER_SEEK, where are we supposed to seek now? (returns absolute position) */ 76 | int ca_decoder_get_seek_offset(CaDecoder *d, uint64_t *ret); 77 | 78 | /* If ca_decoder_step() returned CA_DECODER_SKIP, how many bytes are we supposed to skip? (returns relative number of bytes) */ 79 | int ca_decoder_get_skip_size(CaDecoder *d, uint64_t *ret); 80 | 81 | /* Input: archive stream data */ 82 | int ca_decoder_put_data(CaDecoder *d, const void *p, size_t size, CaOrigin *origin); 83 | int ca_decoder_put_eof(CaDecoder *d); 84 | 85 | /* Output: payload data */ 86 | int ca_decoder_get_payload(CaDecoder *d, const void **ret, size_t *ret_size); 87 | 88 | /* Retrieve information about where we currently are */ 89 | int ca_decoder_current_path(CaDecoder *d, char **ret); 90 | int ca_decoder_current_mode(CaDecoder *d, mode_t *ret); 91 | int ca_decoder_current_target(CaDecoder *d, const char **ret); 92 | int ca_decoder_current_mtime(CaDecoder *d, uint64_t *nsec); 93 | int ca_decoder_current_size(CaDecoder *d, uint64_t *size); 94 | int ca_decoder_current_uid(CaDecoder *d, uid_t *uid); 95 | int ca_decoder_current_gid(CaDecoder *d, gid_t *gid); 96 | int ca_decoder_current_user(CaDecoder *d, const char **user); 97 | int ca_decoder_current_group(CaDecoder *d, const char **user); 98 | int ca_decoder_current_rdev(CaDecoder *d, dev_t *ret); 99 | int ca_decoder_current_offset(CaDecoder *d, uint64_t *ret); 100 | int ca_decoder_current_chattr(CaDecoder *d, unsigned *ret); 101 | int ca_decoder_current_fat_attrs(CaDecoder *d, uint32_t *ret); 102 | int ca_decoder_current_xattr(CaDecoder *d, CaIterate where, const char **ret_name, const void **ret_value, size_t *ret_size); 103 | int ca_decoder_current_quota_projid(CaDecoder *d, uint32_t *ret); 104 | 105 | /* Seeking to positions */ 106 | int ca_decoder_seek_offset(CaDecoder *d, uint64_t offset); 107 | int ca_decoder_seek_path(CaDecoder *d, const char *path); 108 | int ca_decoder_seek_path_offset(CaDecoder *d, const char *path, uint64_t offset); 109 | int ca_decoder_seek_next_sibling(CaDecoder *d); 110 | 111 | /* Statistics */ 112 | int ca_decoder_get_punch_holes_bytes(CaDecoder *d, uint64_t *ret); 113 | int ca_decoder_get_reflink_bytes(CaDecoder *d, uint64_t *ret); 114 | int ca_decoder_get_hardlink_bytes(CaDecoder *d, uint64_t *ret); 115 | 116 | int ca_decoder_current_archive_offset(CaDecoder *d, uint64_t *ret); 117 | 118 | int ca_decoder_enable_archive_digest(CaDecoder *d, bool b); 119 | int ca_decoder_enable_payload_digest(CaDecoder *d, bool b); 120 | int ca_decoder_enable_hardlink_digest(CaDecoder *d, bool b); 121 | 122 | int ca_decoder_get_archive_digest(CaDecoder *d, CaChunkID *ret); 123 | int ca_decoder_get_hardlink_digest(CaDecoder *d, CaChunkID *ret); 124 | int ca_decoder_get_payload_digest(CaDecoder *d, CaChunkID *ret); 125 | 126 | int ca_decoder_try_hardlink(CaDecoder *d, CaFileRoot *root, const char *path); 127 | 128 | #endif 129 | -------------------------------------------------------------------------------- /src/cadigest.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | 5 | #include "cadigest.h" 6 | #include "util.h" 7 | 8 | struct CaDigest { 9 | CaDigestType type; 10 | uint8_t result[CONST_MAX(SHA256_DIGEST_LENGTH, SHA512_DIGEST_LENGTH)]; 11 | 12 | union { 13 | SHA256_CTX sha256; 14 | SHA512_CTX sha512; 15 | }; 16 | }; 17 | 18 | void ca_digest_reset(CaDigest *d) { 19 | if (!d) 20 | return; 21 | 22 | switch (d->type) { 23 | 24 | case CA_DIGEST_SHA256: 25 | SHA256_Init(&d->sha256); 26 | break; 27 | 28 | case CA_DIGEST_SHA512_256: 29 | 30 | /* SHA512/256 is identical to SHA512 truncated to 256 bit, except that the start values are slightly 31 | * different. Since OpenSSL doesn't support them natively, let's hack them in here. As soon as OpenSSL 32 | * learns SHA512/256 natively, let's switch over to that. */ 33 | 34 | SHA512_Init(&d->sha512); 35 | d->sha512.h[0] = UINT64_C(0x22312194fc2bf72c); 36 | d->sha512.h[1] = UINT64_C(0x9f555fa3c84c64c2); 37 | d->sha512.h[2] = UINT64_C(0x2393b86b6f53b151); 38 | d->sha512.h[3] = UINT64_C(0x963877195940eabd); 39 | d->sha512.h[4] = UINT64_C(0x96283ee2a88effe3); 40 | d->sha512.h[5] = UINT64_C(0xbe5e1e2553863992); 41 | d->sha512.h[6] = UINT64_C(0x2b0199fc2c85b8aa); 42 | d->sha512.h[7] = UINT64_C(0x0eb72ddc81c52ca2); 43 | 44 | break; 45 | 46 | default: 47 | assert_not_reached("Unknown hash function"); 48 | } 49 | } 50 | 51 | int ca_digest_new(CaDigestType t, CaDigest **ret) { 52 | CaDigest *d; 53 | 54 | if (t < 0) 55 | return -EINVAL; 56 | if (t >= _CA_DIGEST_TYPE_MAX) 57 | return -EOPNOTSUPP; 58 | if (!ret) 59 | return -EINVAL; 60 | 61 | d = new0(CaDigest, 1); 62 | if (!d) 63 | return -ENOMEM; 64 | 65 | d->type = t; 66 | 67 | ca_digest_reset(d); 68 | 69 | *ret = d; 70 | return 0; 71 | } 72 | 73 | CaDigest *ca_digest_free(CaDigest *d) { 74 | return mfree(d); 75 | } 76 | 77 | void ca_digest_write(CaDigest *d, const void *p, size_t l) { 78 | if (!d) 79 | return; 80 | if (l <= 0) 81 | return; 82 | 83 | assert(p); 84 | 85 | switch (d->type) { 86 | 87 | case CA_DIGEST_SHA256: 88 | SHA256_Update(&d->sha256, p, l); 89 | break; 90 | 91 | case CA_DIGEST_SHA512_256: 92 | SHA512_Update(&d->sha512, p, l); 93 | break; 94 | 95 | default: 96 | assert_not_reached("Unknown hash function"); 97 | } 98 | } 99 | 100 | const void* ca_digest_read(CaDigest *d) { 101 | if (!d) 102 | return NULL; 103 | 104 | switch (d->type) { 105 | 106 | case CA_DIGEST_SHA256: 107 | assert(sizeof(d->result) >= SHA256_DIGEST_LENGTH); 108 | SHA256_Final(d->result, &d->sha256); 109 | break; 110 | 111 | case CA_DIGEST_SHA512_256: 112 | assert(sizeof(d->result) >= SHA512_DIGEST_LENGTH); 113 | SHA512_Final(d->result, &d->sha512); 114 | break; 115 | 116 | default: 117 | assert_not_reached("Unknown hash function"); 118 | 119 | } 120 | 121 | return d->result; 122 | } 123 | 124 | size_t ca_digest_get_size(CaDigest *d) { 125 | if (!d) 126 | return (size_t) -1; 127 | 128 | return ca_digest_type_size(d->type); 129 | } 130 | 131 | CaDigestType ca_digest_get_type(CaDigest *d) { 132 | if (!d) 133 | return _CA_DIGEST_TYPE_INVALID; 134 | 135 | return d->type; 136 | } 137 | 138 | const char *ca_digest_get_name(CaDigest *d) { 139 | if (!d) 140 | return NULL; 141 | 142 | return ca_digest_type_to_string(d->type); 143 | } 144 | 145 | size_t ca_digest_type_size(CaDigestType t) { 146 | 147 | switch (t) { 148 | 149 | case CA_DIGEST_SHA256: 150 | case CA_DIGEST_SHA512_256: 151 | assert(SHA256_DIGEST_LENGTH == SHA512_DIGEST_LENGTH/2); 152 | return SHA256_DIGEST_LENGTH; 153 | 154 | default: 155 | return (size_t) -1; 156 | } 157 | } 158 | 159 | static const char *const table[_CA_DIGEST_TYPE_MAX] = { 160 | [CA_DIGEST_SHA256] = "sha256", 161 | [CA_DIGEST_SHA512_256] = "sha512-256", 162 | }; 163 | 164 | const char *ca_digest_type_to_string(CaDigestType t) { 165 | if (t < 0 || t >= _CA_DIGEST_TYPE_MAX) 166 | return NULL; 167 | 168 | return table[t]; 169 | } 170 | 171 | CaDigestType ca_digest_type_from_string(const char *name) { 172 | CaDigestType t; 173 | 174 | if (!name) 175 | return _CA_DIGEST_TYPE_INVALID; 176 | 177 | if (streq(name, "default")) 178 | return CA_DIGEST_DEFAULT; 179 | 180 | for (t = 0; t < _CA_DIGEST_TYPE_MAX; t++) 181 | if (streq(table[t], name)) 182 | return t; 183 | 184 | return _CA_DIGEST_TYPE_INVALID; 185 | } 186 | 187 | int ca_digest_ensure_allocated(CaDigest **d, CaDigestType t) { 188 | int r; 189 | 190 | if (!d) 191 | return -EINVAL; 192 | if (*d) 193 | return 0; 194 | 195 | r = ca_digest_new(t, d); 196 | if (r < 0) 197 | return r; 198 | 199 | return 1; 200 | } 201 | 202 | int ca_digest_set_type(CaDigest *d, CaDigestType t) { 203 | if (!d) 204 | return -EINVAL; 205 | if (t < 0) 206 | return -EINVAL; 207 | if (t >= _CA_DIGEST_TYPE_MAX) 208 | return -EOPNOTSUPP; 209 | 210 | d->type = t; 211 | ca_digest_reset(d); 212 | 213 | return 0; 214 | } 215 | -------------------------------------------------------------------------------- /src/cadigest.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocadigesthfoo 4 | #define foocadigesthfoo 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | typedef struct CaDigest CaDigest; 11 | 12 | typedef enum CaDigestType { 13 | CA_DIGEST_SHA256, 14 | CA_DIGEST_SHA512_256, 15 | _CA_DIGEST_TYPE_MAX, 16 | CA_DIGEST_DEFAULT = CA_DIGEST_SHA512_256, 17 | _CA_DIGEST_TYPE_INVALID = -1, 18 | } CaDigestType; 19 | 20 | int ca_digest_new(CaDigestType t, CaDigest **ret); 21 | CaDigest *ca_digest_free(CaDigest *d); 22 | 23 | static inline void ca_digest_freep(CaDigest **d) { 24 | if (d) 25 | ca_digest_free(*d); 26 | } 27 | 28 | int ca_digest_ensure_allocated(CaDigest **d, CaDigestType t); 29 | 30 | void ca_digest_write(CaDigest *d, const void *p, size_t l); 31 | 32 | static inline void ca_digest_write_u8(CaDigest *d, uint8_t u) { 33 | ca_digest_write(d, &u, sizeof(u)); 34 | } 35 | 36 | static inline void ca_digest_write_u32(CaDigest *d, uint32_t u) { 37 | u = htole32(u); 38 | ca_digest_write(d, &u, sizeof(u)); 39 | } 40 | 41 | static inline void ca_digest_write_u64(CaDigest *d, uint64_t u) { 42 | u = htole64(u); 43 | ca_digest_write(d, &u, sizeof(u)); 44 | } 45 | 46 | const void* ca_digest_read(CaDigest *d); 47 | 48 | void ca_digest_reset(CaDigest *d); 49 | 50 | size_t ca_digest_get_size(CaDigest *d); 51 | CaDigestType ca_digest_get_type(CaDigest *d); 52 | const char *ca_digest_get_name(CaDigest *d); 53 | 54 | size_t ca_digest_type_size(CaDigestType t); 55 | 56 | const char *ca_digest_type_to_string(CaDigestType t); 57 | CaDigestType ca_digest_type_from_string(const char *name); 58 | 59 | int ca_digest_set_type(CaDigest *d, CaDigestType t); 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/caencoder.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocaencoderhfoo 4 | #define foocaencoderhfoo 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "cachunkid.h" 11 | #include "cacommon.h" 12 | #include "calocation.h" 13 | 14 | typedef struct CaEncoder CaEncoder; 15 | 16 | enum { 17 | CA_ENCODER_FINISHED, /* The end of the stream has been reached */ 18 | CA_ENCODER_NEXT_FILE, /* We started with the next file, and we generated some data */ 19 | CA_ENCODER_DONE_FILE, /* We finished with some file, and possibly generated some data */ 20 | CA_ENCODER_PAYLOAD, /* We just read some payload data, and we generated some data */ 21 | CA_ENCODER_DATA, /* We generated some data */ 22 | }; 23 | 24 | CaEncoder *ca_encoder_new(void); 25 | CaEncoder *ca_encoder_unref(CaEncoder *e); 26 | 27 | int ca_encoder_set_feature_flags(CaEncoder *e, uint64_t flags); 28 | int ca_encoder_get_feature_flags(CaEncoder *e, uint64_t *ret); 29 | 30 | int ca_encoder_get_covering_feature_flags(CaEncoder *e, uint64_t *ret); 31 | 32 | int ca_encoder_set_uid_shift(CaEncoder *e, uid_t u); 33 | int ca_encoder_set_uid_range(CaEncoder *e, uid_t u); 34 | 35 | /* Input: a directory tree, block device node or regular file */ 36 | int ca_encoder_set_base_fd(CaEncoder *e, int fd); 37 | int ca_encoder_get_base_fd(CaEncoder *e); 38 | 39 | int ca_encoder_step(CaEncoder *e); 40 | 41 | /* Output: archive stream data */ 42 | int ca_encoder_get_data(CaEncoder *e, uint64_t suggested_size, const void **ret, size_t *ret_size); 43 | 44 | int ca_encoder_current_path(CaEncoder *e, char **ret); 45 | int ca_encoder_current_mode(CaEncoder *d, mode_t *ret); 46 | int ca_encoder_current_target(CaEncoder *e, const char **ret); 47 | int ca_encoder_current_mtime(CaEncoder *e, uint64_t *nsec); 48 | int ca_encoder_current_size(CaEncoder *e, uint64_t *size); 49 | int ca_encoder_current_uid(CaEncoder *e, uid_t *ret); 50 | int ca_encoder_current_gid(CaEncoder *e, gid_t *ret); 51 | int ca_encoder_current_user(CaEncoder *e, const char **ret); 52 | int ca_encoder_current_group(CaEncoder *e, const char **ret); 53 | int ca_encoder_current_rdev(CaEncoder *e, dev_t *ret); 54 | int ca_encoder_current_chattr(CaEncoder *e, unsigned *ret); 55 | int ca_encoder_current_fat_attrs(CaEncoder *e, uint32_t *ret); 56 | int ca_encoder_current_xattr(CaEncoder *e, CaIterate where, const char **ret_name, const void **ret_value, size_t *ret_size); 57 | int ca_encoder_current_quota_projid(CaEncoder *e, uint32_t *ret); 58 | 59 | int ca_encoder_current_payload_offset(CaEncoder *e, uint64_t *ret); 60 | int ca_encoder_current_archive_offset(CaEncoder *e, uint64_t *ret); 61 | 62 | int ca_encoder_current_location(CaEncoder *e, uint64_t add, CaLocation **ret); 63 | 64 | int ca_encoder_seek_location(CaEncoder *e, CaLocation *location); 65 | 66 | int ca_encoder_enable_archive_digest(CaEncoder *e, bool b); 67 | int ca_encoder_enable_payload_digest(CaEncoder *e, bool b); 68 | int ca_encoder_enable_hardlink_digest(CaEncoder *e, bool b); 69 | 70 | int ca_encoder_get_archive_digest(CaEncoder *e, CaChunkID *ret); 71 | int ca_encoder_get_hardlink_digest(CaEncoder *e, CaChunkID *ret); 72 | int ca_encoder_get_payload_digest(CaEncoder *e, CaChunkID *ret); 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /src/cafileroot.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "cafileroot.h" 4 | #include "util.h" 5 | 6 | int ca_file_root_new(const char *path, int fd, CaFileRoot **ret) { 7 | CaFileRoot *root; 8 | 9 | if (!path && fd < 0) 10 | return -EINVAL; 11 | if (!ret) 12 | return -EINVAL; 13 | 14 | root = new0(CaFileRoot, 1); 15 | if (!root) 16 | return -ENOMEM; 17 | 18 | root->n_ref = 1; 19 | root->fd = fd; 20 | 21 | if (path) { 22 | root->path = strdup(path); 23 | if (!root->path) { 24 | free(root); 25 | return -ENOMEM; 26 | } 27 | } 28 | 29 | *ret = root; 30 | return 0; 31 | } 32 | 33 | CaFileRoot* ca_file_root_ref(CaFileRoot *root) { 34 | if (!root) 35 | return NULL; 36 | 37 | assert_se(root->n_ref > 0); 38 | 39 | root->n_ref++; 40 | 41 | return root; 42 | } 43 | 44 | CaFileRoot* ca_file_root_unref(CaFileRoot *root) { 45 | if (!root) 46 | return NULL; 47 | 48 | assert_se(root->n_ref > 0); 49 | root->n_ref--; 50 | 51 | if (root->n_ref > 0) 52 | return NULL; 53 | 54 | root->fd = -1; 55 | free(root->path); 56 | 57 | return mfree(root); 58 | } 59 | 60 | void ca_file_root_invalidate(CaFileRoot *root) { 61 | if (!root) 62 | return; 63 | 64 | root->fd = -1; 65 | root->path = mfree(root->path); 66 | 67 | root->invalidated = true; 68 | } 69 | -------------------------------------------------------------------------------- /src/cafileroot.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocafileroothfoo 4 | #define foocafileroothfoo 5 | 6 | #include 7 | 8 | typedef struct CaFileRoot { 9 | unsigned n_ref; 10 | char *path; 11 | int fd; 12 | bool invalidated; 13 | } CaFileRoot; 14 | 15 | int ca_file_root_new(const char *path, int fd, CaFileRoot **ret); 16 | 17 | CaFileRoot* ca_file_root_ref(CaFileRoot *root); 18 | CaFileRoot* ca_file_root_unref(CaFileRoot *root); 19 | 20 | void ca_file_root_invalidate(CaFileRoot *root); 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /src/caformat-util.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocaformatutilhfoo 4 | #define foocaformatutilhfoo 5 | 6 | #include 7 | 8 | #include "cadigest.h" 9 | #include "util.h" 10 | 11 | const char *ca_format_type_name(uint64_t u); 12 | 13 | int ca_with_feature_flags_parse_one(const char *name, uint64_t *ret); 14 | int ca_with_feature_flags_format(uint64_t features, char **ret); 15 | 16 | int ca_feature_flags_normalize(uint64_t flags, uint64_t *ret); 17 | int ca_feature_flags_normalize_mask(uint64_t mask, uint64_t *ret); 18 | int ca_feature_flags_time_granularity_nsec(uint64_t flags, uint64_t *ret); 19 | 20 | uint64_t ca_feature_flags_from_chattr(unsigned flags); 21 | unsigned ca_feature_flags_to_chattr(uint64_t flags); 22 | 23 | uint64_t ca_feature_flags_from_fat_attrs(uint32_t flags); 24 | uint32_t ca_feature_flags_to_fat_attrs(uint64_t flags); 25 | 26 | uint64_t ca_feature_flags_from_magic(statfs_f_type_t type); 27 | 28 | int ca_feature_flags_are_normalized(uint64_t f); 29 | 30 | uint64_t ca_feature_flags_from_digest_type(CaDigestType type); 31 | CaDigestType ca_feature_flags_to_digest_type(uint64_t flags); 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /src/cafuse.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef cafusehfoo 4 | #define cafusehfoo 5 | 6 | #include "casync.h" 7 | 8 | int ca_fuse_run(CaSync *s, const char *what, const char *where, bool do_mkdir); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /src/caindex.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocaindexhfoo 4 | #define foocaindexhfoo 5 | 6 | #include "cachunkid.h" 7 | #include "realloc-buffer.h" 8 | 9 | typedef struct CaIndex CaIndex; 10 | 11 | CaIndex *ca_index_new_write(void); /* only (cooked) writing */ 12 | CaIndex *ca_index_new_read(void); /* only (cooked) reading */ 13 | CaIndex *ca_index_new_incremental_write(void); /* incremental cooked writing + raw/byte-wise reading (for uploads) */ 14 | CaIndex *ca_index_new_incremental_read(void); /* incremental raw/byte-wise writing + cooked reading (for downloads) */ 15 | CaIndex *ca_index_unref(CaIndex *i); 16 | static inline void ca_index_unrefp(CaIndex **i) { 17 | ca_index_unref(*i); 18 | } 19 | 20 | int ca_index_set_fd(CaIndex *i, int fd); 21 | int ca_index_set_path(CaIndex *i, const char *path); 22 | 23 | int ca_index_set_make_mode(CaIndex *i, mode_t m); 24 | 25 | int ca_index_open(CaIndex *i); 26 | 27 | int ca_index_install(CaIndex *i); 28 | 29 | int ca_index_write_chunk(CaIndex *i, const CaChunkID *id, uint64_t size); 30 | int ca_index_write_eof(CaIndex *i); 31 | 32 | int ca_index_read_chunk(CaIndex *i, CaChunkID *id, uint64_t *ret_offset_end, uint64_t *ret_size); 33 | 34 | int ca_index_set_position(CaIndex *i, uint64_t position); 35 | int ca_index_get_position(CaIndex *i, uint64_t *ret); 36 | int ca_index_get_available_chunks(CaIndex *i, uint64_t *ret); 37 | 38 | int ca_index_incremental_write(CaIndex *i, const void *data, size_t size); 39 | int ca_index_incremental_eof(CaIndex *i); 40 | 41 | int ca_index_incremental_read(CaIndex *i, ReallocBuffer *buffer); 42 | 43 | int ca_index_set_feature_flags(CaIndex *i, uint64_t flags); 44 | int ca_index_get_feature_flags(CaIndex *i, uint64_t *ret); 45 | 46 | int ca_index_set_chunk_size_min(CaIndex *i, size_t cmin); 47 | int ca_index_set_chunk_size_avg(CaIndex *i, size_t cavg); 48 | int ca_index_set_chunk_size_max(CaIndex *i, size_t cmax); 49 | 50 | int ca_index_get_chunk_size_min(CaIndex *i, size_t *ret); 51 | int ca_index_get_chunk_size_avg(CaIndex *i, size_t *ret); 52 | int ca_index_get_chunk_size_max(CaIndex *i, size_t *ret); 53 | 54 | int ca_index_get_blob_size(CaIndex *i, uint64_t *ret); 55 | int ca_index_get_index_size(CaIndex *i, uint64_t *ret); 56 | int ca_index_get_total_chunks(CaIndex *i, uint64_t *ret); 57 | 58 | int ca_index_seek(CaIndex *i, uint64_t offset, uint64_t *ret_skip); 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /src/calocation.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocalocationhfoo 4 | #define foocalocationhfoo 5 | 6 | #include 7 | #include 8 | 9 | #include "cachunkid.h" 10 | #include "cadigest.h" 11 | #include "cafileroot.h" 12 | #include "canametable.h" 13 | #include "util.h" 14 | 15 | /* Describes the location of some data in a directory hierarchy. This is useful for cache management (i.e. remember 16 | * where to seek to in the directory hierarchy to generate a serialization matching a specific hash), but also for 17 | * tracking data origin for creating file system reflinks. */ 18 | 19 | typedef enum CaLocationDesignator { 20 | CA_LOCATION_ENTRY = 'e', 21 | CA_LOCATION_PAYLOAD = 'p', 22 | CA_LOCATION_FILENAME = 'n', 23 | CA_LOCATION_GOODBYE = 'g', 24 | CA_LOCATION_VOID = 'v', /* Used as placeholder if we have to describe a blob of data from no known location */ 25 | _CA_LOCATION_DESIGNATOR_INVALID = -1, 26 | } CaLocationDesignator; 27 | 28 | static inline bool CA_LOCATION_DESIGNATOR_VALID(CaLocationDesignator d) { 29 | return IN_SET(d, 30 | CA_LOCATION_ENTRY, 31 | CA_LOCATION_PAYLOAD, 32 | CA_LOCATION_FILENAME, 33 | CA_LOCATION_GOODBYE, 34 | CA_LOCATION_VOID); 35 | } 36 | 37 | typedef enum CaLocationWith { 38 | CA_LOCATION_WITH_SIZE = 1U << 0, 39 | CA_LOCATION_WITH_MTIME = 1U << 1, 40 | CA_LOCATION_WITH_FEATURE_FLAGS = 1U << 2, 41 | CA_LOCATION_WITH_ARCHIVE_OFFSET = 1U << 3, 42 | CA_LOCATION_WITH_NAME_TABLE = 1U << 4, 43 | 44 | CA_LOCATION_WITH_ALL = CA_LOCATION_WITH_SIZE|CA_LOCATION_WITH_MTIME|CA_LOCATION_WITH_FEATURE_FLAGS|CA_LOCATION_WITH_ARCHIVE_OFFSET|CA_LOCATION_WITH_NAME_TABLE, 45 | } CaLocationWith; 46 | 47 | /* A location in the serialization of a directory tree. This is considered immutable as soon as it was created 48 | * once. When we change it we make copies. */ 49 | typedef struct CaLocation { 50 | unsigned n_ref; 51 | 52 | /* The following three fields are unconditionally part of the location */ 53 | CaLocationDesignator designator; 54 | char *path; 55 | uint64_t offset; 56 | 57 | /* The following two fields are optional */ 58 | uint64_t size; /* if unspecified UINT64_MAX */ 59 | CaFileRoot *root; /* if unspecified NULL */ 60 | 61 | /* The following is used to detect file changes, and are optional too */ 62 | uint64_t mtime; /* If no mtime is known set to UINT64_MAX */ 63 | uint64_t inode; /* only set if mtime is != UINT64_MAX */ 64 | int generation; /* only valid if generation_valid is true */ 65 | bool generation_valid; 66 | 67 | /* The feature flags used for encoding, so that we don't use cached data created with different settings (if unspecified UINT64_MAX) */ 68 | uint64_t feature_flags; 69 | 70 | /* The archive offset at this location, if that's useful (if unspecified is UINT64_MAX) */ 71 | uint64_t archive_offset; 72 | 73 | /* The following encodes the file name tables so far built of for this node and all its parents, and is 74 | * optional (in which case it is NULL) */ 75 | CaNameTable *name_table; 76 | 77 | /* The formatted version of this location, if requested before */ 78 | char *formatted; 79 | CaLocationWith formatted_with; 80 | } CaLocation; 81 | 82 | int ca_location_new(const char *path, CaLocationDesignator designator, uint64_t offset, uint64_t size, CaLocation **ret); 83 | #define ca_location_new_void(size, ret) ca_location_new(NULL, CA_LOCATION_VOID, 0, size, ret) 84 | int ca_location_copy(CaLocation *l, CaLocation **ret); 85 | 86 | CaLocation* ca_location_unref(CaLocation *l); 87 | CaLocation* ca_location_ref(CaLocation *l); 88 | 89 | DEFINE_TRIVIAL_CLEANUP_FUNC(CaLocation*, ca_location_unref); 90 | 91 | const char* ca_location_format_full(CaLocation *l, CaLocationWith with); 92 | #define ca_location_format(l) ca_location_format_full(l, CA_LOCATION_WITH_SIZE|CA_LOCATION_WITH_MTIME|CA_LOCATION_WITH_FEATURE_FLAGS) 93 | 94 | int ca_location_parse(const char *text, CaLocation **ret); 95 | 96 | int ca_location_patch_size(CaLocation **l, uint64_t size); 97 | int ca_location_patch_root(CaLocation **l, CaFileRoot *root); 98 | 99 | int ca_location_advance(CaLocation **l, uint64_t n_bytes); 100 | 101 | int ca_location_merge(CaLocation **a, CaLocation *b); 102 | 103 | int ca_location_open(CaLocation *l); 104 | 105 | int ca_location_id_make(CaDigest *digest, CaLocation *l, bool include_size, CaChunkID *ret); 106 | 107 | bool ca_location_equal(CaLocation *a, CaLocation *b, CaLocationWith with); 108 | 109 | #endif 110 | -------------------------------------------------------------------------------- /src/camakebst.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "camakebst.h" 4 | #include "util.h" 5 | 6 | /* Permutation function originally by L. Bressel, 2017 */ 7 | 8 | static inline size_t pow_of_2(size_t e) { 9 | return (size_t) 1U << e; 10 | } 11 | 12 | static inline size_t log_of_2(size_t k) { 13 | assert(sizeof(size_t) == sizeof(unsigned long)); 14 | 15 | return sizeof(unsigned long)*8 - __builtin_clzl(k) - 1; 16 | } 17 | 18 | static void make_bst_inner( 19 | const void *input, 20 | size_t n, 21 | size_t size, 22 | size_t e, 23 | void *output, 24 | size_t i) { 25 | 26 | size_t k, p, q; 27 | 28 | if (n == 0) 29 | return; 30 | 31 | assert(input); 32 | assert(size > 0); 33 | assert(e > 0); 34 | assert(output); 35 | 36 | p = pow_of_2(e-1); 37 | q = pow_of_2(e); 38 | 39 | if (n >= p - 1 + p / 2) 40 | k = (q - 2) / 2; 41 | else { 42 | size_t v; 43 | 44 | v = p - 1 + p / 2 - n; 45 | k = (q - 2) / 2 - v; 46 | } 47 | 48 | memcpy((uint8_t*) output + i * size, (const uint8_t*) input + k * size, size); 49 | 50 | /* Prepare left-side subtree */ 51 | make_bst_inner(input, 52 | k, 53 | size, 54 | e - 1, 55 | output, 56 | i*2+1); 57 | 58 | /* Prepare right-side subtree */ 59 | make_bst_inner((const uint8_t*) input + (k + 1) * size, 60 | n - k - 1, 61 | size, 62 | e - 1, 63 | output, 64 | i*2+2); 65 | } 66 | 67 | void ca_make_bst(const void *input, size_t n, size_t size, void *output) { 68 | assert(size > 0); 69 | 70 | /* Generate a binary search tree stored in an array from a sorted array. Specifically, for any given sorted 71 | * array 'input' of 'n' elements of size 'size' permute the array so that the following rule holds: 72 | * 73 | * For each array item with index i, the item at 2*i+1 is smaller and the item 2*i+2 is larger. 74 | * 75 | * This structure permits efficient (meaning: O(log(n)) binary searches: start with item i=0 (i.e. the root of 76 | * the BST), compare the value with the searched item, if smaller proceed at item i*2+1, if larger proceed at 77 | * item i*2+2, and repeat, until either the item is found, or the indexes grow beyond the array size, which 78 | * means the entry does not exist. Effectively this implements bisection, but instead of jumping around wildly 79 | * in the array during a single search we only search with strictly monotonically increasing indexes. 80 | */ 81 | 82 | make_bst_inner(input, n, size, log_of_2(n) + 1, output, 0); 83 | } 84 | -------------------------------------------------------------------------------- /src/camakebst.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocamakebsthfoo 4 | #define foocamakebsthfoo 5 | 6 | #include 7 | 8 | void ca_make_bst(const void *input, size_t n, size_t size, void *output); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /src/camatch.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocamatchhfoo 4 | #define foocamatchhfoo 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | typedef struct CaMatch CaMatch; 11 | 12 | typedef enum CaMatchType { 13 | CA_MATCH_POSITIVE, 14 | CA_MATCH_NEGATIVE, /* Path is prefixed with an exclamation mark: reverses operation */ 15 | CA_MATCH_INNER, /* Item does not match anything, just exists to track children */ 16 | _CA_MATCH_TYPE_MAX, 17 | _CA_MATCH_TYPE_INVALID = -1, 18 | } CaMatchType; 19 | 20 | struct CaMatch { 21 | unsigned n_ref; 22 | 23 | CaMatchType type; 24 | 25 | bool anchored:1; /* Path starts with a slash (or contained them originall, though not at the end): only 26 | * applies to the current directory, not any children */ 27 | bool directory_only:1; /* Path is suffixed with a slash: only matches directories */ 28 | 29 | CaMatch **children; 30 | size_t n_children; 31 | size_t n_allocated; 32 | 33 | char name[]; 34 | }; 35 | 36 | int ca_match_new_from_file(int dir_fd, const char *filename, CaMatch **ret); 37 | int ca_match_new_from_strings(char **path, CaMatch **ret); 38 | 39 | CaMatch* ca_match_ref(CaMatch *match); 40 | CaMatch* ca_match_unref(CaMatch *match); 41 | 42 | static inline void ca_match_unrefp(CaMatch **match) { 43 | if (match) 44 | ca_match_unref(*match); 45 | } 46 | 47 | int ca_match_add_child(CaMatch *match, CaMatch *child); 48 | 49 | int ca_match_merge(CaMatch **a, CaMatch *b); 50 | 51 | int ca_match_normalize(CaMatch **match); 52 | 53 | int ca_match_test(CaMatch *match, const char *name, bool is_directory, CaMatch **ret); 54 | 55 | static inline size_t ca_match_children(CaMatch *match) { 56 | return match ? match->n_children : 0; 57 | } 58 | 59 | void ca_match_dump(FILE *f, CaMatch *match, const char *prefix); 60 | 61 | bool ca_match_equal(CaMatch *a, CaMatch *b); 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /src/canametable.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocanametablehfoo 4 | #define foocanametablehfoo 5 | 6 | #include 7 | #include 8 | 9 | #include "util.h" 10 | #include "realloc-buffer.h" 11 | 12 | /* A name table object. Contains a list of CaNameItem that carry the hash of a filename plus the start and end archive 13 | * offsets of the serializations of these files. The object is considered immutable, unless only a single reference is 14 | * taken. The object also contains a reference to the same table of the parent node. This way it encodes reliably the 15 | * current context of the directory serializations of all nodes and their parents at any moment in time. */ 16 | 17 | typedef struct CaNameItem { 18 | uint64_t hash; /* hash of the filename */ 19 | uint64_t start_offset; /* start offset of the node's serialization in the archive stream */ 20 | uint64_t end_offset; /* end offset */ 21 | } CaNameItem; 22 | 23 | typedef struct CaNameTable { 24 | unsigned n_ref; 25 | struct CaNameTable *parent; /* Parent's name table chained up */ 26 | 27 | uint64_t entry_offset; 28 | 29 | size_t n_items; 30 | size_t n_allocated; 31 | 32 | char *formatted; 33 | 34 | CaNameItem items[]; 35 | } CaNameTable; 36 | 37 | CaNameTable *ca_name_table_new_size(size_t m); 38 | #define ca_name_table_new() ca_name_table_new_size((size_t) -1) 39 | 40 | CaNameTable *ca_name_table_ref(CaNameTable *t); 41 | CaNameTable *ca_name_table_unref(CaNameTable *t); 42 | 43 | DEFINE_TRIVIAL_CLEANUP_FUNC(CaNameTable*, ca_name_table_unref); 44 | 45 | int ca_name_table_make_writable(CaNameTable **t, size_t add); 46 | int ca_name_table_add(CaNameTable **t, CaNameItem **ret); 47 | 48 | int ca_name_table_format_realloc_buffer(CaNameTable *t, ReallocBuffer *buffer); 49 | 50 | char* ca_name_table_format(CaNameTable *t); 51 | int ca_name_table_parse(const char **text, CaNameTable **ret); 52 | 53 | static inline size_t ca_name_table_items(CaNameTable *t) { 54 | return t ? t->n_items : 0; 55 | } 56 | 57 | static inline CaNameItem* ca_name_table_get(CaNameTable *t, size_t i) { 58 | if (i >= ca_name_table_items(t)) 59 | return NULL; 60 | 61 | return t->items + i; 62 | } 63 | 64 | static inline CaNameItem *ca_name_table_last(CaNameTable *t) { 65 | size_t n; 66 | 67 | n = ca_name_table_items(t); 68 | if (n <= 0) 69 | return NULL; 70 | 71 | return t->items + n - 1; 72 | } 73 | 74 | int ca_name_table_make_bst(CaNameTable *t, CaNameTable **ret); 75 | 76 | int ca_name_table_dump(FILE *f, CaNameTable *t); 77 | int ca_name_table_dump_recursive(FILE *f, CaNameTable *t); 78 | 79 | bool ca_name_table_equal(CaNameTable *a, CaNameTable *b); 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /src/canbd.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocanbdfoo 4 | #define foocanbdfoo 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | typedef struct CaBlockDevice CaBlockDevice; 11 | 12 | enum { 13 | CA_BLOCK_DEVICE_CLOSED, 14 | CA_BLOCK_DEVICE_REQUEST, 15 | CA_BLOCK_DEVICE_POLL, 16 | }; 17 | 18 | CaBlockDevice *ca_block_device_new(void); 19 | CaBlockDevice *ca_block_device_unref(CaBlockDevice* d); 20 | static inline void ca_block_device_unrefp(CaBlockDevice **d) { 21 | ca_block_device_unref(*d); 22 | } 23 | 24 | int ca_block_device_set_size(CaBlockDevice *d, uint64_t size); 25 | int ca_block_device_open(CaBlockDevice *d); 26 | 27 | int ca_block_device_step(CaBlockDevice *d); 28 | 29 | int ca_block_device_get_request_offset(CaBlockDevice *d, uint64_t *ret); 30 | int ca_block_device_get_request_size(CaBlockDevice *d, uint64_t *ret); 31 | 32 | int ca_block_device_put_data(CaBlockDevice *d, uint64_t offset, const void *data, size_t size); 33 | 34 | int ca_block_device_poll(CaBlockDevice *d, uint64_t nsec, const sigset_t *ss); 35 | 36 | int ca_block_device_set_path(CaBlockDevice *d, const char *node); 37 | int ca_block_device_get_path(CaBlockDevice *d, const char **ret); 38 | 39 | int ca_block_device_set_friendly_name(CaBlockDevice *d, const char *name); 40 | 41 | int ca_block_device_get_devnum(CaBlockDevice *d, dev_t *ret); 42 | int ca_block_device_get_poll_fd(CaBlockDevice *d); 43 | 44 | int ca_block_device_test_nbd(const char *name); 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /src/caorigin.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocaoriginhfoo 4 | #define foocaoriginhfoo 5 | 6 | #include "calocation.h" 7 | 8 | /* Describes the origin of a data stream, as a series of location objects. This is primarily useful for tracking data 9 | * origins for creating file system reflinks. */ 10 | 11 | typedef struct CaOrigin { 12 | CaLocation *first; 13 | CaLocation **others; 14 | size_t n_items; 15 | size_t n_allocated; 16 | uint64_t n_bytes; 17 | } CaOrigin; 18 | 19 | int ca_origin_new(CaOrigin **ret); 20 | CaOrigin* ca_origin_unref(CaOrigin *origin); 21 | 22 | void ca_origin_flush(CaOrigin *origin); 23 | 24 | int ca_origin_put(CaOrigin *origin, CaLocation *location); 25 | CaLocation* ca_origin_get(CaOrigin *origin, size_t i); 26 | int ca_origin_concat(CaOrigin *origin, CaOrigin *other, uint64_t n_bytes); 27 | 28 | int ca_origin_put_void(CaOrigin *origin, uint64_t n_bytes); 29 | 30 | int ca_origin_advance_items(CaOrigin *origin, size_t n_drop); 31 | int ca_origin_advance_bytes(CaOrigin *origin, uint64_t n_bytes); 32 | 33 | int ca_origin_extract_bytes(CaOrigin *origin, uint64_t n_bytes, CaOrigin **ret); 34 | 35 | int ca_origin_dump(FILE *f, CaOrigin *origin); 36 | 37 | static inline size_t ca_origin_items(CaOrigin *origin) { 38 | return origin ? origin->n_items : 0; 39 | } 40 | 41 | static inline uint64_t ca_origin_bytes(CaOrigin *origin) { 42 | return origin ? origin->n_bytes : 0; 43 | } 44 | 45 | DEFINE_TRIVIAL_CLEANUP_FUNC(CaOrigin*, ca_origin_unref); 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /src/caprotocol-util.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "caprotocol-util.h" 4 | #include "caprotocol.h" 5 | 6 | const char *ca_protocol_type_name(uint64_t u) { 7 | switch (u) { 8 | 9 | case CA_PROTOCOL_HELLO: 10 | return "hello"; 11 | 12 | case CA_PROTOCOL_INDEX: 13 | return "index"; 14 | 15 | case CA_PROTOCOL_INDEX_EOF: 16 | return "index-eof"; 17 | 18 | case CA_PROTOCOL_ARCHIVE: 19 | return "archive"; 20 | 21 | case CA_PROTOCOL_ARCHIVE_EOF: 22 | return "archive-eof"; 23 | 24 | case CA_PROTOCOL_REQUEST: 25 | return "request"; 26 | 27 | case CA_PROTOCOL_CHUNK: 28 | return "chunk"; 29 | 30 | case CA_PROTOCOL_MISSING: 31 | return "missing"; 32 | 33 | case CA_PROTOCOL_GOODBYE: 34 | return "goodbye"; 35 | 36 | case CA_PROTOCOL_ABORT: 37 | return "abort"; 38 | } 39 | 40 | return NULL; 41 | } 42 | -------------------------------------------------------------------------------- /src/caprotocol-util.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | 5 | #include "caprotocol.h" 6 | 7 | const char *ca_protocol_type_name(uint64_t u); 8 | -------------------------------------------------------------------------------- /src/caprotocol.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocaprotocolhfoo 4 | #define foocaprotocolhfoo 5 | 6 | #include 7 | 8 | #include "util.h" 9 | #include "cachunkid.h" 10 | 11 | enum { 12 | CA_PROTOCOL_HELLO = UINT64_C(0x3c71d0948ca5fbee), 13 | CA_PROTOCOL_INDEX = UINT64_C(0xb32a91dd2b3e27f8), 14 | CA_PROTOCOL_INDEX_EOF = UINT64_C(0x4f0932f1043718f5), 15 | CA_PROTOCOL_ARCHIVE = UINT64_C(0x95d6428a69eddcc5), 16 | CA_PROTOCOL_ARCHIVE_EOF = UINT64_C(0x450bef663f24cbad), 17 | CA_PROTOCOL_REQUEST = UINT64_C(0x8ab427e0f89d9210), 18 | CA_PROTOCOL_CHUNK = UINT64_C(0x5213dd180a84bc8c), 19 | CA_PROTOCOL_MISSING = UINT64_C(0xd010f9fac82b7b6c), 20 | CA_PROTOCOL_GOODBYE = UINT64_C(0xad205dbf1a3686c3), 21 | CA_PROTOCOL_ABORT = UINT64_C(0xe7d9136b7efea352), 22 | }; 23 | 24 | /* Protocol description: 25 | * 26 | * Client C connects to server S: 27 | * 28 | * Both C and S immediately send CA_PROTOCOL_HELLO: 29 | * C → S: CA_PROTOCOL_HELLO 30 | * S → C: CA_PROTOCOL_HELLO 31 | * 32 | * On pull: 33 | * S → C: CA_PROTOCOL_INDEX 34 | * S → C: CA_PROTOCOL_INDEX 35 | * … 36 | * S → C: CA_PROTOCOL_INDEX 37 | * S → C: CA_PROTOCOL_INDEX_EOF 38 | * 39 | * Followed by multiple: 40 | * C → S: CA_PROTOCOL_REQUEST 41 | * S → C: CA_PROTOCOL_CHUNK 42 | * 43 | * Finished by: 44 | * C → S: CA_PROTOCOL_GOODBYE (optional) 45 | * 46 | * On push: 47 | * C → S: CA_PROTOCOL_INDEX 48 | * C → S: CA_PROTOCOL_INDEX 49 | * … 50 | * C → S: CA_PROTOCOL_INDEX 51 | * C → S: CA_PROTOCOL_INDEX_EOF 52 | * 53 | * Followed by multiple: 54 | * S → C: CA_PROTOCOL_REQUEST 55 | * C → S: CA_PROTOCOL_CHUNK (or CA_PROTOCOL_MISSING) 56 | * 57 | * Finished by: 58 | * S → C: CA_PROTOCOL_GOODBYE 59 | * 60 | * When a non-recoverable error occurs, either side can send CA_PROTOCOL_ABORTED with an explanation, and terminate the 61 | * connection. 62 | * 63 | */ 64 | 65 | typedef struct CaProtocolHeader { 66 | le64_t size; 67 | le64_t type; 68 | } CaProtocolHeader; 69 | 70 | #define CA_PROTOCOL_SIZE_MIN (sizeof(CaProtocolHeader)) 71 | #define CA_PROTOCOL_SIZE_MAX (16*1024*1024) 72 | 73 | typedef struct CaProtocolHello { 74 | CaProtocolHeader header; 75 | le64_t feature_flags; 76 | } CaProtocolHello; 77 | 78 | enum { 79 | /* Services I provide */ 80 | CA_PROTOCOL_READABLE_STORE = 0x1, /* I provide chunks on request to you */ 81 | CA_PROTOCOL_WRITABLE_STORE = 0x2, /* I can store chunks for you */ 82 | CA_PROTOCOL_READABLE_INDEX = 0x4, /* I provide an index on request to you */ 83 | CA_PROTOCOL_WRITABLE_INDEX = 0x8, /* I can store an index for you */ 84 | CA_PROTOCOL_READABLE_ARCHIVE = 0x10, /* I provide an archive blob to you */ 85 | CA_PROTOCOL_WRITABLE_ARCHIVE = 0x20, /* I can store an archive blob for you */ 86 | 87 | /* Operations I'd like to execute */ 88 | CA_PROTOCOL_PULL_CHUNKS = 0x40, /* I'd like to pull chunks from you */ 89 | CA_PROTOCOL_PULL_INDEX = 0x80, /* I'd like to pull an index from you */ 90 | CA_PROTOCOL_PULL_ARCHIVE = 0x100, /* I'd like to pull an archive from you */ 91 | CA_PROTOCOL_PUSH_CHUNKS = 0x200, /* I'd like to push chunks to you */ 92 | CA_PROTOCOL_PUSH_INDEX = 0x400, /* I'd like to push an index to you */ 93 | CA_PROTOCOL_PUSH_INDEX_CHUNKS = 0x800, /* I'd like you to pull chunks from me, that are declared in the index I just pulled */ 94 | CA_PROTOCOL_PUSH_ARCHIVE = 0x1000, /* I'd like to push an archive to you */ 95 | 96 | CA_PROTOCOL_FEATURE_FLAGS_MAX = 0x1fff, 97 | }; 98 | 99 | typedef struct CaProtocolFile { /* Used for index as well as archive */ 100 | CaProtocolHeader header; 101 | uint8_t data[]; 102 | } CaProtocolFile; 103 | 104 | typedef struct CaProtocolFileEOF { /* Used for index as well as archive */ 105 | CaProtocolHeader header; 106 | } CaProtocolFileEOF; 107 | 108 | typedef struct CaProtocolRequest { 109 | CaProtocolHeader header; 110 | le64_t flags; 111 | uint8_t chunks[]; /* multiple of CA_CHUNK_ID_SIZE */ 112 | } CaProtocolRequest; 113 | 114 | enum { 115 | CA_PROTOCOL_REQUEST_HIGH_PRIORITY = 1, 116 | CA_PROTOCOL_REQUEST_FLAG_MAX = 1, 117 | }; 118 | 119 | typedef struct CaProtocolChunk { 120 | CaProtocolHeader header; 121 | le64_t flags; 122 | uint8_t chunk[CA_CHUNK_ID_SIZE]; 123 | uint8_t data[]; 124 | } CaProtocolChunk; 125 | 126 | enum { 127 | CA_PROTOCOL_CHUNK_COMPRESSED = 1, 128 | CA_PROTOCOL_CHUNK_FLAG_MAX = 1, 129 | }; 130 | 131 | typedef struct CaProtocolMissing { 132 | CaProtocolHeader header; 133 | uint8_t chunk[CA_CHUNK_ID_SIZE]; 134 | } CaProtocolMissing; 135 | 136 | typedef struct CaProtocolGoodbye { 137 | CaProtocolHeader header; 138 | } CaProtocolGoodbye; 139 | 140 | typedef struct CaProtocolAbort { 141 | CaProtocolHeader header; 142 | le64_t error; /* closest errno-style error, or 0 */ 143 | char reason[]; 144 | } CaProtocolAbort; 145 | 146 | #endif 147 | -------------------------------------------------------------------------------- /src/caremote.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocaremotehfoo 4 | #define foocaremotehfoo 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "cachunk.h" 11 | #include "cachunkid.h" 12 | 13 | typedef struct CaRemote CaRemote; 14 | 15 | enum { 16 | CA_REMOTE_POLL, /* Nothing to do, sleep with ca_remote_poll() please! */ 17 | CA_REMOTE_FINISHED, /* Done! */ 18 | CA_REMOTE_STEP, /* Did something, call me again */ 19 | CA_REMOTE_REQUEST, /* push: Got a REQUEST message, please call ca_remote_next_request() to find out more */ 20 | CA_REMOTE_WRITE_INDEX, /* push: Please provide index data now, via ca_remote_write_index() */ 21 | CA_REMOTE_WRITE_ARCHIVE, /* push: Please provide archive data now, via ca_remote_write_archive() */ 22 | CA_REMOTE_CHUNK, /* pull: Got a CHUNK message, ask me if yours is now with ca_remote_request() */ 23 | CA_REMOTE_READ_INDEX, /* pull: Got more INDEX data, retrieve it via ca_remote_read_index() */ 24 | CA_REMOTE_READ_INDEX_EOF, /* pull: INDEX data is now complete */ 25 | CA_REMOTE_READ_ARCHIVE, /* pull: Got more ARCHIVE data, retrieve it via ca_remote_read_archive() */ 26 | CA_REMOTE_READ_ARCHIVE_EOF, /* pull: ARCHIVE data is now complete */ 27 | }; 28 | 29 | enum { 30 | CA_REMOTE_ARG_OPERATION = 0, 31 | CA_REMOTE_ARG_BASE_URL, 32 | CA_REMOTE_ARG_ARCHIVE_URL, 33 | CA_REMOTE_ARG_INDEX_URL, 34 | CA_REMOTE_ARG_WSTORE_URL, /* This should be last except MAX */ 35 | _CA_REMOTE_ARG_MAX, 36 | }; 37 | 38 | CaRemote* ca_remote_new(void); 39 | CaRemote* ca_remote_ref(CaRemote *rr); 40 | CaRemote* ca_remote_unref(CaRemote *rr); 41 | static inline void ca_remote_unrefp(CaRemote **rr) { 42 | ca_remote_unref(*rr); 43 | } 44 | 45 | int ca_remote_set_local_feature_flags(CaRemote *rr, uint64_t flags); 46 | int ca_remote_add_local_feature_flags(CaRemote *rr, uint64_t flags); 47 | int ca_remote_get_local_feature_flags(CaRemote *rr, uint64_t* flags); 48 | int ca_remote_get_remote_feature_flags(CaRemote *rr, uint64_t* flags); 49 | 50 | int ca_remote_set_digest_type(CaRemote *rr, CaDigestType type); 51 | int ca_remote_get_digest_type(CaRemote *rr, CaDigestType *ret); 52 | 53 | int ca_remote_set_log_level(CaRemote *rr, int log_level); 54 | int ca_remote_set_rate_limit_bps(CaRemote *rr, uint64_t rate_limit_bps); 55 | 56 | int ca_remote_set_io_fds(CaRemote *rr, int input_fd, int output_fd); 57 | int ca_remote_get_io_fds(CaRemote *rr, int *ret_input_fd, int *ret_output_fd); 58 | int ca_remote_get_io_events(CaRemote *rr, short *ret_input_events, short *ret_output_events); 59 | 60 | /* int ca_remote_set_base_url(CaRemote *rr, const char *url); */ 61 | int ca_remote_set_archive_url(CaRemote *rr, const char *url); 62 | int ca_remote_set_index_url(CaRemote *rr, const char *url); 63 | int ca_remote_set_store_url(CaRemote *rr, const char *url); 64 | int ca_remote_add_store_url(CaRemote *rr, const char *url); 65 | 66 | int ca_remote_set_cache_path(CaRemote *rr, const char *path); 67 | int ca_remote_set_cache_fd(CaRemote *rr, int fd); 68 | 69 | int ca_remote_set_index_path(CaRemote *rr, const char *path); 70 | int ca_remote_set_index_fd(CaRemote *rr, int fd); 71 | 72 | int ca_remote_set_archive_path(CaRemote *rr, const char *path); 73 | int ca_remote_set_archive_fd(CaRemote *rr, int fd); 74 | 75 | int ca_remote_step(CaRemote *rr); 76 | 77 | int ca_remote_poll(CaRemote *rr, uint64_t timeout_nsec, const sigset_t *ss); 78 | 79 | /* When we are in "pull" mode, interfaces for retrieving chunks, or enqueing requests for them */ 80 | int ca_remote_request(CaRemote *rr, const CaChunkID *chunk_id, bool priority, CaChunkCompression desired_compression, const void **ret, uint64_t *ret_size, CaChunkCompression *ret_effective_compression); 81 | int ca_remote_request_async(CaRemote *rr, const CaChunkID *chunk_id, bool priority); 82 | int ca_remote_next_chunk(CaRemote *rr, CaChunkCompression desired_compression, CaChunkID *ret_id, const void **ret_data, size_t *ret_size, CaChunkCompression *ret_compression); 83 | 84 | /* When we are in "push" mode, interfaces for processing requests and pushing chunks */ 85 | int ca_remote_next_request(CaRemote *rr, CaChunkID *ret); 86 | int ca_remote_can_put_chunk(CaRemote *rr); 87 | int ca_remote_put_chunk(CaRemote *rr, const CaChunkID *chunk_id, CaChunkCompression compression, const void *data, uint64_t size); 88 | int ca_remote_put_missing(CaRemote *rr, const CaChunkID *chunk_id); 89 | 90 | /* pull mode: Read index data */ 91 | int ca_remote_read_index(CaRemote *rr, const void **ret, size_t *ret_size); 92 | 93 | /* pull mode: Read archive data */ 94 | int ca_remote_read_archive(CaRemote *rr, const void **ret, size_t *ret_size); 95 | 96 | /* push mode: Write index data */ 97 | int ca_remote_can_put_index(CaRemote *rr); 98 | int ca_remote_put_index(CaRemote *rr, const void *p, size_t size); 99 | int ca_remote_put_index_eof(CaRemote *rr); 100 | 101 | /* push mode: Write archive data */ 102 | int ca_remote_can_put_archive(CaRemote *rr); 103 | int ca_remote_put_archive(CaRemote *rr, const void *p, size_t size); 104 | int ca_remote_put_archive_eof(CaRemote *rr); 105 | 106 | /* Enqueue a goodbye frame */ 107 | int ca_remote_goodbye(CaRemote *rr); 108 | int ca_remote_abort(CaRemote *rr, int error, const char *message); 109 | 110 | int ca_remote_has_pending_requests(CaRemote *rr); 111 | int ca_remote_has_unwritten(CaRemote *rr); 112 | int ca_remote_has_chunks(CaRemote *rr); 113 | 114 | int ca_remote_forget_chunk(CaRemote *rr, const CaChunkID *id); 115 | 116 | int ca_remote_get_requests(CaRemote *rr, uint64_t *ret); 117 | int ca_remote_get_request_bytes(CaRemote *rr, uint64_t *ret); 118 | 119 | int ca_remote_set_compression_type(CaRemote *rr, CaCompressionType ct); 120 | 121 | #endif 122 | -------------------------------------------------------------------------------- /src/caseed.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocaseedhfoo 4 | #define foocaseedhfoo 5 | 6 | #include 7 | 8 | #include "cachunkid.h" 9 | #include "caorigin.h" 10 | 11 | typedef struct CaSeed CaSeed; 12 | 13 | enum { 14 | CA_SEED_READY = 0, 15 | CA_SEED_STEP = 1, 16 | CA_SEED_NEXT_FILE = 2, 17 | CA_SEED_DONE_FILE = 3, 18 | }; 19 | 20 | CaSeed *ca_seed_new(void); 21 | CaSeed *ca_seed_unref(CaSeed *s); 22 | 23 | int ca_seed_set_base_fd(CaSeed *s, int fd); 24 | int ca_seed_set_base_path(CaSeed *s, const char *path); 25 | 26 | int ca_seed_set_cache_fd(CaSeed *s, int fd); 27 | int ca_seed_set_cache_path(CaSeed *s, const char *path); 28 | 29 | int ca_seed_step(CaSeed *s); 30 | 31 | int ca_seed_get(CaSeed *s, const CaChunkID *chunk_id, const void **ret, size_t *ret_size, CaOrigin **ret_origin); 32 | int ca_seed_has(CaSeed *s, const CaChunkID *chunk_id); 33 | 34 | int ca_seed_get_hardlink_target(CaSeed *s, const CaChunkID *id, char **ret); 35 | 36 | int ca_seed_current_path(CaSeed *seed, char **ret); 37 | int ca_seed_current_mode(CaSeed *seed, mode_t *ret); 38 | 39 | int ca_seed_set_feature_flags(CaSeed *s, uint64_t flags); 40 | 41 | int ca_seed_set_chunk_size(CaSeed *s, size_t cmin, size_t cavg, size_t cmax); 42 | 43 | int ca_seed_set_hardlink(CaSeed *s, bool b); 44 | int ca_seed_set_chunks(CaSeed *s, bool b); 45 | 46 | int ca_seed_get_file_root(CaSeed *s, CaFileRoot **ret); 47 | 48 | int ca_seed_get_requests(CaSeed *s, uint64_t *ret); 49 | int ca_seed_get_request_bytes(CaSeed *s, uint64_t *ret); 50 | int ca_seed_get_seeding_time_nsec(CaSeed *s, uint64_t *ret); 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /src/castore.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocastorehfoo 4 | #define foocastorehfoo 5 | 6 | #include "cachunk.h" 7 | #include "cachunkid.h" 8 | #include "cautil.h" 9 | 10 | typedef struct CaStore CaStore; 11 | typedef struct CaStoreIterator CaStoreIterator; 12 | 13 | CaStore* ca_store_new(void); 14 | CaStore *ca_store_new_cache(void); 15 | CaStore* ca_store_unref(CaStore *store); 16 | static inline void ca_store_unrefp(CaStore **store) { 17 | ca_store_unref(*store); 18 | } 19 | 20 | int ca_store_set_path(CaStore *store, const char *path); 21 | int ca_store_set_compression(CaStore *store, CaChunkCompression c); 22 | int ca_store_set_compression_type(CaStore *store, CaCompressionType compression); 23 | 24 | int ca_store_get(CaStore *store, const CaChunkID *chunk_id, CaChunkCompression desired_compression, const void **ret, uint64_t *ret_size, CaChunkCompression *ret_effective_compression); 25 | int ca_store_has(CaStore *store, const CaChunkID *chunk_id); 26 | int ca_store_put(CaStore *store, const CaChunkID *chunk_id, CaChunkCompression effective_compression, const void *data, uint64_t size); 27 | 28 | int ca_store_get_requests(CaStore *s, uint64_t *ret); 29 | int ca_store_get_request_bytes(CaStore *s, uint64_t *ret); 30 | 31 | int ca_store_set_digest_type(CaStore *s, CaDigestType type); 32 | 33 | CaStoreIterator* ca_store_iterator_new(CaStore *store); 34 | CaStoreIterator* ca_store_iterator_unref(CaStoreIterator *iter); 35 | static inline void ca_store_iterator_unrefp(CaStoreIterator **iter) { 36 | ca_store_iterator_unref(*iter); 37 | } 38 | int ca_store_iterator_next( 39 | CaStoreIterator *iter, 40 | int *rootdir_fd, 41 | const char **subdir, 42 | int *subdir_fd, 43 | const char **chunk); 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /src/cautil.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocautilhfoo 4 | #define foocautilhfoo 5 | 6 | #include 7 | 8 | bool ca_is_url(const char *s); 9 | bool ca_is_ssh_path(const char *s); 10 | 11 | typedef enum CaLocatorClass { 12 | CA_LOCATOR_PATH, 13 | CA_LOCATOR_SSH, 14 | CA_LOCATOR_URL, 15 | _CA_LOCATOR_CLASS_INVALID = -1, 16 | } CaLocatorClass; 17 | 18 | CaLocatorClass ca_classify_locator(const char *s); 19 | 20 | char *ca_strip_file_url(const char *p); 21 | bool ca_locator_has_suffix(const char *p, const char *suffix); 22 | 23 | bool ca_xattr_name_is_valid(const char *s); 24 | bool ca_xattr_name_store(const char *p); 25 | 26 | const char *ca_compressed_chunk_suffix(void); 27 | 28 | int ca_locator_patch_last_component(const char *locator, const char *last_component, char **ret); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /src/chattr.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "util.h" 8 | #include "chattr.h" 9 | 10 | int read_attr_fd(int fd, unsigned *ret) { 11 | assert(fd >= 0); 12 | assert(ret); 13 | 14 | if (ioctl(fd, FS_IOC_GETFLAGS, ret) < 0) { 15 | 16 | if (!ERRNO_IS_UNSUPPORTED(errno)) 17 | return -errno; 18 | 19 | /* If a file system or node type doesn't support chattr flags, then all flags should be considered 0 */ 20 | *ret = 0; 21 | return 0; 22 | } 23 | 24 | return 1; 25 | } 26 | 27 | int write_attr_fd(int fd, unsigned attr) { 28 | assert(fd >= 0); 29 | 30 | if (ioctl(fd, FS_IOC_SETFLAGS, &attr) < 0) { 31 | 32 | /* If we shall write the attributes as 0, and we can't write them because the file system or node type 33 | * doesn't support them, that's fine */ 34 | if (attr == 0 && ERRNO_IS_UNSUPPORTED(errno)) 35 | return 0; 36 | 37 | return -errno; 38 | } 39 | 40 | return 1; 41 | } 42 | 43 | int mask_attr_fd(int fd, unsigned value, unsigned mask) { 44 | unsigned old_attr, new_attr; 45 | int r; 46 | 47 | assert(fd >= 0); 48 | 49 | if (mask == 0) 50 | return 0; 51 | 52 | r = read_attr_fd(fd, &old_attr); 53 | if (r < 0) 54 | return r; 55 | 56 | new_attr = (old_attr & ~mask) | (value & mask); 57 | if (new_attr == old_attr) 58 | return 0; 59 | 60 | return write_attr_fd(fd, new_attr); 61 | } 62 | 63 | int read_fat_attr_fd(int fd, uint32_t *ret) { 64 | assert(fd >= 0); 65 | assert(ret); 66 | 67 | if (ioctl(fd, FAT_IOCTL_GET_ATTRIBUTES, ret) < 0) { 68 | 69 | if (!ERRNO_IS_UNSUPPORTED(errno)) 70 | return -errno; 71 | 72 | *ret = 0; 73 | return 0; 74 | } 75 | 76 | return 1; 77 | } 78 | 79 | int write_fat_attr_fd(int fd, uint32_t attr) { 80 | assert(fd >= 0); 81 | 82 | if (ioctl(fd, FAT_IOCTL_SET_ATTRIBUTES, &attr) < 0) { 83 | 84 | if (attr == 0 && ERRNO_IS_UNSUPPORTED(errno)) 85 | return 0; 86 | 87 | return -errno; 88 | } 89 | 90 | return 1; 91 | } 92 | 93 | int mask_fat_attr_fd(int fd, uint32_t value, uint32_t mask) { 94 | uint32_t old_attr, new_attr; 95 | int r; 96 | 97 | assert(fd >= 0); 98 | 99 | if (mask == 0) 100 | return 0; 101 | 102 | r = read_fat_attr_fd(fd, &old_attr); 103 | if (r < 0) 104 | return r; 105 | 106 | new_attr = (old_attr & ~mask) | (value & mask); 107 | if (new_attr == old_attr) 108 | return 0; 109 | 110 | return write_fat_attr_fd(fd, new_attr); 111 | } 112 | -------------------------------------------------------------------------------- /src/chattr.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foochattrhfoo 4 | #define foochattrhfoo 5 | 6 | #include 7 | 8 | int read_attr_fd(int fd, unsigned *ret); 9 | int write_attr_fd(int fd, unsigned attr); 10 | int mask_attr_fd(int fd, unsigned value, unsigned mask); 11 | 12 | int read_fat_attr_fd(int fd, uint32_t *ret); 13 | int write_fat_attr_fd(int fd, uint32_t attr); 14 | int mask_fat_attr_fd(int fd, uint32_t value, uint32_t mask); 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /src/compressor.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foocompresshorhfoo 4 | #define foocompresshorhfoo 5 | 6 | #include 7 | #include 8 | 9 | #if HAVE_LIBLZMA 10 | # include 11 | #endif 12 | #if HAVE_LIBZ 13 | # include 14 | #endif 15 | #if HAVE_LIBZSTD 16 | # include 17 | #endif 18 | 19 | #include "cacompression.h" 20 | 21 | typedef enum CompressorOperation { 22 | COMPRESSOR_UNINITIALIZED, 23 | COMPRESSOR_ENCODE, 24 | COMPRESSOR_DECODE, 25 | } CompressorOperation; 26 | 27 | typedef struct CompressorContext { 28 | CompressorOperation operation; 29 | CaCompressionType compressor; 30 | 31 | union { 32 | #if HAVE_LIBLZMA 33 | lzma_stream xz; 34 | #endif 35 | #if HAVE_LIBZ 36 | struct z_stream_s gzip; 37 | #endif 38 | #if HAVE_LIBZSTD 39 | struct { 40 | ZSTD_CStream *cstream; 41 | ZSTD_DStream *dstream; 42 | ZSTD_inBuffer input; 43 | ZSTD_outBuffer output; 44 | } zstd; 45 | #endif 46 | }; 47 | } CompressorContext; 48 | 49 | #define COMPRESSOR_CONTEXT_INIT \ 50 | { \ 51 | .operation = COMPRESSOR_UNINITIALIZED, \ 52 | .compressor = _CA_COMPRESSION_TYPE_INVALID, \ 53 | } 54 | 55 | bool compressor_is_supported(CaCompressionType compressor); 56 | int compressor_start_decode(CompressorContext *c, CaCompressionType compressor); 57 | int compressor_start_encode(CompressorContext *c, CaCompressionType compressor); 58 | void compressor_finish(CompressorContext *c); 59 | 60 | int compressor_input(CompressorContext *c, const void *p, size_t sz); 61 | 62 | enum { 63 | COMPRESSOR_EOF, 64 | COMPRESSOR_MORE, 65 | COMPRESSOR_GOOD, 66 | }; 67 | 68 | int compressor_decode(CompressorContext *c, void *p, size_t size, size_t *ret_done); 69 | int compressor_encode(CompressorContext *c, bool finalize, void *p, size_t size, size_t *ret_done); 70 | 71 | int detect_compression(const void *buffer, size_t size); 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /src/copy.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "copy.h" 4 | 5 | #if !HAVE_COPY_FILE_RANGE 6 | # ifndef __NR_copy_file_range 7 | # if defined(__x86_64__) 8 | # define __NR_copy_file_range 326 9 | # elif defined(__i386__) 10 | # define __NR_copy_file_range 377 11 | # elif defined __s390__ 12 | # define __NR_copy_file_range 375 13 | # elif defined __arm__ 14 | # define __NR_copy_file_range 391 15 | # elif defined __aarch64__ 16 | # define __NR_copy_file_range 285 17 | # elif defined __powerpc__ 18 | # define __NR_copy_file_range 379 19 | # else 20 | # warning "__NR_copy_file_range not defined for your architecture" 21 | # endif 22 | # endif 23 | 24 | static inline ssize_t copy_file_range( 25 | int fd_in, loff_t *off_in, 26 | int fd_out, loff_t *off_out, 27 | size_t len, 28 | unsigned flags) { 29 | 30 | # ifdef __NR_copy_file_range 31 | return syscall(__NR_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags); 32 | # else 33 | errno = ENOSYS; 34 | return -1; 35 | # endif 36 | } 37 | #endif 38 | 39 | static ssize_t try_copy_file_range( 40 | int fd_in, loff_t *off_in, 41 | int fd_out, loff_t *off_out, 42 | size_t len, 43 | unsigned flags) { 44 | 45 | static int have = -1; 46 | ssize_t r; 47 | 48 | if (have == false) 49 | return -ENOSYS; 50 | 51 | r = copy_file_range(fd_in, off_in, fd_out, off_out, len, flags); 52 | if (have < 0) 53 | have = r >= 0 || errno != ENOSYS; 54 | if (r >= 0) 55 | return r; 56 | else 57 | return -errno; 58 | } 59 | 60 | int copy_bytes(int fdf, int fdt, uint64_t max_bytes) { 61 | bool try_cfr = true, try_sendfile = true, try_splice = true; 62 | size_t m = SSIZE_MAX; /* that is the maximum that sendfile and c_f_r accept */ 63 | int r; 64 | 65 | assert(fdf >= 0); 66 | assert(fdt >= 0); 67 | 68 | for (;;) { 69 | ssize_t n; 70 | 71 | if (max_bytes != (uint64_t) -1) { 72 | if (max_bytes <= 0) 73 | return 1; /* return > 0 if we hit the max_bytes limit */ 74 | 75 | if (m > max_bytes) 76 | m = max_bytes; 77 | } 78 | 79 | /* First try copy_file_range(), unless we already tried */ 80 | if (try_cfr) { 81 | n = try_copy_file_range(fdf, NULL, fdt, NULL, m, 0u); 82 | if (n < 0) { 83 | if (!IN_SET(n, -EINVAL, -ENOSYS, -EXDEV, -EBADF)) 84 | return n; 85 | 86 | try_cfr = false; 87 | /* use fallback below */ 88 | } else if (n == 0) /* EOF */ 89 | break; 90 | else 91 | /* Success! */ 92 | goto next; 93 | } 94 | 95 | /* First try sendfile(), unless we already tried */ 96 | if (try_sendfile) { 97 | n = sendfile(fdt, fdf, NULL, m); 98 | if (n < 0) { 99 | if (!IN_SET(errno, EINVAL, ENOSYS)) 100 | return -errno; 101 | 102 | try_sendfile = false; 103 | /* use fallback below */ 104 | } else if (n == 0) /* EOF */ 105 | break; 106 | else 107 | /* Success! */ 108 | goto next; 109 | } 110 | 111 | /* Then try splice, unless we already tried */ 112 | if (try_splice) { 113 | n = splice(fdf, NULL, fdt, NULL, m, 0); 114 | if (n < 0) { 115 | if (!IN_SET(errno, EINVAL, ENOSYS)) 116 | return -errno; 117 | 118 | try_splice = false; 119 | /* use fallback below */ 120 | } else if (n == 0) /* EOF */ 121 | break; 122 | else 123 | /* Success! */ 124 | goto next; 125 | } 126 | 127 | /* As a fallback just copy bits by hand */ 128 | { 129 | uint8_t buf[MIN(m, BUFFER_SIZE)]; 130 | 131 | n = read(fdf, buf, sizeof(buf)); 132 | if (n < 0) 133 | return -errno; 134 | if (n == 0) /* EOF */ 135 | break; 136 | 137 | r = loop_write(fdt, buf, (size_t) n); 138 | if (r < 0) 139 | return r; 140 | } 141 | 142 | next: 143 | if (max_bytes != (uint64_t) -1) { 144 | assert(max_bytes >= (uint64_t) n); 145 | max_bytes -= n; 146 | } 147 | /* sendfile accepts at most SSIZE_MAX-offset bytes to copy, 148 | * so reduce our maximum by the amount we already copied, 149 | * but don't go below our copy buffer size, unless we are 150 | * close the limit of bytes we are allowed to copy. */ 151 | m = MAX(MIN(BUFFER_SIZE, max_bytes), m - n); 152 | } 153 | 154 | return 0; /* return 0 if we hit EOF earlier than the size limit */ 155 | } 156 | -------------------------------------------------------------------------------- /src/def.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foodefhfoo 4 | #define foodefhfoo 5 | 6 | #define NODES_MAX 64 7 | 8 | #define BUFFER_SIZE (64U*1024U) 9 | 10 | #if HAVE_SELINUX 11 | #define SUPPORTED_FEATURE_MASK CA_FORMAT_FEATURE_FLAGS_MAX 12 | #else 13 | #define SUPPORTED_FEATURE_MASK (CA_FORMAT_FEATURE_FLAGS_MAX &~ CA_FORMAT_WITH_SELINUX) 14 | #endif 15 | 16 | #define SUPPORTED_WITH_MASK (CA_FORMAT_WITH_MASK & SUPPORTED_FEATURE_MASK) 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /src/dirent-util.c: -------------------------------------------------------------------------------- 1 | /*** 2 | SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | This file is part of systemd. 5 | 6 | Copyright 2010-2012 Lennart Poettering 7 | 8 | systemd is free software; you can redistribute it and/or modify it 9 | under the terms of the GNU Lesser General Public License as published by 10 | the Free Software Foundation; either version 2.1 of the License, or 11 | (at your option) any later version. 12 | 13 | systemd is distributed in the hope that it will be useful, but 14 | WITHOUT ANY WARRANTY; without even the implied warranty of 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | Lesser General Public License for more details. 17 | 18 | You should have received a copy of the GNU Lesser General Public License 19 | along with systemd; If not, see . 20 | ***/ 21 | 22 | #include 23 | #include 24 | 25 | #include "dirent-util.h" 26 | 27 | bool dirent_is_file_with_suffix(const struct dirent *de, const char *suffix) { 28 | assert(de); 29 | 30 | if (!IN_SET(de->d_type, DT_REG, DT_LNK, DT_UNKNOWN)) 31 | return false; 32 | 33 | if (de->d_name[0] == '.') 34 | return false; 35 | 36 | if (!suffix) 37 | return true; 38 | 39 | return endswith(de->d_name, suffix); 40 | } 41 | 42 | struct dirent* readdir_no_dot(DIR *dirp) { 43 | struct dirent* d; 44 | 45 | for (;;) { 46 | d = readdir(dirp); 47 | if (d && dot_or_dot_dot(d->d_name)) 48 | continue; 49 | return d; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/dirent-util.h: -------------------------------------------------------------------------------- 1 | /*** 2 | SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | This file is part of systemd. 5 | 6 | Copyright 2010 Lennart Poettering 7 | 8 | systemd is free software; you can redistribute it and/or modify it 9 | under the terms of the GNU Lesser General Public License as published by 10 | the Free Software Foundation; either version 2.1 of the License, or 11 | (at your option) any later version. 12 | 13 | systemd is distributed in the hope that it will be useful, but 14 | WITHOUT ANY WARRANTY; without even the implied warranty of 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | Lesser General Public License for more details. 17 | 18 | You should have received a copy of the GNU Lesser General Public License 19 | along with systemd; If not, see . 20 | ***/ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #include "util.h" 29 | 30 | bool dirent_is_file_with_suffix(const struct dirent *de, const char *suffix) _pure_; 31 | 32 | struct dirent* readdir_no_dot(DIR *dirp); 33 | 34 | #define FOREACH_DIRENT_ALL(de, d, on_error) \ 35 | for (errno = 0, de = readdir(d);; errno = 0, de = readdir(d)) \ 36 | if (!de) { \ 37 | if (errno > 0) { \ 38 | on_error; \ 39 | } \ 40 | break; \ 41 | } else 42 | -------------------------------------------------------------------------------- /src/fssize.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foofssizehfoo 4 | #define foofssizehfoo 5 | 6 | #include 7 | 8 | int read_file_system_size(int fd, uint64_t *ret); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /src/gc.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #pragma once 4 | 5 | #include "cachunk.h" 6 | #include "castore.h" 7 | 8 | typedef struct CaChunkCollection CaChunkCollection; 9 | 10 | CaChunkCollection* ca_chunk_collection_new(void); 11 | CaChunkCollection* ca_chunk_collection_unref(CaChunkCollection *c); 12 | static inline void ca_chunk_collection_unrefp(CaChunkCollection **c) { 13 | ca_chunk_collection_unref(*c); 14 | } 15 | 16 | int ca_chunk_collection_add_index(CaChunkCollection *coll, const char *path); 17 | int ca_chunk_collection_usage(CaChunkCollection *c, size_t *ret); 18 | int ca_chunk_collection_size(CaChunkCollection *c, size_t *ret); 19 | 20 | enum { 21 | CA_GC_VERBOSE = 1U, 22 | CA_GC_DRY_RUN = 2U, 23 | }; 24 | 25 | int ca_gc_cleanup_unused(CaStore *store, CaChunkCollection *coll, unsigned flags); 26 | -------------------------------------------------------------------------------- /src/gcc-macro.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foogccmacrohfoo 4 | #define foogccmacrohfoo 5 | 6 | #define _printf_(a,b) __attribute__ ((format (printf, a, b))) 7 | #define _sentinel_ __attribute__ ((sentinel)) 8 | #define _unused_ __attribute__ ((unused)) 9 | #define _unused_ __attribute__ ((unused)) 10 | #define _likely_(x) (__builtin_expect(!!(x),1)) 11 | #define _unlikely_(x) (__builtin_expect(!!(x),0)) 12 | #define _malloc_ __attribute__ ((malloc)) 13 | #define _pure_ __attribute__ ((pure)) 14 | #define _packed_ __attribute__ ((packed)) 15 | #define _const_ __attribute__ ((const)) 16 | #ifdef __clang__ 17 | # define _alloc_(...) 18 | #else 19 | # define _alloc_(...) __attribute__ ((alloc_size(__VA_ARGS__))) 20 | #endif 21 | #if __GNUC__ >= 7 22 | #define _fallthrough_ __attribute__((fallthrough)) 23 | #else 24 | #define _fallthrough_ 25 | #endif 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /src/hash-funcs.c: -------------------------------------------------------------------------------- 1 | /*** 2 | This file is part of systemd. 3 | 4 | Copyright 2010 Lennart Poettering 5 | Copyright 2014 Michal Schmidt 6 | 7 | systemd is free software; you can redistribute it and/or modify it 8 | under the terms of the GNU Lesser General Public License as published by 9 | the Free Software Foundation; either version 2.1 of the License, or 10 | (at your option) any later version. 11 | 12 | systemd is distributed in the hope that it will be useful, but 13 | WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | Lesser General Public License for more details. 16 | 17 | You should have received a copy of the GNU Lesser General Public License 18 | along with systemd; If not, see . 19 | ***/ 20 | 21 | #include "hash-funcs.h" 22 | 23 | void string_hash_func(const void *p, struct siphash *state) { 24 | siphash24_compress(p, strlen(p) + 1, state); 25 | } 26 | 27 | int string_compare_func(const void *a, const void *b) { 28 | return strcmp(a, b); 29 | } 30 | 31 | const struct hash_ops string_hash_ops = { 32 | .hash = string_hash_func, 33 | .compare = string_compare_func 34 | }; 35 | 36 | void trivial_hash_func(const void *p, struct siphash *state) { 37 | siphash24_compress(&p, sizeof(p), state); 38 | } 39 | 40 | int trivial_compare_func(const void *a, const void *b) { 41 | return a < b ? -1 : (a > b ? 1 : 0); 42 | } 43 | 44 | const struct hash_ops trivial_hash_ops = { 45 | .hash = trivial_hash_func, 46 | .compare = trivial_compare_func 47 | }; 48 | 49 | void uint64_hash_func(const void *p, struct siphash *state) { 50 | siphash24_compress(p, sizeof(uint64_t), state); 51 | } 52 | 53 | int uint64_compare_func(const void *_a, const void *_b) { 54 | uint64_t a, b; 55 | a = *(const uint64_t*) _a; 56 | b = *(const uint64_t*) _b; 57 | return a < b ? -1 : (a > b ? 1 : 0); 58 | } 59 | 60 | const struct hash_ops uint64_hash_ops = { 61 | .hash = uint64_hash_func, 62 | .compare = uint64_compare_func 63 | }; 64 | -------------------------------------------------------------------------------- /src/hash-funcs.h: -------------------------------------------------------------------------------- 1 | /*** 2 | SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | This file is part of systemd. 5 | 6 | Copyright 2010 Lennart Poettering 7 | Copyright 2014 Michal Schmidt 8 | 9 | systemd is free software; you can redistribute it and/or modify it 10 | under the terms of the GNU Lesser General Public License as published by 11 | the Free Software Foundation; either version 2.1 of the License, or 12 | (at your option) any later version. 13 | 14 | systemd is distributed in the hope that it will be useful, but 15 | WITHOUT ANY WARRANTY; without even the implied warranty of 16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 | Lesser General Public License for more details. 18 | 19 | You should have received a copy of the GNU Lesser General Public License 20 | along with systemd; If not, see . 21 | ***/ 22 | 23 | #pragma once 24 | 25 | #include "util.h" 26 | #include "siphash24.h" 27 | 28 | typedef void (*hash_func_t)(const void *p, struct siphash *state); 29 | typedef int (*compare_func_t)(const void *a, const void *b); 30 | 31 | struct hash_ops { 32 | hash_func_t hash; 33 | compare_func_t compare; 34 | }; 35 | 36 | void string_hash_func(const void *p, struct siphash *state); 37 | int string_compare_func(const void *a, const void *b) _pure_; 38 | extern const struct hash_ops string_hash_ops; 39 | 40 | /* This will compare the passed pointers directly, and will not 41 | * dereference them. This is hence not useful for strings or 42 | * suchlike. */ 43 | void trivial_hash_func(const void *p, struct siphash *state); 44 | int trivial_compare_func(const void *a, const void *b) _const_; 45 | extern const struct hash_ops trivial_hash_ops; 46 | 47 | /* 32bit values we can always just embed in the pointer itself, but 48 | * in order to support 32bit archs we need store 64bit values 49 | * indirectly, since they don't fit in a pointer. */ 50 | void uint64_hash_func(const void *p, struct siphash *state); 51 | int uint64_compare_func(const void *a, const void *b) _pure_; 52 | extern const struct hash_ops uint64_hash_ops; 53 | -------------------------------------------------------------------------------- /src/log.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "log.h" 9 | #include "util.h" 10 | 11 | static int cached_log_level = -1; 12 | 13 | static int level_from_string(const char *str) { 14 | if (STR_IN_SET(str, "emerg", "emergency", "0")) 15 | return LOG_EMERG; 16 | else if (STR_IN_SET(str, "alert", "1")) 17 | return LOG_ALERT; 18 | else if (STR_IN_SET(str, "crit", "critical", "2")) 19 | return LOG_CRIT; 20 | else if (STR_IN_SET(str, "err", "error", "3")) 21 | return LOG_ERR; 22 | else if (STR_IN_SET(str, "warn", "warning", "4")) 23 | return LOG_WARNING; 24 | else if (STR_IN_SET(str, "notice", "5")) 25 | return LOG_NOTICE; 26 | else if (STR_IN_SET(str, "info", "6")) 27 | return LOG_INFO; 28 | else if (STR_IN_SET(str, "debug", "7")) 29 | return LOG_DEBUG; 30 | else 31 | return -EINVAL; 32 | } 33 | 34 | static int get_log_level(void) { 35 | if (cached_log_level < 0) { 36 | const char *e; 37 | 38 | cached_log_level = LOG_INFO; 39 | 40 | e = getenv("CASYNC_LOG_LEVEL"); 41 | if (e) 42 | set_log_level_from_string(e); 43 | } 44 | 45 | return cached_log_level; 46 | } 47 | 48 | void set_log_level(int level) { 49 | cached_log_level = level; 50 | } 51 | 52 | int set_log_level_from_string(const char *str) { 53 | int level; 54 | 55 | level = level_from_string(str); 56 | if (level < 0) 57 | return level; 58 | 59 | cached_log_level = level; 60 | return level; 61 | } 62 | 63 | static int log_fullv( 64 | int level, 65 | int error, 66 | const char *format, 67 | va_list ap) { 68 | 69 | int orig_errno = errno; 70 | const char *fmt; 71 | 72 | if (level > get_log_level()) 73 | return -abs(error); 74 | 75 | if (endswith(format, "\n")) 76 | fmt = format; 77 | else 78 | fmt = strjoina(format, "\n"); 79 | 80 | if (error != 0) 81 | errno = abs(error); 82 | 83 | #pragma GCC diagnostic push 84 | #pragma GCC diagnostic ignored "-Wformat-nonliteral" 85 | vfprintf(stderr, fmt, ap); 86 | #pragma GCC diagnostic pop 87 | 88 | errno = orig_errno; 89 | return -abs(error); 90 | } 91 | 92 | int log_info_errno(int error, const char* format, ...) { 93 | va_list ap; 94 | int r; 95 | 96 | va_start(ap, format); 97 | r = log_fullv(LOG_INFO, error, format, ap); 98 | va_end(ap); 99 | 100 | return r; 101 | } 102 | 103 | int log_error_errno(int error, const char* format, ...) { 104 | va_list ap; 105 | int r; 106 | 107 | va_start(ap, format); 108 | r = log_fullv(LOG_ERR, error, format, ap); 109 | va_end(ap); 110 | 111 | return r; 112 | } 113 | 114 | int log_debug_errno(int error, const char* format, ...) { 115 | va_list ap; 116 | int r; 117 | 118 | va_start(ap, format); 119 | r = log_fullv(LOG_DEBUG, error, format, ap); 120 | va_end(ap); 121 | 122 | return r; 123 | } 124 | -------------------------------------------------------------------------------- /src/log.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #pragma once 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include "gcc-macro.h" 10 | 11 | int log_info_errno(int error, const char* fmt, ...) _printf_(2,3); 12 | int log_error_errno(int error, const char* fmt, ...) _printf_(2,3); 13 | int log_debug_errno(int error, const char* fmt, ...) _printf_(2,3); 14 | 15 | #define log_info(fmt, ...) log_info_errno(0, fmt, ##__VA_ARGS__) 16 | #define log_error(fmt, ...) log_error_errno(0, fmt, ##__VA_ARGS__) 17 | #define log_debug(fmt, ...) log_debug_errno(0, fmt, ##__VA_ARGS__) 18 | 19 | static inline int log_oom(void) { 20 | log_error("Out of memory"); 21 | return -ENOMEM; 22 | } 23 | 24 | #define assert_se(x) \ 25 | do { \ 26 | if (!(x)) { \ 27 | log_error("%s:%d (%s): assertion failed: %s", \ 28 | __FILE__, __LINE__, __PRETTY_FUNCTION__, #x); \ 29 | abort(); \ 30 | } \ 31 | } while(false) 32 | 33 | #define assert_not_reached(x) \ 34 | do { \ 35 | log_error("%s:%d (%s): unreachable code reached: %s", \ 36 | __FILE__, __LINE__, __PRETTY_FUNCTION__, x); \ 37 | abort(); \ 38 | } while(false) 39 | 40 | void set_log_level(int level); 41 | int set_log_level_from_string(const char *str); 42 | -------------------------------------------------------------------------------- /src/mempool.c: -------------------------------------------------------------------------------- 1 | /*** 2 | SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | This file is part of systemd. 5 | 6 | Copyright 2010-2014 Lennart Poettering 7 | Copyright 2014 Michal Schmidt 8 | 9 | systemd is free software; you can redistribute it and/or modify it 10 | under the terms of the GNU Lesser General Public License as published by 11 | the Free Software Foundation; either version 2.1 of the License, or 12 | (at your option) any later version. 13 | 14 | systemd is distributed in the hope that it will be useful, but 15 | WITHOUT ANY WARRANTY; without even the implied warranty of 16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 | Lesser General Public License for more details. 18 | 19 | You should have received a copy of the GNU Lesser General Public License 20 | along with systemd; If not, see . 21 | ***/ 22 | 23 | #include 24 | #include 25 | 26 | #include "mempool.h" 27 | #include "util.h" 28 | 29 | struct pool { 30 | struct pool *next; 31 | unsigned n_tiles; 32 | unsigned n_used; 33 | }; 34 | 35 | void* mempool_alloc_tile(struct mempool *mp) { 36 | unsigned i; 37 | 38 | /* When a tile is released we add it to the list and simply 39 | * place the next pointer at its offset 0. */ 40 | 41 | assert(mp->tile_size >= sizeof(void*)); 42 | assert(mp->at_least > 0); 43 | 44 | if (mp->freelist) { 45 | void *r; 46 | 47 | r = mp->freelist; 48 | mp->freelist = * (void**) mp->freelist; 49 | return r; 50 | } 51 | 52 | if (_unlikely_(!mp->first_pool) || 53 | _unlikely_(mp->first_pool->n_used >= mp->first_pool->n_tiles)) { 54 | unsigned n; 55 | size_t size; 56 | struct pool *p; 57 | 58 | n = mp->first_pool ? mp->first_pool->n_tiles : 0; 59 | n = MAX(mp->at_least, n * 2); 60 | size = PAGE_ALIGN(ALIGN(sizeof(struct pool)) + n*mp->tile_size); 61 | n = (size - ALIGN(sizeof(struct pool))) / mp->tile_size; 62 | 63 | p = malloc(size); 64 | if (!p) 65 | return NULL; 66 | 67 | p->next = mp->first_pool; 68 | p->n_tiles = n; 69 | p->n_used = 0; 70 | 71 | mp->first_pool = p; 72 | } 73 | 74 | i = mp->first_pool->n_used++; 75 | 76 | return ((uint8_t*) mp->first_pool) + ALIGN(sizeof(struct pool)) + i*mp->tile_size; 77 | } 78 | 79 | void* mempool_alloc0_tile(struct mempool *mp) { 80 | void *p; 81 | 82 | p = mempool_alloc_tile(mp); 83 | if (p) 84 | memzero(p, mp->tile_size); 85 | return p; 86 | } 87 | 88 | void mempool_free_tile(struct mempool *mp, void *p) { 89 | * (void**) p = mp->freelist; 90 | mp->freelist = p; 91 | } 92 | 93 | #ifdef VALGRIND 94 | 95 | void mempool_drop(struct mempool *mp) { 96 | struct pool *p = mp->first_pool; 97 | while (p) { 98 | struct pool *n; 99 | n = p->next; 100 | free(p); 101 | p = n; 102 | } 103 | } 104 | 105 | #endif 106 | -------------------------------------------------------------------------------- /src/mempool.h: -------------------------------------------------------------------------------- 1 | /*** 2 | SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | This file is part of systemd. 5 | 6 | Copyright 2011-2014 Lennart Poettering 7 | Copyright 2014 Michal Schmidt 8 | 9 | systemd is free software; you can redistribute it and/or modify it 10 | under the terms of the GNU Lesser General Public License as published by 11 | the Free Software Foundation; either version 2.1 of the License, or 12 | (at your option) any later version. 13 | 14 | systemd is distributed in the hope that it will be useful, but 15 | WITHOUT ANY WARRANTY; without even the implied warranty of 16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 | Lesser General Public License for more details. 18 | 19 | You should have received a copy of the GNU Lesser General Public License 20 | along with systemd; If not, see . 21 | ***/ 22 | 23 | #pragma once 24 | 25 | #include 26 | 27 | struct pool; 28 | 29 | struct mempool { 30 | struct pool *first_pool; 31 | void *freelist; 32 | size_t tile_size; 33 | unsigned at_least; 34 | }; 35 | 36 | void* mempool_alloc_tile(struct mempool *mp); 37 | void* mempool_alloc0_tile(struct mempool *mp); 38 | void mempool_free_tile(struct mempool *mp, void *p); 39 | 40 | #define DEFINE_MEMPOOL(pool_name, tile_type, alloc_at_least) \ 41 | static struct mempool pool_name = { \ 42 | .tile_size = sizeof(tile_type), \ 43 | .at_least = alloc_at_least, \ 44 | } 45 | 46 | 47 | #ifdef VALGRIND 48 | void mempool_drop(struct mempool *mp); 49 | #endif 50 | -------------------------------------------------------------------------------- /src/meson.build: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: LGPL-2.1+ 2 | 3 | util_sources = files(''' 4 | log.c 5 | log.h 6 | time-util.c 7 | time-util.h 8 | util.c 9 | util.h 10 | '''.split()) 11 | 12 | libshared_sources = files(''' 13 | cacache.c 14 | cacache.h 15 | cachunk.c 16 | cachunk.h 17 | cachunker.c 18 | cachunker.h 19 | cachunkid.c 20 | cachunkid.h 21 | cacommon.h 22 | cacompression.c 23 | cacompression.h 24 | cadecoder.c 25 | cadecoder.h 26 | cadigest.c 27 | cadigest.h 28 | caencoder.c 29 | caencoder.h 30 | cafileroot.c 31 | cafileroot.h 32 | caformat-util.c 33 | caformat-util.h 34 | caformat.h 35 | caindex.c 36 | caindex.h 37 | calocation.c 38 | calocation.h 39 | camakebst.c 40 | camakebst.h 41 | camatch.c 42 | camatch.h 43 | canametable.c 44 | canametable.h 45 | canbd.c 46 | canbd.h 47 | caorigin.c 48 | caorigin.h 49 | caprotocol-util.c 50 | caprotocol-util.h 51 | caprotocol.h 52 | caremote.c 53 | caremote.h 54 | caseed.c 55 | caseed.h 56 | castore.c 57 | castore.h 58 | casync.c 59 | casync.h 60 | cautil.c 61 | cautil.h 62 | chattr.c 63 | chattr.h 64 | compressor.c 65 | compressor.h 66 | def.h 67 | dirent-util.c 68 | dirent-util.h 69 | fssize.c 70 | fssize.h 71 | gc.c 72 | gc.h 73 | gcc-macro.h 74 | hash-funcs.c 75 | hash-funcs.h 76 | hashmap.c 77 | hashmap.h 78 | mempool.c 79 | mempool.h 80 | notify.c 81 | notify.h 82 | parse-util.c 83 | parse-util.h 84 | quota-projid.c 85 | quota-projid.h 86 | realloc-buffer.c 87 | realloc-buffer.h 88 | reflink.c 89 | reflink.h 90 | rm-rf.c 91 | rm-rf.h 92 | set.h 93 | siphash24.c 94 | siphash24.h 95 | '''.split()) 96 | 97 | libshared = static_library( 98 | 'shared', 99 | libshared_sources, 100 | util_sources) 101 | 102 | casync_sources = files(''' 103 | casync-tool.c 104 | canbd.c 105 | canbd.h 106 | cafuse.h 107 | signal-handler.c 108 | signal-handler.h 109 | '''.split()) 110 | 111 | if conf.get('HAVE_FUSE') == 1 112 | casync_sources += files('cafuse.c') 113 | endif 114 | 115 | casync_http_sources = files('casync-http.c') 116 | -------------------------------------------------------------------------------- /src/notify.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "notify.h" 8 | #include "util.h" 9 | 10 | int send_notify(const char *text) { 11 | const char *e; 12 | union { 13 | struct sockaddr sa; 14 | struct sockaddr_un un; 15 | } sa = { 16 | .un.sun_family = AF_UNIX, 17 | }; 18 | ssize_t n; 19 | size_t c; 20 | int fd, r; 21 | 22 | if (isempty(text)) 23 | return 0; 24 | 25 | e = getenv("NOTIFY_SOCKET"); 26 | if (!e) 27 | return 0; 28 | 29 | c = strlen(e); 30 | if (c < 2 || c > sizeof(sa.un.sun_path)) 31 | return -EINVAL; 32 | if (!IN_SET(e[0], '/', '@')) 33 | return -EINVAL; 34 | 35 | fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0); 36 | if (fd < 0) 37 | return -errno; 38 | 39 | if (e[0] == '@') { 40 | sa.un.sun_path[0] = 0; 41 | strncpy(sa.un.sun_path + 1, e + 1, sizeof(sa.un.sun_path) - 1); 42 | } else 43 | strncpy(sa.un.sun_path, e, sizeof(sa.un.sun_path)); 44 | 45 | n = sendto(fd, text, strlen(text), MSG_NOSIGNAL, &sa.sa, SOCKADDR_UN_LEN(sa.un)); 46 | if (n < 0) { 47 | r = -errno; 48 | goto finish; 49 | } 50 | 51 | r = 1; 52 | 53 | finish: 54 | safe_close(fd); 55 | return r; 56 | } 57 | -------------------------------------------------------------------------------- /src/notify.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foonotifyhfoo 4 | #define foonotifyhfoo 5 | 6 | int send_notify(const char *text); 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /src/parse-util.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "parse-util.h" 12 | #include "time-util.h" 13 | #include "util.h" 14 | 15 | int parse_size(const char *t, uint64_t *size) { 16 | 17 | struct table { 18 | const char *suffix; 19 | unsigned long long factor; 20 | }; 21 | 22 | static const struct table table[] = { 23 | { "E", 1024ULL*1024ULL*1024ULL*1024ULL*1024ULL*1024ULL }, 24 | { "P", 1024ULL*1024ULL*1024ULL*1024ULL*1024ULL }, 25 | { "T", 1024ULL*1024ULL*1024ULL*1024ULL }, 26 | { "G", 1024ULL*1024ULL*1024ULL }, 27 | { "M", 1024ULL*1024ULL }, 28 | { "K", 1024ULL }, 29 | { "B", 1ULL }, 30 | { "", 1ULL }, 31 | }; 32 | 33 | const char *p; 34 | unsigned long long r = 0; 35 | unsigned n_entries, start_pos = 0; 36 | 37 | assert(t); 38 | assert(size); 39 | 40 | n_entries = ELEMENTSOF(table); 41 | 42 | p = t; 43 | do { 44 | unsigned long long l, tmp; 45 | double frac = 0; 46 | char *e; 47 | unsigned i; 48 | 49 | p += strspn(p, WHITESPACE); 50 | 51 | if (*p == '-') 52 | return -ERANGE; 53 | 54 | errno = 0; 55 | l = strtoull(p, &e, 10); 56 | if (errno > 0) 57 | return -errno; 58 | if (e == p) 59 | return -EINVAL; 60 | 61 | if (*e == '.') { 62 | e++; 63 | 64 | /* strtoull() itself would accept space/+/- */ 65 | if (*e >= '0' && *e <= '9') { 66 | unsigned long long l2; 67 | char *e2; 68 | 69 | l2 = strtoull(e, &e2, 10); 70 | if (errno > 0) 71 | return -errno; 72 | 73 | /* Ignore failure. E.g. 10.M is valid */ 74 | frac = l2; 75 | for (; e < e2; e++) 76 | frac /= 10; 77 | } 78 | } 79 | 80 | e += strspn(e, WHITESPACE); 81 | for (i = start_pos; i < n_entries; i++) 82 | if (startswith(e, table[i].suffix)) 83 | break; 84 | 85 | if (i >= n_entries) 86 | return -EINVAL; 87 | 88 | if (l + (frac > 0) > ULLONG_MAX / table[i].factor) 89 | return -ERANGE; 90 | 91 | tmp = l * table[i].factor + (unsigned long long) (frac * table[i].factor); 92 | if (tmp > ULLONG_MAX - r) 93 | return -ERANGE; 94 | 95 | r += tmp; 96 | if ((unsigned long long) (uint64_t) r != r) 97 | return -ERANGE; 98 | 99 | p = e + strlen(table[i].suffix); 100 | 101 | start_pos = i + 1; 102 | 103 | } while (*p); 104 | 105 | *size = r; 106 | 107 | return 0; 108 | } 109 | 110 | char *format_bytes(char *buf, size_t l, uint64_t t) { 111 | unsigned i; 112 | 113 | /* This only does IEC units so far */ 114 | 115 | static const struct { 116 | const char *suffix; 117 | uint64_t factor; 118 | } table[] = { 119 | { "E", UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024) }, 120 | { "P", UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024) }, 121 | { "T", UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024) }, 122 | { "G", UINT64_C(1024)*UINT64_C(1024)*UINT64_C(1024) }, 123 | { "M", UINT64_C(1024)*UINT64_C(1024) }, 124 | { "K", UINT64_C(1024) }, 125 | }; 126 | 127 | if (t == (uint64_t) -1) 128 | return NULL; 129 | 130 | for (i = 0; i < ELEMENTSOF(table); i++) { 131 | 132 | if (t >= table[i].factor) { 133 | snprintf(buf, l, 134 | "%" PRIu64 ".%" PRIu64 "%s", 135 | t / table[i].factor, 136 | ((t*UINT64_C(10)) / table[i].factor) % UINT64_C(10), 137 | table[i].suffix); 138 | 139 | goto finish; 140 | } 141 | } 142 | 143 | snprintf(buf, l, "%" PRIu64 "B", t); 144 | 145 | finish: 146 | buf[l-1] = 0; 147 | return buf; 148 | 149 | } 150 | -------------------------------------------------------------------------------- /src/parse-util.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef fooparseutilfoo 4 | #define fooparseutilfoo 5 | 6 | #include 7 | 8 | #define FORMAT_BYTES_MAX 128 9 | 10 | int parse_size(const char *t, uint64_t *ret); 11 | 12 | char *format_bytes(char *buf, size_t l, uint64_t t); 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /src/quota-projid.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | 6 | #include "quota-projid.h" 7 | #include "util.h" 8 | 9 | int read_quota_projid(int fd, uint32_t *ret) { 10 | struct fsxattr fa; 11 | 12 | assert(fd >= 0); 13 | assert(ret); 14 | 15 | if (ioctl(fd, FS_IOC_FSGETXATTR, &fa) < 0) { 16 | 17 | if (!ERRNO_IS_UNSUPPORTED(errno)) 18 | return -errno; 19 | 20 | /* If the file system doesn't do project quota we assume it's all owned by project 0 */ 21 | *ret = 0; 22 | return 0; 23 | } 24 | 25 | *ret = fa.fsx_projid; 26 | return 1; 27 | } 28 | 29 | int write_quota_projid(int fd, uint32_t id) { 30 | struct fsxattr fa; 31 | 32 | assert(fd >= 0); 33 | 34 | if (ioctl(fd, FS_IOC_FSGETXATTR, &fa) < 0) { 35 | 36 | /* When the file system doesn't support project IDs, then we consider this as equivalent to all files 37 | * being owned by project 0 */ 38 | if (id == 0 && ERRNO_IS_UNSUPPORTED(errno)) 39 | return 0; 40 | 41 | return -errno; 42 | } 43 | 44 | if (fa.fsx_projid == id) 45 | return 0; 46 | 47 | fa.fsx_projid = id; 48 | 49 | if (ioctl(fd, FS_IOC_FSSETXATTR, &fa) < 0) 50 | return -errno; 51 | 52 | return 0; 53 | } 54 | -------------------------------------------------------------------------------- /src/quota-projid.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef fooquotaprojidhfoo 4 | #define fooquotaprojidhfoo 5 | 6 | #include 7 | 8 | int read_quota_projid(int fd, uint32_t *ret); 9 | int write_quota_projid(int fd, uint32_t id); 10 | 11 | #endif 12 | -------------------------------------------------------------------------------- /src/realloc-buffer.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef fooreallocbufferhfoo 4 | #define fooreallocbufferhfoo 5 | 6 | #include 7 | 8 | #include "util.h" 9 | 10 | typedef struct ReallocBuffer { 11 | void *data; 12 | size_t allocated; 13 | size_t start; 14 | size_t end; 15 | } ReallocBuffer; 16 | 17 | static inline void *realloc_buffer_data(ReallocBuffer *buffer) { 18 | assert(buffer); 19 | assert(buffer->start <= buffer->end); 20 | assert(buffer->end <= buffer->allocated); 21 | assert(buffer->data || buffer->allocated == 0); 22 | 23 | if (!buffer->data) 24 | return buffer; 25 | 26 | return (uint8_t*) buffer->data + buffer->start; 27 | } 28 | 29 | static inline void *realloc_buffer_data_offset(ReallocBuffer *buffer, size_t offset) { 30 | size_t p; 31 | 32 | assert(buffer); 33 | assert(buffer->start <= buffer->end); 34 | assert(buffer->end <= buffer->allocated); 35 | 36 | p = buffer->start + offset; 37 | if (p < buffer->start) /* overflow? */ 38 | return NULL; 39 | if (p > buffer->end) /* out of bounds? */ 40 | return NULL; 41 | 42 | return (uint8_t*) buffer->data + p; 43 | } 44 | 45 | static inline size_t realloc_buffer_size(ReallocBuffer *buffer) { 46 | assert(buffer); 47 | assert(buffer->start <= buffer->end); 48 | assert(buffer->end <= buffer->allocated); 49 | 50 | return buffer->end - buffer->start; 51 | } 52 | 53 | void* realloc_buffer_acquire(ReallocBuffer *b, size_t size); 54 | void* realloc_buffer_acquire0(ReallocBuffer *b, size_t size); 55 | void* realloc_buffer_extend(ReallocBuffer *b, size_t size); 56 | void* realloc_buffer_extend0(ReallocBuffer *b, size_t size); 57 | void* realloc_buffer_append(ReallocBuffer *b, const void *p, size_t size); 58 | 59 | static inline void* realloc_buffer_append_byte(ReallocBuffer *b, uint8_t q) { 60 | return realloc_buffer_append(b, &q, sizeof(q)); 61 | } 62 | 63 | void realloc_buffer_free(ReallocBuffer *b); 64 | 65 | static inline void realloc_buffer_empty(ReallocBuffer *b) { 66 | b->start = b->end = 0; 67 | } 68 | 69 | int realloc_buffer_advance(ReallocBuffer *b, size_t sz); 70 | int realloc_buffer_shorten(ReallocBuffer *b, size_t sz); 71 | int realloc_buffer_truncate(ReallocBuffer *b, size_t sz); 72 | 73 | int realloc_buffer_read_size(ReallocBuffer *b, int fd, size_t add); 74 | 75 | static inline int realloc_buffer_read(ReallocBuffer *b, int fd) { 76 | return realloc_buffer_read_size(b, fd, (size_t) -1); 77 | } 78 | 79 | int realloc_buffer_read_full(ReallocBuffer *b, int fd, size_t limit); 80 | 81 | int realloc_buffer_write(ReallocBuffer *b, int fd); 82 | int realloc_buffer_write_maybe(ReallocBuffer *b, int fd); 83 | 84 | int realloc_buffer_read_target(ReallocBuffer *b, int fd, size_t target_size); 85 | 86 | void* realloc_buffer_steal(ReallocBuffer *b); 87 | 88 | void* realloc_buffer_donate(ReallocBuffer *b, void *p, size_t size); 89 | 90 | int realloc_buffer_printf(ReallocBuffer *b, const char *fmt, ...) _printf_(2,3); 91 | 92 | int realloc_buffer_memchr(ReallocBuffer *buffer, uint8_t c); 93 | 94 | #endif 95 | -------------------------------------------------------------------------------- /src/reflink.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "reflink.h" 10 | #include "util.h" 11 | 12 | #define FS_BLOCK_SIZE 4096U 13 | 14 | #define VALIDATE 1 15 | 16 | #if VALIDATE 17 | static ssize_t pread_try_harder(int fd, void *p, size_t s, off_t o) { 18 | char path[sizeof("/proc/self/fd/") + DECIMAL_STR_MAX(fd)]; 19 | ssize_t n; 20 | int fd_read, r; 21 | 22 | n = pread(fd, p, s, o); 23 | if (n >= 0) 24 | return n; 25 | 26 | r = -errno; 27 | 28 | sprintf(path, "/proc/self/fd/%i", fd); 29 | 30 | fd_read = open(path, O_CLOEXEC|O_RDONLY|O_NOCTTY); 31 | if (fd_read < 0) { 32 | errno = -r; 33 | return r; 34 | } 35 | 36 | n = pread(fd_read, p, s, o); 37 | safe_close(fd_read); 38 | if (n < 0) { 39 | errno = -r; 40 | return r; 41 | } 42 | 43 | return n; 44 | } 45 | #endif 46 | 47 | static void validate(int source_fd, uint64_t source_offset, int destination_fd, uint64_t destination_offset, uint64_t size) { 48 | #if VALIDATE 49 | ssize_t x, y; 50 | uint8_t *buffer1, *buffer2; 51 | 52 | buffer1 = new(uint8_t, size); 53 | assert_se(buffer1); 54 | buffer2 = new(uint8_t, size); 55 | assert_se(buffer2); 56 | 57 | x = pread_try_harder(source_fd, buffer1, size, source_offset); 58 | y = pread_try_harder(destination_fd, buffer2, size, destination_offset); 59 | 60 | assert_se(x == (ssize_t) size); 61 | assert_se(y == (ssize_t) size); 62 | assert_se(memcmp(buffer1, buffer2, size) == 0); 63 | 64 | free(buffer1); 65 | free(buffer2); 66 | #endif 67 | } 68 | 69 | int reflink_fd( 70 | int source_fd, 71 | uint64_t source_offset, 72 | int destination_fd, 73 | uint64_t destination_offset, 74 | uint64_t size, 75 | uint64_t *ret_reflinked) { 76 | 77 | struct stat a, b; 78 | uint64_t add, reflinked; 79 | 80 | /* Creates a reflink on btrfs and other file systems that know the concept. The input parameters are aligned to 81 | * match the fundamental block size (for now assumed to be 4K), and possibly to EOF. */ 82 | 83 | if (source_fd < 0) 84 | return -EBADF; 85 | if (destination_fd < 0) 86 | return -EBADF; 87 | 88 | /* Can only merge blocks starting at a block size boundary */ 89 | if (source_offset % FS_BLOCK_SIZE != destination_offset % FS_BLOCK_SIZE) 90 | return -EBADR; 91 | 92 | /* Overflow checks */ 93 | if (source_offset + size < source_offset) 94 | return -ERANGE; 95 | if (destination_offset + size < destination_offset) 96 | return -ERANGE; 97 | 98 | /* First step, round up start offsets to multiple of 4096 */ 99 | if (source_offset % FS_BLOCK_SIZE > 0) { 100 | add = FS_BLOCK_SIZE - (source_offset % FS_BLOCK_SIZE); 101 | if (add >= size) 102 | return -EBADR; 103 | 104 | source_offset += add; 105 | destination_offset += add; 106 | size -= add; 107 | } 108 | 109 | if (fstat(source_fd, &a) < 0) 110 | return -errno; 111 | if (fstat(destination_fd, &b) < 0) 112 | return -errno; 113 | 114 | /* Never call the ioctls on something that isn't a regular file, as that's not safe (for example, if the fd 115 | * refers to a block or char device of some kind, which overloads the same ioctl numbers) */ 116 | if (S_ISDIR(a.st_mode) || S_ISDIR(b.st_mode)) 117 | return -EISDIR; 118 | if (!S_ISREG(a.st_mode) || !S_ISREG(b.st_mode)) 119 | return -ENOTTY; 120 | 121 | /* Extend to EOF if we can */ 122 | if (source_offset + size >= (uint64_t) a.st_size && 123 | destination_offset + size >= (uint64_t) b.st_size) { 124 | reflinked = size; 125 | size = 0; 126 | } else { 127 | /* Round down size to multiple of 4096 */ 128 | size = (size / FS_BLOCK_SIZE) * FS_BLOCK_SIZE; 129 | if (size <= 0) 130 | return -EBADR; 131 | 132 | reflinked = size; 133 | } 134 | 135 | validate(source_fd, source_offset, destination_fd, destination_offset, reflinked); 136 | 137 | if (ioctl(destination_fd, FICLONERANGE, 138 | &(struct file_clone_range) { 139 | .src_fd = source_fd, 140 | .src_offset = source_offset, 141 | .src_length = size, 142 | .dest_offset = destination_offset, 143 | }) < 0) 144 | return -errno; 145 | 146 | validate(source_fd, source_offset, destination_fd, destination_offset, reflinked); 147 | 148 | if (ret_reflinked) 149 | *ret_reflinked = reflinked; 150 | 151 | return 0; 152 | } 153 | -------------------------------------------------------------------------------- /src/reflink.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef fooreflinkhfoo 4 | #define fooreflinkhfoo 5 | 6 | #include 7 | 8 | int reflink_fd(int source_fd, uint64_t source_offset, int destination_fd, uint64_t destination_offset, uint64_t size, uint64_t *ret_reflinked); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /src/rm-rf.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foormrfhfoo 4 | #define foormrfhfoo 5 | 6 | #include 7 | 8 | typedef enum RemoveFlags { 9 | REMOVE_ONLY_DIRECTORIES = 1, 10 | REMOVE_ROOT = 2, 11 | REMOVE_PHYSICAL = 4, /* if not set, only removes files on tmpfs, never physical file systems */ 12 | REMOVE_SPAN_DEVICES = 8, 13 | REMOVE_UNDO_IMMUTABLE = 16, 14 | } RemoveFlags; 15 | 16 | int rm_rf_children(int fd, RemoveFlags flags, struct stat *root_dev); 17 | int rm_rf(const char *path, RemoveFlags flags); 18 | int rm_rf_at(int dir_fd, const char *path, RemoveFlags flags); 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /src/set.h: -------------------------------------------------------------------------------- 1 | /*** 2 | SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | This file is part of systemd. 5 | 6 | Copyright 2010 Lennart Poettering 7 | 8 | systemd is free software; you can redistribute it and/or modify it 9 | under the terms of the GNU Lesser General Public License as published by 10 | the Free Software Foundation; either version 2.1 of the License, or 11 | (at your option) any later version. 12 | 13 | systemd is distributed in the hope that it will be useful, but 14 | WITHOUT ANY WARRANTY; without even the implied warranty of 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | Lesser General Public License for more details. 17 | 18 | You should have received a copy of the GNU Lesser General Public License 19 | along with systemd; If not, see . 20 | ***/ 21 | 22 | #pragma once 23 | 24 | #include "hashmap.h" 25 | #include "util.h" 26 | 27 | Set *internal_set_new(const struct hash_ops *hash_ops HASHMAP_DEBUG_PARAMS); 28 | #define set_new(ops) internal_set_new(ops HASHMAP_DEBUG_SRC_ARGS) 29 | 30 | static inline Set *set_free(Set *s) { 31 | internal_hashmap_free(HASHMAP_BASE(s)); 32 | return NULL; 33 | } 34 | 35 | static inline Set *set_free_free(Set *s) { 36 | internal_hashmap_free_free(HASHMAP_BASE(s)); 37 | return NULL; 38 | } 39 | 40 | /* no set_free_free_free */ 41 | 42 | static inline Set *set_copy(Set *s) { 43 | return (Set*) internal_hashmap_copy(HASHMAP_BASE(s)); 44 | } 45 | 46 | int internal_set_ensure_allocated(Set **s, const struct hash_ops *hash_ops HASHMAP_DEBUG_PARAMS); 47 | #define set_ensure_allocated(h, ops) internal_set_ensure_allocated(h, ops HASHMAP_DEBUG_SRC_ARGS) 48 | 49 | int set_put(Set *s, const void *key); 50 | /* no set_update */ 51 | /* no set_replace */ 52 | static inline void *set_get(Set *s, void *key) { 53 | return internal_hashmap_get(HASHMAP_BASE(s), key); 54 | } 55 | /* no set_get2 */ 56 | 57 | static inline bool set_contains(Set *s, const void *key) { 58 | return internal_hashmap_contains(HASHMAP_BASE(s), key); 59 | } 60 | 61 | static inline void *set_remove(Set *s, const void *key) { 62 | return internal_hashmap_remove(HASHMAP_BASE(s), key); 63 | } 64 | 65 | /* no set_remove2 */ 66 | /* no set_remove_value */ 67 | int set_remove_and_put(Set *s, const void *old_key, const void *new_key); 68 | /* no set_remove_and_replace */ 69 | int set_merge(Set *s, Set *other); 70 | 71 | static inline int set_reserve(Set *h, unsigned entries_add) { 72 | return internal_hashmap_reserve(HASHMAP_BASE(h), entries_add); 73 | } 74 | 75 | static inline int set_move(Set *s, Set *other) { 76 | return internal_hashmap_move(HASHMAP_BASE(s), HASHMAP_BASE(other)); 77 | } 78 | 79 | static inline int set_move_one(Set *s, Set *other, const void *key) { 80 | return internal_hashmap_move_one(HASHMAP_BASE(s), HASHMAP_BASE(other), key); 81 | } 82 | 83 | static inline unsigned set_size(Set *s) { 84 | return internal_hashmap_size(HASHMAP_BASE(s)); 85 | } 86 | 87 | static inline bool set_isempty(Set *s) { 88 | return set_size(s) == 0; 89 | } 90 | 91 | static inline unsigned set_buckets(Set *s) { 92 | return internal_hashmap_buckets(HASHMAP_BASE(s)); 93 | } 94 | 95 | bool set_iterate(Set *s, Iterator *i, void **value); 96 | 97 | static inline void set_clear(Set *s) { 98 | internal_hashmap_clear(HASHMAP_BASE(s)); 99 | } 100 | 101 | static inline void set_clear_free(Set *s) { 102 | internal_hashmap_clear_free(HASHMAP_BASE(s)); 103 | } 104 | 105 | /* no set_clear_free_free */ 106 | 107 | static inline void *set_steal_first(Set *s) { 108 | return internal_hashmap_steal_first(HASHMAP_BASE(s)); 109 | } 110 | 111 | /* no set_steal_first_key */ 112 | /* no set_first_key */ 113 | 114 | static inline void *set_first(Set *s) { 115 | return internal_hashmap_first(HASHMAP_BASE(s)); 116 | } 117 | 118 | /* no set_next */ 119 | 120 | static inline char **set_get_strv(Set *s) { 121 | return internal_hashmap_get_strv(HASHMAP_BASE(s)); 122 | } 123 | 124 | int set_consume(Set *s, void *value); 125 | int set_put_strdup(Set *s, const char *p); 126 | int set_put_strdupv(Set *s, char **l); 127 | 128 | #define SET_FOREACH(e, s, i) \ 129 | for ((i) = ITERATOR_FIRST; set_iterate((s), &(i), (void**)&(e)); ) 130 | 131 | #define SET_FOREACH_MOVE(e, d, s) \ 132 | for (; ({ e = set_first(s); assert_se(!e || set_move_one(d, s, e) >= 0); e; }); ) 133 | 134 | DEFINE_TRIVIAL_CLEANUP_FUNC(Set*, set_free); 135 | DEFINE_TRIVIAL_CLEANUP_FUNC(Set*, set_free_free); 136 | 137 | #define _cleanup_set_free_ _cleanup_(set_freep) 138 | #define _cleanup_set_free_free_ _cleanup_(set_free_freep) 139 | -------------------------------------------------------------------------------- /src/signal-handler.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "signal-handler.h" 4 | #include "util.h" 5 | 6 | volatile sig_atomic_t quit = false; 7 | 8 | void block_exit_handler(int how, sigset_t *old) { 9 | sigset_t ss; 10 | 11 | assert_se(sigemptyset(&ss) >= 0); 12 | assert_se(sigaddset(&ss, SIGINT) >= 0); 13 | assert_se(sigdelset(&ss, SIGTERM) >= 0); 14 | assert_se(sigaddset(&ss, SIGHUP) >= 0); 15 | assert_se(sigprocmask(how, &ss, old) >= 0); 16 | } 17 | 18 | void exit_signal_handler(int signo) { 19 | quit = true; 20 | } 21 | 22 | void install_exit_handler(void (*handler)(int)) { 23 | const struct sigaction sa = { 24 | .sa_handler = handler ?: exit_signal_handler, 25 | }; 26 | 27 | assert_se(sigaction(SIGINT, &sa, NULL) >= 0); 28 | assert_se(sigaction(SIGTERM, &sa, NULL) >= 0); 29 | assert_se(sigaction(SIGHUP, &sa, NULL) >= 0); 30 | } 31 | 32 | int sync_poll_sigset(CaSync *s) { 33 | sigset_t ss; 34 | int r; 35 | 36 | /* Block SIGTERM/SIGINT for now */ 37 | block_exit_handler(SIG_BLOCK, &ss); 38 | 39 | if (quit) /* Check if we are supposed to quit, if so, do so now */ 40 | r = -ESHUTDOWN; 41 | else { 42 | /* Wait for an event, temporarily and atomically unblocking SIGTERM/SIGINT while doing so */ 43 | r = ca_sync_poll(s, UINT64_MAX, &ss); 44 | if ((r == -EINTR || r >= 0) && quit) 45 | r = -ESHUTDOWN; 46 | } 47 | 48 | /* Unblock SIGTERM/SIGINT again */ 49 | block_exit_handler(SIG_UNBLOCK, NULL); 50 | 51 | return r; 52 | } 53 | 54 | void disable_sigpipe(void) { 55 | static const struct sigaction sa = { 56 | .sa_handler = SIG_IGN, 57 | .sa_flags = SA_RESTART, 58 | }; 59 | 60 | assert_se(sigaction(SIGPIPE, &sa, NULL) >= 0); 61 | } 62 | -------------------------------------------------------------------------------- /src/signal-handler.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef foosignalhandlerhfoo 4 | 5 | #include 6 | #include 7 | 8 | #include "casync.h" 9 | 10 | extern volatile sig_atomic_t quit; 11 | 12 | void exit_signal_handler(int signo); 13 | 14 | void install_exit_handler(void (*handler)(int)); 15 | void block_exit_handler(int how, sigset_t *old); 16 | 17 | int sync_poll_sigset(CaSync *s); 18 | 19 | void disable_sigpipe(void); 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /src/siphash24.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: CC0-1.0 */ 2 | 3 | #ifndef foosiphash24hfoo 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | struct siphash { 11 | uint64_t v0; 12 | uint64_t v1; 13 | uint64_t v2; 14 | uint64_t v3; 15 | uint64_t padding; 16 | size_t inlen; 17 | }; 18 | 19 | void siphash24_init(struct siphash *state, const uint8_t k[16]); 20 | void siphash24_compress(const void *in, size_t inlen, struct siphash *state); 21 | #define siphash24_compress_byte(byte, state) siphash24_compress((const uint8_t[]) { (byte) }, 1, (state)) 22 | 23 | uint64_t siphash24_finalize(struct siphash *state); 24 | 25 | uint64_t siphash24(const void *in, size_t inlen, const uint8_t k[16]); 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /src/time-util.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "util.h" 4 | #include "time-util.h" 5 | 6 | char *format_timespan(char *buf, size_t l, uint64_t t, uint64_t accuracy) { 7 | static const struct { 8 | const char *suffix; 9 | uint64_t nsec; 10 | } table[] = { 11 | { "y", NSEC_PER_YEAR }, 12 | { "month", NSEC_PER_MONTH }, 13 | { "w", NSEC_PER_WEEK }, 14 | { "d", NSEC_PER_DAY }, 15 | { "h", NSEC_PER_HOUR }, 16 | { "min", NSEC_PER_MINUTE }, 17 | { "s", NSEC_PER_SEC }, 18 | { "ms", NSEC_PER_MSEC }, 19 | { "us", NSEC_PER_USEC }, 20 | { "ns", 1 }, 21 | }; 22 | 23 | size_t i; 24 | char *p = buf; 25 | bool something = false; 26 | 27 | assert(buf); 28 | assert(l > 0); 29 | 30 | if (t == NSEC_INFINITY) { 31 | strncpy(p, "infinity", l-1); 32 | p[l-1] = 0; 33 | return p; 34 | } 35 | 36 | if (t <= 0) { 37 | strncpy(p, "0", l-1); 38 | p[l-1] = 0; 39 | return p; 40 | } 41 | 42 | /* The result of this function can be parsed with parse_sec */ 43 | 44 | for (i = 0; i < ELEMENTSOF(table); i++) { 45 | int k = 0; 46 | size_t n; 47 | bool done = false; 48 | uint64_t a, b; 49 | 50 | if (t <= 0) 51 | break; 52 | 53 | if (t < accuracy && something) 54 | break; 55 | 56 | if (t < table[i].nsec) 57 | continue; 58 | 59 | if (l <= 1) 60 | break; 61 | 62 | a = t / table[i].nsec; 63 | b = t % table[i].nsec; 64 | 65 | /* Let's see if we should shows this in dot notation */ 66 | if (t < NSEC_PER_MINUTE && b > 0) { 67 | uint64_t cc; 68 | signed char j; 69 | 70 | j = 0; 71 | for (cc = table[i].nsec; cc > 1; cc /= 10) 72 | j++; 73 | 74 | for (cc = accuracy; cc > 1; cc /= 10) { 75 | b /= 10; 76 | j--; 77 | } 78 | 79 | if (j > 0) { 80 | k = snprintf(p, l, 81 | "%s%" PRIu64 ".%0*" PRIu64 "%s", 82 | p > buf ? " " : "", 83 | a, 84 | j, 85 | b, 86 | table[i].suffix); 87 | 88 | t = 0; 89 | done = true; 90 | } 91 | } 92 | 93 | /* No? Then let's show it normally */ 94 | if (!done) { 95 | k = snprintf(p, l, 96 | "%s%" PRIu64 "%s", 97 | p > buf ? " " : "", 98 | a, 99 | table[i].suffix); 100 | 101 | t = b; 102 | } 103 | 104 | n = MIN((size_t) k, l); 105 | 106 | l -= n; 107 | p += n; 108 | 109 | something = true; 110 | } 111 | 112 | *p = 0; 113 | 114 | return buf; 115 | } 116 | -------------------------------------------------------------------------------- /src/time-util.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #ifndef footimeutilfoo 4 | #define footimeutilfoo 5 | 6 | #include 7 | #include 8 | 9 | #define NSEC_INFINITY ((uint64_t) -1) 10 | 11 | #define NSEC_PER_SEC ((uint64_t) UINT64_C(1000000000)) 12 | #define NSEC_PER_MSEC ((uint64_t) UINT64_C(1000000)) 13 | #define NSEC_PER_USEC ((uint64_t) UINT64_C(1000)) 14 | 15 | #define NSEC_PER_MINUTE ((uint64_t) (UINT64_C(60)*NSEC_PER_SEC)) 16 | #define NSEC_PER_HOUR ((uint64_t) (UINT64_C(60)*NSEC_PER_MINUTE)) 17 | #define NSEC_PER_DAY ((uint64_t) (UINT64_C(24)*NSEC_PER_HOUR)) 18 | #define NSEC_PER_WEEK ((uint64_t) (UINT64_C(7)*NSEC_PER_DAY)) 19 | #define NSEC_PER_MONTH ((uint64_t) (UINT64_C(2629800)*NSEC_PER_SEC)) 20 | #define NSEC_PER_YEAR ((uint64_t) (UINT64_C(31557600)*NSEC_PER_SEC)) 21 | 22 | static inline uint64_t timespec_to_nsec(struct timespec t) { 23 | 24 | if (t.tv_sec == (time_t) -1 && 25 | t.tv_nsec == (long) -1) 26 | return UINT64_MAX; 27 | 28 | return (uint64_t) t.tv_sec * UINT64_C(1000000000) + (uint64_t) t.tv_nsec; 29 | } 30 | 31 | static inline struct timespec nsec_to_timespec(uint64_t u) { 32 | 33 | if (u == UINT64_MAX) 34 | return (struct timespec) { 35 | .tv_sec = (time_t) -1, 36 | .tv_nsec = (long) -1, 37 | }; 38 | 39 | return (struct timespec) { 40 | .tv_sec = u / UINT64_C(1000000000), 41 | .tv_nsec = u % UINT64_C(1000000000) 42 | }; 43 | } 44 | 45 | #define NSEC_TO_TIMESPEC_INIT(u) \ 46 | { .tv_sec = u == UINT64_MAX ? (time_t) -1 : (time_t) (u / UINT64_C(1000000000)), \ 47 | .tv_nsec = u == UINT64_MAX ? (long) -1 : (long) (u % UINT64_C(1000000000)) } 48 | 49 | static inline uint64_t now(clockid_t id) { 50 | struct timespec ts; 51 | 52 | if (clock_gettime(id, &ts) < 0) 53 | return UINT64_MAX; 54 | 55 | return timespec_to_nsec(ts); 56 | } 57 | 58 | char *format_timespan(char *buf, size_t l, uint64_t t, uint64_t accuracy); 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /src/udev-util.h: -------------------------------------------------------------------------------- 1 | #ifndef fooudevutilhfoo 2 | #define fooudevutilhfoo 3 | 4 | /* SPDX-License-Identifier: LGPL-2.1+ */ 5 | 6 | #include "util.h" 7 | 8 | #include 9 | 10 | DEFINE_TRIVIAL_CLEANUP_FUNC(struct udev*, udev_unref); 11 | DEFINE_TRIVIAL_CLEANUP_FUNC(struct udev_device*, udev_device_unref); 12 | DEFINE_TRIVIAL_CLEANUP_FUNC(struct udev_monitor*, udev_monitor_unref); 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /test-files/.gitignore: -------------------------------------------------------------------------------- 1 | /acl 2 | /immutable 3 | /nocow 4 | /reflink 5 | /sparse 6 | /xattr 7 | -------------------------------------------------------------------------------- /test-files/a symlink with umläüts: -------------------------------------------------------------------------------- 1 | fööbär -------------------------------------------------------------------------------- /test-files/bar: -------------------------------------------------------------------------------- 1 | foo -------------------------------------------------------------------------------- /test-files/file: -------------------------------------------------------------------------------- 1 | 0000000 561c 47ed d7c9 274c 7874 f1b3 2739 297b 2 | 0000010 8574 0122 07ad dc0a 1cf5 f1a5 3ddd a4a0 3 | 0000020 b7cd 19e9 675e 3a50 4ef6 00bf 0ee2 67aa 4 | 0000030 02e7 df7b b757 1dfb ef38 0f95 4e08 0184 5 | 0000040 a821 8fc6 5473 e4bc 2c04 5dcc 8fce 9655 6 | 0000050 06d8 d6c6 3171 3f53 ac9c 57b1 cb0a 822e 7 | 0000060 a870 e61d 158f 2640 926f 7c44 71f2 758e 8 | 0000070 78d3 ad00 3614 1783 18bf 4920 9416 966d 9 | 0000080 590d f2cb e258 ec66 00c2 eb05 9c9d 1257 10 | 0000090 7f2f 2d3d 1424 e0b3 afe0 4447 2fe1 e4e3 11 | -------------------------------------------------------------------------------- /test-files/gęślą/000000/file: -------------------------------------------------------------------------------- 1 | 0000000 a3fd 3a42 f90c fecf 443b 6eaa 3bfc 0010 2 | 0000010 8240 4b3d 1fbd ba98 68dd b21a 11b0 0a2d 3 | 0000020 24c7 d1a3 67a6 3aaa a272 522e 8cb4 219a 4 | 0000030 4d60 e501 5422 f27f 267a 1a35 3508 b6eb 5 | 0000040 a9eb 09c3 08fe 4732 9da3 e876 50c5 7b32 6 | 0000050 f682 b994 bfee 81af adde 7fba a211 4f3a 7 | 0000060 3c9a bc7b 0797 1746 8c2f 756a 6d4c 6bc3 8 | 0000070 9730 06cb 3fae 88a8 2e62 c5d3 21b9 018a 9 | 0000080 8ad4 533f fdd1 5026 30d1 b7f2 f549 d62d 10 | 0000090 e505 428e 75a4 dd1a 5a59 f5cd 58ba cdbd 11 | -------------------------------------------------------------------------------- /test-files/gęślą/a/file: -------------------------------------------------------------------------------- 1 | 0000000 62ad 0761 c5c4 bc9a 5a50 66e9 3677 13e1 2 | 0000010 7542 f354 33c1 48c0 e36a 6a05 30aa 897b 3 | 0000020 b84c 22cc 5c8e 472e 6af9 df1d 52bf 0829 4 | 0000030 e520 7f85 f91a 1084 6593 1ca8 7c14 eb7b 5 | 0000040 618b d14f fef0 4866 7100 b4d1 19a7 5ab4 6 | 0000050 50db a796 74b6 ca1b d6cd c2f4 6898 6abb 7 | 0000060 5402 0ba2 a442 a7e1 b465 381e d611 c6d5 8 | 0000070 4ae8 c8f5 143f a2f0 b931 8829 1a35 8068 9 | 0000080 47a7 0f7c 91db cf64 445a 3e0b c281 784d 10 | 0000090 4fa3 1b5a edfc 9a06 235d e1a5 50f1 9b0c 11 | -------------------------------------------------------------------------------- /test-files/gęślą/b/file: -------------------------------------------------------------------------------- 1 | 0000000 2625 7356 e6ae 5555 2fe4 75bb d4eb 7534 2 | 0000010 730f d304 b214 35f7 d352 9af1 5747 a8c9 3 | 0000020 6500 d5f3 15cc 28d5 4821 cb88 9139 684f 4 | 0000030 5122 8308 8719 072c b584 f1df 1112 7e8a 5 | 0000040 7279 3df6 bab2 1648 244b 151f 0538 1e82 6 | 0000050 22c1 0f43 d080 8ba2 6c9c b586 8231 274b 7 | 0000060 1568 4d7c c20c 7f2d 2974 359f a636 7784 8 | 0000070 4836 173f 8056 6608 e782 d487 c97c 843a 9 | 0000080 7765 dba5 8abe 732d 37e2 2c7a 16f8 a429 10 | 0000090 9bd3 ecc5 1641 d33a 9891 d60d 439f 273a 11 | -------------------------------------------------------------------------------- /test-files/gęślą/file: -------------------------------------------------------------------------------- 1 | 0000000 9dd7 cbd7 1f5b 106a 56a8 d98a b602 b3f5 2 | 0000010 0c9c 7bd9 5eff 2797 9f4d b453 4ec2 11a3 3 | 0000020 96a2 7bbe cfdc cf00 b844 65d3 15b1 cc14 4 | 0000030 631b 2065 4ea7 6cb8 1c8d 7a61 c1ae b674 5 | 0000040 f01c 6b06 e216 55b2 3a2c 858d 4e2d a9d6 6 | 0000050 5117 ed3e d6f6 db45 2d1b ea34 fabb 4e0d 7 | 0000060 ec1b 844f d71d a175 f339 9c96 dfeb 624f 8 | 0000070 7d72 03f8 63e6 7764 1445 8637 4514 2f49 9 | 0000080 bad9 acfa 8a82 2af8 d196 b1d0 c1b8 a385 10 | 0000090 9f40 36ce 1d88 e518 6e1d 8926 d8cd 655b 11 | -------------------------------------------------------------------------------- /test-files/jaźn/000000/file: -------------------------------------------------------------------------------- 1 | 0000000 3b05 e4b1 8e04 3e41 5108 b0e8 939c 439f 2 | 0000010 1466 265c 7e87 b205 ac91 d2e3 da56 9af3 3 | 0000020 bbc3 25c6 e49e 9a34 6beb a247 0add 6aff 4 | 0000030 4c47 f8cf ddec cbd7 27b6 6fd9 54e6 6dda 5 | 0000040 0b9f bafa 685f 2e5d 21fc b9c7 ffbc 3771 6 | 0000050 47d0 f18d 8de1 9912 afa2 5b77 c5b8 51c0 7 | 0000060 bf91 b063 7d21 e552 d6a6 a38e 3d6a d23f 8 | 0000070 3ad1 2511 b6b6 f69b 5876 6205 620e a9f1 9 | 0000080 b575 1bba 0e9c 6721 bf6f 22fe 1e9c 5f38 10 | 0000090 77da 3976 64d2 0d2d 424b ab76 8b48 f791 11 | -------------------------------------------------------------------------------- /test-files/jaźn/a/file: -------------------------------------------------------------------------------- 1 | 0000000 c03f 9f69 7141 3752 7bec 11a0 3ede 5aa8 2 | 0000010 59e7 766e a81a 3a4d f06e 99ce 1e68 7977 3 | 0000020 6845 3f61 2b23 6fbc 0c50 1b7b 2e83 6b9a 4 | 0000030 65f1 c916 f816 5b8f 3552 4553 1b55 b0aa 5 | 0000040 07de 0f4e be88 13aa 81e7 4655 f3e5 5366 6 | 0000050 c9eb f166 e939 44c3 bf88 7eef 2dac 2cbc 7 | 0000060 dbde 1fdb 1a14 59a6 c3df 4a36 ad18 84c1 8 | 0000070 cf00 8cae 2b80 f3a0 9cf7 2917 baa2 5d58 9 | 0000080 40b0 0ab8 c00b 97ba 5ae5 a3b7 37ba 5513 10 | 0000090 31ba a341 2ee8 e557 fbd1 2913 ebae 8f0d 11 | -------------------------------------------------------------------------------- /test-files/jaźn/b/file: -------------------------------------------------------------------------------- 1 | 0000000 180b 0dcf 5952 7d6e c355 5abb 9df3 62ee 2 | 0000010 e8fe 54b4 e57c a847 1c2c 9b37 cd94 56cb 3 | 0000020 9390 71d7 0aa1 53bc 2afd 36b3 4d2c 53cc 4 | 0000030 0d59 7d2a 1ae1 17c5 881a dd98 4764 5159 5 | 0000040 b6d0 e2f1 28c5 4759 3c10 30ed f831 02e8 6 | 0000050 c382 54ea 9f07 6831 41fd c059 8e38 68b9 7 | 0000060 fa16 e9f0 5053 a7c0 dca5 ad7a 0fed 1c04 8 | 0000070 40a0 9433 fe90 da6a d115 e8e9 0337 6e5c 9 | 0000080 9c3c fc6a 325f 750e e647 93ca e083 21c3 10 | 0000090 006c 5fa0 5c1d f8b8 3dae e84c 18a8 091f 11 | -------------------------------------------------------------------------------- /test-files/jaźn/file: -------------------------------------------------------------------------------- 1 | 0000000 1fe4 e62c a57c f767 8005 c126 c773 8284 2 | 0000010 6112 eb49 1123 3cec 521b 0606 476e d8f7 3 | 0000020 32a1 cafa 4f68 dbb1 68c9 d930 58ff cd3b 4 | 0000030 5490 d362 f1c4 955b 463f b5fd 891f 2fb4 5 | 0000040 3d4f 9c51 e30a 806c 9f6d 3327 0732 1d0e 6 | 0000050 e4f6 dfde 547d b082 245a a4a2 0916 08ea 7 | 0000060 3feb 2391 5ee8 9d43 c539 d394 a1c2 fbea 8 | 0000070 345b b7da cd75 7090 abc6 8080 3435 56ad 9 | 0000080 60ce e0e2 e856 09ba 15e0 eee1 de64 9cd4 10 | 0000090 4dd7 5b5b 948c 59f7 bf37 c63f c504 f13a 11 | -------------------------------------------------------------------------------- /test-files/large: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/systemd/casync/e6817a79d89b48e1c6083fb1868a28f1afb32505/test-files/large -------------------------------------------------------------------------------- /test-files/test-files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -ex 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | cd "$(dirname "$0")" 5 | if [ "$1" != "clean" ]; then 6 | test -e thisisafifo || mkfifo thisisafifo 7 | test -e ablockdevice || mknod ablockdevice b 0 0 8 | test -e achardevice || mknod achardevice c 0 0 9 | test -e immutable || ( touch immutable && chattr +i immutable ) 10 | test -e nocow || ( touch nocow && chattr +C nocow ) 11 | test -e acl || ( touch acl && setfacl -nm u:nobody:rw,u:root:rw acl ) 12 | test -e sparse || dd if=/dev/urandom of=sparse count=2 bs=1 seek=9999 13 | test -e reflink || ( cp --reflink=auto large reflink && 14 | dd if=/dev/urandom of=reflink seek=102400 bs=1 count=1 conv=notrunc ) 15 | test -s xattr || ( touch xattr && setfattr -n user.foo -v bar xattr && setfattr -n user.quux -v piep xattr ) 16 | else 17 | chattr -i test-files/immutable || : 18 | rm -f test-files/thisisafifo 19 | rm -f test-files/ablockdevice 20 | rm -f test-files/achardevice 21 | rm -f test-files/immutable 22 | rm -f test-files/nocow 23 | rm -f test-files/acl 24 | rm -f test-files/sparse 25 | rm -f test-files/xattr 26 | fi 27 | -------------------------------------------------------------------------------- /test-files/äöüß: -------------------------------------------------------------------------------- 1 | mief 2 | -------------------------------------------------------------------------------- /test-files/żażółcić/000000/file: -------------------------------------------------------------------------------- 1 | 0000000 f319 52b5 7d07 cf7e 0f3b 853e b8cf 4bff 2 | 0000010 1deb fc55 37ad e318 550f f9f7 17ab 9807 3 | 0000020 ac54 2990 8495 019e a0f5 bde0 43d7 77ae 4 | 0000030 ed4a a553 eef0 a5e1 c2df df30 beb8 e13f 5 | 0000040 daf4 301a 32b3 ba15 23d4 6cbb 1ac7 5e47 6 | 0000050 c7bb 939d ac7e 18c7 137f 69f4 6b36 25ff 7 | 0000060 89bb 9fbe 9492 c10e 4735 b851 767a 8515 8 | 0000070 f462 f037 aa32 23da d0cd 7467 9eaf be84 9 | 0000080 59e7 9d49 ca26 015d cbdf 2acc 9b69 1fb1 10 | 0000090 7cd3 9c91 6de1 3a09 f414 3cec cc94 4419 11 | -------------------------------------------------------------------------------- /test-files/żażółcić/a/file: -------------------------------------------------------------------------------- 1 | 0000000 3714 33b4 cca2 e457 96e9 fe5f 2973 dc7d 2 | 0000010 75e9 a1b4 e8bd 000d ef2e 836a e278 bd20 3 | 0000020 6f92 709e 5165 a46a 72b4 82e6 84fc 8da4 4 | 0000030 ac98 07c9 a4d5 2125 a125 7111 1694 f08f 5 | 0000040 63d0 3aed fcdb 7d15 84fe 8cb2 0906 de9f 6 | 0000050 54d9 4659 adbe e3d6 1cf4 45d4 e900 9bdc 7 | 0000060 562c d24b 0040 a5c6 3ea1 6eca 9927 6d08 8 | 0000070 9b1a f440 b6e4 422c b2af b08e b72d 8bc1 9 | 0000080 5363 a1e7 5ac1 065f 99fc 9664 d07f 6686 10 | 0000090 6030 e748 9304 236d 8b3c 648e d563 43c4 11 | -------------------------------------------------------------------------------- /test-files/żażółcić/b/file: -------------------------------------------------------------------------------- 1 | 0000000 1639 c025 8084 ae98 d67f 40d6 e046 444d 2 | 0000010 c84b f5ef 5f05 6b0e d5ea d293 b116 ce0e 3 | 0000020 f40a ac8e de84 2414 3b2d 2fa2 af5f 567c 4 | 0000030 e876 93a9 a279 276d 5b47 1647 86fc 152c 5 | 0000040 df1d 8b1f 4014 5297 213d ae27 4b4b 1b90 6 | 0000050 e51e 20e3 759c 9646 54a3 10de 1694 27b5 7 | 0000060 21a8 ae18 d379 98ce ce90 fd24 a5a3 b121 8 | 0000070 8ebb 119f b22f acf2 62af f8e6 2dea 2764 9 | 0000080 efcb 6720 6367 02b4 075f b350 a3ce 2c91 10 | 0000090 e5b8 c61a 027b 8226 3a83 2099 3df5 762a 11 | -------------------------------------------------------------------------------- /test-files/żażółcić/file: -------------------------------------------------------------------------------- 1 | 0000000 79b6 6bf3 255c c2b3 fcbf 128e 6389 f728 2 | 0000010 7ceb c982 eafa d708 c98b 8da5 974e d055 3 | 0000020 e7c8 f2aa af81 5bfa 092e db64 5f90 6fcb 4 | 0000030 efc9 5743 7cac 6130 c133 97eb dace 7798 5 | 0000040 887a 5324 8059 e066 75de 6fb8 ae9c ad9d 6 | 0000050 46a9 1d71 2b47 f46c a4db a324 34e5 5b2b 7 | 0000060 d75f 867d 4c9e b098 e45c 010c d492 958c 8 | 0000070 e18b 730c 4750 5e7d 0f8d 2269 019d 68e3 9 | 0000080 ec79 178b 7c55 6a75 004f 4598 6ead 5f78 10 | 0000090 5251 4f8c 3f3a beed dcfb 1b32 9c8b 4773 11 | -------------------------------------------------------------------------------- /test/fuzz/fuzz-compress.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | 6 | #include "compressor.h" 7 | #include "fuzz.h" 8 | #include "log.h" 9 | #include "util.h" 10 | 11 | typedef struct header { 12 | uint32_t alg; 13 | uint32_t reserved[5]; /* Extra space to keep fuzz cases stable in case we need to 14 | * add stuff in the future. */ 15 | uint8_t data[]; 16 | } header; 17 | 18 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { 19 | _cleanup_free_ void *buf = NULL; 20 | int r; 21 | 22 | if (size < offsetof(header, data) + 1) 23 | return 0; 24 | 25 | /* We don't want to fill the logs with messages about parse errors. 26 | * Disable most logging if not running standalone */ 27 | if (!getenv("CASYNC_LOG_LEVEL")) 28 | set_log_level(LOG_CRIT); 29 | 30 | const header *h = (struct header*) data; 31 | const size_t data_len = size - offsetof(header, data); 32 | 33 | _cleanup_(compressor_finish) CompressorContext c = COMPRESSOR_CONTEXT_INIT; 34 | 35 | r = compressor_start_decode(&c, h->alg); 36 | if (r < 0) { 37 | log_debug_errno(r, "compressor_start_decode failed: %m"); 38 | return 0; 39 | } 40 | 41 | log_info("Using compression %d, data size=%zu", h->alg, data_len); 42 | 43 | size_t out_size = MAX(size, 128u), /* Make the buffer a bit larger for very small data */ 44 | ret_done; 45 | buf = malloc(out_size); 46 | if (!buf) { 47 | log_oom(); 48 | return 0; 49 | } 50 | 51 | r = compressor_input(&c, h->data, data_len); 52 | if (r < 0) { 53 | log_debug_errno(r, "compressor_input failed: %m"); 54 | return 0; 55 | } 56 | 57 | r = compressor_decode(&c, buf, out_size, &ret_done); 58 | if (r < 0) { 59 | log_debug_errno(r, "compressor_decode failed: %m"); 60 | return 0; 61 | } 62 | 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /test/fuzz/fuzz-main.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | 5 | #include "fuzz.h" 6 | #include "util.h" 7 | 8 | #define EXIT_TEST_SKIP 77 9 | 10 | /* This is a test driver for fuzzers that provides a main function 11 | * for regression testing outside of oss-fuzz (https://github.com/google/oss-fuzz) 12 | * 13 | * It reads files named on the command line and passes them one by one 14 | * into the fuzzer that it is compiled into. */ 15 | 16 | /* This one was borrowed from 17 | * https://github.com/google/oss-fuzz/blob/646fca1b506b056db3a60d32c4a1a7398f171c94/infra/base-images/base-runner/bad_build_check#L19 18 | */ 19 | 20 | int main(int argc, char **argv) { 21 | int i; 22 | 23 | for (i = 1; i < argc; i++) { 24 | const char *name = argv[i]; 25 | char buf[4096]; 26 | ssize_t size; 27 | 28 | _cleanup_(safe_fclosep) FILE *f = fopen(name, "r"); 29 | if (!f) { 30 | log_error_errno(errno, "Failed to open %s: %m", name); 31 | return EXIT_FAILURE; 32 | } 33 | 34 | size = fread(buf, 1, sizeof(buf), f); 35 | if (size < 0) { 36 | log_error_errno(errno, "Failed to read %s: %m", name); 37 | return EXIT_FAILURE; 38 | } 39 | 40 | printf("%s... ", name); 41 | fflush(stdout); 42 | if (LLVMFuzzerTestOneInput((uint8_t*)buf, size) == EXIT_TEST_SKIP) 43 | return EXIT_TEST_SKIP; 44 | printf("ok\n"); 45 | } 46 | 47 | return EXIT_SUCCESS; 48 | } 49 | -------------------------------------------------------------------------------- /test/fuzz/fuzz.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | #pragma once 3 | 4 | #include 5 | #include 6 | 7 | /* The entry point into the fuzzer */ 8 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); 9 | -------------------------------------------------------------------------------- /test/fuzz/meson.build: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: LGPL-2.1+ 2 | 3 | fuzzers = [ 4 | [['test/fuzz/fuzz-compress.c']], 5 | ] 6 | 7 | fuzz_main_c = files('fuzz-main.c') 8 | -------------------------------------------------------------------------------- /test/http-server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | PORT = 4321 5 | 6 | import http.server 7 | import os 8 | import socket 9 | import socketserver 10 | import sys 11 | import time 12 | os.chdir(sys.argv[1]) 13 | 14 | if len(sys.argv) >= 3: 15 | PORT = int(sys.argv[2]) 16 | 17 | def send_notify(text): 18 | 19 | if text is None or text == "": 20 | return 21 | 22 | e = os.getenv("NOTIFY_SOCKET") 23 | if e is None: 24 | return 25 | 26 | assert len(e) >= 2 27 | assert e[0] == '/' or e[0] == '@' 28 | 29 | fd = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) 30 | fd.connect("\0" + e[1:] if e[0] == '@' else e) 31 | fd.send(bytes(text, 'utf-8')) 32 | 33 | class AllowReuseAddressServer(socketserver.TCPServer): 34 | allow_reuse_address = True 35 | 36 | def server_activate(self): 37 | super().server_activate() 38 | send_notify("READY=1") 39 | 40 | httpd = AllowReuseAddressServer(("", PORT), http.server.SimpleHTTPRequestHandler) 41 | 42 | httpd.serve_forever() 43 | -------------------------------------------------------------------------------- /test/meson-check-help.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -eu 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | # output width 5 | if "$1" --help | grep -v 'default:' | grep -E -q '.{80}.'; then 6 | echo "$(basename "$1") --help output is too wide:" 7 | "$1" --help | awk 'length > 80' | grep -E --color=yes '.{80}' 8 | exit 1 9 | fi 10 | 11 | # no --help output to stdout 12 | if "$1" --help 2>&1 1>/dev/null | grep .; then 13 | echo "$(basename "$1") --help prints to stderr" 14 | exit 2 15 | fi 16 | 17 | # error output to stderr 18 | if ! "$1" --no-such-parameter 2>&1 1>/dev/null | grep -q .; then 19 | echo "$(basename "$1") with an unknown parameter does not print to stderr" 20 | exit 3 21 | fi 22 | -------------------------------------------------------------------------------- /test/meson.build: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: LGPL-2.1+ 2 | 3 | notify_wait_sources = [ 4 | files('notify-wait.c'), 5 | util_sources, 6 | ] 7 | 8 | subdir('fuzz') 9 | -------------------------------------------------------------------------------- /test/pseudo-ssh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | # A tool that we can use like ssh, but instead of connecting anywhere simply 5 | # invokes the specified command locally. To be used via $CASYNC_SSH_PATH for 6 | # testing the remoting protocol without actually doing anything remote. 7 | 8 | shift 1 9 | exec "$@" 10 | -------------------------------------------------------------------------------- /test/report-holes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # SPDX-License-Identifier: CC0-1.0 3 | 4 | import os 5 | import sys 6 | import errno 7 | 8 | def report(fname): 9 | fd = os.open(fname, os.O_RDONLY) 10 | len = os.lseek(fd, 0, os.SEEK_END) 11 | offset = 0 12 | while offset < len: 13 | start = os.lseek(fd, offset, os.SEEK_HOLE) 14 | if start == len: 15 | break 16 | try: 17 | offset = os.lseek(fd, start, os.SEEK_DATA) 18 | except OSError as e: 19 | if e.errno == errno.ENXIO: 20 | offset = len 21 | else: 22 | raise 23 | print(f'found hole between 0x{start:08X} and 0x{offset:08X} ({offset - start} bytes)') 24 | 25 | if __name__ == '__main__': 26 | for name in sys.argv[1:]: 27 | report(name) 28 | -------------------------------------------------------------------------------- /test/semaphore-run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | DEVPKGS=( 5 | pkg-config 6 | liblzma-dev 7 | libcurl4-openssl-dev 8 | libssl-dev 9 | libacl1-dev 10 | libfuse-dev 11 | zlib1g-dev 12 | libzstd-dev 13 | libudev-dev 14 | ) 15 | 16 | echo 17 | echo "============= Installing amd64 build dependencies ==============" 18 | set -eu 19 | ( 20 | set -x 21 | 22 | sudo rm /etc/apt/sources.list.d/* 23 | 24 | sudo apt install -y software-properties-common 25 | sudo add-apt-repository -y ppa:jonathonf/python-3.6 26 | sudo add-apt-repository -y ppa:ginggs/backports 27 | sudo apt -y update 28 | sudo apt -y --no-install-recommends install \ 29 | python3.6 \ 30 | squashfs-tools 31 | 32 | sudo dpkg --add-architecture i386 33 | sudo apt-get update 34 | 35 | sudo apt -y --no-install-recommends install \ 36 | rsync \ 37 | python3-pip \ 38 | python3-setuptools \ 39 | python3-wheel \ 40 | python-sphinx \ 41 | "${DEVPKGS[@]}" 42 | python3.6 -m pip install --user meson ninja 43 | ) 44 | 45 | echo 46 | echo "============= Building amd64 ==============" 47 | ( 48 | set -x 49 | meson build 50 | ninja -C build 51 | ) 52 | 53 | echo 54 | echo "============= Running amd64 tests as user ==============" 55 | ( 56 | set -x 57 | ninja -C build test 58 | ) 59 | 60 | echo 61 | echo "============= Running amd64 tests as root ==============" 62 | ( 63 | set -x 64 | sudo CASYNC_TEST_NBD=0 $(which ninja) -C build test 65 | ) 66 | 67 | echo 68 | echo "============= Installing i386 build dependencies ==============" 69 | 70 | # library -dev packages are not co-installable for multiple architectures, 71 | # so this can't go into the setup step 72 | ( 73 | set -x 74 | # both arch versions provide /usr/bin/curl-config, which can't go well 75 | sudo apt remove -y libcurl4-openssl-dev 76 | sudo apt-get install -y --no-install-recommends \ 77 | gcc-multilib \ 78 | libgcc-5-dev:i386 \ 79 | "${DEVPKGS[@]/%/:i386}" 80 | ) 81 | 82 | echo 83 | echo "============= Building i386 ==============" 84 | ( 85 | set -x 86 | export PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu 87 | CC=gcc-5 CFLAGS=-m32 LDFLAGS=-m32 meson build-i386 88 | ninja -C build-i386 89 | ) 90 | 91 | echo 92 | echo "============= Running i386 tests as user ==============" 93 | ( 94 | set -x 95 | linux32 ninja -C build-i386 test 96 | ) 97 | 98 | echo 99 | echo "============= Running i386 tests as root ==============" 100 | ( 101 | set -x 102 | sudo CASYNC_TEST_NBD=0 linux32 $(which ninja) -C build-i386 test 103 | ) 104 | -------------------------------------------------------------------------------- /test/test-cache.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | SCRATCH_DIR=${TMPDIR:-/var/tmp}/test-casync.$RANDOM 5 | mkdir -p $SCRATCH_DIR/src 6 | 7 | mkdir $SCRATCH_DIR/src/casync 8 | if [ $UID = 0 ]; then 9 | cp -a @top_srcdir@/{test-files,src} $SCRATCH_DIR/src/casync/ 10 | else 11 | # If we lack privileges we use rsync rather than cp to copy, as it will just skip over device nodes 12 | rsync -a --exclude=.cacac @top_srcdir@/{test-files,src} $SCRATCH_DIR/src/casync/ 13 | fi 14 | 15 | cd $SCRATCH_DIR/src 16 | 17 | @top_builddir@/casync list >$SCRATCH_DIR/test.list 18 | @top_builddir@/casync mtree >$SCRATCH_DIR/test.mtree 19 | @top_builddir@/casync digest >$SCRATCH_DIR/test.digest 20 | 21 | # Make three versions of the caidx: one with no cache, one with an unpopulated cache, and one with a populated cache 22 | @top_builddir@/casync make $SCRATCH_DIR/test1.caidx 23 | @top_builddir@/casync make --cache $SCRATCH_DIR/test.cache $SCRATCH_DIR/test2.caidx 24 | @top_builddir@/casync make --cache $SCRATCH_DIR/test.cache $SCRATCH_DIR/test3.caidx 25 | 26 | cmp $SCRATCH_DIR/test1.caidx $SCRATCH_DIR/test2.caidx 27 | cmp $SCRATCH_DIR/test1.caidx $SCRATCH_DIR/test3.caidx 28 | 29 | for f in test1 test2 test3 ; do 30 | @top_builddir@/casync list $SCRATCH_DIR/$f.caidx >$SCRATCH_DIR/$f.list 31 | @top_builddir@/casync mtree $SCRATCH_DIR/$f.caidx >$SCRATCH_DIR/$f.mtree 32 | @top_builddir@/casync digest $SCRATCH_DIR/$f.caidx >$SCRATCH_DIR/$f.digest 33 | 34 | diff -q $SCRATCH_DIR/test.list $SCRATCH_DIR/$f.list 35 | diff -q $SCRATCH_DIR/test.mtree $SCRATCH_DIR/$f.mtree 36 | diff -q $SCRATCH_DIR/test.digest $SCRATCH_DIR/$f.digest 37 | done 38 | 39 | # Make some changes: one in the beginning, one in the middle, one at the end 40 | mkdir casync/000-early 41 | touch casync/000-early/testa 42 | mkdir casync/ggg-middle 43 | touch casync/ggg-middle/testb 44 | mkdir casync/zzz-late 45 | touch casync/zzz-late/testc 46 | 47 | # Also remove a file 48 | rm casync/src/notify.* 49 | 50 | # And update an existing one 51 | chmod +w casync/src/compressor.c 52 | echo xxx >>casync/src/compressor.c 53 | 54 | @top_builddir@/casync list >$SCRATCH_DIR/testx.list 55 | @top_builddir@/casync mtree >$SCRATCH_DIR/testx.mtree 56 | @top_builddir@/casync digest >$SCRATCH_DIR/testx.digest 57 | 58 | @top_builddir@/casync make $SCRATCH_DIR/test4.caidx 59 | @top_builddir@/casync make --cache $SCRATCH_DIR/test.cache $SCRATCH_DIR/test5.caidx 60 | @top_builddir@/casync make --cache $SCRATCH_DIR/test.cache $SCRATCH_DIR/test6.caidx 61 | @top_builddir@/casync make $SCRATCH_DIR/test7.caidx 62 | 63 | cmp $SCRATCH_DIR/test4.caidx $SCRATCH_DIR/test5.caidx 64 | cmp $SCRATCH_DIR/test4.caidx $SCRATCH_DIR/test6.caidx 65 | cmp $SCRATCH_DIR/test4.caidx $SCRATCH_DIR/test7.caidx 66 | 67 | for f in test4 test5 test6 test7; do 68 | @top_builddir@/casync list $SCRATCH_DIR/$f.caidx >$SCRATCH_DIR/$f.list 69 | @top_builddir@/casync mtree $SCRATCH_DIR/$f.caidx >$SCRATCH_DIR/$f.mtree 70 | @top_builddir@/casync digest $SCRATCH_DIR/$f.caidx >$SCRATCH_DIR/$f.digest 71 | 72 | diff -q $SCRATCH_DIR/testx.list $SCRATCH_DIR/$f.list 73 | diff -q $SCRATCH_DIR/testx.mtree $SCRATCH_DIR/$f.mtree 74 | diff -q $SCRATCH_DIR/testx.digest $SCRATCH_DIR/$f.digest 75 | done 76 | 77 | chmod -R u+rwx $SCRATCH_DIR 78 | rm -rf $SCRATCH_DIR 79 | -------------------------------------------------------------------------------- /test/test-cachunk.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | 5 | #include "cachunk.h" 6 | #include "def.h" 7 | 8 | static void test_chunk_file(void) { 9 | uint8_t buffer[BUFFER_SIZE*4]; 10 | _cleanup_(realloc_buffer_free) ReallocBuffer rb = {}, rb2 = {}; 11 | const char *d; 12 | char *path; 13 | _cleanup_(safe_closep) int fd = -1; 14 | int r; 15 | 16 | assert(var_tmp_dir(&d) >= 0); 17 | path = strjoina(d, "/chunk-test.XXXXXX"); 18 | 19 | assert_se(dev_urandom(buffer, sizeof(buffer)) >= 0); 20 | 21 | fd = mkostemp(path, O_RDWR|O_CLOEXEC); 22 | assert_se(fd >= 0); 23 | assert_se(unlink(path) >= 0); 24 | 25 | r = ca_save_and_compress_fd(fd, CA_COMPRESSION_DEFAULT, buffer, sizeof(buffer)); 26 | assert_se(r >= 0); 27 | 28 | assert_se(lseek(fd, 0, SEEK_SET) == 0); 29 | 30 | r = ca_load_and_decompress_fd(fd, &rb); 31 | safe_close(r >= 0); 32 | 33 | assert_se(realloc_buffer_size(&rb) == sizeof(buffer)); 34 | assert_se(memcmp(realloc_buffer_data(&rb), buffer, sizeof(buffer)) == 0); 35 | 36 | realloc_buffer_empty(&rb); 37 | 38 | r = ca_compress(CA_COMPRESSION_DEFAULT, buffer, sizeof(buffer), &rb); 39 | assert_se(r >= 0); 40 | 41 | assert_se(lseek(fd, 0, SEEK_SET) == 0); 42 | assert_se(ftruncate(fd, 0) == 0); 43 | 44 | r = ca_save_and_decompress_fd(fd, realloc_buffer_data(&rb), realloc_buffer_size(&rb)); 45 | assert_se(r >= 0); 46 | 47 | realloc_buffer_empty(&rb); 48 | 49 | assert_se(lseek(fd, 0, SEEK_SET) == 0); 50 | 51 | r = ca_load_and_compress_fd(fd, CA_COMPRESSION_DEFAULT, &rb); 52 | assert_se(r >= 0); 53 | 54 | r = ca_decompress(realloc_buffer_data(&rb), realloc_buffer_size(&rb), &rb2); 55 | assert_se(r >= 0); 56 | 57 | assert_se(realloc_buffer_size(&rb2) == sizeof(buffer)); 58 | assert_se(memcmp(realloc_buffer_data(&rb2), buffer, sizeof(buffer)) == 0); 59 | } 60 | 61 | int main(int argc, char *argv[]) { 62 | 63 | test_chunk_file(); 64 | 65 | return 0; 66 | } 67 | -------------------------------------------------------------------------------- /test/test-cachunker-histogram.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "time-util.h" 9 | #include "util.h" 10 | #include "cachunker.h" 11 | 12 | #define BUFFER_SIZE (64*1024) 13 | #define CHUNKS_MAX 1000000 14 | #define THREADS_MAX 16 15 | #define RUNTIME_NSEC UINT64_C(2000000000) 16 | 17 | struct thread_info { 18 | pthread_t id; 19 | CaChunker chunker; 20 | unsigned *histogram; 21 | unsigned n_chunks; 22 | int random_fd; 23 | uint64_t until; 24 | }; 25 | 26 | static void* process(void *q) { 27 | struct thread_info *t = q; 28 | size_t previous = 0; 29 | 30 | for (;;) { 31 | uint8_t buffer[BUFFER_SIZE], *p; 32 | ssize_t l; 33 | 34 | if (t->until < now(CLOCK_MONOTONIC)) 35 | return NULL; 36 | 37 | l = read(t->random_fd, buffer, sizeof(buffer)); 38 | assert_se(l == sizeof(buffer)); 39 | 40 | p = buffer; 41 | for (;;) { 42 | size_t n; 43 | 44 | n = ca_chunker_scan(&t->chunker, p, l); 45 | if (n == (size_t) -1) { 46 | previous += l; 47 | break; 48 | } 49 | 50 | assert_se(n <= (size_t) l); 51 | 52 | assert_se(previous + n >= t->chunker.chunk_size_min); 53 | assert_se(previous + n <= t->chunker.chunk_size_max); 54 | 55 | t->histogram[previous + n] ++; 56 | t->n_chunks ++; 57 | 58 | p += n; 59 | l -= n; 60 | 61 | previous = 0; 62 | } 63 | } 64 | } 65 | 66 | static void draw(unsigned *histogram, size_t n) { 67 | #define N_BUCKETS 30 68 | #define N_HEIGHT 69 69 | 70 | unsigned buckets[N_BUCKETS] = {}; 71 | unsigned highest = 0; 72 | size_t i; 73 | 74 | for (i = 0; i < n; i++) 75 | buckets[i * N_BUCKETS / n] += histogram[i]; 76 | 77 | for (i = 0; i < N_BUCKETS; i++) 78 | if (buckets[i] > highest) 79 | highest = buckets[i]; 80 | 81 | for (i = 0; i < N_BUCKETS; i++) { 82 | unsigned k, j; 83 | 84 | k = buckets[i] * N_HEIGHT / highest; 85 | 86 | printf("%10zu ", (i+1) * n / N_BUCKETS -1); 87 | 88 | for (j = 0; j < k; j++) 89 | putchar('#'); 90 | 91 | putchar('\n'); 92 | } 93 | } 94 | 95 | static void run(size_t pick, size_t *ret_avg) { 96 | CaChunker chunker = CA_CHUNKER_INIT; 97 | unsigned *histogram, n_chunks = 0; 98 | struct thread_info threads[THREADS_MAX] = {}; 99 | uint64_t until, sum = 0; 100 | size_t i; 101 | int fd, r; 102 | 103 | ca_chunker_set_size(&chunker, 0, pick, 0); 104 | 105 | histogram = new0(unsigned, chunker.chunk_size_max+1); 106 | assert_se(histogram); 107 | 108 | log_info("Min/Avg/Max = %zu/%zu/%zu (discriminator=%zu)", 109 | chunker.chunk_size_min, 110 | chunker.chunk_size_avg, 111 | chunker.chunk_size_max, 112 | chunker.discriminator); 113 | 114 | fd = open("/dev/urandom", O_CLOEXEC|O_RDONLY); 115 | assert_se(fd >= 0); 116 | 117 | until = now(CLOCK_MONOTONIC) + RUNTIME_NSEC; 118 | 119 | for (i = 0; i < THREADS_MAX; i++) { 120 | threads[i].chunker = chunker; 121 | 122 | threads[i].histogram = new0(unsigned, chunker.chunk_size_max+1); 123 | assert_se(threads[i].histogram); 124 | 125 | threads[i].random_fd = fd; 126 | threads[i].until = until; 127 | 128 | r = pthread_create(&threads[i].id, NULL, process, threads + i); 129 | assert_se(r == 0); 130 | } 131 | 132 | for (i = 0; i < THREADS_MAX; i++) { 133 | size_t j; 134 | 135 | r = pthread_join(threads[i].id, NULL); 136 | assert_se(r == 0); 137 | 138 | for (j = 0; j <= chunker.chunk_size_max; j++) { 139 | histogram[j] += threads[i].histogram[j]; 140 | sum += threads[i].histogram[j] * j; 141 | } 142 | 143 | n_chunks += threads[i].n_chunks; 144 | 145 | free(threads[i].histogram); 146 | } 147 | 148 | log_info("Generated %u chunks.", n_chunks); 149 | log_info("Effective average is %" PRIu64 ".", sum / n_chunks); 150 | 151 | *ret_avg = sum / n_chunks; 152 | 153 | draw(histogram, chunker.chunk_size_max+1); 154 | 155 | free(histogram); 156 | safe_close(fd); 157 | 158 | return; 159 | } 160 | 161 | int main(int argc, char* argv[]) { 162 | size_t avg, start, end, step; 163 | 164 | if (argc > 1) { 165 | start = 4096; 166 | end = 1024*1024; 167 | step = 4*1024; 168 | } else { 169 | start = CA_CHUNK_SIZE_AVG_DEFAULT; 170 | end = CA_CHUNK_SIZE_AVG_DEFAULT+1; 171 | step = 1; 172 | } 173 | 174 | for (avg = start; avg < end; avg += step) { 175 | 176 | size_t effective_avg; 177 | double factor; 178 | 179 | run(avg, &effective_avg); 180 | 181 | factor = (double) effective_avg / (double) avg; 182 | 183 | printf("%zu\t%zu\t%g\n", avg, effective_avg, factor); 184 | log_error("Asked for average: %zu — Got average: %zu — Factor: %g", avg, effective_avg, factor); 185 | } 186 | 187 | return 0; 188 | } 189 | -------------------------------------------------------------------------------- /test/test-cadigest.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "time-util.h" 4 | #include "util.h" 5 | #include "cadigest.h" 6 | 7 | #define TEST_SPEED_RUNTIME_NSEC (5U*NSEC_PER_SEC) 8 | 9 | static void test_speed(CaDigestType t) { 10 | 11 | uint32_t megabyte[1024*1024/sizeof(uint32_t)]; 12 | size_t k, c = 0; 13 | CaDigest *d; 14 | uint64_t n; 15 | 16 | /* Generate 1MB test data */ 17 | srand(0); 18 | for (k = 0; k < ELEMENTSOF(megabyte); k++) 19 | megabyte[k] = rand(); 20 | 21 | assert_se(ca_digest_new(t, &d) >= 0); 22 | 23 | n = now(CLOCK_MONOTONIC); 24 | 25 | while (n + TEST_SPEED_RUNTIME_NSEC > now(CLOCK_MONOTONIC)) { 26 | ca_digest_write(d, megabyte, sizeof(megabyte)); 27 | c++; 28 | } 29 | 30 | printf("%s: %zu MB/s\n", ca_digest_type_to_string(t), (size_t) ((c * NSEC_PER_SEC) / TEST_SPEED_RUNTIME_NSEC)); 31 | 32 | ca_digest_free(d); 33 | } 34 | 35 | int main(int argc, char *argv[]) { 36 | CaDigest *d; 37 | CaDigestType t; 38 | 39 | assert_se(ca_digest_new(CA_DIGEST_SHA256, &d) >= 0); 40 | 41 | assert_se(memcmp(ca_digest_read(d), (const uint8_t[]) { 42 | 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 43 | 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 44 | 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 45 | 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55 }, 32) == 0); 46 | 47 | ca_digest_reset(d); 48 | ca_digest_write(d, "foobar", 6); 49 | 50 | assert_se(memcmp(ca_digest_read(d), (const uint8_t[]) { 51 | 0xc3, 0xab, 0x8f, 0xf1, 0x37, 0x20, 0xe8, 0xad, 52 | 0x90, 0x47, 0xdd, 0x39, 0x46, 0x6b, 0x3c, 0x89, 53 | 0x74, 0xe5, 0x92, 0xc2, 0xfa, 0x38, 0x3d, 0x4a, 54 | 0x39, 0x60, 0x71, 0x4c, 0xae, 0xf0, 0xc4, 0xf2 }, 32) == 0); 55 | 56 | 57 | ca_digest_reset(d); 58 | 59 | ca_digest_write(d, "foo", 3); 60 | ca_digest_write(d, "bar", 3); 61 | 62 | assert_se(memcmp(ca_digest_read(d), (const uint8_t[]) { 63 | 0xc3, 0xab, 0x8f, 0xf1, 0x37, 0x20, 0xe8, 0xad, 64 | 0x90, 0x47, 0xdd, 0x39, 0x46, 0x6b, 0x3c, 0x89, 65 | 0x74, 0xe5, 0x92, 0xc2, 0xfa, 0x38, 0x3d, 0x4a, 66 | 0x39, 0x60, 0x71, 0x4c, 0xae, 0xf0, 0xc4, 0xf2 }, 32) == 0); 67 | 68 | d = ca_digest_free(d); 69 | 70 | assert_se(ca_digest_new(CA_DIGEST_SHA512_256, &d) >= 0); 71 | 72 | assert_se(memcmp(ca_digest_read(d), (const uint8_t[]) { 73 | 0xc6, 0x72, 0xb8, 0xd1, 0xef, 0x56, 0xed, 0x28, 74 | 0xab, 0x87, 0xc3, 0x62, 0x2c, 0x51, 0x14, 0x06, 75 | 0x9b, 0xdd, 0x3a, 0xd7, 0xb8, 0xf9, 0x73, 0x74, 76 | 0x98, 0xd0, 0xc0, 0x1e, 0xce, 0xf0, 0x96, 0x7a }, 32) == 0); 77 | 78 | ca_digest_reset(d); 79 | ca_digest_write(d, "foobar", 6); 80 | 81 | assert_se(memcmp(ca_digest_read(d), (const uint8_t[]) { 82 | 0xd0, 0x14, 0xc7, 0x52, 0xbc, 0x2b, 0xe8, 0x68, 83 | 0xe1, 0x63, 0x30, 0xf4, 0x7e, 0x0c, 0x31, 0x6a, 84 | 0x59, 0x67, 0xbc, 0xbc, 0x9c, 0x28, 0x6a, 0x45, 85 | 0x77, 0x61, 0xd7, 0x05, 0x5b, 0x92, 0x14, 0xce }, 32) == 0); 86 | 87 | ca_digest_reset(d); 88 | 89 | ca_digest_write(d, "foo", 3); 90 | ca_digest_write(d, "bar", 3); 91 | 92 | assert_se(memcmp(ca_digest_read(d), (const uint8_t[]) { 93 | 0xd0, 0x14, 0xc7, 0x52, 0xbc, 0x2b, 0xe8, 0x68, 94 | 0xe1, 0x63, 0x30, 0xf4, 0x7e, 0x0c, 0x31, 0x6a, 95 | 0x59, 0x67, 0xbc, 0xbc, 0x9c, 0x28, 0x6a, 0x45, 96 | 0x77, 0x61, 0xd7, 0x05, 0x5b, 0x92, 0x14, 0xce }, 32) == 0); 97 | 98 | d = ca_digest_free(d); 99 | 100 | for (t = 0; t < _CA_DIGEST_TYPE_MAX; t++) 101 | test_speed(t); 102 | 103 | return 0; 104 | } 105 | -------------------------------------------------------------------------------- /test/test-caindex.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | 5 | #include "util.h" 6 | #include "caindex.h" 7 | 8 | int main(int argc, char*argv[]) { 9 | _cleanup_(ca_index_unrefp) CaIndex* index = NULL; 10 | int r; 11 | 12 | if (argc != 2) { 13 | fprintf(stderr, "Expected an index file as argument.\n"); 14 | return EXIT_FAILURE; 15 | } 16 | 17 | assert_se(index = ca_index_new_read()); 18 | assert_se(ca_index_set_path(index, argv[1]) >= 0); 19 | assert_se(ca_index_open(index) >= 0); 20 | 21 | for (;;) { 22 | CaChunkID id; 23 | uint64_t size; 24 | char ids[CA_CHUNK_ID_FORMAT_MAX]; 25 | 26 | r = ca_index_read_chunk(index, &id, NULL, &size); 27 | assert_se(r >= 0); 28 | 29 | if (r == 0) 30 | break; 31 | 32 | printf("%s (%" PRIu64 ")\n", ca_chunk_id_format(&id, ids), size); 33 | } 34 | 35 | printf("EOF\n"); 36 | return 0; 37 | } 38 | -------------------------------------------------------------------------------- /test/test-calc-digest.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "cadigest.h" 11 | #include "util.h" 12 | 13 | int main(int argc, char *argv[]) { 14 | CaDigest *digest = NULL; 15 | CaDigestType type; 16 | int fd = -1, r; 17 | const char *q, *path, *dt; 18 | size_t l; 19 | char *p = NULL; 20 | 21 | if (argc > 3) { 22 | log_error("Expected a two arguments: digest and file name."); 23 | r = -EINVAL; 24 | goto finish; 25 | } 26 | 27 | path = argc == 3 ? argv[2] : NULL; 28 | dt = argc >= 2 ? argv[1] : NULL; 29 | 30 | if (dt) { 31 | type = ca_digest_type_from_string(dt); 32 | if (type < 0) { 33 | log_error("Failed to parse digest name: %s", dt); 34 | r = -EINVAL; 35 | goto finish; 36 | } 37 | } else 38 | type = CA_DIGEST_SHA512_256; 39 | 40 | r = ca_digest_new(type, &digest); 41 | if (r < 0) { 42 | log_error("Failed to set up digest %s: %m", dt); 43 | goto finish; 44 | } 45 | 46 | if (path) { 47 | fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); 48 | if (fd < 0) { 49 | log_error_errno(errno, "Failed to open %s: %m", path); 50 | goto finish; 51 | } 52 | } else 53 | fd = STDIN_FILENO; 54 | 55 | for (;;) { 56 | uint8_t buffer[64*1024]; 57 | ssize_t n; 58 | 59 | n = read(fd, buffer, sizeof(buffer)); 60 | if (n < 0) { 61 | log_error_errno(errno, "Failed to read: %m"); 62 | goto finish; 63 | } 64 | if (n == 0) /* EOF */ 65 | break; 66 | 67 | ca_digest_write(digest, buffer, (size_t) n); 68 | } 69 | 70 | q = ca_digest_read(digest); 71 | l = ca_digest_get_size(digest); 72 | 73 | p = hexmem(q, l); 74 | if (!p) { 75 | r = log_oom(); 76 | goto finish; 77 | } 78 | 79 | fputs(p, stdout); 80 | fputc('\n', stdout); 81 | 82 | r = 0; 83 | 84 | finish: 85 | ca_digest_free(digest); 86 | 87 | if (fd > 2) 88 | safe_close(fd); 89 | 90 | free(p); 91 | 92 | return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; 93 | } 94 | -------------------------------------------------------------------------------- /test/test-calocation.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "cachunkid.h" 4 | #include "cadigest.h" 5 | #include "calocation.h" 6 | #include "def.h" 7 | #include "time-util.h" 8 | #include "util.h" 9 | #include "caformat.h" 10 | 11 | int main(int argc, char *argv[]) { 12 | _cleanup_(ca_name_table_unrefp) CaNameTable *ntp = NULL, *nt = NULL, *nt2 = NULL; 13 | _cleanup_(ca_location_unrefp) CaLocation *loc = NULL, *loc2 = NULL; 14 | _cleanup_(ca_digest_freep) CaDigest *digest = NULL; 15 | CaChunkID id, id2; 16 | CaNameItem *item; 17 | const char *x; 18 | 19 | assert_se(ca_digest_new(CA_DIGEST_DEFAULT, &digest) >= 0); 20 | 21 | assert_se(ntp = ca_name_table_new_size(1)); 22 | ntp->entry_offset = 0; 23 | assert_se(ca_name_table_add(&ntp, &item) >= 0); 24 | *item = (CaNameItem) { .hash = 0x70, .start_offset = 0, .end_offset = 10 }; 25 | 26 | assert_se(nt = ca_name_table_new_size(3)); 27 | nt->entry_offset = 1; 28 | nt->parent = ca_name_table_ref(ntp); 29 | assert_se(ca_name_table_add(&nt, &item) >= 0); 30 | *item = (CaNameItem) { .hash = 0xa0, .start_offset = 1, .end_offset = 2 }; 31 | assert_se(ca_name_table_add(&nt, &item) >= 0); 32 | *item = (CaNameItem) { .hash = 0xa1, .start_offset = 2, .end_offset = 5 }; 33 | assert_se(ca_name_table_add(&nt, &item) >= 0); 34 | *item = (CaNameItem) { .hash = 0xa2, .start_offset = 5, .end_offset = 9 }; 35 | 36 | assert_se(x = ca_name_table_format(nt)); 37 | assert_se(ca_name_table_parse(&x, &nt2) >= 0); 38 | assert_se(ca_name_table_equal(nt, nt2)); 39 | 40 | assert_se(ca_location_new("foo/quux", CA_LOCATION_PAYLOAD, 4711, 815, &loc) >= 0); 41 | 42 | loc->feature_flags = CA_FORMAT_WITH_BEST; 43 | loc->mtime = 1517231408U * NSEC_PER_SEC; 44 | loc->inode = 123456; 45 | loc->generation = 2345; 46 | loc->generation_valid = true; 47 | loc->name_table = ca_name_table_ref(nt); 48 | loc->archive_offset = 87654; 49 | 50 | assert_se(ca_location_patch_size(&loc, 333) >= 0); 51 | assert_se(ca_location_advance(&loc, 7) >= 0); 52 | assert_se(ca_location_id_make(digest, loc, true, &id) >= 0); 53 | 54 | assert_se(ca_location_parse(ca_location_format_full(loc, CA_LOCATION_WITH_ALL), &loc2) >= 0); 55 | 56 | assert_se(ca_location_equal(loc, loc2, CA_LOCATION_WITH_ALL)); 57 | assert_se(ca_location_id_make(digest, loc2, true, &id2) >= 0); 58 | assert_se(ca_chunk_id_equal(&id, &id2)); 59 | 60 | assert_se(streq(loc2->path, "foo/quux")); 61 | assert_se(loc2->designator == CA_LOCATION_PAYLOAD); 62 | assert_se(loc2->offset == 4711 + 7); 63 | assert_se(loc2->size == 333 - 7); 64 | assert_se(loc2->feature_flags == CA_FORMAT_WITH_BEST); 65 | assert_se(loc2->mtime == 1517231408U * NSEC_PER_SEC); 66 | assert_se(loc2->inode == 123456); 67 | assert_se(loc2->generation == 2345); 68 | assert_se(loc2->generation_valid); 69 | assert_se(loc2->archive_offset == 87654 + 7); 70 | 71 | return 0; 72 | } 73 | -------------------------------------------------------------------------------- /test/test-camakebst.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "camakebst.h" 4 | #include "util.h" 5 | 6 | #define TEST_MAX 9999 7 | 8 | static size_t find_bst(const int b[], size_t n, int x) { 9 | size_t i = 0; 10 | 11 | for (;;) { 12 | if (i >= n) 13 | return (size_t) -1; 14 | 15 | if (b[i] == x) 16 | return i; 17 | 18 | if (x < b[i]) 19 | i = 2*i + 1; 20 | else if (x > b[i]) 21 | i = 2*i + 2; 22 | } 23 | } 24 | 25 | static void test_makebst_size(size_t n) { 26 | size_t i; 27 | int a[MAX(1U, n)], b[MAX(1U, n)]; 28 | 29 | for (i = 0; i < n; i++) { 30 | a[i] = (int) i; 31 | b[i] = -1; 32 | } 33 | 34 | ca_make_bst(a, n, sizeof(int), b); 35 | 36 | for (i = 0; i < n; i++) { 37 | assert_se(i*2+1 >= n || b[i] > b[i*2+1]); 38 | assert_se(i*2+2 >= n || b[i] < b[i*2+2]); 39 | } 40 | 41 | for (i = 0; i < n; i++) { 42 | size_t j; 43 | 44 | j = find_bst(b, n, (int) i); 45 | assert_se(j != (size_t) -1); 46 | 47 | assert_se(b[j] == (int) i); 48 | } 49 | 50 | assert_se(find_bst(b, n, -2) == (size_t) -1); 51 | assert_se(find_bst(b, n, -1) == (size_t) -1); 52 | assert_se(find_bst(b, n, n) == (size_t) -1); 53 | assert_se(find_bst(b, n, n+1) == (size_t) -1); 54 | } 55 | 56 | int main(int argc, char *argv[]) { 57 | size_t i; 58 | 59 | for (i = 0; i < TEST_MAX; i++) 60 | test_makebst_size(i); 61 | 62 | return 0; 63 | } 64 | -------------------------------------------------------------------------------- /test/test-caorigin.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "caorigin.h" 4 | 5 | int main(int argc, char *argv[]) { 6 | CaOrigin *o, *p; 7 | CaLocation *a, *b, *c, *d; 8 | 9 | assert_se(ca_location_new("foo", CA_LOCATION_ENTRY, 0, 22, &a) >= 0); 10 | assert_se(ca_location_new("foo", CA_LOCATION_PAYLOAD, 0, 55, &b) >= 0); 11 | assert_se(ca_location_new("foo", CA_LOCATION_PAYLOAD, 55, 77, &c) >= 0); 12 | assert_se(ca_location_new("quux", CA_LOCATION_ENTRY, 0, 33, &d) >= 0); 13 | 14 | assert_se(ca_origin_new(&o) >= 0); 15 | assert_se(ca_origin_put(o, a) >= 0); 16 | assert_se(ca_origin_put(o, b) >= 0); 17 | assert_se(ca_origin_put(o, c) >= 0); 18 | assert_se(ca_origin_put(o, d) >= 0); 19 | 20 | assert_se(ca_origin_dump(NULL, o) >= 0); 21 | assert_se(ca_origin_advance_bytes(o, 1) >= 0); 22 | assert_se(ca_origin_dump(NULL, o) >= 0); 23 | assert_se(ca_origin_advance_bytes(o, 20) >= 0); 24 | assert_se(ca_origin_dump(NULL, o) >= 0); 25 | assert_se(ca_origin_advance_bytes(o, 2) >= 0); 26 | assert_se(ca_origin_dump(NULL, o) >= 0); 27 | 28 | assert_se(ca_origin_concat(o, o, UINT64_MAX) >= 0); 29 | assert_se(ca_origin_dump(NULL, o) >= 0); 30 | 31 | assert_se(ca_origin_new(&p) >= 0); 32 | assert_se(ca_origin_put(p, b) >= 0); 33 | assert_se(ca_origin_put(p, b) >= 0); 34 | assert_se(ca_origin_put(p, c) >= 0); 35 | assert_se(ca_origin_dump(NULL, p) >= 0); 36 | 37 | assert_se(ca_origin_concat(o, p, UINT64_MAX) >= 0); 38 | assert_se(ca_origin_dump(NULL, o) >= 0); 39 | 40 | assert_se(ca_origin_concat(o, p, 56) >= 0); 41 | assert_se(ca_origin_dump(NULL, o) >= 0); 42 | 43 | ca_origin_unref(o); 44 | ca_origin_unref(p); 45 | ca_location_unref(a); 46 | ca_location_unref(b); 47 | ca_location_unref(c); 48 | ca_location_unref(d); 49 | 50 | return 0; 51 | } 52 | -------------------------------------------------------------------------------- /test/test-casync.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | #include 5 | 6 | #include "caformat.h" 7 | #include "casync.h" 8 | #include "rm-rf.h" 9 | #include "util.h" 10 | 11 | int main(int argc, char *argv[]) { 12 | const char *d; 13 | char *teststore, *testindex, *testtree; 14 | CaSync *s; 15 | int r, base_fd; 16 | CaChunkID digest; 17 | char t[CA_CHUNK_ID_FORMAT_MAX]; 18 | uint64_t flags; 19 | 20 | assert(var_tmp_dir(&d) >= 0); 21 | 22 | r = asprintf(&teststore, "%s/teststore.%" PRIx64, d, random_u64()); 23 | assert_se(r >= 0); 24 | r = asprintf(&testindex, "%s/testindex.%" PRIx64, d, random_u64()); 25 | assert_se(r >= 0); 26 | r = asprintf(&testtree, "%s/testtree.%" PRIx64, d, random_u64()); 27 | assert_se(r >= 0); 28 | 29 | assert_se(s = ca_sync_new_encode()); 30 | 31 | flags = CA_FORMAT_DEFAULT; 32 | 33 | if (geteuid() != 0) 34 | flags &= ~CA_FORMAT_WITH_PRIVILEGED; 35 | 36 | assert_se(ca_sync_set_feature_flags(s, flags) >= 0); 37 | 38 | base_fd = open(".", O_RDONLY|O_CLOEXEC|O_DIRECTORY); 39 | assert_se(base_fd >= 0); 40 | assert_se(ca_sync_set_base_fd(s, base_fd) >= 0); 41 | 42 | assert_se(ca_sync_enable_archive_digest(s, true) >= 0); 43 | assert_se(ca_sync_set_store_path(s, teststore) >= 0); 44 | assert_se(ca_sync_set_index_path(s, testindex) >= 0); 45 | 46 | for (;;) { 47 | r = ca_sync_step(s); 48 | assert_se(r >= 0); 49 | 50 | switch (r) { 51 | 52 | case CA_SYNC_FINISHED: { 53 | assert_se(ca_sync_get_archive_digest(s, &digest) >= 0); 54 | printf("%s\n", ca_chunk_id_format(&digest, t)); 55 | goto step2; 56 | } 57 | 58 | case CA_SYNC_STEP: 59 | case CA_SYNC_PAYLOAD: 60 | case CA_SYNC_NEXT_FILE: 61 | case CA_SYNC_DONE_FILE: 62 | break; 63 | 64 | default: 65 | assert_se(false); 66 | } 67 | } 68 | 69 | step2: 70 | ca_sync_unref(s); 71 | assert_se(s = ca_sync_new_decode()); 72 | 73 | (void) mkdir(testtree, 0777); 74 | base_fd = open(testtree, O_RDONLY|O_CLOEXEC|O_DIRECTORY); 75 | 76 | assert_se(base_fd >= 0); 77 | assert_se(ca_sync_set_base_fd(s, base_fd) >= 0); 78 | 79 | assert_se(ca_sync_enable_archive_digest(s, true) >= 0); 80 | assert_se(ca_sync_set_store_path(s, teststore) >= 0); 81 | assert_se(ca_sync_set_index_path(s, testindex) >= 0); 82 | 83 | for (;;) { 84 | r = ca_sync_step(s); 85 | assert_se(r >= 0); 86 | 87 | switch (r) { 88 | 89 | case CA_SYNC_FINISHED: { 90 | assert_se(ca_sync_get_archive_digest(s, &digest) >= 0); 91 | printf("%s\n", ca_chunk_id_format(&digest, t)); 92 | goto finish; 93 | } 94 | 95 | case CA_SYNC_STEP: 96 | case CA_SYNC_PAYLOAD: 97 | case CA_SYNC_NEXT_FILE: 98 | case CA_SYNC_DONE_FILE: 99 | break; 100 | 101 | default: 102 | assert_se(false); 103 | } 104 | } 105 | 106 | finish: 107 | ca_sync_unref(s); 108 | 109 | assert_se(unlink(testindex) == 0); 110 | assert_se(rm_rf(teststore, REMOVE_ROOT|REMOVE_PHYSICAL) == 0); 111 | assert_se(rm_rf(testtree, REMOVE_ROOT|REMOVE_PHYSICAL) == 0); 112 | 113 | free(teststore); 114 | free(testindex); 115 | free(testtree); 116 | 117 | return 0; 118 | } 119 | -------------------------------------------------------------------------------- /test/test-feature-flags.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include "caformat-util.h" 4 | #include "caformat.h" 5 | #include "util.h" 6 | 7 | int main(int argc, char *argv[]) { 8 | 9 | uint64_t i, normalized; 10 | int r; 11 | 12 | /* Make sure that the with mask is a subset of all currently defined bits */ 13 | assert((CA_FORMAT_WITH_MASK & ~CA_FORMAT_FEATURE_FLAGS_MAX) == UINT64_C(0)); 14 | 15 | /* Make sure that the various with flag subsets are actually subsets of the with mask */ 16 | assert((CA_FORMAT_WITH_BEST & ~CA_FORMAT_WITH_MASK) == UINT64_C(0)); 17 | assert((CA_FORMAT_WITH_UNIX & ~CA_FORMAT_WITH_MASK) == UINT64_C(0)); 18 | assert((CA_FORMAT_WITH_FAT & ~CA_FORMAT_WITH_MASK) == UINT64_C(0)); 19 | assert((CA_FORMAT_WITH_CHATTR & ~CA_FORMAT_WITH_MASK) == UINT64_C(0)); 20 | assert((CA_FORMAT_WITH_FAT_ATTRS & ~CA_FORMAT_WITH_MASK) == UINT64_C(0)); 21 | assert((CA_FORMAT_WITH_PRIVILEGED & ~CA_FORMAT_WITH_MASK) == UINT64_C(0)); 22 | assert((CA_FORMAT_WITH_FUSE & ~CA_FORMAT_WITH_MASK) == UINT64_C(0)); 23 | 24 | /* Make sure that if we normalize the full mask we arrive at the best mask (modulo the NODUMP flag, as that's 25 | * masked by CA_FORMAT_EXCLUDE_NODUMP) */ 26 | assert_se(ca_feature_flags_normalize(CA_FORMAT_FEATURE_FLAGS_MAX, &normalized) >= 0); 27 | assert_se((normalized & CA_FORMAT_WITH_MASK) == (CA_FORMAT_WITH_BEST & ~CA_FORMAT_WITH_FLAG_NODUMP)); 28 | 29 | assert_se(ca_feature_flags_are_normalized(CA_FORMAT_WITH_BEST) > 0); 30 | assert_se(ca_feature_flags_are_normalized(CA_FORMAT_WITH_UNIX) > 0); 31 | assert_se(ca_feature_flags_are_normalized(CA_FORMAT_WITH_FAT) > 0); 32 | assert_se(ca_feature_flags_are_normalized(CA_FORMAT_DEFAULT) > 0); 33 | 34 | for (i = 0; i < sizeof(uint64_t) * 8; i++) { 35 | uint64_t flag = UINT64_C(1) << i, flag2; 36 | _cleanup_free_ char *s = NULL; 37 | 38 | r = ca_with_feature_flags_format(flag, &s); 39 | 40 | /* This has to succeed whenever the bit is valid at all */ 41 | assert_se((r >= 0) == !!(flag & CA_FORMAT_FEATURE_FLAGS_MAX)); 42 | if (r < 0) { 43 | assert_se(r == -EINVAL); 44 | continue; 45 | } 46 | 47 | /* If this is not a with mask, the result should be the empty string, but only then */ 48 | assert_se(!(flag & CA_FORMAT_WITH_MASK) == isempty(s)); 49 | if (isempty(s)) 50 | continue; 51 | 52 | assert_se(ca_with_feature_flags_parse_one(s, &flag2) == 0); 53 | assert_se(flag2 == flag); 54 | } 55 | 56 | return 0; 57 | } 58 | -------------------------------------------------------------------------------- /test/test-fuse.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | PARAMS="--with=fuse" 5 | 6 | CASYNC_PROTOCOL_PATH=@top_builddir@ 7 | export CASYNC_PROTOCOL_PATH 8 | 9 | SCRATCH_DIR=${TMPDIR:-/var/tmp}/test-casync.$RANDOM 10 | mkdir -p $SCRATCH_DIR/src 11 | 12 | if [ $UID = 0 ]; then 13 | cp -a @top_srcdir@ $SCRATCH_DIR/src 14 | else 15 | # If we lack privileges we use rsync rather than cp to copy, as it will just skip over device nodes 16 | rsync --exclude=.cacac -a @top_srcdir@ $SCRATCH_DIR/src 17 | fi 18 | 19 | @top_builddir@/casync $PARAMS digest $SCRATCH_DIR/src > $SCRATCH_DIR/test.digest 20 | 21 | @top_builddir@/casync $PARAMS make $SCRATCH_DIR/test.caidx $SCRATCH_DIR/src 22 | @top_builddir@/casync $PARAMS digest $SCRATCH_DIR/test.caidx > $SCRATCH_DIR/test.caidx.digest 23 | 24 | diff -q $SCRATCH_DIR/test.digest $SCRATCH_DIR/test.caidx.digest 25 | 26 | @top_builddir@/casync $PARAMS extract $SCRATCH_DIR/test.caidx $SCRATCH_DIR/extract 27 | @top_builddir@/casync $PARAMS digest $SCRATCH_DIR/extract > $SCRATCH_DIR/extract.digest 28 | 29 | diff -q $SCRATCH_DIR/test.digest $SCRATCH_DIR/extract.digest 30 | 31 | if [ $UID = 0 ]; then 32 | modprobe fuse ||: 33 | if [ -e /dev/fuse ]; then 34 | MOUNT_PID=`@top_builddir@/notify-wait @top_builddir@/casync $PARAMS mount $SCRATCH_DIR/test.caidx $SCRATCH_DIR/mount` 35 | 36 | @top_builddir@/casync $PARAMS digest $SCRATCH_DIR/extract > $SCRATCH_DIR/mount.digest 37 | 38 | diff -q $SCRATCH_DIR/test.digest $SCRATCH_DIR/mount.digest 39 | 40 | kill $MOUNT_PID 41 | umount $SCRATCH_DIR/mount ||: 42 | fi 43 | fi 44 | 45 | rm -rf $SCRATCH_DIR 46 | -------------------------------------------------------------------------------- /test/test-nbd.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | DIGEST=${1:-sha512-256} 5 | PARAMS="-v --digest=$DIGEST" 6 | 7 | export CASYNC_PROTOCOL_PATH=@top_builddir@ 8 | 9 | SCRATCH_DIR=${TMPDIR:-/var/tmp}/test-casync.$RANDOM 10 | mkdir -p $SCRATCH_DIR 11 | 12 | dd if=/dev/urandom of=$SCRATCH_DIR/blob bs=102400 count=80 13 | 14 | @top_builddir@/casync $PARAMS digest $SCRATCH_DIR/blob > $SCRATCH_DIR/test.digest 15 | @top_builddir@/test-calc-digest $DIGEST $SCRATCH_DIR/blob > $SCRATCH_DIR/test.digest2 16 | 17 | @top_builddir@/casync $PARAMS make $SCRATCH_DIR/test.caibx $SCRATCH_DIR/blob 18 | @top_builddir@/casync $PARAMS digest $SCRATCH_DIR/test.caibx > $SCRATCH_DIR/test.caibx.digest 19 | 20 | diff -q $SCRATCH_DIR/test.digest $SCRATCH_DIR/test.digest2 21 | diff -q $SCRATCH_DIR/test.digest $SCRATCH_DIR/test.caibx.digest 22 | 23 | @top_builddir@/casync $PARAMS extract $SCRATCH_DIR/test.caibx $SCRATCH_DIR/blob2 24 | @top_builddir@/casync $PARAMS extract $SCRATCH_DIR/test.caibx --seed=$SCRATCH_DIR/blob2 $SCRATCH_DIR/blob3 25 | 26 | @top_builddir@/test-calc-digest $DIGEST $SCRATCH_DIR/blob2 > $SCRATCH_DIR/extract.digest 27 | @top_builddir@/test-calc-digest $DIGEST $SCRATCH_DIR/blob3 > $SCRATCH_DIR/extract2.digest 28 | 29 | diff -q $SCRATCH_DIR/test.digest $SCRATCH_DIR/extract.digest 30 | diff -q $SCRATCH_DIR/test.digest $SCRATCH_DIR/extract2.digest 31 | 32 | if [ $UID = 0 ]; then 33 | modprobe nbd ||: 34 | 35 | if [ -e /dev/nbd0 -a ${CASYNC_TEST_NBD:-1} = 1 ]; then 36 | MKDEV_PID=`@top_builddir@/notify-wait @top_builddir@/casync $PARAMS mkdev $SCRATCH_DIR/test.caibx $SCRATCH_DIR/test-node` 37 | 38 | dd if=$SCRATCH_DIR/test-node bs=102400 count=80 | @top_builddir@/test-calc-digest $DIGEST > $SCRATCH_DIR/mkdev.digest 39 | 40 | diff -q $SCRATCH_DIR/test.digest $SCRATCH_DIR/mkdev.digest 41 | 42 | kill $MKDEV_PID 43 | fi 44 | fi 45 | 46 | rm -rf $SCRATCH_DIR 47 | -------------------------------------------------------------------------------- /test/test-script-gzip.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | # This is the same as test-script.sh, except that we force the compression 5 | # algorithm to be gzip. 6 | 7 | exec @top_builddir@/test-script.sh default gzip 8 | -------------------------------------------------------------------------------- /test/test-script-sha256.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | # This is the same as test-script.sh, except that we force the digest to be 5 | # sha256, in order to detect potential incompatibilities with the remoting 6 | # feature. 7 | 8 | exec @top_builddir@/test-script.sh sha256 9 | -------------------------------------------------------------------------------- /test/test-script-xz.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | # This is the same as test-script.sh, except that we force the compression 5 | # algorithm to be xz. 6 | 7 | exec @top_builddir@/test-script.sh default xz 8 | -------------------------------------------------------------------------------- /test/test-util.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | 3 | #include 4 | 5 | #include "util.h" 6 | 7 | #define PART1 4096 8 | #define PART2 4096 9 | #define PART3 8192 10 | #define PART4 3000 11 | #define PART5 13 12 | 13 | int main(int argc, char *argv[]) { 14 | uint8_t buffer[PART1 + PART2 + PART3 + PART4 + PART5]; 15 | uint8_t buffer2[sizeof(buffer)]; 16 | char *fn; 17 | int fd, p[2]; 18 | uint64_t n_punched; 19 | const char *d; 20 | 21 | assert_se(tmp_dir(&d) >= 0); 22 | fn = strjoina(d, "/zeroXXXXXX"); 23 | 24 | memzero(buffer, PART1); 25 | dev_urandom(buffer + PART1, PART2); 26 | memzero(buffer + PART1 + PART2, PART3); 27 | dev_urandom(buffer + PART1 + PART2 + PART3, PART4); 28 | memzero(buffer + PART1 + PART2 + PART3 + PART4, PART5); 29 | 30 | fd = mkostemp(fn, O_CLOEXEC); 31 | assert_se(fd >= 0); 32 | assert_se(unlink(fn) == 0); 33 | 34 | assert_se(loop_write_with_holes(fd, buffer, sizeof(buffer), &n_punched) >= 0); 35 | assert_se(n_punched == PART1 + PART3); 36 | 37 | assert_se(lseek(fd, 0, SEEK_SET) == 0); 38 | assert_se(loop_read(fd, buffer2, sizeof(buffer2)) == sizeof(buffer2)); 39 | assert_se(memcmp(buffer, buffer2, sizeof(buffer)) == 0); 40 | 41 | memzero(buffer + PART1 + 1, PART2 - 2); 42 | assert_se(lseek(fd, PART1-1, SEEK_SET) == PART1-1); 43 | assert_se(loop_write_with_holes(fd, buffer + PART1 - 1, PART2 + 2, &n_punched) >= 0); 44 | 45 | assert_se(lseek(fd, 0, SEEK_SET) == 0); 46 | assert_se(loop_read(fd, buffer2, sizeof(buffer2)) == sizeof(buffer2)); 47 | assert_se(memcmp(buffer, buffer2, sizeof(buffer)) == 0); 48 | 49 | fd = safe_close(fd); 50 | 51 | assert_se(pipe2(p, O_CLOEXEC) >= 0); 52 | 53 | assert_se(loop_write_with_holes(p[1], buffer, MIN(sizeof(buffer), (size_t) PIPE_BUF), &n_punched) >= 0); 54 | assert_se(n_punched == 0); 55 | 56 | p[1] = safe_close(p[1]); 57 | 58 | assert_se(loop_read(p[0], buffer2, sizeof(buffer2)) == MIN((ssize_t) sizeof(buffer2), PIPE_BUF)); 59 | 60 | p[0] = safe_close(p[0]); 61 | 62 | assert_se(memcmp(buffer, buffer2, MIN(sizeof(buffer), (size_t) PIPE_BUF)) == 0); 63 | 64 | return 0; 65 | } 66 | -------------------------------------------------------------------------------- /tools/oss-fuzz.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # SPDX-License-Identifier: LGPL-2.1+ 3 | 4 | set -ex 5 | 6 | export LC_CTYPE=C.UTF-8 7 | 8 | export CC=${CC:-clang} 9 | export CXX=${CXX:-clang++} 10 | clang_version="$($CC --version | sed -nr 's/.*version ([^ ]+?) .*/\1/p' | sed -r 's/-$//')" 11 | 12 | SANITIZER=${SANITIZER:-address -fsanitize-address-use-after-scope} 13 | flags="-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=$SANITIZER -fsanitize-coverage=trace-pc-guard,trace-cmp" 14 | 15 | clang_lib="/usr/lib64/clang/${clang_version}/lib/linux" 16 | [ -d "$clang_lib" ] || clang_lib="/usr/lib/clang/${clang_version}/lib/linux" 17 | 18 | export CFLAGS=${CFLAGS:-$flags} 19 | export CXXFLAGS=${CXXFLAGS:-$flags} 20 | export LDFLAGS=${LDFLAGS:--L${clang_lib}} 21 | 22 | export WORK=${WORK:-$(pwd)} 23 | export OUT=${OUT:-$(pwd)/out} 24 | mkdir -p $OUT 25 | 26 | build=$WORK/build 27 | rm -rf $build 28 | mkdir -p $build 29 | 30 | fuzzflag="oss-fuzz=true" 31 | if [ -z "$FUZZING_ENGINE" ]; then 32 | fuzzflag="llvm-fuzz=true" 33 | fi 34 | 35 | meson $build -D$fuzzflag \ 36 | -Db_lundef=false \ 37 | -Dlibzstd=disabled \ 38 | -Dman=false 39 | ninja -C $build fuzzers 40 | 41 | # The seed corpus is a separate flat archive for each fuzzer, 42 | # with a fixed name ${fuzzer}_seed_corpus.zip. 43 | for d in "$(dirname "$0")/../test/fuzz/fuzz-"*/; do 44 | [ -d "$d" ] || continue 45 | zip -jqr $OUT/$(basename "$d")_seed_corpus.zip "$d" 46 | done 47 | 48 | find $build -maxdepth 1 -type f -executable -name "fuzz-*" -exec mv {} $OUT \; 49 | # cp test/fuzz/*.options $OUT 50 | --------------------------------------------------------------------------------