├── .gitignore ├── Makefile ├── README.md ├── ReleaseNotes.md ├── kmod ├── .gitignore ├── Makefile ├── scoutfs-kmod.spec.in └── src │ ├── Kconfig │ ├── Makefile │ ├── Makefile.kernelcompat │ ├── acl.c │ ├── acl.h │ ├── alloc.c │ ├── alloc.h │ ├── attr_x.c │ ├── attr_x.h │ ├── avl.c │ ├── avl.h │ ├── block.c │ ├── block.h │ ├── btree.c │ ├── btree.h │ ├── client.c │ ├── client.h │ ├── cmp.h │ ├── counters.c │ ├── counters.h │ ├── data.c │ ├── data.h │ ├── dir.c │ ├── dir.h │ ├── endian_swap.h │ ├── export.c │ ├── export.h │ ├── ext.c │ ├── ext.h │ ├── fence.c │ ├── fence.h │ ├── file.c │ ├── file.h │ ├── forest.c │ ├── forest.h │ ├── format.h │ ├── hash.h │ ├── inode.c │ ├── inode.h │ ├── ioctl.c │ ├── ioctl.h │ ├── item.c │ ├── item.h │ ├── kernelcompat.c │ ├── kernelcompat.h │ ├── key.h │ ├── lock.c │ ├── lock.h │ ├── lock_server.c │ ├── lock_server.h │ ├── msg.c │ ├── msg.h │ ├── net.c │ ├── net.h │ ├── omap.c │ ├── omap.h │ ├── options.c │ ├── options.h │ ├── per_task.c │ ├── per_task.h │ ├── quorum.c │ ├── quorum.h │ ├── quota.c │ ├── quota.h │ ├── recov.c │ ├── recov.h │ ├── scoutfs_trace.c │ ├── scoutfs_trace.h │ ├── server.c │ ├── server.h │ ├── sort_priv.c │ ├── sort_priv.h │ ├── spbm.c │ ├── spbm.h │ ├── srch.c │ ├── srch.h │ ├── super.c │ ├── super.h │ ├── sysfs.c │ ├── sysfs.h │ ├── totl.c │ ├── totl.h │ ├── trace │ ├── quota.h │ └── wkic.h │ ├── trans.c │ ├── trans.h │ ├── triggers.c │ ├── triggers.h │ ├── tseq.c │ ├── tseq.h │ ├── util.h │ ├── volopt.c │ ├── volopt.h │ ├── wkic.c │ ├── wkic.h │ ├── xattr.c │ └── xattr.h ├── tests ├── .gitignore ├── .xfstests-branch ├── Makefile ├── README.md ├── fenced-local-force-unmount.sh ├── funcs │ ├── exec.sh │ ├── filter.sh │ ├── fs.sh │ ├── require.sh │ └── tap.sh ├── golden │ ├── archive-light-cycle │ ├── basic-bad-mounts │ ├── basic-block-counts │ ├── basic-posix-acl │ ├── basic-posix-consistency │ ├── basic-truncate │ ├── block-stale-reads │ ├── change-devices │ ├── client-unmount-recovery │ ├── createmany-large-names │ ├── createmany-parallel │ ├── createmany-parallel-mounts │ ├── createmany-rename-large-dir │ ├── cross-mount-data-free │ ├── data-prealloc │ ├── dirent-consistency │ ├── enospc │ ├── export-get-name-parent │ ├── export-lookup-evict-race │ ├── fallocate │ ├── fence-and-reclaim │ ├── format-version-forward-back │ ├── get-referring-entries │ ├── inode-deletion │ ├── inode-items-updated │ ├── large-fragmented-free │ ├── lock-ex-race-processes │ ├── lock-pr-cw-conflict │ ├── lock-recover-invalidate │ ├── lock-refleak │ ├── lock-rever-invalidate │ ├── lock-revoke-getcwd │ ├── lock-shrink-consistency │ ├── lock-shrink-read-race │ ├── mkdir-rename-rmdir │ ├── mmap │ ├── mount-unmount-race │ ├── move-blocks │ ├── o_tmpfile │ ├── offline-extent-waiting │ ├── orphan-inodes │ ├── persistent-item-vers │ ├── projects │ ├── quorum-heartbeat-timeout │ ├── quota │ ├── renameat2-noreplace │ ├── resize-devices │ ├── retention-basic │ ├── setattr_more │ ├── setup-error-teardown │ ├── simple-inode-index │ ├── simple-readdir │ ├── simple-release-extents │ ├── simple-staging │ ├── simple-xattr-unit │ ├── srch-basic-functionality │ ├── srch-safe-merge-pos │ ├── stage-multi-part │ ├── stage-release-race-alloc │ ├── totl-xattr-tag │ └── xfstests ├── run-tests.sh ├── sequence ├── src │ ├── bulk_create_paths.c │ ├── create_xattr_loop.c │ ├── createmany.c │ ├── dumb_renameat2.c │ ├── dumb_setxattr.c │ ├── find_xattrs.c │ ├── fragmented_data_extents.c │ ├── handle_cat.c │ ├── handle_fsetxattr.c │ ├── mmap_stress.c │ ├── mmap_validate.c │ ├── o_tmpfile_linkat.c │ ├── o_tmpfile_umask.c │ └── stage_tmpfile.c └── tests │ ├── archive-light-cycle.sh │ ├── basic-bad-mounts.sh │ ├── basic-block-counts.sh │ ├── basic-posix-acl.sh │ ├── basic-posix-consistency.sh │ ├── basic-truncate.sh │ ├── block-stale-reads.sh │ ├── change-devices.sh │ ├── client-unmount-recovery.sh │ ├── createmany-large-names.sh │ ├── createmany-parallel-mounts.sh │ ├── createmany-parallel.sh │ ├── createmany-rename-large-dir.sh │ ├── cross-mount-data-free.sh │ ├── data-prealloc.sh │ ├── dirent-consistency.sh │ ├── enospc.sh │ ├── export-get-name-parent.sh │ ├── export-lookup-evict-race.sh │ ├── fallocate.sh │ ├── fence-and-reclaim.sh │ ├── format-version-forward-back.sh │ ├── get-referring-entries.sh │ ├── inode-deletion.sh │ ├── inode-items-updated.sh │ ├── large-fragmented-free.sh │ ├── lock-ex-race-processes.sh │ ├── lock-pr-cw-conflict.sh │ ├── lock-recover-invalidate.sh │ ├── lock-refleak.sh │ ├── lock-revoke-getcwd.sh │ ├── lock-shrink-consistency.sh │ ├── lock-shrink-read-race.sh │ ├── mkdir-rename-rmdir.sh │ ├── mmap.sh │ ├── mount-unmount-race.sh │ ├── move-blocks.sh │ ├── o_tmpfile.sh │ ├── offline-extent-waiting.sh │ ├── orphan-inodes.sh │ ├── persistent-item-vers.sh │ ├── projects.sh │ ├── quorum-heartbeat-timeout.sh │ ├── quota.sh │ ├── renameat2-noreplace.sh │ ├── resize-devices.sh │ ├── retention-basic.sh │ ├── setattr_more.sh │ ├── setup-error-teardown.sh │ ├── simple-inode-index.sh │ ├── simple-readdir.sh │ ├── simple-release-extents.sh │ ├── simple-staging.sh │ ├── simple-xattr-unit.sh │ ├── srch-basic-functionality.sh │ ├── srch-safe-merge-pos.sh │ ├── stage-multi-part.sh │ ├── stage-release-race-alloc.sh │ ├── totl-xattr-tag.sh │ └── xfstests.sh └── utils ├── .gitignore ├── Makefile ├── fenced ├── scoutfs-fenced ├── scoutfs-fenced.conf.example └── scoutfs-fenced.service ├── man ├── scoutfs-corruption.7 ├── scoutfs-fenced.8 ├── scoutfs.5 └── scoutfs.8 ├── scoutfs-utils.spec.in ├── sparse.sh ├── src ├── attr_x.c ├── avl.c ├── avl.h ├── bitmap.c ├── bitmap.h ├── bitops.h ├── blkid.c ├── blkid.h ├── bloom.c ├── bloom.h ├── btree.c ├── btree.h ├── change_format_version.c ├── change_quorum_config.c ├── cmd.c ├── cmd.h ├── cmp.h ├── counters.c ├── crc.c ├── crc.h ├── dev.c ├── dev.h ├── df.c ├── endian_swap.h ├── fiemap.c ├── get_allocated_inos.c ├── get_referring_entries.c ├── hash.h ├── ino_path.c ├── key.h ├── leaf_item_hash.c ├── leaf_item_hash.h ├── list.h ├── listxattr_hidden.c ├── lk_rbtree_wrapper.h ├── main.c ├── mkfs.c ├── mode_types.c ├── mode_types.h ├── move_blocks.c ├── name_hash.h ├── parse.c ├── parse.h ├── prepare_empty_data_device.c ├── print.c ├── quorum.c ├── quorum.h ├── quota.c ├── rand.c ├── rand.h ├── rbtree.c ├── rbtree.h ├── rbtree_augmented.h ├── rbtree_types.h ├── read_xattr_index.c ├── read_xattr_totals.c ├── resize_devices.c ├── search_xattrs.c ├── setattr.c ├── sparse.h ├── srch.c ├── srch.h ├── stage_release.c ├── stat.c ├── util.c ├── util.h ├── waiting.c └── walk_inodes.c └── tex ├── .gitignore ├── Makefile ├── scoutfs.tex ├── usenix2019.sty └── usenix2019.tex /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/versity/scoutfs/9741d40e10bb92cf9b77658cc7a282a67e403a6c/.gitignore -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Typically development is done in each subdir, but we have a tiny 3 | # makefile here to make it easy to run simple targets across all the 4 | # subdirs. 5 | # 6 | 7 | SUBDIRS := kmod utils tests 8 | NOTTESTS := kmod utils 9 | 10 | all clean: $(SUBDIRS) FORCE 11 | dist: $(NOTTESTS) FORCE 12 | 13 | $(SUBDIRS): FORCE 14 | $(MAKE) -C $@ $(MAKECMDGOALS) 15 | 16 | all: 17 | FORCE: 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | scoutfs is a clustered in-kernel Linux filesystem designed to support 4 | large archival systems. It features additional interfaces and metadata 5 | so that archive agents can perform their maintenance workflows without 6 | walking all the files in the namespace. Its cluster support lets 7 | deployments add nodes to satisfy archival tier bandwidth targets. 8 | 9 | The design goal is to reach file populations in the trillions, with the 10 | archival bandwidth to match, while remaining operational and responsive. 11 | 12 | Highlights of the design and implementation include: 13 | 14 | * Fully consistent POSIX semantics between nodes 15 | * Atomic transactions to maintain consistent persistent structures 16 | * Integrated archival metadata replaces syncing to external databases 17 | * Dynamic seperation of resources lets nodes write in parallel 18 | * 64bit throughout; no limits on file or directory sizes or counts 19 | * Open GPLv2 implementation 20 | 21 | # Community Mailing List 22 | 23 | Please join us on the open scoutfs-devel@scoutfs.org [mailing list 24 | hosted on Google Groups](https://groups.google.com/a/scoutfs.org/forum/#!forum/scoutfs-devel) 25 | -------------------------------------------------------------------------------- /kmod/.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.ko 3 | *.mod.c 4 | *.cmd 5 | *~ 6 | src/.tmp_versions/ 7 | src/Module.symvers 8 | src/modules.order 9 | cscope.* 10 | *.spec 11 | *.sw[po] 12 | rpmbuild/ 13 | rpms/ 14 | 15 | scoutfs-*.git*/ 16 | scoutfs-kmod-*.tar 17 | -------------------------------------------------------------------------------- /kmod/Makefile: -------------------------------------------------------------------------------- 1 | ALL: module 2 | 3 | # default to building against the installed source for the running kernel 4 | ifeq ($(SK_KSRC),) 5 | SK_KSRC := $(shell echo /lib/modules/`uname -r`/build) 6 | endif 7 | 8 | # fail if sparse fails if we find it 9 | ifeq ($(shell sparse && echo found),found) 10 | SP = 11 | else 12 | SP = @: 13 | endif 14 | 15 | SCOUTFS_GIT_DESCRIBE ?= \ 16 | $(shell git describe --all --abbrev=6 --long 2>/dev/null || \ 17 | echo no-git) 18 | 19 | ESCAPED_GIT_DESCRIBE := \ 20 | $(shell echo $(SCOUTFS_GIT_DESCRIBE) |sed -e 's/\//\\\//g') 21 | 22 | RPM_GITHASH ?= $(shell git rev-parse --short HEAD) 23 | 24 | SCOUTFS_ARGS := SCOUTFS_GIT_DESCRIBE=$(SCOUTFS_GIT_DESCRIBE) \ 25 | RPM_GITHASH=$(RPM_GITHASH) \ 26 | CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src \ 27 | EXTRA_CFLAGS="-Werror" 28 | 29 | # - We use the git describe from tags to set up the RPM versioning 30 | RPM_VERSION := $(shell git describe --long --tags | awk -F '-' '{gsub(/^v/,""); print $$1}') 31 | TARFILE = scoutfs-kmod-$(RPM_VERSION).tar 32 | 33 | 34 | .PHONY: .FORCE 35 | 36 | all: module 37 | 38 | module: 39 | $(MAKE) $(SCOUTFS_ARGS) 40 | $(SP) $(MAKE) C=2 CF="-D__CHECK_ENDIAN__" $(SCOUTFS_ARGS) 41 | 42 | 43 | modules_install: 44 | $(MAKE) $(SCOUTFS_ARGS) modules_install 45 | 46 | 47 | %.spec: %.spec.in .FORCE 48 | sed -e 's/@@VERSION@@/$(RPM_VERSION)/g' \ 49 | -e 's/@@GITHASH@@/$(RPM_GITHASH)/g' \ 50 | -e 's/@@GITDESCRIBE@@/$(ESCAPED_GIT_DESCRIBE)/g' < $< > $@+ 51 | mv $@+ $@ 52 | 53 | 54 | dist: scoutfs-kmod.spec 55 | git archive --format=tar --prefix scoutfs-kmod-$(RPM_VERSION)/ HEAD^{tree} > $(TARFILE) 56 | @ tar rf $(TARFILE) --transform="s@\(.*\)@scoutfs-kmod-$(RPM_VERSION)/\1@" scoutfs-kmod.spec 57 | 58 | clean: 59 | $(MAKE) $(SCOUTFS_ARGS) clean 60 | -------------------------------------------------------------------------------- /kmod/src/Kconfig: -------------------------------------------------------------------------------- 1 | config SCOUTFS_FS 2 | tristate "scoutfs filesystem" 3 | help 4 | scoutfs is a clustered file system that stores data in large 5 | blocks in shared block storage. 6 | 7 | To compile this file system support as a module, choose M here. The 8 | module will be called scoutfs. 9 | 10 | If unsure, say N. 11 | -------------------------------------------------------------------------------- /kmod/src/Makefile: -------------------------------------------------------------------------------- 1 | obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o 2 | 3 | CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" 4 | 5 | CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include 6 | 7 | # add EXTRA_CFLAGS defines for kernel compat 8 | -include $(src)/Makefile.kernelcompat 9 | 10 | scoutfs-y += \ 11 | acl.o \ 12 | attr_x.o \ 13 | avl.o \ 14 | alloc.o \ 15 | block.o \ 16 | btree.o \ 17 | client.o \ 18 | counters.o \ 19 | data.o \ 20 | dir.o \ 21 | export.o \ 22 | ext.o \ 23 | fence.o \ 24 | file.o \ 25 | forest.o \ 26 | inode.o \ 27 | ioctl.o \ 28 | item.o \ 29 | kernelcompat.o \ 30 | lock.o \ 31 | lock_server.o \ 32 | msg.o \ 33 | net.o \ 34 | omap.o \ 35 | options.o \ 36 | per_task.o \ 37 | quorum.o \ 38 | quota.o \ 39 | recov.o \ 40 | scoutfs_trace.o \ 41 | server.o \ 42 | sort_priv.o \ 43 | spbm.o \ 44 | srch.o \ 45 | super.o \ 46 | sysfs.o \ 47 | totl.o \ 48 | trans.o \ 49 | triggers.o \ 50 | tseq.o \ 51 | volopt.o \ 52 | wkic.o \ 53 | xattr.o 54 | 55 | # 56 | # The raw types aren't available in userspace headers. Make sure all 57 | # the types we use in the headers are the exported __ versions. 58 | # 59 | # XXX dunno how we're really supposed to do this in kbuild 60 | # 61 | .PHONY: $(src)/check_exported_types 62 | $(src)/check_exported_types: 63 | @if egrep '\<[us](8|16|32|64\>)' $(src)/format.h $(src)/ioctl.h; then \ 64 | echo "no raw types in exported headers, preface with __"; \ 65 | exit 1; \ 66 | fi 67 | @if egrep '\<__packed\>' $(src)/format.h $(src)/ioctl.h; then \ 68 | echo "no __packed allowed in exported headers"; \ 69 | exit 1; \ 70 | fi 71 | 72 | extra-y += check_exported_types 73 | -------------------------------------------------------------------------------- /kmod/src/acl.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_ACL_H_ 2 | #define _SCOUTFS_ACL_H_ 3 | 4 | struct posix_acl *scoutfs_get_acl(struct inode *inode, int type); 5 | struct posix_acl *scoutfs_get_acl_locked(struct inode *inode, int type, struct scoutfs_lock *lock); 6 | int scoutfs_set_acl(struct inode *inode, struct posix_acl *acl, int type); 7 | int scoutfs_set_acl_locked(struct inode *inode, struct posix_acl *acl, int type, 8 | struct scoutfs_lock *lock, struct list_head *ind_locks); 9 | #ifdef KC_XATTR_STRUCT_XATTR_HANDLER 10 | int scoutfs_acl_get_xattr(const struct xattr_handler *, struct dentry *dentry, 11 | struct inode *inode, const char *name, void *value, 12 | size_t size); 13 | int scoutfs_acl_set_xattr(const struct xattr_handler *, 14 | KC_VFS_NS_DEF 15 | struct dentry *dentry, 16 | struct inode *inode, const char *name, const void *value, 17 | size_t size, int flags); 18 | #else 19 | int scoutfs_acl_get_xattr(struct dentry *dentry, const char *name, void *value, size_t size, 20 | int type); 21 | int scoutfs_acl_set_xattr(struct dentry *dentry, const char *name, const void *value, size_t size, 22 | int flags, int type); 23 | #endif 24 | int scoutfs_acl_chmod_locked(struct inode *inode, struct iattr *attr, 25 | struct scoutfs_lock *lock, struct list_head *ind_locks); 26 | int scoutfs_init_acl_locked(struct inode *inode, struct inode *dir, 27 | struct scoutfs_lock *lock, struct scoutfs_lock *dir_lock, 28 | struct list_head *ind_locks); 29 | #endif 30 | -------------------------------------------------------------------------------- /kmod/src/attr_x.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_ATTR_X_H_ 2 | #define _SCOUTFS_ATTR_X_H_ 3 | 4 | #include 5 | #include 6 | #include "ioctl.h" 7 | 8 | int scoutfs_get_attr_x(struct inode *inode, struct scoutfs_ioctl_inode_attr_x *iax); 9 | int scoutfs_set_attr_x(struct inode *inode, struct scoutfs_ioctl_inode_attr_x *iax); 10 | 11 | #endif 12 | -------------------------------------------------------------------------------- /kmod/src/avl.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_AVL_H_ 2 | #define _SCOUTFS_AVL_H_ 3 | 4 | #include "format.h" 5 | 6 | typedef int (*scoutfs_avl_compare_t)(void *arg, 7 | struct scoutfs_avl_node *node); 8 | 9 | struct scoutfs_avl_node * 10 | scoutfs_avl_search(struct scoutfs_avl_root *root, 11 | scoutfs_avl_compare_t compare, void *arg, int *cmp_ret, 12 | struct scoutfs_avl_node **par, 13 | struct scoutfs_avl_node **next, 14 | struct scoutfs_avl_node **prev); 15 | struct scoutfs_avl_node *scoutfs_avl_first(struct scoutfs_avl_root *root); 16 | struct scoutfs_avl_node *scoutfs_avl_last(struct scoutfs_avl_root *root); 17 | struct scoutfs_avl_node *scoutfs_avl_next(struct scoutfs_avl_root *root, 18 | struct scoutfs_avl_node *node); 19 | struct scoutfs_avl_node *scoutfs_avl_prev(struct scoutfs_avl_root *root, 20 | struct scoutfs_avl_node *node); 21 | void scoutfs_avl_insert(struct scoutfs_avl_root *root, 22 | struct scoutfs_avl_node *parent, 23 | struct scoutfs_avl_node *node, int cmp); 24 | void scoutfs_avl_delete(struct scoutfs_avl_root *root, 25 | struct scoutfs_avl_node *node); 26 | void scoutfs_avl_relocate(struct scoutfs_avl_root *root, 27 | struct scoutfs_avl_node *to, 28 | struct scoutfs_avl_node *from); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /kmod/src/block.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_BLOCK_H_ 2 | #define _SCOUTFS_BLOCK_H_ 3 | 4 | struct scoutfs_block_writer { 5 | spinlock_t lock; 6 | struct list_head dirty_list; 7 | u64 nr_dirty_blocks; 8 | }; 9 | 10 | struct scoutfs_block { 11 | u64 blkno; 12 | void *data; 13 | void *priv; 14 | }; 15 | 16 | struct scoutfs_block_saved_refs { 17 | struct scoutfs_block_ref refs[2]; 18 | }; 19 | 20 | #define DECLARE_SAVED_REFS(name) \ 21 | struct scoutfs_block_saved_refs name = {{{0,}}} 22 | 23 | int scoutfs_block_check_stale(struct super_block *sb, int ret, 24 | struct scoutfs_block_saved_refs *saved, 25 | struct scoutfs_block_ref *a, struct scoutfs_block_ref *b); 26 | 27 | int scoutfs_block_read_ref(struct super_block *sb, struct scoutfs_block_ref *ref, u32 magic, 28 | struct scoutfs_block **bl_ret); 29 | void scoutfs_block_put(struct super_block *sb, struct scoutfs_block *bl); 30 | 31 | void scoutfs_block_writer_init(struct super_block *sb, 32 | struct scoutfs_block_writer *wri); 33 | int scoutfs_block_dirty_ref(struct super_block *sb, struct scoutfs_alloc *alloc, 34 | struct scoutfs_block_writer *wri, struct scoutfs_block_ref *ref, 35 | u32 magic, struct scoutfs_block **bl_ret, 36 | u64 dirty_blkno, u64 *ref_blkno); 37 | int scoutfs_block_writer_write(struct super_block *sb, 38 | struct scoutfs_block_writer *wri); 39 | void scoutfs_block_writer_forget_all(struct super_block *sb, 40 | struct scoutfs_block_writer *wri); 41 | void scoutfs_block_writer_forget(struct super_block *sb, 42 | struct scoutfs_block_writer *wri, 43 | struct scoutfs_block *bl); 44 | bool scoutfs_block_writer_has_dirty(struct super_block *sb, 45 | struct scoutfs_block_writer *wri); 46 | u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, 47 | struct scoutfs_block_writer *wri); 48 | 49 | int scoutfs_block_read_sm(struct super_block *sb, 50 | struct block_device *bdev, u64 blkno, 51 | struct scoutfs_block_header *hdr, size_t len, 52 | __le32 *blk_crc); 53 | int scoutfs_block_write_sm(struct super_block *sb, 54 | struct block_device *bdev, u64 blkno, 55 | struct scoutfs_block_header *hdr, size_t len); 56 | 57 | int scoutfs_block_setup(struct super_block *sb); 58 | void scoutfs_block_destroy(struct super_block *sb); 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /kmod/src/client.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_CLIENT_H_ 2 | #define _SCOUTFS_CLIENT_H_ 3 | 4 | int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, 5 | u64 *ino, u64 *nr); 6 | int scoutfs_client_get_log_trees(struct super_block *sb, 7 | struct scoutfs_log_trees *lt); 8 | int scoutfs_client_commit_log_trees(struct super_block *sb, 9 | struct scoutfs_log_trees *lt); 10 | int scoutfs_client_get_roots(struct super_block *sb, 11 | struct scoutfs_net_roots *roots); 12 | u64 *scoutfs_client_bulk_alloc(struct super_block *sb); 13 | int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq); 14 | int scoutfs_client_lock_request(struct super_block *sb, 15 | struct scoutfs_net_lock *nl); 16 | int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, 17 | struct scoutfs_net_lock *nl); 18 | int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, 19 | struct scoutfs_net_lock_recover *nlr); 20 | int scoutfs_client_srch_get_compact(struct super_block *sb, 21 | struct scoutfs_srch_compact *sc); 22 | int scoutfs_client_srch_commit_compact(struct super_block *sb, 23 | struct scoutfs_srch_compact *res); 24 | int scoutfs_client_get_log_merge(struct super_block *sb, 25 | struct scoutfs_log_merge_request *req); 26 | int scoutfs_client_commit_log_merge(struct super_block *sb, 27 | struct scoutfs_log_merge_complete *comp); 28 | int scoutfs_client_send_omap_response(struct super_block *sb, u64 id, 29 | struct scoutfs_open_ino_map *map); 30 | int scoutfs_client_open_ino_map(struct super_block *sb, u64 group_nr, 31 | struct scoutfs_open_ino_map *map); 32 | int scoutfs_client_get_volopt(struct super_block *sb, struct scoutfs_volume_options *volopt); 33 | int scoutfs_client_set_volopt(struct super_block *sb, struct scoutfs_volume_options *volopt); 34 | int scoutfs_client_clear_volopt(struct super_block *sb, struct scoutfs_volume_options *volopt); 35 | int scoutfs_client_resize_devices(struct super_block *sb, struct scoutfs_net_resize_devices *nrd); 36 | int scoutfs_client_statfs(struct super_block *sb, struct scoutfs_net_statfs *nst); 37 | 38 | void scoutfs_client_net_shutdown(struct super_block *sb); 39 | int scoutfs_client_setup(struct super_block *sb); 40 | void scoutfs_client_destroy(struct super_block *sb); 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /kmod/src/cmp.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_CMP_H_ 2 | #define _SCOUTFS_CMP_H_ 3 | 4 | /* 5 | * A generic ternary comparison macro with strict type checking. 6 | */ 7 | #define scoutfs_cmp(a, b) \ 8 | ({ \ 9 | __typeof__(a) _a = (a); \ 10 | __typeof__(b) _b = (b); \ 11 | int _ret; \ 12 | \ 13 | (void) (&_a == &_b); \ 14 | _ret = _a < _b ? -1 : _a > _b ? 1 : 0; \ 15 | _ret; \ 16 | }) 17 | 18 | static inline int scoutfs_cmp_u64s(u64 a, u64 b) 19 | { 20 | return a < b ? -1 : a > b ? 1 : 0; 21 | } 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /kmod/src/dir.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_DIR_H_ 2 | #define _SCOUTFS_DIR_H_ 3 | 4 | #include "format.h" 5 | #include "lock.h" 6 | 7 | extern const struct file_operations scoutfs_dir_fops; 8 | #ifdef KC_LINUX_HAVE_RHEL_IOPS_WRAPPER 9 | extern const struct inode_operations_wrapper scoutfs_dir_iops; 10 | #else 11 | extern const struct inode_operations scoutfs_dir_iops; 12 | #endif 13 | extern const struct inode_operations scoutfs_symlink_iops; 14 | 15 | extern const struct dentry_operations scoutfs_dentry_ops; 16 | 17 | struct scoutfs_link_backref_entry { 18 | struct list_head head; 19 | u64 dir_ino; 20 | u64 dir_pos; 21 | u16 name_len; 22 | u8 d_type; 23 | bool last; 24 | struct scoutfs_dirent dent; 25 | /* the full name is allocated and stored in dent.name[] */ 26 | }; 27 | 28 | int scoutfs_dir_get_backref_path(struct super_block *sb, u64 ino, u64 dir_ino, 29 | u64 dir_pos, struct list_head *list); 30 | void scoutfs_dir_free_backref_path(struct super_block *sb, 31 | struct list_head *list); 32 | 33 | int scoutfs_dir_add_next_linkrefs(struct super_block *sb, u64 ino, u64 dir_ino, u64 dir_pos, 34 | int count, struct list_head *list); 35 | 36 | int scoutfs_symlink_drop(struct super_block *sb, u64 ino, 37 | struct scoutfs_lock *lock, u64 i_size); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /kmod/src/endian_swap.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_ENDIAN_SWAP_H_ 2 | #define _SCOUTFS_ENDIAN_SWAP_H_ 3 | 4 | #define le64_to_be64(x) cpu_to_be64(le64_to_cpu(x)) 5 | #define le32_to_be32(x) cpu_to_be32(le32_to_cpu(x)) 6 | #define le16_to_be16(x) cpu_to_be16(le16_to_cpu(x)) 7 | 8 | #define be64_to_le64(x) cpu_to_le64(be64_to_cpu(x)) 9 | #define be32_to_le32(x) cpu_to_le32(be32_to_cpu(x)) 10 | #define be16_to_le16(x) cpu_to_le16(be16_to_cpu(x)) 11 | 12 | #endif 13 | -------------------------------------------------------------------------------- /kmod/src/export.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_EXPORT_H_ 2 | #define _SCOUTFS_EXPORT_H_ 3 | 4 | #include 5 | 6 | extern const struct export_operations scoutfs_export_ops; 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /kmod/src/ext.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_EXT_H_ 2 | #define _SCOUTFS_EXT_H_ 3 | 4 | struct scoutfs_extent { 5 | u64 start; 6 | u64 len; 7 | u64 map; 8 | u8 flags; 9 | }; 10 | 11 | struct scoutfs_ext_ops { 12 | int (*next)(struct super_block *sb, void *arg, 13 | u64 start, u64 len, struct scoutfs_extent *ext); 14 | int (*insert)(struct super_block *sb, void *arg, 15 | u64 start, u64 len, u64 map, u8 flags); 16 | int (*remove)(struct super_block *sb, void *arg, u64 start, u64 len, 17 | u64 map, u8 flags); 18 | 19 | bool insert_overlap_warn; 20 | }; 21 | 22 | bool scoutfs_ext_can_merge(struct scoutfs_extent *left, 23 | struct scoutfs_extent *right); 24 | 25 | int scoutfs_ext_next(struct super_block *sb, struct scoutfs_ext_ops *ops, 26 | void *arg, u64 start, u64 len, struct scoutfs_extent *ext); 27 | int scoutfs_ext_insert(struct super_block *sb, struct scoutfs_ext_ops *ops, 28 | void *arg, u64 start, u64 len, u64 map, u8 flags); 29 | int scoutfs_ext_remove(struct super_block *sb, struct scoutfs_ext_ops *ops, 30 | void *arg, u64 start, u64 len); 31 | int scoutfs_ext_alloc(struct super_block *sb, struct scoutfs_ext_ops *ops, 32 | void *arg, u64 start, u64 len, u64 limit, 33 | struct scoutfs_extent *ext); 34 | int scoutfs_ext_set(struct super_block *sb, struct scoutfs_ext_ops *ops, 35 | void *arg, u64 start, u64 len, u64 map, u8 flags); 36 | bool scoutfs_ext_inside(u64 start, u64 len, struct scoutfs_extent *out); 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /kmod/src/fence.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_FENCE_H_ 2 | #define _SCOUTFS_FENCE_H_ 3 | 4 | enum { 5 | SCOUTFS_FENCE_CLIENT_RECOVERY, 6 | SCOUTFS_FENCE_CLIENT_RECONNECT, 7 | SCOUTFS_FENCE_QUORUM_BLOCK_LEADER, 8 | }; 9 | 10 | int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason); 11 | int scoutfs_fence_next(struct super_block *sb, u64 *rid, int *reason, bool *error); 12 | int scoutfs_fence_reason_pending(struct super_block *sb, int reason); 13 | int scoutfs_fence_free(struct super_block *sb, u64 rid); 14 | int scoutfs_fence_wait_fenced(struct super_block *sb, long timeout_jiffies); 15 | 16 | int scoutfs_fence_setup(struct super_block *sb); 17 | void scoutfs_fence_stop(struct super_block *sb); 18 | void scoutfs_fence_destroy(struct super_block *sb); 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /kmod/src/file.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_FILE_H_ 2 | #define _SCOUTFS_FILE_H_ 3 | 4 | #ifdef KC_LINUX_HAVE_FOP_AIO_READ 5 | ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, 6 | unsigned long nr_segs, loff_t pos); 7 | ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, 8 | unsigned long nr_segs, loff_t pos); 9 | #else 10 | ssize_t scoutfs_file_read_iter(struct kiocb *, struct iov_iter *); 11 | ssize_t scoutfs_file_write_iter(struct kiocb *, struct iov_iter *); 12 | #endif 13 | int scoutfs_permission(KC_VFS_NS_DEF 14 | struct inode *inode, int mask); 15 | loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence); 16 | 17 | #endif /* _SCOUTFS_FILE_H_ */ 18 | -------------------------------------------------------------------------------- /kmod/src/forest.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_FOREST_H_ 2 | #define _SCOUTFS_FOREST_H_ 3 | 4 | struct scoutfs_alloc; 5 | struct scoutfs_block_writer; 6 | struct scoutfs_block; 7 | struct scoutfs_lock; 8 | 9 | #include "btree.h" 10 | 11 | /* caller gives an item to the callback */ 12 | enum { 13 | FIC_FS_ROOT = (1 << 0), 14 | FIC_FINALIZED = (1 << 1), 15 | }; 16 | typedef int (*scoutfs_forest_item_cb)(struct super_block *sb, struct scoutfs_key *key, u64 seq, 17 | u8 flags, void *val, int val_len, int fic, void *arg); 18 | 19 | int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, 20 | struct scoutfs_key *next); 21 | int scoutfs_forest_read_items(struct super_block *sb, 22 | struct scoutfs_key *key, 23 | struct scoutfs_key *bloom_key, 24 | struct scoutfs_key *start, 25 | struct scoutfs_key *end, 26 | scoutfs_forest_item_cb cb, void *arg); 27 | int scoutfs_forest_read_items_roots(struct super_block *sb, struct scoutfs_net_roots *roots, 28 | struct scoutfs_key *key, struct scoutfs_key *bloom_key, 29 | struct scoutfs_key *start, struct scoutfs_key *end, 30 | scoutfs_forest_item_cb cb, void *arg); 31 | int scoutfs_forest_set_bloom_bits(struct super_block *sb, 32 | struct scoutfs_lock *lock); 33 | void scoutfs_forest_set_max_seq(struct super_block *sb, u64 max_seq); 34 | int scoutfs_forest_get_max_seq(struct super_block *sb, 35 | struct scoutfs_super_block *super, 36 | u64 *seq); 37 | int scoutfs_forest_insert_list(struct super_block *sb, 38 | struct scoutfs_btree_item_list *lst); 39 | int scoutfs_forest_srch_add(struct super_block *sb, u64 hash, u64 ino, u64 id); 40 | 41 | void scoutfs_forest_inc_inode_count(struct super_block *sb); 42 | void scoutfs_forest_dec_inode_count(struct super_block *sb); 43 | int scoutfs_forest_inode_count(struct super_block *sb, struct scoutfs_super_block *super, 44 | u64 *inode_count); 45 | 46 | void scoutfs_forest_init_btrees(struct super_block *sb, 47 | struct scoutfs_alloc *alloc, 48 | struct scoutfs_block_writer *wri, 49 | struct scoutfs_log_trees *lt); 50 | void scoutfs_forest_get_btrees(struct super_block *sb, 51 | struct scoutfs_log_trees *lt); 52 | 53 | /* > 0 error codes */ 54 | #define SCOUTFS_DELTA_COMBINED 1 /* src val was combined, drop src */ 55 | #define SCOUTFS_DELTA_COMBINED_NULL 2 /* combined val has no data, drop both */ 56 | int scoutfs_forest_combine_deltas(struct scoutfs_key *key, void *dst, int dst_len, 57 | void *src, int src_len); 58 | 59 | int scoutfs_forest_setup(struct super_block *sb); 60 | void scoutfs_forest_start(struct super_block *sb); 61 | void scoutfs_forest_stop(struct super_block *sb); 62 | void scoutfs_forest_destroy(struct super_block *sb); 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /kmod/src/hash.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_HASH_H_ 2 | #define _SCOUTFS_HASH_H_ 3 | 4 | /* 5 | * We're using FNV1a for now. It's fine. Ish. 6 | * 7 | * The longer term plan is xxh3 but it looks like it'll take just a bit 8 | * more time to be declared stable and then it needs to be ported to the 9 | * kernel. 10 | * 11 | * - https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html 12 | * - https://github.com/Cyan4973/xxHash/releases/tag/v0.7.4 13 | */ 14 | 15 | static inline u32 fnv1a32(const void *data, unsigned int len) 16 | { 17 | u32 hash = 0x811c9dc5; 18 | 19 | while (len--) { 20 | hash ^= *(u8 *)(data++); 21 | hash *= 0x01000193; 22 | } 23 | 24 | return hash; 25 | } 26 | 27 | static inline u64 fnv1a64(const void *data, unsigned int len) 28 | { 29 | u64 hash = 0xcbf29ce484222325ULL; 30 | 31 | while (len--) { 32 | hash ^= *(u8 *)(data++); 33 | hash *= 0x100000001b3ULL; 34 | } 35 | 36 | return hash; 37 | } 38 | 39 | static inline u32 scoutfs_hash32(const void *data, unsigned int len) 40 | { 41 | return fnv1a32(data, len); 42 | } 43 | 44 | static inline u64 scoutfs_hash64(const void *data, unsigned int len) 45 | { 46 | return fnv1a64(data, len); 47 | } 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /kmod/src/item.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_ITEM_H_ 2 | #define _SCOUTFS_ITEM_H_ 3 | 4 | int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, 5 | void *val, int val_len, struct scoutfs_lock *lock); 6 | int scoutfs_item_lookup_smaller_zero(struct super_block *sb, struct scoutfs_key *key, 7 | void *val, int val_len, struct scoutfs_lock *lock); 8 | int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key *key, 9 | void *val, int val_len, 10 | struct scoutfs_lock *lock); 11 | int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, 12 | struct scoutfs_key *last, void *val, int val_len, 13 | struct scoutfs_lock *lock); 14 | int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, 15 | struct scoutfs_lock *lock); 16 | int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, 17 | void *val, int val_len, struct scoutfs_lock *lock); 18 | int scoutfs_item_create_force(struct super_block *sb, struct scoutfs_key *key, 19 | void *val, int val_len, 20 | struct scoutfs_lock *lock, struct scoutfs_lock *primary); 21 | int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, 22 | void *val, int val_len, struct scoutfs_lock *lock); 23 | int scoutfs_item_delta(struct super_block *sb, struct scoutfs_key *key, 24 | void *val, int val_len, struct scoutfs_lock *lock); 25 | int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, 26 | struct scoutfs_lock *lock); 27 | int scoutfs_item_delete_force(struct super_block *sb, struct scoutfs_key *key, 28 | struct scoutfs_lock *lock, struct scoutfs_lock *primary); 29 | 30 | u64 scoutfs_item_dirty_pages(struct super_block *sb); 31 | int scoutfs_item_write_dirty(struct super_block *sb); 32 | int scoutfs_item_write_done(struct super_block *sb); 33 | bool scoutfs_item_range_cached(struct super_block *sb, 34 | struct scoutfs_key *start, 35 | struct scoutfs_key *end, bool *dirty); 36 | void scoutfs_item_invalidate(struct super_block *sb, struct scoutfs_key *start, 37 | struct scoutfs_key *end); 38 | 39 | int scoutfs_item_setup(struct super_block *sb); 40 | void scoutfs_item_destroy(struct super_block *sb); 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /kmod/src/kernelcompat.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include "kernelcompat.h" 5 | 6 | #ifdef KC_SHRINKER_SHRINK 7 | #include 8 | /* 9 | * If a target doesn't have that .{count,scan}_objects() interface then 10 | * we have a .shrink() helper that performs the shrink work in terms of 11 | * count/scan. 12 | */ 13 | int kc_shrink_wrapper_fn(struct shrinker *shrink, struct shrink_control *sc) 14 | { 15 | struct kc_shrinker_wrapper *wrapper = container_of(shrink, struct kc_shrinker_wrapper, shrink); 16 | unsigned long nr; 17 | unsigned long rc; 18 | 19 | if (sc->nr_to_scan != 0) { 20 | rc = wrapper->scan_objects(shrink, sc); 21 | /* translate magic values to the equivalent for older kernels */ 22 | if (rc == SHRINK_STOP) 23 | return -1; 24 | else if (rc == SHRINK_EMPTY) 25 | return 0; 26 | } 27 | 28 | nr = wrapper->count_objects(shrink, sc); 29 | 30 | return min_t(unsigned long, nr, INT_MAX); 31 | } 32 | #endif 33 | 34 | #ifndef KC_CURRENT_TIME_INODE 35 | struct timespec64 kc_current_time(struct inode *inode) 36 | { 37 | struct timespec64 now; 38 | unsigned gran; 39 | 40 | getnstimeofday64(&now); 41 | 42 | if (unlikely(!inode->i_sb)) { 43 | WARN(1, "current_time() called with uninitialized super_block in the inode"); 44 | return now; 45 | } 46 | 47 | gran = inode->i_sb->s_time_gran; 48 | 49 | /* Avoid division in the common cases 1 ns and 1 s. */ 50 | if (gran == 1) { 51 | /* nothing */ 52 | } else if (gran == NSEC_PER_SEC) { 53 | now.tv_nsec = 0; 54 | } else if (gran > 1 && gran < NSEC_PER_SEC) { 55 | now.tv_nsec -= now.tv_nsec % gran; 56 | } else { 57 | WARN(1, "illegal file time granularity: %u", gran); 58 | } 59 | 60 | return now; 61 | } 62 | #endif 63 | 64 | #ifndef KC_GENERIC_FILE_BUFFERED_WRITE 65 | ssize_t 66 | kc_generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, 67 | unsigned long nr_segs, loff_t pos, loff_t *ppos, 68 | size_t count, ssize_t written) 69 | { 70 | ssize_t status; 71 | struct iov_iter i; 72 | 73 | iov_iter_init(&i, WRITE, iov, nr_segs, count); 74 | status = kc_generic_perform_write(iocb, &i, pos); 75 | 76 | if (likely(status >= 0)) { 77 | written += status; 78 | *ppos = pos + status; 79 | } 80 | 81 | return written ? written : status; 82 | } 83 | #endif 84 | -------------------------------------------------------------------------------- /kmod/src/lock_server.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_LOCK_SERVER_H_ 2 | #define _SCOUTFS_LOCK_SERVER_H_ 3 | 4 | int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, 5 | struct scoutfs_net_lock_recover *nlr); 6 | int scoutfs_lock_server_finished_recovery(struct super_block *sb); 7 | int scoutfs_lock_server_request(struct super_block *sb, u64 rid, 8 | u64 net_id, struct scoutfs_net_lock *nl); 9 | int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid); 10 | int scoutfs_lock_server_response(struct super_block *sb, u64 rid, 11 | struct scoutfs_net_lock *nl); 12 | int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); 13 | 14 | int scoutfs_lock_server_setup(struct super_block *sb); 15 | void scoutfs_lock_server_destroy(struct super_block *sb); 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /kmod/src/msg.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "msg.h" 5 | 6 | /* 7 | * This can be called with pre-emption disabled if the caller is printing 8 | * the contents of formated per-cpu key string buffers. 9 | */ 10 | void scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, 11 | const char *fmt, ...) 12 | { 13 | struct va_format vaf; 14 | va_list args; 15 | 16 | va_start(args, fmt); 17 | 18 | vaf.fmt = fmt; 19 | vaf.va = &args; 20 | 21 | printk("%sscoutfs "SCSBF"%s: %pV\n", prefix, SCSB_ARGS(sb), str, &vaf); 22 | 23 | va_end(args); 24 | } 25 | -------------------------------------------------------------------------------- /kmod/src/msg.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_MSG_H_ 2 | #define _SCOUTFS_MSG_H_ 3 | 4 | #include 5 | #include "key.h" 6 | #include "counters.h" 7 | #include "super.h" 8 | 9 | void __printf(4, 5) scoutfs_msg(struct super_block *sb, const char *prefix, 10 | const char *str, const char *fmt, ...); 11 | 12 | #define scoutfs_msg_check(sb, pref, str, fmt, args...) \ 13 | do { \ 14 | BUILD_BUG_ON(fmt[sizeof(fmt) - 2] == '\n'); \ 15 | scoutfs_msg(sb, pref, str, fmt, ##args); \ 16 | } while (0) 17 | 18 | #define scoutfs_err(sb, fmt, args...) \ 19 | scoutfs_msg_check(sb, KERN_ERR, " error", fmt, ##args) 20 | 21 | #define scoutfs_warn(sb, fmt, args...) \ 22 | scoutfs_msg_check(sb, KERN_WARNING, " warning", fmt, ##args) 23 | 24 | #define scoutfs_info(sb, fmt, args...) \ 25 | scoutfs_msg_check(sb, KERN_INFO, "", fmt, ##args) 26 | 27 | #define scoutfs_tprintk(sb, fmt, args...) \ 28 | trace_printk(SCSBF " " fmt "\n", SCSB_ARGS(sb), ##args); 29 | 30 | #define scoutfs_bug_on(sb, cond, fmt, args...) \ 31 | do { \ 32 | if (cond) { \ 33 | scoutfs_err(sb, "(" __stringify(cond) "), " fmt, ##args); \ 34 | BUG(); \ 35 | } \ 36 | } while (0) \ 37 | 38 | /* 39 | * Each message is only generated once per volume. Remounting resets 40 | * the messages. 41 | */ 42 | #define scoutfs_corruption(sb, which, counter, fmt, args...) \ 43 | do { \ 44 | __typeof__(sb) _sb = (sb); \ 45 | struct scoutfs_sb_info *_sbi = SCOUTFS_SB(_sb); \ 46 | unsigned int _bit = (which); \ 47 | \ 48 | if (WARN_ON_ONCE(_bit >= SC_NR_SOURCES)) \ 49 | break; \ 50 | \ 51 | scoutfs_inc_counter(_sb, counter); \ 52 | if (!test_and_set_bit(_bit, _sbi->corruption_messages_once)) { \ 53 | scoutfs_err(_sb, "corruption (see scoutfs-corruption(5)): " \ 54 | #which ": " fmt, ##args); \ 55 | dump_stack(); \ 56 | } \ 57 | } while (0) \ 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /kmod/src/omap.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_OMAP_H_ 2 | #define _SCOUTFS_OMAP_H_ 3 | 4 | int scoutfs_omap_set(struct super_block *sb, u64 ino); 5 | bool scoutfs_omap_test(struct super_block *sb, u64 ino); 6 | void scoutfs_omap_clear(struct super_block *sb, u64 ino); 7 | int scoutfs_omap_client_handle_request(struct super_block *sb, u64 id, 8 | struct scoutfs_open_ino_map_args *args); 9 | void scoutfs_omap_calc_group_nrs(u64 ino, u64 *group_nr, int *bit_nr); 10 | 11 | int scoutfs_omap_add_rid(struct super_block *sb, u64 rid); 12 | int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid); 13 | int scoutfs_omap_finished_recovery(struct super_block *sb); 14 | int scoutfs_omap_server_handle_request(struct super_block *sb, u64 rid, u64 id, 15 | struct scoutfs_open_ino_map_args *args); 16 | int scoutfs_omap_server_handle_response(struct super_block *sb, u64 rid, 17 | struct scoutfs_open_ino_map *resp_map); 18 | void scoutfs_omap_server_shutdown(struct super_block *sb); 19 | 20 | int scoutfs_omap_setup(struct super_block *sb); 21 | void scoutfs_omap_destroy(struct super_block *sb); 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /kmod/src/options.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_OPTIONS_H_ 2 | #define _SCOUTFS_OPTIONS_H_ 3 | 4 | #include 5 | #include 6 | #include "format.h" 7 | 8 | struct scoutfs_mount_options { 9 | u64 data_prealloc_blocks; 10 | bool data_prealloc_contig_only; 11 | unsigned int log_merge_wait_timeout_ms; 12 | char *metadev_path; 13 | unsigned int orphan_scan_delay_ms; 14 | int quorum_slot_nr; 15 | u64 quorum_heartbeat_timeout_ms; 16 | }; 17 | 18 | void scoutfs_options_read(struct super_block *sb, struct scoutfs_mount_options *opts); 19 | int scoutfs_options_show(struct seq_file *seq, struct dentry *root); 20 | 21 | int scoutfs_options_early_setup(struct super_block *sb, char *options); 22 | int scoutfs_options_setup(struct super_block *sb); 23 | void scoutfs_options_stop(struct super_block *sb); 24 | void scoutfs_options_destroy(struct super_block *sb); 25 | 26 | #endif /* _SCOUTFS_OPTIONS_H_ */ 27 | -------------------------------------------------------------------------------- /kmod/src/per_task.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_PER_TASK_H_ 2 | #define _SCOUTFS_PER_TASK_H_ 3 | 4 | struct scoutfs_per_task { 5 | spinlock_t lock; 6 | struct list_head list; 7 | }; 8 | 9 | struct scoutfs_per_task_entry { 10 | struct list_head head; 11 | struct task_struct *task; 12 | void *ptr; 13 | }; 14 | 15 | #define SCOUTFS_DECLARE_PER_TASK_ENTRY(name) \ 16 | struct scoutfs_per_task_entry name = { \ 17 | .head = LIST_HEAD_INIT((name).head), \ 18 | } 19 | 20 | 21 | void *scoutfs_per_task_get(struct scoutfs_per_task *pt); 22 | void scoutfs_per_task_add(struct scoutfs_per_task *pt, 23 | struct scoutfs_per_task_entry *ent, void *ptr); 24 | bool scoutfs_per_task_add_excl(struct scoutfs_per_task *pt, 25 | struct scoutfs_per_task_entry *ent, void *ptr); 26 | bool scoutfs_per_task_del(struct scoutfs_per_task *pt, 27 | struct scoutfs_per_task_entry *ent); 28 | void scoutfs_per_task_init(struct scoutfs_per_task *pt); 29 | void scoutfs_per_task_init_entry(struct scoutfs_per_task_entry *ent); 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /kmod/src/quorum.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_QUORUM_H_ 2 | #define _SCOUTFS_QUORUM_H_ 3 | 4 | int scoutfs_quorum_server_sin(struct super_block *sb, struct sockaddr_in *sin); 5 | 6 | u8 scoutfs_quorum_votes_needed(struct super_block *sb); 7 | void scoutfs_quorum_slot_sin(struct scoutfs_quorum_config *qconf, int i, 8 | struct sockaddr_in *sin); 9 | 10 | int scoutfs_quorum_fence_leaders(struct super_block *sb, struct scoutfs_quorum_config *qconf, 11 | u64 term); 12 | 13 | int scoutfs_quorum_setup(struct super_block *sb); 14 | void scoutfs_quorum_shutdown(struct super_block *sb); 15 | void scoutfs_quorum_destroy(struct super_block *sb); 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /kmod/src/quota.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_QUOTA_H_ 2 | #define _SCOUTFS_QUOTA_H_ 3 | 4 | #include "ioctl.h" 5 | 6 | /* 7 | * Each rule's name can be in the ruleset's rbtree associated with the 8 | * source attr that it selects. This lets checks only test rules that 9 | * the inputs could match. The 'i' field indicates which name is in the 10 | * tree so we can find the containing rule. 11 | * 12 | * This is mostly private to quota.c but we expose it for tracing. 13 | */ 14 | struct squota_rule { 15 | u64 limit; 16 | u8 prio; 17 | u8 op; 18 | u8 rule_flags; 19 | struct squota_rule_name { 20 | struct rb_node node; 21 | u64 val; 22 | u8 source; 23 | u8 flags; 24 | u8 i; 25 | } names[3]; 26 | }; 27 | 28 | /* private to quota.c, only here for tracing */ 29 | struct squota_input { 30 | u64 attrs[SQ_NS__NR_SELECT]; 31 | u8 op; 32 | }; 33 | 34 | int scoutfs_quota_check_inode(struct super_block *sb, struct inode *dir); 35 | int scoutfs_quota_check_data(struct super_block *sb, struct inode *inode); 36 | 37 | int scoutfs_quota_get_rules(struct super_block *sb, u64 *iterator, 38 | struct scoutfs_ioctl_quota_rule *irules, int nr); 39 | int scoutfs_quota_mod_rule(struct super_block *sb, bool is_add, 40 | struct scoutfs_ioctl_quota_rule *irule); 41 | 42 | void scoutfs_quota_get_lock_range(struct scoutfs_key *start, struct scoutfs_key *end); 43 | void scoutfs_quota_invalidate(struct super_block *sb); 44 | 45 | int scoutfs_quota_setup(struct super_block *sb); 46 | void scoutfs_quota_destroy(struct super_block *sb); 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /kmod/src/recov.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_RECOV_H_ 2 | #define _SCOUTFS_RECOV_H_ 3 | 4 | enum { 5 | SCOUTFS_RECOV_GREETING = ( 1 << 0), 6 | SCOUTFS_RECOV_LOCKS = ( 1 << 1), 7 | 8 | SCOUTFS_RECOV_INVALID = (~0 << 2), 9 | SCOUTFS_RECOV_ALL = (~SCOUTFS_RECOV_INVALID), 10 | }; 11 | 12 | int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which); 13 | int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_block *), 14 | unsigned int timeout_ms); 15 | int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which); 16 | bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which); 17 | u64 scoutfs_recov_next_pending(struct super_block *sb, u64 rid, int which); 18 | void scoutfs_recov_shutdown(struct super_block *sb); 19 | 20 | int scoutfs_recov_setup(struct super_block *sb); 21 | void scoutfs_recov_destroy(struct super_block *sb); 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /kmod/src/scoutfs_trace.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Versity Software, Inc. All rights reserved. 3 | * 4 | * This program is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public 6 | * License v2 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | */ 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "super.h" 22 | #include "format.h" 23 | #include "inode.h" 24 | #include "dir.h" 25 | #include "msg.h" 26 | 27 | #define CREATE_TRACE_POINTS 28 | #include "scoutfs_trace.h" 29 | -------------------------------------------------------------------------------- /kmod/src/sort_priv.c: -------------------------------------------------------------------------------- 1 | /* 2 | * A copy of sort() from upstream with a priv argument that's passed 3 | * to comparison, like list_sort(). 4 | */ 5 | 6 | /* ------------------------ */ 7 | 8 | /* 9 | * A fast, small, non-recursive O(nlog n) sort for the Linux kernel 10 | * 11 | * Jan 23 2005 Matt Mackall 12 | */ 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "sort_priv.h" 19 | 20 | /** 21 | * sort - sort an array of elements 22 | * @priv: caller's pointer to pass to comparison and swap functions 23 | * @base: pointer to data to sort 24 | * @num: number of elements 25 | * @size: size of each element 26 | * @cmp_func: pointer to comparison function 27 | * @swap_func: pointer to swap function or NULL 28 | * 29 | * This function does a heapsort on the given array. You may provide a 30 | * swap_func function optimized to your element type. 31 | * 32 | * Sorting time is O(n log n) both on average and worst-case. While 33 | * qsort is about 20% faster on average, it suffers from exploitable 34 | * O(n*n) worst-case behavior and extra memory requirements that make 35 | * it less suitable for kernel use. 36 | */ 37 | 38 | void sort_priv(void *priv, void *base, size_t num, size_t size, 39 | int (*cmp_func)(void *priv, const void *, const void *), 40 | void (*swap_func)(void *priv, void *, void *, int size)) 41 | { 42 | /* pre-scale counters for performance */ 43 | int i = (num/2 - 1) * size, n = num * size, c, r; 44 | 45 | /* heapify */ 46 | for ( ; i >= 0; i -= size) { 47 | for (r = i; r * 2 + size < n; r = c) { 48 | c = r * 2 + size; 49 | if (c < n - size && 50 | cmp_func(priv, base + c, base + c + size) < 0) 51 | c += size; 52 | if (cmp_func(priv, base + r, base + c) >= 0) 53 | break; 54 | swap_func(priv, base + r, base + c, size); 55 | } 56 | } 57 | 58 | /* sort */ 59 | for (i = n - size; i > 0; i -= size) { 60 | swap_func(priv, base, base + i, size); 61 | for (r = 0; r * 2 + size < i; r = c) { 62 | c = r * 2 + size; 63 | if (c < i - size && 64 | cmp_func(priv, base + c, base + c + size) < 0) 65 | c += size; 66 | if (cmp_func(priv, base + r, base + c) >= 0) 67 | break; 68 | swap_func(priv, base + r, base + c, size); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /kmod/src/sort_priv.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_SORT_PRIV_H_ 2 | #define _SCOUTFS_SORT_PRIV_H_ 3 | 4 | void sort_priv(void *priv, void *base, size_t num, size_t size, 5 | int (*cmp_func)(void *priv, const void *, const void *), 6 | void (*swap_func)(void *priv, void *, void *, int size)); 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /kmod/src/spbm.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_SPBM_H_ 2 | #define _SCOUTFS_SPBM_H_ 3 | 4 | struct scoutfs_spbm { 5 | struct rb_root root; 6 | }; 7 | 8 | void scoutfs_spbm_init(struct scoutfs_spbm *spbm); 9 | bool scoutfs_spbm_empty(struct scoutfs_spbm *spbm); 10 | void scoutfs_spbm_destroy(struct scoutfs_spbm *spbm); 11 | 12 | int scoutfs_spbm_set(struct scoutfs_spbm *spbm, u64 bit); 13 | int scoutfs_spbm_test(struct scoutfs_spbm *spbm, u64 bit); 14 | void scoutfs_spbm_clear(struct scoutfs_spbm *spbm, u64 bit); 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /kmod/src/srch.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_SRCH_H_ 2 | #define _SCOUTFS_SRCH_H_ 3 | 4 | struct scoutfs_block; 5 | 6 | struct scoutfs_srch_rb_root { 7 | struct rb_root root; 8 | struct rb_node *last; 9 | unsigned long nr; 10 | }; 11 | 12 | struct scoutfs_srch_rb_node { 13 | struct rb_node node; 14 | u64 ino; 15 | u64 id; 16 | }; 17 | 18 | #define scoutfs_srch_foreach_rb_node(snode, node, sroot) \ 19 | for (node = rb_first(&(sroot)->root); \ 20 | node && (snode = container_of(node, struct scoutfs_srch_rb_node, \ 21 | node), 1); \ 22 | node = rb_next(node)) 23 | 24 | int scoutfs_srch_add(struct super_block *sb, 25 | struct scoutfs_alloc *alloc, 26 | struct scoutfs_block_writer *wri, 27 | struct scoutfs_srch_file *sfl, 28 | struct scoutfs_block **bl_ret, 29 | u64 hash, u64 ino, u64 id); 30 | 31 | void scoutfs_srch_destroy_rb_root(struct scoutfs_srch_rb_root *sroot); 32 | int scoutfs_srch_search_xattrs(struct super_block *sb, 33 | struct scoutfs_srch_rb_root *sroot, 34 | u64 hash, u64 ino, u64 last_ino, bool *done); 35 | 36 | int scoutfs_srch_rotate_log(struct super_block *sb, 37 | struct scoutfs_alloc *alloc, 38 | struct scoutfs_block_writer *wri, 39 | struct scoutfs_btree_root *root, 40 | struct scoutfs_srch_file *sfl, bool force); 41 | int scoutfs_srch_get_compact(struct super_block *sb, 42 | struct scoutfs_alloc *alloc, 43 | struct scoutfs_block_writer *wri, 44 | struct scoutfs_btree_root *root, 45 | u64 rid, struct scoutfs_srch_compact *sc); 46 | int scoutfs_srch_update_compact(struct super_block *sb, 47 | struct scoutfs_alloc *alloc, 48 | struct scoutfs_block_writer *wri, 49 | struct scoutfs_btree_root *root, u64 rid, 50 | struct scoutfs_srch_compact *sc); 51 | int scoutfs_srch_commit_compact(struct super_block *sb, 52 | struct scoutfs_alloc *alloc, 53 | struct scoutfs_block_writer *wri, 54 | struct scoutfs_btree_root *root, u64 rid, 55 | struct scoutfs_srch_compact *res, 56 | struct scoutfs_alloc_list_head *av, 57 | struct scoutfs_alloc_list_head *fr); 58 | int scoutfs_srch_cancel_compact(struct super_block *sb, 59 | struct scoutfs_alloc *alloc, 60 | struct scoutfs_block_writer *wri, 61 | struct scoutfs_btree_root *root, u64 rid, 62 | struct scoutfs_alloc_list_head *av, 63 | struct scoutfs_alloc_list_head *fr); 64 | 65 | void scoutfs_srch_destroy(struct super_block *sb); 66 | int scoutfs_srch_setup(struct super_block *sb); 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /kmod/src/sysfs.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_SYSFS_H_ 2 | #define _SCOUTFS_SYSFS_H_ 3 | 4 | #include 5 | 6 | /* 7 | * We have some light wrappers around sysfs attributes to make it safe 8 | * to tear down the attributes before freeing the data they describe. 9 | */ 10 | 11 | #define SCOUTFS_ATTR_RO(_name) \ 12 | static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RO(_name) 13 | #define SCOUTFS_ATTR_RW(_name) \ 14 | static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RW(_name) 15 | 16 | #define SCOUTFS_ATTR_PTR(_name) \ 17 | &scoutfs_attr_##_name.attr 18 | 19 | struct scoutfs_sysfs_attrs { 20 | struct super_block *sb; 21 | char *name; 22 | struct completion comp; 23 | 24 | struct kobject kobj; 25 | struct kobj_type ktype; 26 | }; 27 | 28 | #define SCOUTFS_SYSFS_ATTRS(kobj) \ 29 | container_of(kobj, struct scoutfs_sysfs_attrs, kobj) 30 | 31 | #define SCOUTFS_SYSFS_ATTRS_SB(kobj) \ 32 | (SCOUTFS_SYSFS_ATTRS(kobj)->sb) 33 | 34 | #define DECLARE_SCOUTFS_SYSFS_ATTRS(name, kobj) \ 35 | struct scoutfs_sysfs_attrs *ssa = SCOUTFS_SYSFS_ATTRS(kobj) 36 | 37 | void scoutfs_sysfs_init_attrs(struct super_block *sb, 38 | struct scoutfs_sysfs_attrs *ssa); 39 | int scoutfs_sysfs_create_attrs_parent(struct super_block *sb, 40 | struct kobject *parent, 41 | struct scoutfs_sysfs_attrs *ssa, 42 | struct attribute **attrs, char *fmt, ...); 43 | #define scoutfs_sysfs_create_attrs(sb, ssa, attrs, fmt, args...) \ 44 | scoutfs_sysfs_create_attrs_parent(sb, scoutfs_sysfs_sb_dir(sb), \ 45 | ssa, attrs, fmt, ##args) 46 | 47 | void scoutfs_sysfs_destroy_attrs(struct super_block *sb, 48 | struct scoutfs_sysfs_attrs *ssa); 49 | 50 | struct kobject *scoutfs_sysfs_sb_dir(struct super_block *sb); 51 | 52 | int scoutfs_setup_sysfs(struct super_block *sb); 53 | void scoutfs_destroy_sysfs(struct super_block *sb); 54 | 55 | int __init scoutfs_sysfs_init(void); 56 | void scoutfs_sysfs_exit(void); 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /kmod/src/totl.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_TOTL_H_ 2 | #define _SCOUTFS_TOTL_H_ 3 | 4 | #include "key.h" 5 | 6 | struct scoutfs_totl_merging { 7 | u64 fs_seq; 8 | u64 fs_total; 9 | u64 fs_count; 10 | u64 fin_seq; 11 | u64 fin_total; 12 | s64 fin_count; 13 | u64 log_seq; 14 | u64 log_total; 15 | s64 log_count; 16 | }; 17 | 18 | void scoutfs_totl_set_range(struct scoutfs_key *start, struct scoutfs_key *end); 19 | void scoutfs_totl_merge_init(struct scoutfs_totl_merging *merg); 20 | void scoutfs_totl_merge_contribute(struct scoutfs_totl_merging *merg, 21 | u64 seq, u8 flags, void *val, int val_len, int fic); 22 | void scoutfs_totl_merge_resolve(struct scoutfs_totl_merging *merg, __u64 *total, __u64 *count); 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /kmod/src/trans.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_TRANS_H_ 2 | #define _SCOUTFS_TRANS_H_ 3 | 4 | void scoutfs_trans_write_func(struct work_struct *work); 5 | int scoutfs_trans_sync(struct super_block *sb, int wait); 6 | int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, 7 | int datasync); 8 | void scoutfs_trans_restart_sync_deadline(struct super_block *sb); 9 | 10 | int scoutfs_hold_trans(struct super_block *sb, bool allocing); 11 | bool scoutfs_trans_held(void); 12 | void scoutfs_release_trans(struct super_block *sb); 13 | u64 scoutfs_trans_sample_seq(struct super_block *sb); 14 | 15 | int scoutfs_trans_get_log_trees(struct super_block *sb); 16 | bool scoutfs_trans_has_dirty(struct super_block *sb); 17 | 18 | int scoutfs_setup_trans(struct super_block *sb); 19 | void scoutfs_shutdown_trans(struct super_block *sb); 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /kmod/src/triggers.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_TRIGGERS_H_ 2 | #define _SCOUTFS_TRIGGERS_H_ 3 | 4 | enum scoutfs_trigger { 5 | SCOUTFS_TRIGGER_BLOCK_REMOVE_STALE, 6 | SCOUTFS_TRIGGER_SRCH_COMPACT_LOGS_PAD_SAFE, 7 | SCOUTFS_TRIGGER_SRCH_FORCE_LOG_ROTATE, 8 | SCOUTFS_TRIGGER_SRCH_MERGE_STOP_SAFE, 9 | SCOUTFS_TRIGGER_STATFS_LOCK_PURGE, 10 | SCOUTFS_TRIGGER_NR, 11 | }; 12 | 13 | bool scoutfs_trigger_test_and_clear(struct super_block *sb, unsigned int t); 14 | 15 | #define scoutfs_trigger(sb, which) \ 16 | scoutfs_trigger_test_and_clear(sb, SCOUTFS_TRIGGER_##which) 17 | 18 | int scoutfs_setup_triggers(struct super_block *sb); 19 | void scoutfs_destroy_triggers(struct super_block *sb); 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /kmod/src/tseq.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_TSEQ_H_ 2 | #define _SCOUTFS_TSEQ_H_ 3 | 4 | #include 5 | 6 | struct scoutfs_tseq_entry; 7 | typedef void (*scoutfs_tseq_show_t)(struct seq_file *m, 8 | struct scoutfs_tseq_entry *ent); 9 | 10 | struct scoutfs_tseq_tree { 11 | spinlock_t lock; 12 | struct rb_root root; 13 | scoutfs_tseq_show_t show; 14 | }; 15 | 16 | struct scoutfs_tseq_entry { 17 | struct rb_node node; 18 | loff_t pos; 19 | loff_t total; 20 | }; 21 | 22 | void scoutfs_tseq_tree_init(struct scoutfs_tseq_tree *tree, 23 | scoutfs_tseq_show_t show); 24 | void scoutfs_tseq_add(struct scoutfs_tseq_tree *tree, 25 | struct scoutfs_tseq_entry *ent); 26 | void scoutfs_tseq_del(struct scoutfs_tseq_tree *tree, 27 | struct scoutfs_tseq_entry *ent); 28 | 29 | struct dentry *scoutfs_tseq_create(const char *name, struct dentry *parent, 30 | struct scoutfs_tseq_tree *tree); 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /kmod/src/util.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_UTIL_H_ 2 | #define _SCOUTFS_UTIL_H_ 3 | 4 | /* 5 | * Little utility helpers that probably belong upstream. 6 | */ 7 | 8 | static inline void down_write_two(struct rw_semaphore *a, 9 | struct rw_semaphore *b) 10 | { 11 | BUG_ON(a == b); 12 | 13 | if (a > b) 14 | swap(a, b); 15 | 16 | down_write(a); 17 | down_write_nested(b, SINGLE_DEPTH_NESTING); 18 | } 19 | 20 | /* 21 | * When returning shrinker counts from scan_objects, we should steer 22 | * clear of the magic SHRINK_STOP and SHRINK_EMPTY values, which are near 23 | * ~0UL values. Hence, we cap count to ~0L, which is arbitarily high 24 | * enough to avoid it. 25 | */ 26 | static inline long shrinker_min_long(long count) 27 | { 28 | return min(count, LONG_MAX); 29 | } 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /kmod/src/volopt.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_VOLOPT_H_ 2 | #define _SCOUTFS_VOLOPT_H_ 3 | 4 | int scoutfs_volopt_setup(struct super_block *sb); 5 | void scoutfs_volopt_destroy(struct super_block *sb); 6 | 7 | #endif 8 | -------------------------------------------------------------------------------- /kmod/src/wkic.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_WKIC_H_ 2 | #define _SCOUTFS_WKIC_H_ 3 | 4 | #include "format.h" 5 | 6 | typedef int (*wkic_iter_cb_t)(struct scoutfs_key *key, void *val, unsigned int val_len, 7 | void *cb_arg); 8 | 9 | int scoutfs_wkic_iterate(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *last, 10 | struct scoutfs_key *range_start, struct scoutfs_key *range_end, 11 | wkic_iter_cb_t cb, void *cb_arg); 12 | int scoutfs_wkic_iterate_stable(struct super_block *sb, struct scoutfs_key *key, 13 | struct scoutfs_key *last, struct scoutfs_key *range_start, 14 | struct scoutfs_key *range_end, wkic_iter_cb_t cb, void *cb_arg); 15 | 16 | int scoutfs_wkic_setup(struct super_block *sb); 17 | void scoutfs_wkic_destroy(struct super_block *sb); 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /kmod/src/xattr.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_XATTR_H_ 2 | #define _SCOUTFS_XATTR_H_ 3 | 4 | struct scoutfs_xattr_prefix_tags { 5 | unsigned long hide:1, 6 | indx:1, 7 | srch:1, 8 | totl:1; 9 | }; 10 | 11 | extern const struct xattr_handler *scoutfs_xattr_handlers[]; 12 | 13 | int scoutfs_xattr_get_locked(struct inode *inode, const char *name, void *buffer, size_t size, 14 | struct scoutfs_lock *lck); 15 | int scoutfs_xattr_set_locked(struct inode *inode, const char *name, size_t name_len, 16 | const void *value, size_t size, int flags, 17 | const struct scoutfs_xattr_prefix_tags *tgs, 18 | struct scoutfs_lock *lck, struct scoutfs_lock *totl_lock, 19 | struct list_head *ind_locks); 20 | 21 | ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size); 22 | ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, 23 | size_t size, __u32 *hash_pos, __u64 *id_pos, 24 | bool e_range, bool show_hidden); 25 | int scoutfs_xattr_drop(struct super_block *sb, u64 ino, 26 | struct scoutfs_lock *lock); 27 | 28 | int scoutfs_xattr_parse_tags(const char *name, unsigned int name_len, 29 | struct scoutfs_xattr_prefix_tags *tgs); 30 | 31 | void scoutfs_xattr_init_totl_key(struct scoutfs_key *key, u64 *name); 32 | int scoutfs_xattr_combine_totl(void *dst, int dst_len, void *src, int src_len); 33 | 34 | void scoutfs_xattr_indx_get_range(struct scoutfs_key *start, struct scoutfs_key *end); 35 | void scoutfs_xattr_init_indx_key(struct scoutfs_key *key, u8 major, u64 minor, u64 ino, u64 xid); 36 | void scoutfs_xattr_get_indx_key(struct scoutfs_key *key, u8 *major, u64 *minor, u64 *ino, u64 *xid); 37 | void scoutfs_xattr_set_indx_key_xid(struct scoutfs_key *key, u64 xid); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | src/*.d 2 | src/createmany 3 | src/dumb_renameat2 4 | src/dumb_setxattr 5 | src/handle_cat 6 | src/handle_fsetxattr 7 | src/bulk_create_paths 8 | src/find_xattrs 9 | src/stage_tmpfile 10 | src/create_xattr_loop 11 | src/o_tmpfile_umask 12 | src/o_tmpfile_linkat 13 | src/mmap_stress 14 | src/mmap_validate 15 | -------------------------------------------------------------------------------- /tests/.xfstests-branch: -------------------------------------------------------------------------------- 1 | v2022.05.01-2-g787cd20 2 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS := -Wall -O2 -Werror -D_FILE_OFFSET_BITS=64 -fno-strict-aliasing -I ../kmod/src 2 | SHELL := /usr/bin/bash 3 | 4 | # each binary command is built from a single .c file 5 | BIN := src/createmany \ 6 | src/dumb_renameat2 \ 7 | src/dumb_setxattr \ 8 | src/handle_cat \ 9 | src/handle_fsetxattr \ 10 | src/bulk_create_paths \ 11 | src/stage_tmpfile \ 12 | src/find_xattrs \ 13 | src/create_xattr_loop \ 14 | src/fragmented_data_extents \ 15 | src/o_tmpfile_umask \ 16 | src/o_tmpfile_linkat \ 17 | src/mmap_stress \ 18 | src/mmap_validate 19 | 20 | DEPS := $(wildcard src/*.d) 21 | 22 | all: $(BIN) 23 | 24 | ifneq ($(DEPS),) 25 | -include $(DEPS) 26 | endif 27 | 28 | src/mmap_stress: LIBS+=-lpthread 29 | 30 | $(BIN): %: %.c Makefile 31 | gcc $(CFLAGS) -MD -MP -MF $*.d $< -o $@ $(LIBS) 32 | 33 | .PHONY: clean 34 | clean: 35 | @rm -f $(BIN) $(DEPS) 36 | 37 | # 38 | # Make sure we only have all three items needed for each test: entry in 39 | # sequence, test script in tests/, and output in golden/. 40 | # 41 | .PHONY: check-test-files 42 | check-test-files: 43 | @for t in $$(grep -v "^#" sequence); do \ 44 | test -e "tests/$$t" || \ 45 | echo "no test for list entry: $$t"; \ 46 | t=$${t%%.sh}; \ 47 | test -e "golden/$$t" || \ 48 | echo "no output for list entry: $$t"; \ 49 | done; \ 50 | for t in golden/*; do \ 51 | t=$$(basename "$$t"); \ 52 | grep -q "^$$t.sh$$" sequence || \ 53 | echo "output not in list: $$t"; \ 54 | done; \ 55 | for t in tests/*; do \ 56 | t=$$(basename "$$t"); \ 57 | test "$$t" == "list" && continue; \ 58 | grep -q "^$$t$$" sequence || \ 59 | echo "test not in list: $$t"; \ 60 | done 61 | -------------------------------------------------------------------------------- /tests/fenced-local-force-unmount.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | # 4 | # This fencing script is used for testing clusters of multiple mounts on 5 | # a single host. It finds mounts to fence by looking for their rids and 6 | # only knows how to "fence" by using forced unmount. 7 | # 8 | 9 | echo "$0 running rid '$SCOUTFS_FENCED_REQ_RID' ip '$SCOUTFS_FENCED_REQ_IP' args '$@'" 10 | 11 | log() { 12 | echo "$@" > /dev/stderr 13 | exit 1 14 | } 15 | 16 | echo_fail() { 17 | echo "$@" > /dev/stderr 18 | exit 1 19 | } 20 | 21 | rid="$SCOUTFS_FENCED_REQ_RID" 22 | 23 | for fs in /sys/fs/scoutfs/*; do 24 | [ ! -d "$fs" ] && continue 25 | 26 | fs_rid="$(cat $fs/rid)" || \ 27 | echo_fail "failed to get rid in $fs" 28 | if [ "$fs_rid" != "$rid" ]; then 29 | continue 30 | fi 31 | 32 | nr="$(cat $fs/data_device_maj_min)" || \ 33 | echo_fail "failed to get data device major:minor in $fs" 34 | 35 | mnts=$(findmnt -l -n -t scoutfs -o TARGET -S $nr) || \ 36 | echo_fail "findmnt -t scoutfs -S $nr failed" 37 | for mnt in $mnts; do 38 | umount -f "$mnt" || \ 39 | echo_fail "umout -f $mnt failed" 40 | done 41 | done 42 | 43 | exit 0 44 | -------------------------------------------------------------------------------- /tests/funcs/exec.sh: -------------------------------------------------------------------------------- 1 | 2 | t_status_msg() 3 | { 4 | echo "$*" > "$T_TMPDIR/status.msg" 5 | } 6 | 7 | export T_PASS_STATUS=100 8 | export T_SKIP_STATUS=101 9 | export T_FAIL_STATUS=102 10 | export T_SKIP_PERMITTED_STATUS=103 11 | export T_FIRST_STATUS="$T_PASS_STATUS" 12 | export T_LAST_STATUS="$T_SKIP_PERMITTED_STATUS" 13 | 14 | t_pass() 15 | { 16 | exit $T_PASS_STATUS 17 | } 18 | 19 | t_skip() 20 | { 21 | t_status_msg "$@" 22 | exit $T_SKIP_STATUS 23 | } 24 | 25 | # 26 | # This exit code is *reserved* for tests that are up-front never going to work 27 | # in certain cases. This should be expressly documented per-case and made 28 | # abundantly clear before merging. The test itself should document its case. 29 | # 30 | t_skip_permitted() 31 | { 32 | t_status_msg "$@" 33 | exit $T_SKIP_PERMITTED_STATUS 34 | } 35 | 36 | t_fail() 37 | { 38 | t_status_msg "$@" 39 | exit $T_FAIL_STATUS 40 | } 41 | 42 | # 43 | # Quietly run a command during a test. If it succeeds then we have a 44 | # log of its execution but its output isn't included in the test's 45 | # compared output. If it fails then the test fails. 46 | # 47 | t_quiet() 48 | { 49 | echo "# $*" >> "$T_TMPDIR/quiet.log" 50 | "$@" >> "$T_TMPDIR/quiet.log" 2>&1 || \ 51 | t_fail "quiet command failed" 52 | } 53 | 54 | # 55 | # Quietly run a command during a test. The output is logged but only 56 | # the return code is printed, presumably because the output contains 57 | # a lot of invocation specific text that is difficult to filter. 58 | # 59 | t_rc() 60 | { 61 | echo "# $*" >> "$T_TMP.rc.log" 62 | "$@" >> "$T_TMP.rc.log" 2>&1 63 | echo "rc: $?" 64 | } 65 | 66 | # 67 | # redirect test output back to the output of the invoking script intead 68 | # of the compared output. 69 | # 70 | t_restore_output() 71 | { 72 | exec >&6 2>&1 73 | } 74 | 75 | # 76 | # redirect a command's output back to the compared output after the 77 | # test has restored its output 78 | # 79 | t_compare_output() 80 | { 81 | "$@" >&7 2>&1 82 | } 83 | 84 | # 85 | # usually bash prints an annoying output message when jobs 86 | # are killed. We can avoid that by redirecting stderr for 87 | # the bash process when it reaps the jobs that are killed. 88 | # 89 | t_silent_kill() { 90 | exec {ERR}>&2 2>/dev/null 91 | kill "$@" 92 | wait "$@" 93 | exec 2>&$ERR {ERR}>&- 94 | } 95 | -------------------------------------------------------------------------------- /tests/funcs/require.sh: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Make sure that all the base command arguments are found in the path. 4 | # This isn't strictly necessary as the test will naturally fail if the 5 | # command isn't found, but it's nice to fail fast and clearly 6 | # communicate why. 7 | # 8 | t_require_commands() { 9 | local c 10 | 11 | for c in "$@"; do 12 | which "$c" >/dev/null 2>&1 || \ 13 | t_fail "command $c not found in path" 14 | done 15 | } 16 | 17 | # 18 | # make sure that we have at least this many mounts 19 | # 20 | t_require_mounts() { 21 | local req="$1" 22 | 23 | test "$T_NR_MOUNTS" -ge "$req" || \ 24 | t_skip "$req mounts required, only have $T_NR_MOUNTS" 25 | } 26 | 27 | # 28 | # Require that the meta device be at least the size string argument, as 29 | # parsed by numfmt using single char base 2 suffixes (iec).. 64G, etc. 30 | # 31 | t_require_meta_size() { 32 | local dev="$T_META_DEVICE" 33 | local req_iec="$1" 34 | local req_bytes=$(numfmt --from=iec --to=none $req_iec) 35 | local dev_bytes=$(blockdev --getsize64 $dev) 36 | local dev_iec=$(numfmt --from=auto --to=iec $dev_bytes) 37 | 38 | test "$dev_bytes" -ge "$req_bytes" || \ 39 | t_skip "$dev must be at least $req_iec, is $dev_iec" 40 | } 41 | -------------------------------------------------------------------------------- /tests/funcs/tap.sh: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Generate TAP format test results 4 | # 5 | 6 | t_tap_header() 7 | { 8 | local runid=$1 9 | local sequence=( $(echo $tests) ) 10 | local count=${#sequence[@]} 11 | 12 | # avoid recreating the same TAP result over again - harness sets this 13 | [[ -z "$runid" ]] && runid="*test*" 14 | 15 | cat > $T_RESULTS/scoutfs.tap <> $T_RESULTS/scoutfs.tap 36 | echo "#" >> $T_RESULTS/scoutfs.tap 37 | } 38 | 39 | t_tap_progress() 40 | { 41 | ( 42 | local i=$(( testcount + 1 )) 43 | local testname=$1 44 | local result=$2 45 | 46 | local diff="" 47 | local dmsg="" 48 | 49 | if [[ -s "$T_RESULTS/tmp/${testname}/dmesg.new" ]]; then 50 | dmsg="1" 51 | fi 52 | 53 | if ! cmp -s golden/${testname} $T_RESULTS/output/${testname}; then 54 | diff="1" 55 | fi 56 | 57 | if [[ "${result}" == "100" ]] && [[ -z "${dmsg}" ]] && [[ -z "${diff}" ]]; then 58 | echo "ok ${i} - ${testname}" 59 | elif [[ "${result}" == "103" ]]; then 60 | echo "ok ${i} - ${testname}" 61 | echo "# ${testname} ** skipped - permitted **" 62 | else 63 | echo "not ok ${i} - ${testname}" 64 | case ${result} in 65 | 101) 66 | echo "# ${testname} ** skipped **" 67 | ;; 68 | 102) 69 | echo "# ${testname} ** failed **" 70 | ;; 71 | esac 72 | 73 | if [[ -n "${diff}" ]]; then 74 | echo "#" 75 | echo "# diff:" 76 | echo "#" 77 | diff -u golden/${testname} $T_RESULTS/output/${testname} | expand | sed 's/^/# /' 78 | fi 79 | 80 | if [[ -n "${dmsg}" ]]; then 81 | echo "#" 82 | echo "# dmesg:" 83 | echo "#" 84 | cat "$T_RESULTS/tmp/${testname}/dmesg.new" | sed 's/^/# /' 85 | fi 86 | fi 87 | ) >> $T_RESULTS/scoutfs.tap 88 | } 89 | -------------------------------------------------------------------------------- /tests/golden/archive-light-cycle: -------------------------------------------------------------------------------- 1 | == calculate number of files 2 | == create per mount dirs 3 | == generate phase scripts 4 | == round 1: create 5 | == round 1: online 6 | == round 1: verify 7 | == round 1: release 8 | == round 1: offline 9 | == round 1: stage 10 | == round 1: online 11 | == round 1: verify 12 | == round 1: release 13 | == round 1: offline 14 | == round 1: unlink 15 | == round 2: create 16 | == round 2: online 17 | == round 2: verify 18 | == round 2: release 19 | == round 2: offline 20 | == round 2: stage 21 | == round 2: online 22 | == round 2: verify 23 | == round 2: release 24 | == round 2: offline 25 | == round 2: unlink 26 | == round 3: create 27 | == round 3: online 28 | == round 3: verify 29 | == round 3: release 30 | == round 3: offline 31 | == round 3: stage 32 | == round 3: online 33 | == round 3: verify 34 | == round 3: release 35 | == round 3: offline 36 | == round 3: unlink 37 | -------------------------------------------------------------------------------- /tests/golden/basic-bad-mounts: -------------------------------------------------------------------------------- 1 | == prepare devices, mount point, and logs 2 | == bad devices, bad options 3 | == swapped devices 4 | == both meta devices 5 | == both data devices 6 | == good volume, bad option and good options 7 | -------------------------------------------------------------------------------- /tests/golden/basic-block-counts: -------------------------------------------------------------------------------- 1 | == single block write 2 | online: 1 3 | offline: 0 4 | st_blocks: 8 5 | == single block overwrite 6 | online: 1 7 | offline: 0 8 | st_blocks: 8 9 | == append 10 | online: 2 11 | offline: 0 12 | st_blocks: 16 13 | == release 14 | online: 0 15 | offline: 2 16 | st_blocks: 16 17 | == duplicate release 18 | online: 0 19 | offline: 2 20 | st_blocks: 16 21 | == duplicate release past i_size 22 | online: 0 23 | offline: 2 24 | st_blocks: 16 25 | == stage 26 | online: 2 27 | offline: 0 28 | st_blocks: 16 29 | == duplicate stage 30 | online: 2 31 | offline: 0 32 | st_blocks: 16 33 | == larger file 34 | online: 256 35 | offline: 0 36 | st_blocks: 2048 37 | == partial truncate 38 | online: 128 39 | offline: 0 40 | st_blocks: 1024 41 | == single sparse block 42 | online: 1 43 | offline: 0 44 | st_blocks: 8 45 | == empty file 46 | online: 0 47 | offline: 0 48 | st_blocks: 0 49 | == non-regular file 50 | online: 0 51 | offline: 0 52 | st_blocks: 0 53 | == cleanup 54 | -------------------------------------------------------------------------------- /tests/golden/basic-posix-consistency: -------------------------------------------------------------------------------- 1 | == root inode updates flow back and forth 2 | == stat of created file matches 3 | == written file contents match 4 | == overwritten file contents match 5 | == appended file contents match 6 | == fiemap matches after racey appends 7 | == unlinked file isn't found 8 | == symlink targets match 9 | /mnt/test/test/basic-posix-consistency/file.targ 10 | /mnt/test/test/basic-posix-consistency/file.targ 11 | /mnt/test/test/basic-posix-consistency/file.targ2 12 | /mnt/test/test/basic-posix-consistency/file.targ2 13 | == new xattrs are visible 14 | # file: /mnt/test/test/basic-posix-consistency/file 15 | user.xat="1" 16 | 17 | # file: /mnt/test/test/basic-posix-consistency/file 18 | user.xat="1" 19 | 20 | == modified xattrs are updated 21 | # file: /mnt/test/test/basic-posix-consistency/file 22 | user.xat="2" 23 | 24 | # file: /mnt/test/test/basic-posix-consistency/file 25 | user.xat="2" 26 | 27 | == deleted xattrs 28 | /mnt/test/test/basic-posix-consistency/file: user.xat: No such attribute 29 | /mnt/test/test/basic-posix-consistency/file: user.xat: No such attribute 30 | == readdir after modification 31 | one 32 | two 33 | three 34 | four 35 | one 36 | two 37 | three 38 | four 39 | two 40 | four 41 | two 42 | four 43 | == can delete empty dir 44 | == some easy rename cases 45 | --- file between dirs 46 | --- file within dir 47 | --- dir within dir 48 | --- overwrite file 49 | --- can't overwrite non-empty dir 50 | mv: cannot move '/mnt/test/test/basic-posix-consistency/dir/c/clobber' to '/mnt/test/test/basic-posix-consistency/dir/a/dir': Directory not empty 51 | --- can overwrite empty dir 52 | --- can rename into root 53 | == path resoluion 54 | == inode indexes match after syncing existing 55 | == inode indexes match after copying and syncing 56 | == inode indexes match after removing and syncing 57 | == concurrent creates make one file 58 | one-file 59 | == cleanup 60 | -------------------------------------------------------------------------------- /tests/golden/basic-truncate: -------------------------------------------------------------------------------- 1 | == truncate writes zeroed partial end of file block 2 | 0000000 0a79 0a79 0a79 0a79 0a79 0a79 0a79 0a79 3 | * 4 | 0006144 0000 0000 0000 0000 0000 0000 0000 0000 5 | * 6 | 0012288 7 | -------------------------------------------------------------------------------- /tests/golden/block-stale-reads: -------------------------------------------------------------------------------- 1 | == Issue scoutfs df to force block reads to trigger stale invalidation/retry 2 | counter block_cache_remove_stale changed 3 | -------------------------------------------------------------------------------- /tests/golden/change-devices: -------------------------------------------------------------------------------- 1 | == make tmp sparse data dev files 2 | == make scratch fs 3 | == small new data device fails 4 | rc: 1 5 | == check sees data device errors 6 | rc: 1 7 | rc: 0 8 | == preparing while mounted fails 9 | rc: 1 10 | == preparing without recovery fails 11 | rc: 1 12 | == check sees metadata errors 13 | rc: 1 14 | rc: 1 15 | == preparing with file data fails 16 | rc: 1 17 | == preparing after emptied 18 | rc: 0 19 | == checks pass 20 | rc: 0 21 | rc: 0 22 | == using prepared 23 | == preparing larger and resizing 24 | rc: 0 25 | equal_prepared 26 | large_prepared 27 | resized larger test rc: 0 28 | == cleanup 29 | -------------------------------------------------------------------------------- /tests/golden/client-unmount-recovery: -------------------------------------------------------------------------------- 1 | == 60s of unmounting non-quorum clients during recovery 2 | -------------------------------------------------------------------------------- /tests/golden/createmany-large-names: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/versity/scoutfs/9741d40e10bb92cf9b77658cc7a282a67e403a6c/tests/golden/createmany-large-names -------------------------------------------------------------------------------- /tests/golden/createmany-parallel: -------------------------------------------------------------------------------- 1 | Run createmany in /mnt/test/test/createmany-parallel/0 2 | Run createmany in /mnt/test/test/createmany-parallel/1 3 | Run createmany in /mnt/test/test/createmany-parallel/2 4 | Run createmany in /mnt/test/test/createmany-parallel/3 5 | -------------------------------------------------------------------------------- /tests/golden/createmany-parallel-mounts: -------------------------------------------------------------------------------- 1 | == measure initial createmany 2 | == measure initial createmany 3 | == measure two concurrent createmany runs 4 | == cleanup 5 | -------------------------------------------------------------------------------- /tests/golden/createmany-rename-large-dir: -------------------------------------------------------------------------------- 1 | == create large directory with 1220608 files 2 | == randomly renaming 5000 files 3 | -------------------------------------------------------------------------------- /tests/golden/cross-mount-data-free: -------------------------------------------------------------------------------- 1 | == repeated cross-mount alloc+free, totalling 2x free 2 | == remove empty test file 3 | -------------------------------------------------------------------------------- /tests/golden/dirent-consistency: -------------------------------------------------------------------------------- 1 | == create per node dirs 2 | == touch files on each node 3 | == recreate the files 4 | == turn the files into directories 5 | == rename parent dirs 6 | == rename parent dirs back 7 | == create some hard links 8 | == recreate one of the hard links 9 | == delete the remaining hard link 10 | == race to blow everything away 11 | -------------------------------------------------------------------------------- /tests/golden/enospc: -------------------------------------------------------------------------------- 1 | == prepare directories and files 2 | == fallocate until enospc 3 | == remove all the files and verify free data blocks 4 | == make small meta fs 5 | == create large xattrs until we fill up metadata 6 | == remove files with xattrs after enospc 7 | == make sure we can create again 8 | == cleanup small meta fs 9 | -------------------------------------------------------------------------------- /tests/golden/export-get-name-parent: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/versity/scoutfs/9741d40e10bb92cf9b77658cc7a282a67e403a6c/tests/golden/export-get-name-parent -------------------------------------------------------------------------------- /tests/golden/export-lookup-evict-race: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/versity/scoutfs/9741d40e10bb92cf9b77658cc7a282a67e403a6c/tests/golden/export-lookup-evict-race -------------------------------------------------------------------------------- /tests/golden/fallocate: -------------------------------------------------------------------------------- 1 | == creating reasonably large per-mount files 2 | == 10s of racing cold reads and fallocate nop 3 | == cleaning up files 4 | -------------------------------------------------------------------------------- /tests/golden/fence-and-reclaim: -------------------------------------------------------------------------------- 1 | == make sure all mounts can see each other 2 | == force unmount one client, connection timeout, fence nop, mount 3 | == force unmount all non-server, connection timeout, fence nop, mount 4 | == force unmount server, quorum elects new leader, fence nop, mount 5 | == force unmount everything, new server fences all previous 6 | -------------------------------------------------------------------------------- /tests/golden/format-version-forward-back: -------------------------------------------------------------------------------- 1 | == ensuring utils and module for old versions 2 | == unmounting test fs and removing test module 3 | == testing combinations of old and new format versions 4 | == restoring test module and mount 5 | -------------------------------------------------------------------------------- /tests/golden/get-referring-entries: -------------------------------------------------------------------------------- 1 | == root inode returns nothing 2 | == crazy large unused inode does nothing 3 | == basic entry 4 | file 5 | == rename 6 | renamed 7 | == hard link 8 | file 9 | link 10 | == removal 11 | == different dirs 12 | == file types 13 | type b name block 14 | type c name char 15 | type d name dir 16 | type f name file 17 | type l name symlink 18 | == all name lengths work 19 | -------------------------------------------------------------------------------- /tests/golden/inode-deletion: -------------------------------------------------------------------------------- 1 | == basic unlink deletes 2 | ino found in dseq index 3 | ino not found in dseq index 4 | == local open-unlink waits for close to delete 5 | contents after rm: contents 6 | ino found in dseq index 7 | ino not found in dseq index 8 | == multiple local opens are protected 9 | contents after rm 1: contents 10 | contents after rm 2: contents 11 | ino found in dseq index 12 | ino not found in dseq index 13 | == remote unopened unlink deletes 14 | ino not found in dseq index 15 | ino not found in dseq index 16 | == unlink wait for open on other mount 17 | mount 0 contents after mount 1 rm: contents 18 | ino found in dseq index 19 | ino found in dseq index 20 | stat: cannot stat '/mnt/test/test/inode-deletion/file': No such file or directory 21 | ino not found in dseq index 22 | ino not found in dseq index 23 | == lots of deletions use one open map 24 | == open files survive remote scanning orphans 25 | mount 0 contents after mount 1 remounted: contents 26 | ino not found in dseq index 27 | ino not found in dseq index 28 | -------------------------------------------------------------------------------- /tests/golden/inode-items-updated: -------------------------------------------------------------------------------- 1 | == create files and sync 2 | == modify files 3 | == mount and unmount 4 | == verify files 5 | -------------------------------------------------------------------------------- /tests/golden/large-fragmented-free: -------------------------------------------------------------------------------- 1 | == setting longer hung task timeout 2 | == creating fragmented extents 3 | == unlink file with moved extents to free extents per block 4 | == cleanup 5 | -------------------------------------------------------------------------------- /tests/golden/lock-ex-race-processes: -------------------------------------------------------------------------------- 1 | === setup files === 2 | === ping-pong xattr ops === 3 | -------------------------------------------------------------------------------- /tests/golden/lock-pr-cw-conflict: -------------------------------------------------------------------------------- 1 | == race writing and index walking 2 | -------------------------------------------------------------------------------- /tests/golden/lock-recover-invalidate: -------------------------------------------------------------------------------- 1 | == starting background invalidating read/write load 2 | == 60s of lock recovery during invalidating load 3 | == stopping background load 4 | -------------------------------------------------------------------------------- /tests/golden/lock-refleak: -------------------------------------------------------------------------------- 1 | == make test dir 2 | == do enough stuff to make lock leaks visible 3 | == make sure nothing has leaked 4 | -------------------------------------------------------------------------------- /tests/golden/lock-rever-invalidate: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/versity/scoutfs/9741d40e10bb92cf9b77658cc7a282a67e403a6c/tests/golden/lock-rever-invalidate -------------------------------------------------------------------------------- /tests/golden/lock-revoke-getcwd: -------------------------------------------------------------------------------- 1 | === getcwd after lock revocation 2 | trigger statfs_lock_purge armed: 1 3 | -------------------------------------------------------------------------------- /tests/golden/lock-shrink-consistency: -------------------------------------------------------------------------------- 1 | === setup test file === 2 | # file: /mnt/test/test/lock-shrink-consistency/dir/file 3 | user.test="aaa" 4 | 5 | === commit dirty trans and revoke lock === 6 | trigger statfs_lock_purge armed: 1 7 | trigger statfs_lock_purge after it fired: 0 8 | === change xattr on other mount === 9 | # file: /mnt/test/test/lock-shrink-consistency/dir/file 10 | user.test="bbb" 11 | 12 | === verify new xattr under new lock on first mount === 13 | # file: /mnt/test/test/lock-shrink-consistency/dir/file 14 | user.test="bbb" 15 | 16 | -------------------------------------------------------------------------------- /tests/golden/lock-shrink-read-race: -------------------------------------------------------------------------------- 1 | === setup 2 | === spin reading and shrinking 3 | -------------------------------------------------------------------------------- /tests/golden/mkdir-rename-rmdir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/versity/scoutfs/9741d40e10bb92cf9b77658cc7a282a67e403a6c/tests/golden/mkdir-rename-rmdir -------------------------------------------------------------------------------- /tests/golden/mmap: -------------------------------------------------------------------------------- 1 | == mmap_stress 2 | thread 0 complete 3 | thread 1 complete 4 | thread 2 complete 5 | thread 3 complete 6 | thread 4 complete 7 | == basic mmap/read/write consistency checks 8 | == mmap read from offline extent 9 | 0: offset: 0 length: 2 flags: O.L 10 | extents: 1 11 | 1 12 | 00000200: ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea ................ 13 | 0 14 | 0: offset: 0 length: 2 flags: ..L 15 | extents: 1 16 | == mmap write to an offline extent 17 | 0: offset: 0 length: 2 flags: O.L 18 | extents: 1 19 | 1 20 | 0 21 | 0: offset: 0 length: 2 flags: ..L 22 | extents: 1 23 | 00000000 ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea |................| 24 | 00000010 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 11 |................| 25 | 00000020 ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea ea |................| 26 | 00000030 27 | == done 28 | -------------------------------------------------------------------------------- /tests/golden/mount-unmount-race: -------------------------------------------------------------------------------- 1 | == create per mount files 2 | == 30s of racing random mount/umount 3 | == mounting any unmounted 4 | -------------------------------------------------------------------------------- /tests/golden/move-blocks: -------------------------------------------------------------------------------- 1 | == build test files 2 | == wrapped offsets should fail 3 | ioctl failed on '/mnt/test/test/move-blocks/to': Value too large for defined data type (75) 4 | scoutfs: move-blocks failed: Value too large for defined data type (75) 5 | ioctl failed on '/mnt/test/test/move-blocks/to': Value too large for defined data type (75) 6 | scoutfs: move-blocks failed: Value too large for defined data type (75) 7 | == specifying same file fails 8 | ioctl failed on '/mnt/test/test/move-blocks/hardlink': Invalid argument (22) 9 | scoutfs: move-blocks failed: Invalid argument (22) 10 | == specifying files in other file systems fails 11 | ioctl failed on '/mnt/test/test/move-blocks/to': Invalid cross-device link (18) 12 | scoutfs: move-blocks failed: Invalid cross-device link (18) 13 | == offsets must be multiples of 4KB 14 | ioctl failed on '/mnt/test/test/move-blocks/to': Invalid argument (22) 15 | scoutfs: move-blocks failed: Invalid argument (22) 16 | ioctl failed on '/mnt/test/test/move-blocks/to': Invalid argument (22) 17 | scoutfs: move-blocks failed: Invalid argument (22) 18 | ioctl failed on '/mnt/test/test/move-blocks/to': Invalid argument (22) 19 | scoutfs: move-blocks failed: Invalid argument (22) 20 | == can't move onto existing extent 21 | ioctl failed on '/mnt/test/test/move-blocks/to': Invalid argument (22) 22 | scoutfs: move-blocks failed: Invalid argument (22) 23 | == can't move between files with offline extents 24 | ioctl failed on '/mnt/test/test/move-blocks/to': No data available (61) 25 | scoutfs: move-blocks failed: No data available (61) 26 | ioctl failed on '/mnt/test/test/move-blocks/to': No data available (61) 27 | scoutfs: move-blocks failed: No data available (61) 28 | == basic moves work 29 | == moving final partial block sets partial i_size 30 | 123 31 | == moving updates inode fields 32 | == moving blocks backwards works 33 | == combine many files into one 34 | -------------------------------------------------------------------------------- /tests/golden/o_tmpfile: -------------------------------------------------------------------------------- 1 | == non-acl O_TMPFILE creation honors umask 2 | umask 022 3 | fstat after open(0777): 0100755 4 | stat after linkat: 0100755 5 | umask 077 6 | fstat after open(0777): 0100700 7 | stat after linkat: 0100700 8 | == stage from tmpfile 9 | total file size 33669120 10 | 00000000 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 |AAAAAAAAAAAAAAAA| 11 | * 12 | 00400000 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 |BBBBBBBBBBBBBBBB| 13 | * 14 | 00801000 43 43 43 43 43 43 43 43 43 43 43 43 43 43 43 43 |CCCCCCCCCCCCCCCC| 15 | * 16 | 00c03000 44 44 44 44 44 44 44 44 44 44 44 44 44 44 44 44 |DDDDDDDDDDDDDDDD| 17 | * 18 | 01006000 45 45 45 45 45 45 45 45 45 45 45 45 45 45 45 45 |EEEEEEEEEEEEEEEE| 19 | * 20 | 0140a000 46 46 46 46 46 46 46 46 46 46 46 46 46 46 46 46 |FFFFFFFFFFFFFFFF| 21 | * 22 | 0180f000 47 47 47 47 47 47 47 47 47 47 47 47 47 47 47 47 |GGGGGGGGGGGGGGGG| 23 | * 24 | 01c15000 48 48 48 48 48 48 48 48 48 48 48 48 48 48 48 48 |HHHHHHHHHHHHHHHH| 25 | * 26 | 0201c000 27 | -------------------------------------------------------------------------------- /tests/golden/offline-extent-waiting: -------------------------------------------------------------------------------- 1 | == create files 2 | == waiter shows up in ioctl 3 | offline waiting should be empty: 4 | 0 5 | offline waiting should now have one known entry: 6 | == multiple waiters on same block listed once 7 | offline waiting still has one known entry: 8 | == different blocks show up 9 | offline waiting now has two known entries: 10 | == staging wakes everyone 11 | offline waiting should be empty again: 12 | 0 13 | == interruption does no harm 14 | offline waiting should now have one known entry: 15 | offline waiting should be empty again: 16 | 0 17 | == EIO injection for waiting readers works 18 | offline waiting should now have two known entries: 19 | 2 20 | data_wait_err found 2 waiters. 21 | offline waiting should now have 0 known entries: 22 | 0 23 | dd: error reading '/mnt/test/test/offline-extent-waiting/dir/file': Input/output error 24 | 0+0 records in 25 | 0+0 records out 26 | dd: error reading '/mnt/test/test/offline-extent-waiting/dir/file': Input/output error 27 | 0+0 records in 28 | 0+0 records out 29 | offline waiting should be empty again: 30 | 0 31 | == readahead while offline does no harm 32 | == waiting on interesting blocks works 33 | offline waiting is empty at block 0 34 | 0 35 | offline waiting is empty at block 1 36 | 0 37 | offline waiting is empty at block 128 38 | 0 39 | offline waiting is empty at block 129 40 | 0 41 | offline waiting is empty at block 254 42 | 0 43 | offline waiting is empty at block 255 44 | 0 45 | == contents match when staging blocks forward 46 | == contents match when staging blocks backwards 47 | == truncate to same size doesn't wait 48 | offline wating should be empty: 49 | 0 50 | == truncating does wait 51 | truncate should be waiting for first block: 52 | trunate should no longer be waiting: 53 | 0 54 | == writing waits 55 | should be waiting for write 56 | == cleanup 57 | -------------------------------------------------------------------------------- /tests/golden/orphan-inodes: -------------------------------------------------------------------------------- 1 | == test our inode existance function 2 | == unlinked and opened inodes still exist 3 | == orphan from failed evict deletion is picked up 4 | == orphaned inos in all mounts all deleted 5 | == 30s of racing evict deletion, orphan scanning, and open by handle 6 | -------------------------------------------------------------------------------- /tests/golden/persistent-item-vers: -------------------------------------------------------------------------------- 1 | == advance lock version by creating unrelated files 2 | == create before file version 3 | == verify before version, touch after version 4 | == verify after version 5 | -------------------------------------------------------------------------------- /tests/golden/projects: -------------------------------------------------------------------------------- 1 | == default new files don't have project 2 | 0 3 | == set new project on files and dirs 4 | 8675309 5 | 8675309 6 | == non-root can see id 7 | 8675309 8 | == can use IDs around long width limits 9 | 2147483647 10 | 2147483648 11 | 4294967295 12 | 9223372036854775807 13 | 9223372036854775808 14 | 18446744073709551615 15 | == created files and dirs inherit project id 16 | 8675309 17 | 8675309 18 | == inheritance continues 19 | 8675309 20 | == clearing project id stops inheritance 21 | 0 22 | 0 23 | == o_tmpfile creations inherit dir 24 | 8675309 25 | -------------------------------------------------------------------------------- /tests/golden/quorum-heartbeat-timeout: -------------------------------------------------------------------------------- 1 | == bad timeout values fail 2 | == bad mount option fails 3 | == mount option 4 | == sysfs 5 | == reset all options 6 | -------------------------------------------------------------------------------- /tests/golden/quota: -------------------------------------------------------------------------------- 1 | == prepare dir with write perm for test ids 2 | == test assumes starting with no rules, empty list 3 | == add rule 4 | 7 13,L,- 15,L,- 17,L,- I 33 - 5 | == list is empty again after delete 6 | == can change limits without deleting 7 | 1 1,L,- 1,L,- 1,L,- I 100 - 8 | 1 1,L,- 1,L,- 1,L,- I 101 - 9 | 1 1,L,- 1,L,- 1,L,- I 99 - 10 | == wipe and restore rules in bulk 11 | 7 15,L,- 0,L,- 0,L,- I 33 - 12 | 7 14,L,- 0,L,- 0,L,- I 33 - 13 | 7 13,L,- 0,L,- 0,L,- I 33 - 14 | 7 12,L,- 0,L,- 0,L,- I 33 - 15 | 7 11,L,- 0,L,- 0,L,- I 33 - 16 | 7 10,L,- 0,L,- 0,L,- I 33 - 17 | 7 15,L,- 0,L,- 0,L,- I 33 - 18 | 7 14,L,- 0,L,- 0,L,- I 33 - 19 | 7 13,L,- 0,L,- 0,L,- I 33 - 20 | 7 12,L,- 0,L,- 0,L,- I 33 - 21 | 7 11,L,- 0,L,- 0,L,- I 33 - 22 | 7 10,L,- 0,L,- 0,L,- I 33 - 23 | == default rule prevents file creation 24 | touch: cannot touch '/mnt/test/test/quota/dir/file': Disk quota exceeded 25 | == decreasing totl allows file creation again 26 | == attr selecting rules prevent creation 27 | touch: cannot touch '/mnt/test/test/quota/dir/file': Disk quota exceeded 28 | touch: cannot touch '/mnt/test/test/quota/dir/file': Disk quota exceeded 29 | == multi attr selecting doesn't prevent partial 30 | touch: cannot touch '/mnt/test/test/quota/dir/file': Disk quota exceeded 31 | == op differentiates 32 | == higher priority rule applies 33 | touch: cannot touch '/mnt/test/test/quota/dir/file': Disk quota exceeded 34 | == data rules with total and count prevent write and fallocate 35 | dd: error writing '/mnt/test/test/quota/dir/file': Disk quota exceeded 36 | fallocate: fallocate failed: Disk quota exceeded 37 | dd: error writing '/mnt/test/test/quota/dir/file': Disk quota exceeded 38 | fallocate: fallocate failed: Disk quota exceeded 39 | == added rules work after bulk restore 40 | touch: cannot touch '/mnt/test/test/quota/dir/file': Disk quota exceeded 41 | == cleanup 42 | -------------------------------------------------------------------------------- /tests/golden/renameat2-noreplace: -------------------------------------------------------------------------------- 1 | === renameat2 noreplace flag test 2 | === run two asynchronous calls to renameat2 NOREPLACE 3 | -------------------------------------------------------------------------------- /tests/golden/resize-devices: -------------------------------------------------------------------------------- 1 | == make initial small fs 2 | == 0s do nothing 3 | == shrinking fails 4 | resize_devices ioctl failed: Invalid argument (22) 5 | scoutfs: resize-devices failed: Invalid argument (22) 6 | resize_devices ioctl failed: Invalid argument (22) 7 | scoutfs: resize-devices failed: Invalid argument (22) 8 | resize_devices ioctl failed: Invalid argument (22) 9 | scoutfs: resize-devices failed: Invalid argument (22) 10 | == existing sizes do nothing 11 | == growing outside device fails 12 | resize_devices ioctl failed: Invalid argument (22) 13 | scoutfs: resize-devices failed: Invalid argument (22) 14 | resize_devices ioctl failed: Invalid argument (22) 15 | scoutfs: resize-devices failed: Invalid argument (22) 16 | resize_devices ioctl failed: Invalid argument (22) 17 | scoutfs: resize-devices failed: Invalid argument (22) 18 | == resizing meta works 19 | == resizing data works 20 | == shrinking back fails 21 | resize_devices ioctl failed: Invalid argument (22) 22 | scoutfs: resize-devices failed: Invalid argument (22) 23 | resize_devices ioctl failed: Invalid argument (22) 24 | scoutfs: resize-devices failed: Invalid argument (22) 25 | == resizing again does nothing 26 | == resizing to full works 27 | == cleanup extra fs 28 | -------------------------------------------------------------------------------- /tests/golden/retention-basic: -------------------------------------------------------------------------------- 1 | == setting retention on dir fails 2 | attr_x ioctl failed on '/mnt/test/test/retention-basic': Invalid argument (22) 3 | scoutfs: set-attr-x failed: Invalid argument (22) 4 | == set retention 5 | == get-attr-x shows retention 6 | 1 7 | == unpriv can't clear retention 8 | attr_x ioctl failed on '/mnt/test/test/retention-basic/file-1': Operation not permitted (1) 9 | scoutfs: set-attr-x failed: Operation not permitted (1) 10 | == can set hidden scoutfs xattr in retention 11 | == setting user. xattr fails in retention 12 | setfattr: /mnt/test/test/retention-basic/file-1: Operation not permitted 13 | == file deletion fails in retention 14 | rm: cannot remove '/mnt/test/test/retention-basic/file-1': Operation not permitted 15 | == file rename fails in retention 16 | mv: cannot move '/mnt/test/test/retention-basic/file-1' to '/mnt/test/test/retention-basic/file-2': Operation not permitted 17 | == file write fails in retention 18 | date: write error: Operation not permitted 19 | == file truncate fails in retention 20 | truncate: failed to truncate '/mnt/test/test/retention-basic/file-1' at 0 bytes: Operation not permitted 21 | == setattr fails in retention 22 | touch: setting times of '/mnt/test/test/retention-basic/file-1': Operation not permitted 23 | == clear retention 24 | == file write 25 | == file rename 26 | == setattr 27 | == xattr deletion 28 | == cleanup 29 | -------------------------------------------------------------------------------- /tests/golden/setattr_more: -------------------------------------------------------------------------------- 1 | == 0 data_version arg fails 2 | setattr: data version must not be 0 3 | Try `setattr --help' or `setattr --usage' for more information. 4 | == args must specify size and offline 5 | setattr: must provide size if using --offline option 6 | Try `setattr --help' or `setattr --usage' for more information. 7 | == only works on regular files 8 | failed to open '/mnt/test/test/setattr_more/dir': Is a directory (21) 9 | scoutfs: setattr failed: Is a directory (21) 10 | setattr_more ioctl failed on '/mnt/test/test/setattr_more/char': Inappropriate ioctl for device (25) 11 | scoutfs: setattr failed: Inappropriate ioctl for device (25) 12 | == non-zero file size fails 13 | setattr_more ioctl failed on '/mnt/test/test/setattr_more/file': Invalid argument (22) 14 | scoutfs: setattr failed: Invalid argument (22) 15 | == non-zero file data_version fails 16 | setattr_more ioctl failed on '/mnt/test/test/setattr_more/file': Invalid argument (22) 17 | scoutfs: setattr failed: Invalid argument (22) 18 | == large size is set 19 | 578437695752307201 20 | == large data_version is set 21 | 578437695752307201 22 | == large ctime is set 23 | 1972-02-19 00:06:25.999999999 +0000 24 | == large offline extents are created 25 | 0: offset: 0 0 length: 10007 flags: O.L 26 | extents: 1 27 | == correct offline extent length 28 | 976563 29 | == omitting data_version should not fail 30 | -------------------------------------------------------------------------------- /tests/golden/setup-error-teardown: -------------------------------------------------------------------------------- 1 | == interrupt waiting mount 2 | -------------------------------------------------------------------------------- /tests/golden/simple-inode-index: -------------------------------------------------------------------------------- 1 | == dirs shouldn't appear in data_seq queries 2 | == two created files are present and come after each other 3 | found first 4 | found second 5 | == unlinked entries must not be present 6 | == dirty inodes can not be present 7 | == changing metadata must increase meta seq 8 | == changing contents must increase data seq 9 | == make sure dirtying doesn't livelock walk 10 | == concurrent update attempts maintain single entries 11 | -------------------------------------------------------------------------------- /tests/golden/simple-staging: -------------------------------------------------------------------------------- 1 | == create/release/stage single block file 2 | 0: offset: 0 0 length: 1 flags: O.L 3 | extents: 1 4 | == create/release/stage larger file 5 | 0: offset: 0 0 length: 4096 flags: O.L 6 | extents: 1 7 | == multiple release,drop_cache,stage cycles 8 | == release+stage shouldn't change stat, data seq or vers 9 | == stage does change meta_seq 10 | == can't use stage to extend online file 11 | stage: must provide file version with --data-version 12 | Try `stage --help' or `stage --usage' for more information. 13 | == wrapped region fails 14 | stage returned -1, not 8192: error Invalid argument (22) 15 | scoutfs: stage failed: Input/output error (5) 16 | == non-block aligned offset fails 17 | stage returned -1, not 4095: error Invalid argument (22) 18 | scoutfs: stage failed: Input/output error (5) 19 | 0: offset: 0 0 length: 1 flags: O.L 20 | extents: 1 21 | == non-block aligned len within block fails 22 | stage returned -1, not 1024: error Invalid argument (22) 23 | scoutfs: stage failed: Input/output error (5) 24 | 0: offset: 0 0 length: 1 flags: O.L 25 | extents: 1 26 | == partial final block that writes to i_size does work 27 | == zero length stage doesn't bring blocks online 28 | 0: offset: 0 0 length: 100 flags: O.L 29 | extents: 1 30 | == stage of non-regular file fails 31 | ioctl failed: Inappropriate ioctl for device (25) 32 | stage: must provide file version with --data-version 33 | Try `stage --help' or `stage --usage' for more information. 34 | -------------------------------------------------------------------------------- /tests/golden/simple-xattr-unit: -------------------------------------------------------------------------------- 1 | === XATTR_ flag combinations 2 | dumb_setxattr -p /mnt/test/test/simple-xattr-unit/file -n user.test -v val -c -r 3 | returned -1 errno 22 (Invalid argument) 4 | dumb_setxattr -p /mnt/test/test/simple-xattr-unit/file -n user.test -v val -r 5 | returned -1 errno 61 (No data available) 6 | dumb_setxattr -p /mnt/test/test/simple-xattr-unit/file -n user.test -v val -c 7 | returned 0 8 | dumb_setxattr -p /mnt/test/test/simple-xattr-unit/file -n user.test -v val -c 9 | returned -1 errno 17 (File exists) 10 | dumb_setxattr -p /mnt/test/test/simple-xattr-unit/file -n user.test -v val -r 11 | returned 0 12 | === bad lengths 13 | setfattr: /mnt/test/test/simple-xattr-unit/file: Operation not supported 14 | setfattr: /mnt/test/test/simple-xattr-unit/file: Numerical result out of range 15 | setfattr: /mnt/test/test/simple-xattr-unit/file: Numerical result out of range 16 | setfattr: /mnt/test/test/simple-xattr-unit/file: Argument list too long 17 | === good length boundaries 18 | === 500 random lengths 19 | === alternate val size between interesting sizes 20 | -------------------------------------------------------------------------------- /tests/golden/srch-basic-functionality: -------------------------------------------------------------------------------- 1 | == create new xattrs 2 | == update existing xattr 3 | == remove an xattr 4 | == remove xattr with files 5 | == trigger small log merges by rotating single block with unmount 6 | == create entries in current log 7 | == delete small fraction 8 | == remove files 9 | == create entries that exceed one log 10 | == delete fractions in phases 11 | == remove files 12 | == create entries for exceed search entry limit 13 | == delete half 14 | == entirely remove third batch 15 | -------------------------------------------------------------------------------- /tests/golden/srch-safe-merge-pos: -------------------------------------------------------------------------------- 1 | == initialize per-mount values 2 | == arm compaction triggers 3 | trigger srch_compact_logs_pad_safe armed: 1 4 | trigger srch_merge_stop_safe armed: 1 5 | trigger srch_compact_logs_pad_safe armed: 1 6 | trigger srch_merge_stop_safe armed: 1 7 | trigger srch_compact_logs_pad_safe armed: 1 8 | trigger srch_merge_stop_safe armed: 1 9 | trigger srch_compact_logs_pad_safe armed: 1 10 | trigger srch_merge_stop_safe armed: 1 11 | trigger srch_compact_logs_pad_safe armed: 1 12 | trigger srch_merge_stop_safe armed: 1 13 | == compact more often 14 | == create padded sorted inputs by forcing log rotation 15 | trigger srch_force_log_rotate armed: 1 16 | trigger srch_force_log_rotate armed: 1 17 | trigger srch_force_log_rotate armed: 1 18 | trigger srch_force_log_rotate armed: 1 19 | trigger srch_compact_logs_pad_safe armed: 1 20 | trigger srch_force_log_rotate armed: 1 21 | trigger srch_force_log_rotate armed: 1 22 | trigger srch_force_log_rotate armed: 1 23 | trigger srch_force_log_rotate armed: 1 24 | trigger srch_compact_logs_pad_safe armed: 1 25 | trigger srch_force_log_rotate armed: 1 26 | trigger srch_force_log_rotate armed: 1 27 | trigger srch_force_log_rotate armed: 1 28 | trigger srch_force_log_rotate armed: 1 29 | trigger srch_compact_logs_pad_safe armed: 1 30 | trigger srch_force_log_rotate armed: 1 31 | trigger srch_force_log_rotate armed: 1 32 | trigger srch_force_log_rotate armed: 1 33 | trigger srch_force_log_rotate armed: 1 34 | trigger srch_compact_logs_pad_safe armed: 1 35 | == compaction of padded should stop at safe 36 | == verify no compaction errors 37 | == cleanup 38 | -------------------------------------------------------------------------------- /tests/golden/stage-multi-part: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/versity/scoutfs/9741d40e10bb92cf9b77658cc7a282a67e403a6c/tests/golden/stage-multi-part -------------------------------------------------------------------------------- /tests/golden/stage-release-race-alloc: -------------------------------------------------------------------------------- 1 | == create initial files 2 | == race stage and release 3 | == cleanup 4 | -------------------------------------------------------------------------------- /tests/golden/totl-xattr-tag: -------------------------------------------------------------------------------- 1 | == single file 2 | 1.2.3 = 1, 1 3 | 4.5.6 = 1, 1 4 | == multiple files add up 5 | 1.2.3 = 2, 2 6 | 4.5.6 = 2, 2 7 | == removing xattr updates total 8 | 1.2.3 = 2, 2 9 | 4.5.6 = 1, 1 10 | == updating xattr updates total 11 | 1.2.3 = 11, 2 12 | 4.5.6 = 1, 1 13 | == removing files update total 14 | 1.2.3 = 10, 1 15 | == multiple files/names in one transaction 16 | 1.2.3 = 55, 10 17 | == testing invalid names 18 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 19 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 20 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 21 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 22 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 23 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 24 | == testing invalid values 25 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 26 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 27 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 28 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 29 | setfattr: /mnt/test/test/totl-xattr-tag/invalid: Invalid argument 30 | == larger population that could merge 31 | -------------------------------------------------------------------------------- /tests/sequence: -------------------------------------------------------------------------------- 1 | export-get-name-parent.sh 2 | basic-block-counts.sh 3 | basic-bad-mounts.sh 4 | basic-posix-acl.sh 5 | inode-items-updated.sh 6 | simple-inode-index.sh 7 | simple-staging.sh 8 | simple-release-extents.sh 9 | simple-readdir.sh 10 | get-referring-entries.sh 11 | fallocate.sh 12 | basic-truncate.sh 13 | data-prealloc.sh 14 | setattr_more.sh 15 | offline-extent-waiting.sh 16 | move-blocks.sh 17 | projects.sh 18 | large-fragmented-free.sh 19 | format-version-forward-back.sh 20 | enospc.sh 21 | mmap.sh 22 | srch-safe-merge-pos.sh 23 | srch-basic-functionality.sh 24 | simple-xattr-unit.sh 25 | retention-basic.sh 26 | totl-xattr-tag.sh 27 | quota.sh 28 | lock-refleak.sh 29 | lock-shrink-consistency.sh 30 | lock-shrink-read-race.sh 31 | lock-pr-cw-conflict.sh 32 | lock-revoke-getcwd.sh 33 | lock-recover-invalidate.sh 34 | export-lookup-evict-race.sh 35 | createmany-parallel.sh 36 | createmany-large-names.sh 37 | createmany-rename-large-dir.sh 38 | stage-release-race-alloc.sh 39 | stage-multi-part.sh 40 | o_tmpfile.sh 41 | basic-posix-consistency.sh 42 | dirent-consistency.sh 43 | mkdir-rename-rmdir.sh 44 | lock-ex-race-processes.sh 45 | cross-mount-data-free.sh 46 | persistent-item-vers.sh 47 | setup-error-teardown.sh 48 | resize-devices.sh 49 | change-devices.sh 50 | fence-and-reclaim.sh 51 | quorum-heartbeat-timeout.sh 52 | orphan-inodes.sh 53 | mount-unmount-race.sh 54 | client-unmount-recovery.sh 55 | createmany-parallel-mounts.sh 56 | archive-light-cycle.sh 57 | block-stale-reads.sh 58 | inode-deletion.sh 59 | renameat2-noreplace.sh 60 | xfstests.sh 61 | -------------------------------------------------------------------------------- /tests/src/create_xattr_loop.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | static void exit_usage(void) 12 | { 13 | printf(" -h/-? output this usage message and exit\n" 14 | " -c number of xattrs to create\n" 15 | " -n xattr name prefix, -NR is appended\n" 16 | " -p string with path to file with xattrs\n" 17 | " -s xattr value size\n"); 18 | exit(1); 19 | } 20 | 21 | int main(int argc, char **argv) 22 | { 23 | char *pref = NULL; 24 | char *path = NULL; 25 | char *val; 26 | char *name; 27 | unsigned long long count = 0; 28 | unsigned long long size = 0; 29 | unsigned long long i; 30 | int ret; 31 | int c; 32 | 33 | while ((c = getopt(argc, argv, "+c:n:p:s:")) != -1) { 34 | 35 | switch (c) { 36 | case 'c': 37 | count = strtoull(optarg, NULL, 0); 38 | break; 39 | case 'n': 40 | pref = strdup(optarg); 41 | break; 42 | case 'p': 43 | path = strdup(optarg); 44 | break; 45 | case 's': 46 | size = strtoull(optarg, NULL, 0); 47 | break; 48 | case '?': 49 | printf("unknown argument: %c\n", optind); 50 | case 'h': 51 | exit_usage(); 52 | } 53 | } 54 | 55 | if (count == 0) { 56 | printf("specify count of xattrs to create with -c\n"); 57 | exit(1); 58 | } 59 | 60 | if (count == ULLONG_MAX) { 61 | printf("invalid -c count\n"); 62 | exit(1); 63 | } 64 | 65 | if (size == 0) { 66 | printf("specify xattrs value size with -s\n"); 67 | exit(1); 68 | } 69 | 70 | if (size == ULLONG_MAX || size < 2) { 71 | printf("invalid -s size\n"); 72 | exit(1); 73 | } 74 | 75 | if (path == NULL) { 76 | printf("specify path to file with -p\n"); 77 | exit(1); 78 | } 79 | 80 | if (pref == NULL) { 81 | printf("specify xattr name prefix string with -n\n"); 82 | exit(1); 83 | } 84 | 85 | ret = snprintf(NULL, 0, "%s-%llu", pref, ULLONG_MAX) + 1; 86 | name = malloc(ret); 87 | if (!name) { 88 | printf("couldn't allocate xattr name buffer\n"); 89 | exit(1); 90 | } 91 | 92 | val = malloc(size); 93 | if (!val) { 94 | printf("couldn't allocate xattr value buffer\n"); 95 | exit(1); 96 | } 97 | 98 | memset(val, 'a', size - 1); 99 | val[size - 1] = '\0'; 100 | 101 | for (i = 0; i < count; i++) { 102 | sprintf(name, "%s-%llu", pref, i); 103 | 104 | ret = setxattr(path, name, val, size, 0); 105 | if (ret) { 106 | printf("returned %d errno %d (%s)\n", 107 | ret, errno, strerror(errno)); 108 | return 1; 109 | } 110 | } 111 | 112 | return 0; 113 | } 114 | -------------------------------------------------------------------------------- /tests/src/dumb_renameat2.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #ifndef RENAMEAT2_EXIST 7 | #include 8 | #include 9 | 10 | #if !defined(SYS_renameat2) && defined(__x86_64__) 11 | #define SYS_renameat2 316 /* from arch/x86/entry/syscalls/syscall_64.tbl */ 12 | #endif 13 | 14 | static int renameat2(int olddfd, const char *old_dir, 15 | int newdfd, const char *new_dir, 16 | unsigned int flags) 17 | { 18 | #ifdef SYS_renameat2 19 | return syscall(SYS_renameat2, olddfd, old_dir, newdfd, new_dir, flags); 20 | #else 21 | errno = ENOSYS; 22 | return -1; 23 | #endif 24 | } 25 | #endif 26 | 27 | #ifndef RENAME_NOREPLACE 28 | #define RENAME_NOREPLACE (1 << 0) /* Don't overwrite newpath of rename */ 29 | #endif 30 | #ifndef RENAME_EXCHANGE 31 | #define RENAME_EXCHANGE (1 << 1) /* Exchange oldpath and newpath */ 32 | #endif 33 | #ifndef RENAME_WHITEOUT 34 | #define RENAME_WHITEOUT (1 << 2) /* Whiteout oldpath */ 35 | #endif 36 | 37 | static void exit_usage(char **argv) 38 | { 39 | fprintf(stderr, 40 | "usage: %s [-n|-x|-w] old_path new_path\n" 41 | " -n noreplace\n" 42 | " -x exchange\n" 43 | " -w whiteout\n", argv[0]); 44 | 45 | exit(1); 46 | } 47 | 48 | int main(int argc, char **argv) 49 | { 50 | const char *old_path = NULL; 51 | const char *new_path = NULL; 52 | unsigned int flags = 0; 53 | int ret; 54 | int c; 55 | 56 | for (c = 1; c < argc; c++) { 57 | if (argv[c][0] == '-') { 58 | switch (argv[c][1]) { 59 | case 'n': 60 | flags |= RENAME_NOREPLACE; 61 | break; 62 | case 'x': 63 | flags |= RENAME_EXCHANGE; 64 | break; 65 | case 'w': 66 | flags |= RENAME_WHITEOUT; 67 | break; 68 | default: 69 | exit_usage(argv); 70 | } 71 | } else if (!old_path) { 72 | old_path = argv[c]; 73 | } else if (!new_path) { 74 | new_path = argv[c]; 75 | } else { 76 | exit_usage(argv); 77 | } 78 | } 79 | 80 | if (!old_path || !new_path) { 81 | printf("specify the correct directory path\n"); 82 | errno = ENOENT; 83 | return 1; 84 | } 85 | 86 | ret = renameat2(AT_FDCWD, old_path, AT_FDCWD, new_path, flags); 87 | if (ret == -1) { 88 | perror("Error"); 89 | return 1; 90 | } 91 | 92 | return 0; 93 | } 94 | -------------------------------------------------------------------------------- /tests/src/handle_cat.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Given a scoutfs mountpoint and an inode number, open the inode by 3 | * handle and print the contents to stdout. 4 | * 5 | * Copyright (C) 2018 Versity Software, Inc. All rights reserved. 6 | * 7 | * This program is free software; you can redistribute it and/or 8 | * modify it under the terms of the GNU General Public 9 | * License v2 as published by the Free Software Foundation. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * General Public License for more details. 15 | */ 16 | 17 | #ifndef _GNU_SOURCE 18 | #define _GNU_SOURCE 19 | #endif 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #define FILEID_SCOUTFS 0x81 32 | #define FILEID_SCOUTFS_WITH_PARENT 0x82 33 | 34 | struct our_handle { 35 | struct file_handle handle; 36 | /* 37 | * scoutfs file handle can be ino or ino/parent. The 38 | * handle_type field of struct file_handle denotes which 39 | * version is in use. We only use the ino variant here. 40 | */ 41 | __le64 scoutfs_ino; 42 | }; 43 | 44 | #define SZ 4096 45 | char buf[SZ]; 46 | 47 | int main(int argc, char **argv) 48 | { 49 | int fd, mntfd, bytes; 50 | char *mnt; 51 | uint64_t ino; 52 | struct our_handle handle; 53 | 54 | if (argc < 3) { 55 | printf("%s \n", argv[0]); 56 | return 1; 57 | } 58 | 59 | mnt = argv[1]; 60 | ino = strtoull(argv[2], NULL, 10); 61 | 62 | mntfd = open(mnt, O_RDONLY); 63 | if (mntfd == -1) { 64 | perror("while opening mountpoint"); 65 | return 1; 66 | } 67 | 68 | handle.handle.handle_bytes = sizeof(struct our_handle); 69 | handle.handle.handle_type = FILEID_SCOUTFS; 70 | handle.scoutfs_ino = htole64(ino); 71 | 72 | fd = open_by_handle_at(mntfd, &handle.handle, O_RDONLY); 73 | if (fd == -1) { 74 | perror("while opening inode by handle"); 75 | return 1; 76 | } 77 | 78 | while ((bytes = read(fd, buf, SZ)) > 0) 79 | write(STDOUT_FILENO, buf, bytes); 80 | 81 | close(fd); 82 | close(mntfd); 83 | return 0; 84 | } 85 | -------------------------------------------------------------------------------- /tests/src/o_tmpfile_linkat.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023 Versity Software, Inc. All rights reserved. 3 | * 4 | * This program is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public 6 | * License v2 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | */ 13 | 14 | #ifndef _GNU_SOURCE 15 | #define _GNU_SOURCE 16 | #endif 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | static void linkat_tmpfile(char *dir, char *lpath) 29 | { 30 | char proc_self[PATH_MAX]; 31 | int ret; 32 | int fd; 33 | 34 | fd = open(dir, O_RDWR | O_TMPFILE, 0777); 35 | if (fd < 0) { 36 | perror("open(O_TMPFILE)"); 37 | exit(1); 38 | } 39 | 40 | snprintf(proc_self, sizeof(proc_self), "/proc/self/fd/%d", fd); 41 | 42 | ret = linkat(AT_FDCWD, proc_self, AT_FDCWD, lpath, AT_SYMLINK_FOLLOW); 43 | if (ret < 0) { 44 | perror("linkat"); 45 | exit(1); 46 | } 47 | 48 | close(fd); 49 | } 50 | 51 | /* 52 | * Use O_TMPFILE and linkat to create a new visible file, used to test 53 | * the O_TMPFILE creation path by inspecting the created file. 54 | */ 55 | int main(int argc, char **argv) 56 | { 57 | char *lpath; 58 | char *dir; 59 | 60 | if (argc < 3) { 61 | printf("%s \n", argv[0]); 62 | return 1; 63 | } 64 | 65 | dir = argv[1]; 66 | lpath = argv[2]; 67 | 68 | linkat_tmpfile(dir, lpath); 69 | 70 | return 0; 71 | } 72 | -------------------------------------------------------------------------------- /tests/src/o_tmpfile_umask.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Show the modes of files as we create them with O_TMPFILE and link 3 | * them into the namespace. 4 | * 5 | * Copyright (C) 2022 Versity Software, Inc. All rights reserved. 6 | * 7 | * This program is free software; you can redistribute it and/or 8 | * modify it under the terms of the GNU General Public 9 | * License v2 as published by the Free Software Foundation. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * General Public License for more details. 15 | */ 16 | 17 | #ifndef _GNU_SOURCE 18 | #define _GNU_SOURCE 19 | #endif 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | static void linkat_tmpfile_modes(char *dir, char *lpath, mode_t mode) 32 | { 33 | char proc_self[PATH_MAX]; 34 | struct stat st; 35 | int ret; 36 | int fd; 37 | 38 | umask(mode); 39 | printf("umask 0%o\n", mode); 40 | 41 | fd = open(dir, O_RDWR | O_TMPFILE, 0777); 42 | if (fd < 0) { 43 | perror("open(O_TMPFILE)"); 44 | exit(1); 45 | } 46 | 47 | ret = fstat(fd, &st); 48 | if (ret < 0) { 49 | perror("fstat"); 50 | exit(1); 51 | } 52 | 53 | printf("fstat after open(0777): 0%o\n", st.st_mode); 54 | 55 | snprintf(proc_self, sizeof(proc_self), "/proc/self/fd/%d", fd); 56 | 57 | ret = linkat(AT_FDCWD, proc_self, AT_FDCWD, lpath, AT_SYMLINK_FOLLOW); 58 | if (ret < 0) { 59 | perror("linkat"); 60 | exit(1); 61 | } 62 | 63 | close(fd); 64 | 65 | ret = stat(lpath, &st); 66 | if (ret < 0) { 67 | perror("fstat"); 68 | exit(1); 69 | } 70 | 71 | printf("stat after linkat: 0%o\n", st.st_mode); 72 | 73 | ret = unlink(lpath); 74 | if (ret < 0) { 75 | perror("unlink"); 76 | exit(1); 77 | } 78 | } 79 | 80 | int main(int argc, char **argv) 81 | { 82 | char *lpath; 83 | char *dir; 84 | 85 | if (argc < 3) { 86 | printf("%s \n", argv[0]); 87 | return 1; 88 | } 89 | 90 | dir = argv[1]; 91 | lpath = argv[2]; 92 | 93 | linkat_tmpfile_modes(dir, lpath, 022); 94 | linkat_tmpfile_modes(dir, lpath, 077); 95 | 96 | return 0; 97 | } 98 | -------------------------------------------------------------------------------- /tests/tests/basic-bad-mounts.sh: -------------------------------------------------------------------------------- 1 | 2 | mount_fail() 3 | { 4 | local mnt=${!#} 5 | 6 | echo "mounting $@" >> $T_TMP.mount.out 7 | mount -t scoutfs "$@" >> $T_TMP.mount.out 2>&1 8 | if [ $? == 0 ]; then 9 | umount "$mnt" || t_fail "couldn't unmount" 10 | t_fail "bad mount succeeded" 11 | fi 12 | } 13 | 14 | echo "== prepare devices, mount point, and logs" 15 | SCR="$T_TMPDIR/mnt.scratch" 16 | mkdir -p "$SCR" 17 | > $T_TMP.mount.out 18 | scoutfs mkfs -f -Q 0,127.0.0.1,53000 "$T_EX_META_DEV" "$T_EX_DATA_DEV" > $T_TMP.mkfs.out 2>&1 \ 19 | || t_fail "mkfs failed" 20 | 21 | echo "== bad devices, bad options" 22 | mount_fail -o _bad /dev/null /dev/null "$SCR" 23 | 24 | echo "== swapped devices" 25 | mount_fail -o metadev_path=$T_EX_DATA_DEV,quorum_slot_nr=0 "$T_EX_META_DEV" "$SCR" 26 | 27 | echo "== both meta devices" 28 | mount_fail -o metadev_path=$T_EX_META_DEV,quorum_slot_nr=0 "$T_EX_META_DEV" "$SCR" 29 | 30 | echo "== both data devices" 31 | mount_fail -o metadev_path=$T_EX_DATA_DEV,quorum_slot_nr=0 "$T_EX_DATA_DEV" "$SCR" 32 | 33 | echo "== good volume, bad option and good options" 34 | mount_fail -o _bad,metadev_path=$T_EX_META_DEV,quorum_slot_nr=0 "$T_EX_DATA_DEV" "$SCR" 35 | 36 | t_pass 37 | -------------------------------------------------------------------------------- /tests/tests/basic-block-counts.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Test basic correctness of tracking online, offline, and st_blocks 3 | # counts. 4 | # 5 | 6 | t_require_commands scoutfs dd truncate touch mkdir rm rmdir 7 | 8 | # if vers is "stat" then we ask stat_more for the data_version 9 | release_vers() { 10 | local file="$1" 11 | local vers="$2" 12 | local offset="$3" 13 | local length="$4" 14 | 15 | if [ "$vers" == "stat" ]; then 16 | vers=$(scoutfs stat -s data_version "$file") 17 | fi 18 | 19 | scoutfs release "$file" -V "$vers" -o "$offset" -l "$length" 20 | } 21 | 22 | # if vers is "stat" then we ask stat_more for the data_version 23 | stage_vers() { 24 | local file="$1" 25 | local vers="$2" 26 | local offset="$3" 27 | local length="$4" 28 | local contents="$5" 29 | 30 | if [ "$vers" == "stat" ]; then 31 | vers=$(scoutfs stat -s data_version "$file") 32 | fi 33 | 34 | scoutfs stage "$contents" "$file" -V "$vers" -o "$offset" -l "$length" 35 | } 36 | 37 | echo_blocks() 38 | { 39 | echo "online:" $(scoutfs stat -s online_blocks "$1") 40 | echo "offline:" $(scoutfs stat -s offline_blocks "$1") 41 | echo "st_blocks:" $(stat -c '%b' "$1") 42 | } 43 | 44 | FILE="$T_D0/file" 45 | DIR="$T_D0/dir" 46 | 47 | echo "== single block write" 48 | dd if=/dev/zero of="$FILE" bs=4K count=1 status=none 49 | echo_blocks "$FILE" 50 | 51 | echo "== single block overwrite" 52 | dd if=/dev/zero of="$FILE" bs=4K count=1 conv=notrunc status=none 53 | echo_blocks "$FILE" 54 | 55 | echo "== append" 56 | dd if=/dev/zero of="$FILE" bs=4K count=1 conv=notrunc oflag=append status=none 57 | echo_blocks "$FILE" 58 | 59 | echo "== release" 60 | release_vers "$FILE" stat 0 8K 61 | echo_blocks "$FILE" 62 | 63 | echo "== duplicate release" 64 | release_vers "$FILE" stat 0 8K 65 | echo_blocks "$FILE" 66 | 67 | echo "== duplicate release past i_size" 68 | release_vers "$FILE" stat 0 64K 69 | echo_blocks "$FILE" 70 | 71 | echo "== stage" 72 | stage_vers "$FILE" stat 0 8192 /dev/zero 73 | echo_blocks "$FILE" 74 | 75 | echo "== duplicate stage" 76 | stage_vers "$FILE" stat 0 8192 /dev/zero 77 | echo_blocks "$FILE" 78 | 79 | echo "== larger file" 80 | dd if=/dev/zero of="$FILE" bs=1M count=1 status=none 81 | echo_blocks "$FILE" 82 | 83 | echo "== partial truncate" 84 | truncate -s 512K "$FILE" 85 | echo_blocks "$FILE" 86 | 87 | echo "== single sparse block" 88 | rm -f "$FILE" 89 | dd if=/dev/zero of="$FILE" bs=4K count=1 seek=1K status=none 90 | echo_blocks "$FILE" 91 | 92 | echo "== empty file" 93 | rm -f "$FILE" 94 | touch "$FILE" 95 | echo_blocks "$FILE" 96 | 97 | echo "== non-regular file" 98 | mkdir "$DIR" 99 | echo_blocks "$DIR" 100 | 101 | echo "== cleanup" 102 | rm -f "$FILE" 103 | rmdir "$DIR" 104 | 105 | t_pass 106 | -------------------------------------------------------------------------------- /tests/tests/basic-truncate.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Test basic correctness of truncate. 3 | # 4 | 5 | t_require_commands yes dd od truncate 6 | 7 | FILE="$T_D0/file" 8 | 9 | # 10 | # We forgot to write a dirty block that zeroed the tail of a partial 11 | # final block as we truncated past it. 12 | # 13 | echo "== truncate writes zeroed partial end of file block" 14 | yes 2>/dev/null | dd of="$FILE" bs=8K count=1 status=none iflag=fullblock 15 | sync 16 | 17 | # not passing iflag=fullblock causes the file occasionally to just be 18 | # 4K, so just to be safe we should at least check size once 19 | test `stat --printf="%s\n" "$FILE"` -eq 8192 || t_fail "test file incorrect start size" 20 | 21 | truncate -s 6K "$FILE" 22 | truncate -s 12K "$FILE" 23 | echo 3 > /proc/sys/vm/drop_caches 24 | od -Ad -x "$FILE" 25 | 26 | t_pass 27 | -------------------------------------------------------------------------------- /tests/tests/block-stale-reads.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Exercise stale block reading. 3 | # 4 | # It would be very difficult to manipulate the allocators, cache, and 5 | # persistent blocks to create stable block reading scenarios. Instead 6 | # we use triggers to exercise how readers encounter stale blocks. 7 | # 8 | # Trigger retries in the block cache by calling scoutfs df 9 | # which in turn will call scoutfs_ioctl_alloc_detail. This 10 | # is guaranteed to exist, which will force block cache reads. 11 | 12 | echo "== Issue scoutfs df to force block reads to trigger stale invalidation/retry" 13 | nr=0 14 | 15 | old=$(t_counter block_cache_remove_stale $nr) 16 | t_trigger_arm_silent block_remove_stale $nr 17 | 18 | scoutfs df -p "$T_M0" > /dev/null 19 | 20 | t_counter_diff_changed block_cache_remove_stale $old $nr 21 | 22 | t_pass 23 | -------------------------------------------------------------------------------- /tests/tests/client-unmount-recovery.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Unmount Server and unmount a client as it's replaying to a remaining server 3 | # 4 | 5 | majority_nr=$(t_majority_count) 6 | quorum_nr=$T_QUORUM 7 | 8 | test "$quorum_nr" == "$majority_nr" && \ 9 | t_skip "all quorum members make up majority, need more mounts to unmount" 10 | 11 | test "$T_NR_MOUNTS" -lt "$T_QUORUM" && \ 12 | t_skip "Need enough non-quorum clients to unmount" 13 | 14 | for i in $(t_fs_nrs); do 15 | mounted[$i]=1 16 | done 17 | 18 | LENGTH=60 19 | echo "== ${LENGTH}s of unmounting non-quorum clients during recovery" 20 | END=$((SECONDS + LENGTH)) 21 | while [ "$SECONDS" -lt "$END" ]; do 22 | sv=$(t_server_nr) 23 | rid=$(t_mount_rid $sv) 24 | echo "sv $sv rid $rid" >> "$T_TMP.log" 25 | sync 26 | t_umount $sv & 27 | 28 | for i in $(t_fs_nrs); do 29 | if [ "$i" -ge "$quorum_nr" ]; then 30 | t_umount $i & 31 | echo "umount $i rid $rid quo $quorum_nr" \ 32 | >> $T_TMP.log 33 | mounted[$i]=0 34 | fi 35 | done 36 | 37 | wait 38 | 39 | t_mount $sv & 40 | for i in $(t_fs_nrs); do 41 | if [ "${mounted[$i]}" == 0 ]; then 42 | t_mount $i & 43 | fi 44 | done 45 | 46 | wait 47 | 48 | declare RID_LIST=$(cat /sys/fs/scoutfs/*/rid | sort -u) 49 | read -a rid_arr <<< $RID_LIST 50 | 51 | declare LOCK_LIST=$(cut -d' ' -f 5 /sys/kernel/debug/scoutfs/*/server_locks | sort -u) 52 | read -a lock_arr <<< $LOCK_LIST 53 | 54 | for i in "${lock_arr[@]}"; do 55 | if [[ ! " ${rid_arr[*]} " =~ " $i " ]]; then 56 | echo -e "RID($i) exists" >> $T_TMP.log 57 | echo -e "rid_arr:\n${rid_arr[@]}" >> $T_TMP.log 58 | echo -e "lock_arr:\n${lock_arr[@]}" >> $T_TMP.log 59 | t_fail "RID($i): exists when not mounted" 60 | fi 61 | done 62 | done 63 | 64 | t_pass 65 | -------------------------------------------------------------------------------- /tests/tests/createmany-large-names.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Create a lot of files with large names. In the past this caught bugs 3 | # in the btree code as it stored manifest entries with large keys that 4 | # stored directory entry names. Now keys are a small fixed size so this 5 | # has less of an effect. 6 | # 7 | 8 | t_require_commands createmany 9 | 10 | DIRS="0 1 2 3" 11 | COUNT=100000 12 | for i in $DIRS; do 13 | d="$T_D0/$i" 14 | mkdir -p "$d" 15 | # Use an absurdly long file name to blow the dirent key sizes out 16 | ./src/createmany -o $d/file_$(printf "a%.0s" {1..195})_$i $COUNT \ 17 | >> $T_TMP.log & 18 | done 19 | 20 | wait 21 | 22 | t_pass 23 | -------------------------------------------------------------------------------- /tests/tests/createmany-parallel-mounts.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Test clustered parallel createmany 3 | # 4 | 5 | t_require_commands mkdir createmany bc 6 | t_require_mounts 2 7 | 8 | COUNT=50000 9 | 10 | # 11 | # Prep dirs for test. We have per-directory inode number allocators so 12 | # by putting each createmany in a per-mount dir they get their own inode 13 | # number region and cluster locks. 14 | # 15 | echo "== measure initial createmany" 16 | mkdir -p $T_D0/dir/0 17 | mkdir $T_D1/dir/1 18 | 19 | echo "== measure initial createmany" 20 | START=$(date +%s.%N) 21 | createmany -o "$T_D0/file_" $COUNT >> $T_TMP.full 22 | sync 23 | END=$(date +%s.%N) 24 | SINGLE=$(echo "$END - $START" | bc) 25 | 26 | echo "== measure two concurrent createmany runs" 27 | START=$(date +%s.%N) 28 | (cd $T_D0/dir/0; createmany -o ./file_ $COUNT > /dev/null) & 29 | pids="$!" 30 | (cd $T_D1/dir/1; createmany -o ./file_ $COUNT > /dev/null) & 31 | pids="$pids $!" 32 | for p in $pids; do 33 | wait $p 34 | done 35 | sync 36 | END=$(date +%s.%N) 37 | BOTH=$(echo "$END - $START" | bc) 38 | 39 | echo both $BOTH >> $T_TMP.full 40 | 41 | # Multi node still adds significant overhead, even with our CW locks 42 | # being effectively local node for this test. Different hardware 43 | # setups might have a different amount of skew on the result as 44 | # well. Cover for this with a sufficiently large safety factor so 45 | # we're not needlessly tripping up testing. We will still easily 46 | # exceed this factor should the CW locked items go back to fully 47 | # synchronized operation. 48 | FACTOR=200 49 | if [ $(echo "$BOTH > ( $SINGLE * $FACTOR )" | bc) == "1" ]; then 50 | t_fail "both createmany took $BOTH sec, more than $FACTOR x single $SINGLE sec" 51 | fi 52 | 53 | echo "== cleanup" 54 | find $T_D0/dir -delete 55 | 56 | t_pass 57 | -------------------------------------------------------------------------------- /tests/tests/createmany-parallel.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Run createmany in parallel, make sure we don't crash or throw errors 3 | # 4 | 5 | t_require_commands createmany 6 | 7 | DIRS="0 1 2 3" 8 | COUNT=1000 9 | for i in $DIRS; do 10 | d="$T_D0/$i" 11 | echo "Run createmany in $d" | t_filter_fs 12 | mkdir -p "$d" 13 | createmany -o "$d/file_$i" $COUNT >> $T_TMP.log & 14 | done 15 | 16 | wait 17 | 18 | for i in $DIRS; do 19 | rm -fr "$T_D0/$i" 20 | done 21 | 22 | t_pass 23 | -------------------------------------------------------------------------------- /tests/tests/createmany-rename-large-dir.sh: -------------------------------------------------------------------------------- 1 | # 2 | # We've had bugs where client transactions weren't properly being 3 | # committed as their allocators ran out of resources for the amount of 4 | # log btree dirtying they'd need to do for the size of their dirty item 5 | # cache pages. 6 | # 7 | # This stresses those heuristics by trying to maximize the number of 8 | # btree blocks dirtied for each dirty item cache page. We create an 9 | # enormous directory and then randomly rename entries in it. 10 | # 11 | # With the bad client commit heuristics this would reliably fail before 12 | # a few thousand renames. 13 | # 14 | 15 | t_require_commands createmany mv mkdir 16 | 17 | # 298 dirents per 64k block, 4096 max meta avail -- often fewer 18 | NR=$((298 * 4096)) 19 | RENAMES=5000 20 | 21 | DIR="$T_D0/dir/" 22 | 23 | echo "== create large directory with $NR files" 24 | t_quiet mkdir -p "$DIR" 25 | ./src/createmany -o "$DIR/f-" $NR > "$T_TMP.createmany.stdout" 26 | 27 | echo "== randomly renaming $RENAMES files" 28 | for i in $(seq 1 $RENAMES); do 29 | rnd=$(((1$RANDOM$RANDOM) % NR)) 30 | orig="$DIR/f-$rnd" 31 | tmp="$DIR/f-$rnd.$i" 32 | 33 | mv "$orig" "$tmp" 34 | mv "$tmp" "$orig" 35 | done 36 | 37 | t_quiet rm -rf "$DIR" 38 | 39 | t_pass 40 | -------------------------------------------------------------------------------- /tests/tests/cross-mount-data-free.sh: -------------------------------------------------------------------------------- 1 | # 2 | # cross mount freeing 3 | # 4 | # We should be able to continually allocate on one node and free 5 | # on another and have free blocks flow without seeing premature 6 | # enospc failures. 7 | # 8 | 9 | t_require_commands stat fallocate truncate 10 | t_require_mounts 2 11 | 12 | echo "== repeated cross-mount alloc+free, totalling 2x free" 13 | free_blocks=$(stat -f -c "%a" "$T_M0") 14 | file_blocks=$((free_blocks / 10)) 15 | iter=$((free_blocks * 2 / file_blocks)) 16 | file_size=$((file_blocks * 4096)) 17 | 18 | for i in $(seq 1 $iter); do 19 | fallocate -l $file_size "$T_D0/file" 20 | truncate -s 0 "$T_D1/file" 21 | done 22 | 23 | echo "== remove empty test file" 24 | t_quiet rm $T_D0/file 25 | 26 | t_pass 27 | -------------------------------------------------------------------------------- /tests/tests/dirent-consistency.sh: -------------------------------------------------------------------------------- 1 | # 2 | # basic dirent consistency 3 | # 4 | 5 | t_require_mounts 2 6 | 7 | # atime isn't consistent 8 | compare_find_stat_on_all_mounts() { 9 | local path 10 | local i 11 | 12 | for i in $(t_fs_nrs); do 13 | eval path="\$T_D${i}/dir" 14 | find $path | sort | xargs stat | t_filter_fs | \ 15 | grep -v "^Access: [0-9]*" 2>&1 > $T_TMP.stat.$i 16 | done 17 | 18 | for i in $(t_fs_nrs); do 19 | diff -u $T_TMP.stat.0 $T_TMP.stat.$i || \ 20 | t_fail "node $i find output differed from node 0" 21 | done 22 | 23 | } 24 | 25 | echo "== create per node dirs" 26 | for i in $(t_fs_nrs); do 27 | eval path="\$T_D${i}/dir/$i" 28 | mkdir -p $path 29 | done 30 | 31 | echo "== touch files on each node" 32 | for i in $(t_fs_nrs); do 33 | eval path="\$T_D${i}/dir/$i/$i" 34 | touch $path 35 | done 36 | compare_find_stat_on_all_mounts 37 | 38 | echo "== recreate the files" 39 | for i in $(t_fs_nrs); do 40 | eval path="\$T_D${i}/dir/$i/$i" 41 | rm -f $path 42 | touch $path 43 | done 44 | compare_find_stat_on_all_mounts 45 | 46 | echo "== turn the files into directories" 47 | for i in $(t_fs_nrs); do 48 | eval path="\$T_D${i}/dir/$i/$i" 49 | rm -f $path 50 | mkdir $path 51 | done 52 | compare_find_stat_on_all_mounts 53 | 54 | echo "== rename parent dirs" 55 | for i in $(t_fs_nrs); do 56 | eval before="\$T_D${i}/dir/$i" 57 | eval after="\$T_D${i}/dir/$i-renamed" 58 | mv $before $after 59 | done 60 | compare_find_stat_on_all_mounts 61 | 62 | echo "== rename parent dirs back" 63 | for i in $(t_fs_nrs); do 64 | eval before="\$T_D${i}/dir/$i-renamed" 65 | eval after="\$T_D${i}/dir/$i" 66 | mv $before $after 67 | done 68 | compare_find_stat_on_all_mounts 69 | 70 | echo "== create some hard links" 71 | for i in $(t_fs_nrs); do 72 | eval path="\$T_D${i}/dir/$i/$i.file" 73 | touch $path 74 | for link in $(seq 1 3); do 75 | ln $path $path-$link 76 | done 77 | done 78 | compare_find_stat_on_all_mounts 79 | 80 | echo "== recreate one of the hard links" 81 | for i in $(t_fs_nrs); do 82 | eval path="\$T_D${i}/dir/$i/$i.file-3" 83 | rm -f $path 84 | touch $path 85 | done 86 | compare_find_stat_on_all_mounts 87 | 88 | echo "== delete the remaining hard link" 89 | for i in $(t_fs_nrs); do 90 | eval path="\$T_D${i}/dir/$i/$i.file-2" 91 | rm -f $path 92 | done 93 | compare_find_stat_on_all_mounts 94 | 95 | echo "== race to blow everything away" 96 | pids="" 97 | echo "[nodes are racing to log std(out|err) now..]" >> $T_TMP.log 98 | for i in $(t_fs_nrs); do 99 | eval path="\$T_D${i}/dir" 100 | rm -rf "$path/*" >> $T_TMP.log 2>&1 & 101 | pids="$pids $!" 102 | done 103 | # failure's fine 104 | wait $pids 105 | compare_find_stat_on_all_mounts 106 | 107 | t_pass 108 | -------------------------------------------------------------------------------- /tests/tests/export-get-name-parent.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Test operation of scoutfs_get_name and scoutfs_get_parent. We can do 3 | # this by creating a directory, recording it's inode number then 4 | # opening it by handle after a remount. 5 | # 6 | 7 | t_require_commands mkdir stat handle_cat 8 | 9 | DIR="$T_D0/dir" 10 | 11 | mkdir -p "$DIR" 12 | ino=$(stat -c "%i" "$DIR") 13 | 14 | t_umount_all 15 | t_mount_all 16 | 17 | t_quiet handle_cat "$T_M0" "$ino" 18 | 19 | t_pass 20 | -------------------------------------------------------------------------------- /tests/tests/export-lookup-evict-race.sh: -------------------------------------------------------------------------------- 1 | # 2 | # test racing fh_to_dentry with evict from lock invalidation. We've 3 | # had deadlocks between the ordering of iget and evict when they acquire 4 | # cluster locks. 5 | # 6 | 7 | t_require_commands touch stat handle_cat 8 | t_require_mounts 2 9 | 10 | CPUS=$(getconf _NPROCESSORS_ONLN) 11 | NR=$((CPUS * 4)) 12 | END=$((SECONDS + 30)) 13 | 14 | touch "$T_D0/file" 15 | ino=$(stat -c "%i" "$T_D0/file") 16 | 17 | while test $SECONDS -lt $END; do 18 | for i in $(seq 1 $NR); do 19 | fs=$((RANDOM % T_NR_MOUNTS)) 20 | eval dir="\$T_D${fs}" 21 | write=$((RANDOM & 1)) 22 | 23 | if [ "$write" == 1 ]; then 24 | touch "$dir/file" & 25 | else 26 | handle_cat "$dir" "$ino" & 27 | fi 28 | done 29 | wait 30 | done 31 | 32 | t_pass 33 | -------------------------------------------------------------------------------- /tests/tests/fallocate.sh: -------------------------------------------------------------------------------- 1 | 2 | t_require_commands fallocate cat 3 | 4 | echo "== creating reasonably large per-mount files" 5 | for n in $(t_fs_nrs); do 6 | eval path="\$T_D${n}/file-$n" 7 | 8 | LC_ALL=C fallocate -l 128MiB "$path" || \ 9 | t_fail "initial creating fallocate failed" 10 | done 11 | 12 | # 13 | # we had lock inversions between read and fallocate, dropping 14 | # the cache each time forces waiting for IO during the calls 15 | # with the inverted locks held so we have a better chance 16 | # of the deadlock happening. 17 | # 18 | DURATION=10 19 | echo "== ${DURATION}s of racing cold reads and fallocate nop" 20 | END=$((SECONDS + DURATION)) 21 | while [ $SECONDS -le $END ]; do 22 | 23 | echo 3 > /proc/sys/vm/drop_caches 24 | 25 | for n in $(t_fs_nrs); do 26 | eval path="\$T_D${n}/file-$n" 27 | 28 | LC_ALL=C fallocate -o 0 -l 4KiB "$path" & 29 | cat "$path" > /dev/null & 30 | done 31 | 32 | wait || t_fail "fallocate or cat failed" 33 | done 34 | 35 | echo "== cleaning up files" 36 | rm -f "$T_D0"/file-* 37 | 38 | t_pass 39 | -------------------------------------------------------------------------------- /tests/tests/get-referring-entries.sh: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Test _GET_REFERRING_ENTRIES ioctl via the get-referring-entries cli 4 | # command 5 | # 6 | 7 | # consistently print only entry names 8 | filter_names() { 9 | exec cut -d ' ' -f 8- | sort 10 | } 11 | 12 | # print entries with type characters to match find. not happy with hard 13 | # coding, but abi won't change much. 14 | filter_types() { 15 | exec cut -d ' ' -f 5- | \ 16 | sed \ 17 | -e 's/type 1 /type p /' \ 18 | -e 's/type 2 /type c /' \ 19 | -e 's/type 4 /type d /' \ 20 | -e 's/type 6 /type b /' \ 21 | -e 's/type 8 /type f /' \ 22 | -e 's/type 10 /type l /' \ 23 | -e 's/type 12 /type s /' \ 24 | | \ 25 | sort 26 | } 27 | 28 | n_chars() { 29 | local n="$1" 30 | printf 'A%.0s' $(eval echo {1..\$n}) 31 | } 32 | 33 | GRE="scoutfs get-referring-entries -p $T_M0" 34 | 35 | echo "== root inode returns nothing" 36 | $GRE 1 37 | 38 | echo "== crazy large unused inode does nothing" 39 | $GRE 4611686018427387904 # 1 << 62 40 | 41 | echo "== basic entry" 42 | touch $T_D0/file 43 | ino=$(stat -c '%i' $T_D0/file) 44 | $GRE $ino | filter_names 45 | 46 | echo "== rename" 47 | mv $T_D0/file $T_D0/renamed 48 | $GRE $ino | filter_names 49 | 50 | echo "== hard link" 51 | mv $T_D0/renamed $T_D0/file 52 | ln $T_D0/file $T_D0/link 53 | $GRE $ino | filter_names 54 | 55 | echo "== removal" 56 | rm $T_D0/file $T_D0/link 57 | $GRE $ino 58 | 59 | echo "== different dirs" 60 | touch $T_D0/file 61 | ino=$(stat -c '%i' $T_D0/file) 62 | for i in $(seq 1 10); do 63 | mkdir $T_D0/dir-$i 64 | ln $T_D0/file $T_D0/dir-$i/file-$i 65 | done 66 | diff -u <(find $T_D0 -type f -printf '%f\n' | sort) <($GRE $ino | filter_names) 67 | rm $T_D0/file 68 | 69 | echo "== file types" 70 | mkdir $T_D0/dir 71 | touch $T_D0/dir/file 72 | mkdir $T_D0/dir/dir 73 | ln -s $T_D0/dir/file $T_D0/dir/symlink 74 | mknod $T_D0/dir/char c 1 3 # null 75 | mknod $T_D0/dir/block b 7 0 # loop0 76 | for name in $(ls -UA $T_D0/dir | sort); do 77 | ino=$(stat -c '%i' $T_D0/dir/$name) 78 | $GRE $ino | filter_types 79 | done 80 | rm -rf $T_D0/dir 81 | 82 | echo "== all name lengths work" 83 | mkdir $T_D0/dir 84 | touch $T_D0/dir/file 85 | ino=$(stat -c '%i' $T_D0/dir/file) 86 | name="" 87 | > $T_TMP.unsorted 88 | for i in $(seq 1 255); do 89 | name+="a" 90 | echo "$name" >> $T_TMP.unsorted 91 | ln $T_D0/dir/file $T_D0/dir/$name 92 | done 93 | sort $T_TMP.unsorted > $T_TMP.sorted 94 | rm $T_D0/dir/file 95 | $GRE $ino | filter_names > $T_TMP.gre 96 | diff -u $T_TMP.sorted $T_TMP.gre 97 | rm -rf $T_D0/dir 98 | 99 | t_pass 100 | -------------------------------------------------------------------------------- /tests/tests/inode-items-updated.sh: -------------------------------------------------------------------------------- 1 | # 2 | # test inode item updating 3 | # 4 | # Our inode updating pattern involves updating in-memory inode 5 | # structures and then explicitly migrating those to dirty persistent 6 | # items. If we forget to update the persistent items then modifications 7 | # to the in-memory inode can be lost as the inode is evicted. 8 | # 9 | # We test this by modifying inodes, unmounting, and comparing the 10 | # mounted inodes to the inodes before the unmount. 11 | # 12 | 13 | t_require_commands mkdir stat touch find setfattr mv dd scoutfs 14 | 15 | DIR="$T_D0/dir" 16 | 17 | stat_paths() 18 | { 19 | while read path; do 20 | echo "=== $path ===" 21 | # XXX atime isn't consistent :/ 22 | stat "$path" 2>&1 | grep -v "Access: " 23 | scoutfs stat "$path" 2>&1 24 | done 25 | } 26 | 27 | t_quiet mkdir -p "$DIR" 28 | 29 | echo "== create files and sync" 30 | dd if=/dev/zero of="$DIR/truncate" bs=4096 count=1 status=none 31 | dd if=/dev/zero of="$DIR/stage" bs=4096 count=1 status=none 32 | vers=$(scoutfs stat -s data_version "$DIR/stage") 33 | scoutfs release "$DIR/stage" -V $vers -o 0 -l 4K 34 | dd if=/dev/zero of="$DIR/release" bs=4096 count=1 status=none 35 | touch "$DIR/write_end" 36 | mkdir "$DIR"/{mknod_dir,link_dir,unlink_dir,symlink_dir,rename_dir} 37 | touch $DIR/setattr 38 | touch $DIR/xattr_set 39 | sync; sync 40 | 41 | echo "== modify files" 42 | truncate -s 0 "$DIR/truncate" 43 | vers=$(scoutfs stat -s data_version "$DIR/stage") 44 | scoutfs stage /dev/zero "$DIR/stage" -V $vers -o 0 -l 4096 45 | vers=$(scoutfs stat -s data_version "$DIR/release") 46 | scoutfs release "$DIR/release" -V $vers -o 0 -l 4K 47 | dd if=/dev/zero of="$DIR/write_end" bs=4096 count=1 status=none conv=notrunc 48 | touch $DIR/mknod_dir/mknod_file 49 | touch $DIR/link_dir/link_targ 50 | ln $DIR/link_dir/link_targ $DIR/link_dir/link_file 51 | touch $DIR/unlink_dir/unlink_file 52 | rm -f $DIR/unlink_dir/unlink_file 53 | touch $DIR/symlink_dir/symlink_targ 54 | ln -s $DIR/symlink_dir/symlink_targ $DIR/symlink_dir/symlink_file 55 | touch $DIR/rename_dir/rename_from 56 | mv $DIR/rename_dir/rename_from $DIR/rename_dir/rename_to 57 | touch -m --date=@1234 $DIR/setattr 58 | setfattr -n user.test -v val $DIR/xattr_set 59 | 60 | find "$DIR"/* > $T_TMP.paths 61 | echo $DIR/unlink_dir/unlink_file >> $T_TMP.paths 62 | echo $DIR/rename_dir/rename_from >> $T_TMP.paths 63 | stat_paths < $T_TMP.paths > $T_TMP.before 64 | 65 | echo "== mount and unmount" 66 | t_umount_all 67 | t_mount_all 68 | 69 | echo "== verify files" 70 | stat_paths < $T_TMP.paths > $T_TMP.after 71 | diff -u $T_TMP.before $T_TMP.after 72 | 73 | t_pass 74 | -------------------------------------------------------------------------------- /tests/tests/large-fragmented-free.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Make sure the server can handle a transaction with a data_freed whose 3 | # blocks all hit different btree blocks in the main free list. It 4 | # probably has to be merged in multiple commits. 5 | # 6 | 7 | t_require_commands fragmented_data_extents 8 | 9 | EXTENTS_PER_BTREE_BLOCK=600 10 | EXTENTS_PER_LIST_BLOCK=8192 11 | FREED_EXTENTS=$((EXTENTS_PER_BTREE_BLOCK * EXTENTS_PER_LIST_BLOCK)) 12 | 13 | # 14 | # This test specifically creates a pathologically sparse file that will 15 | # be as expensive as possible to free. This is usually fine on 16 | # dedicated or reasonable hardware, but trying to run this in 17 | # virtualized debug kernels can take a very long time. This test is 18 | # about making sure that the server doesn't fail, not that the platform 19 | # can handle the scale of work that our btree formats happen to require 20 | # while execution is bogged down with use-after-free memory reference 21 | # tracking. So we give the test a lot more breathing room before 22 | # deciding that its hung. 23 | # 24 | echo "== setting longer hung task timeout" 25 | if [ -w /proc/sys/kernel/hung_task_timeout_secs ]; then 26 | secs=$(cat /proc/sys/kernel/hung_task_timeout_secs) 27 | test "$secs" -gt 0 || \ 28 | t_fail "confusing value '$secs' from /proc/sys/kernel/hung_task_timeout_secs" 29 | restore_hung_task_timeout() 30 | { 31 | echo "$secs" > /proc/sys/kernel/hung_task_timeout_secs 32 | } 33 | trap restore_hung_task_timeout EXIT 34 | echo "$((secs * 5))" > /proc/sys/kernel/hung_task_timeout_secs 35 | fi 36 | 37 | echo "== creating fragmented extents" 38 | fragmented_data_extents $FREED_EXTENTS $EXTENTS_PER_BTREE_BLOCK "$T_D0/alloc" "$T_D0/move" 39 | 40 | echo "== unlink file with moved extents to free extents per block" 41 | rm -f "$T_D0/move" 42 | 43 | echo "== cleanup" 44 | rm -f "$T_D0/alloc" 45 | 46 | t_pass 47 | -------------------------------------------------------------------------------- /tests/tests/lock-ex-race-processes.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Multi-mount, multi-process EX locking test. This has uncovered at 3 | # least one race between the downconvert thread and local processes 4 | # wanting a lock. 5 | # 6 | 7 | t_require_commands setfattr 8 | 9 | DIR="$T_D0/dir" 10 | FILES=4 11 | COUNT=250 12 | 13 | echo "=== setup files ===" 14 | mkdir -p $T_D0/dir 15 | for f in $(seq 1 $FILES); do 16 | touch $T_D0/dir/file-$f 17 | done 18 | 19 | echo "=== ping-pong xattr ops ===" 20 | pids="" 21 | for f in $(seq 1 $FILES); do 22 | for m in $(t_fs_nrs); do 23 | eval file="\$T_D${m}/dir/file-$f" 24 | (for i in $(seq 1 $COUNT); do 25 | setfattr -n user.test -v mount-$m $file; 26 | done) & 27 | pids="$pids $!" 28 | done 29 | done 30 | wait $pids 31 | 32 | t_pass 33 | -------------------------------------------------------------------------------- /tests/tests/lock-pr-cw-conflict.sh: -------------------------------------------------------------------------------- 1 | # 2 | # make sure pr/cw don't conflict 3 | # 4 | 5 | t_require_commands scoutfs 6 | 7 | FILE="$T_D0/file" 8 | 9 | echo "== race writing and index walking" 10 | for i in $(seq 1 10); do 11 | dd if=/dev/zero of="$FILE" bs=4K count=1 status=none conv=notrunc & 12 | scoutfs walk-inodes -p "$T_M0" -- data_seq 0 -1 > /dev/null & 13 | wait 14 | done 15 | 16 | t_pass 17 | -------------------------------------------------------------------------------- /tests/tests/lock-recover-invalidate.sh: -------------------------------------------------------------------------------- 1 | # 2 | # trigger server failover and lock recovery during heavy invalidating 3 | # load on multiple mounts 4 | # 5 | 6 | majority_nr=$(t_majority_count) 7 | quorum_nr=$T_QUORUM 8 | 9 | test "$quorum_nr" == "$majority_nr" && \ 10 | t_skip "need remaining majority when leader unmounted" 11 | 12 | test "$T_NR_MOUNTS" -lt "$((quorum_nr + 2))" && \ 13 | t_skip "need at least 2 non-quorum load mounts" 14 | 15 | echo "== starting background invalidating read/write load" 16 | touch "$T_D0/file" 17 | load_pids="" 18 | for i in $(t_fs_nrs); do 19 | if [ "$i" -ge "$quorum_nr" ]; then 20 | eval path="\$T_D${i}/file" 21 | 22 | (while true; do touch $path > /dev/null 2>&1; done) & 23 | load_pids="$load_pids $!" 24 | (while true; do stat $path > /dev/null 2>&1; done) & 25 | load_pids="$load_pids $!" 26 | fi 27 | done 28 | 29 | # had it reproduce in ~40s on wimpy debug kernel guests 30 | LENGTH=60 31 | echo "== ${LENGTH}s of lock recovery during invalidating load" 32 | END=$((SECONDS + LENGTH)) 33 | while [ "$SECONDS" -lt "$END" ]; do 34 | sv=$(t_server_nr) 35 | t_umount $sv 36 | t_mount $sv 37 | # new server had to process greeting for mount to finish 38 | done 39 | 40 | echo "== stopping background load" 41 | t_silent_kill $load_pids 42 | 43 | t_pass 44 | -------------------------------------------------------------------------------- /tests/tests/lock-refleak.sh: -------------------------------------------------------------------------------- 1 | # 2 | # make sure we don't leak lock refs 3 | # 4 | # We've had bugs where we leak lock references. We perform a bunch 5 | # of operations and if they're leaking we should see user counts 6 | # related to the number of iterations. The test assumes that the 7 | # system is relatively idle and that they're won't be significant 8 | # other users of the locks. 9 | # 10 | 11 | t_require_commands mkdir touch stat setfattr getfattr cp mv rm cat awk 12 | 13 | DIR="$T_D0/dir" 14 | 15 | echo "== make test dir" 16 | mkdir "$DIR" 17 | 18 | echo "== do enough stuff to make lock leaks visible" 19 | for i in $(seq 1 20); do 20 | t_quiet touch "$DIR/file" 21 | t_quiet stat "$DIR/file" 22 | t_quiet setfattr -n "user.name" -v "$i" "$DIR/file" 23 | t_quiet getfattr --absolute-names -d "$DIR/file" 24 | echo "pants" >> "$DIR/file" 25 | t_quiet cp "$DIR/file" "$DIR/copied" 26 | t_quiet mv "$DIR/copied" "$DIR/moved" 27 | t_quiet truncate -s 0 "$DIR/moved" 28 | t_quiet rm -f "$DIR/moved" 29 | done 30 | 31 | # start 2.2.0.0.0.0 end 2.2.255.18446744073709551615.18446744073709551615.255 refresh_gen 1 mode 2 waiters: rd 0 wr 0 wo 0 users: rd 0 wr 1 wo 0 32 | # users are fields 18, 20, 22 33 | 34 | echo "== make sure nothing has leaked" 35 | awk '($18 > 10 || $20 > 10 || $22 > 10) { 36 | print $i, "might have leaked:", $0 37 | }' < "$(t_debugfs_path)/client_locks" 38 | 39 | t_pass 40 | -------------------------------------------------------------------------------- /tests/tests/lock-revoke-getcwd.sh: -------------------------------------------------------------------------------- 1 | # 2 | # make sure lock revocation doesn't confuse getcwd 3 | # 4 | 5 | DIR="$T_D0/dir" 6 | 7 | t_quiet mkdir -p "$DIR" 8 | 9 | echo "=== getcwd after lock revocation" 10 | cd "$DIR" 11 | t_trigger_arm statfs_lock_purge 12 | stat -f "$T_M0" > /dev/null 13 | strace -e getcwd pwd 2>&1 | grep -i enoent 14 | ls -la /proc/self/cwd | grep "(deleted)" 15 | cd - > /dev/null 16 | 17 | t_pass 18 | -------------------------------------------------------------------------------- /tests/tests/lock-shrink-consistency.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Test that lock shrinking properly invalidates metadata so that future 3 | # locks see new data. 4 | # 5 | 6 | t_require_commands getfattr 7 | t_require_mounts 2 8 | 9 | GETFATTR="getfattr --absolute-names" 10 | 11 | # put the inode in its own lock in a new dir with new ino allocation 12 | echo "=== setup test file ===" 13 | t_quiet mkdir -p $T_D0/dir 14 | touch $T_D0/dir/file 15 | setfattr -n user.test -v aaa $T_D0/dir/file 16 | $GETFATTR -n user.test $T_D0/dir/file 2>&1 | t_filter_fs 17 | 18 | echo "=== commit dirty trans and revoke lock ===" 19 | t_trigger_arm statfs_lock_purge 20 | stat -f "$T_M0" > /dev/null 21 | t_quiet sync 22 | t_trigger_show statfs_lock_purge "after it fired" 23 | 24 | echo "=== change xattr on other mount ===" 25 | setfattr -n user.test -v bbb $T_D1/dir/file 26 | $GETFATTR -n user.test $T_D1/dir/file 2>&1 | t_filter_fs 27 | 28 | # This forces the shrinking node to recreate the lock resource. If our 29 | # lock shrinker isn't properly invalidating metadata, we'd get the old 30 | # xattr value here. 31 | echo "=== verify new xattr under new lock on first mount ===" 32 | $GETFATTR -n user.test $T_D0/dir/file 2>&1 | t_filter_fs 33 | 34 | t_pass 35 | -------------------------------------------------------------------------------- /tests/tests/lock-shrink-read-race.sh: -------------------------------------------------------------------------------- 1 | # 2 | # We had a lock server refcounting bug that could let one thread get a 3 | # reference on a lock struct that was being freed by another thread. We 4 | # were able to reproduce this by having all clients try and produce a 5 | # lot of read and null requests. 6 | # 7 | # This will manfiest as a hung lock and timed out test runs, probably 8 | # with hung task messages on the console. Depending on how the race 9 | # turns out, it can trigger KASAN warnings in 10 | # process_waiting_requests(). 11 | # 12 | 13 | READERS_PER=3 14 | SECS=30 15 | 16 | echo "=== setup" 17 | touch "$T_D0/file" 18 | 19 | echo "=== spin reading and shrinking" 20 | END=$((SECONDS + SECS)) 21 | for m in $(t_fs_nrs); do 22 | eval file="\$T_D${m}/file" 23 | 24 | # lots of tasks reading as fast as they can 25 | for t in $(seq 1 $READERS_PER); do 26 | (while [ $SECONDS -lt $END ]; do 27 | stat $file > /dev/null 28 | done) & 29 | done 30 | # one task shrinking (triggering null requests) and reading 31 | (while [ $SECONDS -lt $END ]; do 32 | stat $file > /dev/null 33 | t_trigger_arm_silent statfs_lock_purge $m 34 | stat -f "$file" > /dev/null 35 | done) & 36 | done 37 | 38 | wait 39 | 40 | t_pass 41 | -------------------------------------------------------------------------------- /tests/tests/mkdir-rename-rmdir.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Sequentially perform operations on a dir (mkdir; rename*2; rmdir) on 3 | # all possible combinations of different mounts that could perform the 4 | # operations. 5 | # 6 | # We're testing that the tracking of the entry key in our cached dirents 7 | # stays consitent with the persistent entry items as they're modified 8 | # around the cluster. 9 | # 10 | 11 | t_require_commands mkdir mv rmdir 12 | 13 | NR_OPS=4 14 | 15 | unset op_mnt 16 | for op in $(seq 0 $NR_OPS); do 17 | op_mnt[$op]=0 18 | done 19 | 20 | if [ $T_NR_MOUNTS -gt $NR_OPS ]; then 21 | NR_MNTS=$NR_OPS 22 | else 23 | NR_MNTS=$T_NR_MOUNTS 24 | fi 25 | 26 | while : ; do 27 | # sequentially perform each op from its mount dir 28 | for op in $(seq 0 $((NR_OPS - 1))); do 29 | m=${op_mnt[$op]} 30 | eval dir="\$T_D${m}/dir" 31 | 32 | case "$op" in 33 | 0) mkdir "$dir" ;; 34 | 1) mv "$dir" "$dir-1" ;; 35 | 2) mv "$dir-1" "$dir-2" ;; 36 | 3) rmdir "$dir-2" ;; 37 | esac 38 | 39 | if [ $? != 0 ]; then 40 | t_fail "${op_mnt[*]} failed at op $op" 41 | fi 42 | done 43 | 44 | # advance through mnt nrs for each op 45 | i=0 46 | while [ $i -lt $NR_OPS ]; do 47 | ((op_mnt[$i]++)) 48 | if [ ${op_mnt[$i]} -ge $NR_MNTS ]; then 49 | op_mnt[$i]=0 50 | ((i++)) 51 | else 52 | break 53 | fi 54 | done 55 | 56 | # done when the last op's mnt nr wrapped 57 | [ $i -ge $NR_OPS ] && break 58 | done 59 | 60 | t_pass 61 | -------------------------------------------------------------------------------- /tests/tests/mmap.sh: -------------------------------------------------------------------------------- 1 | # 2 | # test mmap() and normal read/write consistency between different nodes 3 | # 4 | 5 | t_require_commands mmap_stress mmap_validate scoutfs xfs_io 6 | 7 | echo "== mmap_stress" 8 | mmap_stress 8192 2000 "$T_D0/mmap_stress" "$T_D1/mmap_stress" "$T_D2/mmap_stress" "$T_D3/mmap_stress" "$T_D4/mmap_stress" | sed 's/:.*//g' | sort 9 | 10 | echo "== basic mmap/read/write consistency checks" 11 | mmap_validate 256 1000 "$T_D0/mmap_val1" "$T_D1/mmap_val1" 12 | mmap_validate 8192 1000 "$T_D0/mmap_val2" "$T_D1/mmap_val2" 13 | mmap_validate 88400 1000 "$T_D0/mmap_val3" "$T_D1/mmap_val3" 14 | 15 | echo "== mmap read from offline extent" 16 | F="$T_D0/mmap-offline" 17 | touch "$F" 18 | xfs_io -c "pwrite -S 0xEA 0 8192" "$F" > /dev/null 19 | cp "$F" "${F}-stage" 20 | vers=$(scoutfs stat -s data_version "$F") 21 | scoutfs release "$F" -V "$vers" -o 0 -l 8192 22 | scoutfs get-fiemap -L "$F" 23 | xfs_io -c "mmap -rwx 0 8192" \ 24 | -c "mread -v 512 16" "$F" & 25 | sleep 1 26 | # should be 1 - data waiting 27 | jobs | wc -l 28 | scoutfs stage "${F}-stage" "$F" -V "$vers" -o 0 -l 8192 29 | # xfs_io thread will output 16 bytes of read data 30 | sleep 1 31 | # should be 0 - no more waiting jobs, xfs_io should have exited 32 | jobs | wc -l 33 | scoutfs get-fiemap -L "$F" 34 | 35 | echo "== mmap write to an offline extent" 36 | # reuse the same file 37 | scoutfs release "$F" -V "$vers" -o 0 -l 8192 38 | scoutfs get-fiemap -L "$F" 39 | xfs_io -c "mmap -rwx 0 8192" \ 40 | -c "mwrite -S 0x11 528 16" "$F" & 41 | sleep 1 42 | # should be 1 job waiting 43 | jobs | wc -l 44 | scoutfs stage "${F}-stage" "$F" -V "$vers" -o 0 -l 8192 45 | # no output here from write 46 | sleep 1 47 | # should be 0 - no more waiting jobs, xfs_io should have exited 48 | jobs | wc -l 49 | scoutfs get-fiemap -L "$F" 50 | # read back contents to assure write changed the file 51 | dd status=none if="$F" bs=1 count=48 skip=512 | hexdump -C 52 | 53 | echo "== done" 54 | t_pass 55 | -------------------------------------------------------------------------------- /tests/tests/mount-unmount-race.sh: -------------------------------------------------------------------------------- 1 | # 2 | # stress concurrent mounting and unmounting across mounts 3 | # 4 | # At the start of the test all mounts are mounted. Each iteration 5 | # randomly decides to change each mount or to leave it alone. 6 | # 7 | # Each iteration create dirty items across the mounts randomly, giving 8 | # unmount some work to do. 9 | # 10 | # For this test to be meaningful it needs multiple mounts beyond the 11 | # quorum majority which can be racing to mount and unmount. A 12 | # reasonable config would be 5 mounts with 3 quorum members. But the 13 | # test will run with whatever count it finds. 14 | # 15 | # The test assumes that the first mounts are the quorum members. 16 | # 17 | 18 | majority_nr=$(t_majority_count) 19 | quorum_nr=$T_QUORUM 20 | 21 | cur_quorum=$quorum_nr 22 | test "$cur_quorum" == "$majority_nr" && \ 23 | t_skip "all quorum members make up majority, need more mounts to unmount" 24 | 25 | echo "== create per mount files" 26 | for i in $(t_fs_nrs); do 27 | eval path="\$T_D${i}/$i" 28 | mkdir -p "$path" 29 | touch "$path/$i" 30 | mounted[$i]=1 31 | done 32 | 33 | LENGTH=30 34 | echo "== ${LENGTH}s of racing random mount/umount" 35 | END=$((SECONDS + LENGTH)) 36 | while [ "$SECONDS" -lt "$END" ]; do 37 | 38 | # give some mounts dirty data 39 | for i in $(t_fs_nrs); do 40 | eval path="\$T_D${i}/$i" 41 | dirty=$((RANDOM % 2)) 42 | if [ "${mounted[$i]}" == 1 -a "$dirty" == 1 ]; then 43 | touch "$path/$i" 44 | fi 45 | done 46 | 47 | pids="" 48 | for i in $(t_fs_nrs); do 49 | 50 | change=$((RANDOM % 2)) 51 | if [ "$change" == 0 ]; then 52 | continue; 53 | fi 54 | 55 | if [ "${mounted[$i]}" == 1 ]; then 56 | # 57 | # can always unmount non-quorum mounts, 58 | # can only unmount quorum members beyond majority 59 | # 60 | if [ "$i" -ge "$quorum_nr" -o \ 61 | "$cur_quorum" -gt "$majority_nr" ]; then 62 | t_umount $i & 63 | pid=$! 64 | echo "umount $i pid $pid quo $cur_quorum" \ 65 | >> $T_TMP.log 66 | pids="$pids $pid" 67 | mounted[$i]=0 68 | if [ "$i" -lt "$quorum_nr" ]; then 69 | (( cur_quorum-- )) 70 | fi 71 | fi 72 | else 73 | t_mount $i & 74 | pid=$! 75 | pids="$pids $pid" 76 | echo "mount $i pid $pid quo $cur_quorum" >> $T_TMP.log 77 | mounted[$i]=1 78 | if [ "$i" -lt "$quorum_nr" ]; then 79 | (( cur_quorum++ )) 80 | fi 81 | fi 82 | done 83 | 84 | echo "waiting (secs $SECONDS)" >> $T_TMP.log 85 | for p in $pids; do 86 | wait $p 87 | rc=$? 88 | if [ "$rc" != 0 ]; then 89 | echo "waiting for pid $p returned $rc" 90 | t_fail "background mount/umount returned error" 91 | fi 92 | done 93 | echo "done waiting (secs $SECONDS))" >> $T_TMP.log 94 | done 95 | 96 | echo "== mounting any unmounted" 97 | for i in $(t_fs_nrs); do 98 | if [ "${mounted[$i]}" == 0 ]; then 99 | t_mount $i 100 | fi 101 | done 102 | 103 | t_pass 104 | -------------------------------------------------------------------------------- /tests/tests/o_tmpfile.sh: -------------------------------------------------------------------------------- 1 | # 2 | # basic tests of O_TMPFILE 3 | # 4 | 5 | t_require_commands stage_tmpfile hexdump 6 | 7 | echo "== non-acl O_TMPFILE creation honors umask" 8 | o_tmpfile_umask "$T_D0" "$T_D0/umask-file" 9 | 10 | echo "== stage from tmpfile" 11 | DEST_FILE="$T_D0/dest_file" 12 | stage_tmpfile $T_D0 $DEST_FILE 13 | hexdump -C "$DEST_FILE" 14 | rm -f "$DEST_FILE" 15 | 16 | t_pass 17 | -------------------------------------------------------------------------------- /tests/tests/persistent-item-vers.sh: -------------------------------------------------------------------------------- 1 | # 2 | # make sure we increment item vers from persistent items 3 | # 4 | # Make sure that new locks are given a write_version that is greater 5 | # than all existing items so that new items with greater versions are 6 | # preferred over old versions with lesser versions. 7 | # 8 | # We create an inode in a mount that has been granted multiple locks, 9 | # which could lead to a greater version. We then modify the item very 10 | # early in a mount so that it's lock could have a lesser version. 11 | # 12 | # This was all written to test a bug where the write_version was 13 | # statically initialized as the module was inserted. 14 | # 15 | 16 | t_require_commands mkdir createmany touch stat sleep diff 17 | 18 | echo "== advance lock version by creating unrelated files" 19 | mkdir "$T_D0/a" 20 | createmany -o "$T_D0/a/file-" 10240 2>&1 > /dev/null 21 | mkdir "$T_D0/b" 22 | createmany -o "$T_D0/b/file-" 10240 2>&1 > /dev/null 23 | 24 | echo "== create before file version" 25 | touch "$T_D0/file" 26 | stat "$T_D0/file" | grep Modify: > "$T_TMP.before" 27 | 28 | # remount, possibly wiping the lock server's write_version 29 | t_reinsert_remount_all 30 | 31 | echo "== verify before version, touch after version" 32 | stat "$T_D0/file" | grep Modify: > "$T_TMP.b" 33 | diff -u "$T_TMP.before" "$T_TMP.b" 34 | sleep 1 35 | touch "$T_D0/file" 36 | stat "$T_D0/file" | grep Modify: > "$T_TMP.after" 37 | 38 | # remount and make sure we got the newest version 39 | t_remount_all 40 | 41 | echo "== verify after version" 42 | stat "$T_D0/file" | grep Modify: > "$T_TMP.a" 43 | diff -u "$T_TMP.after" "$T_TMP.a" 44 | 45 | t_pass 46 | -------------------------------------------------------------------------------- /tests/tests/projects.sh: -------------------------------------------------------------------------------- 1 | 2 | # notable id to recognize in output 3 | ID=8675309 4 | 5 | echo "== default new files don't have project" 6 | touch "$T_D0/file" 7 | scoutfs get-attr-x -p "$T_D0/file" 8 | 9 | echo "== set new project on files and dirs" 10 | mkdir "$T_D0/dir" 11 | scoutfs set-attr-x -p $ID "$T_D0/file" 12 | scoutfs set-attr-x -p $ID "$T_D0/dir" 13 | scoutfs get-attr-x -p "$T_D0/file" 14 | scoutfs get-attr-x -p "$T_D0/dir" 15 | 16 | echo "== non-root can see id" 17 | chmod 644 "$T_D0/file" 18 | setpriv --ruid=12345 --euid=12345 scoutfs get-attr-x -p "$T_D0/file" 19 | 20 | echo "== can use IDs around long width limits" 21 | touch "$T_D0/ids" 22 | for id in 0x7FFFFFFF 0x80000000 0xFFFFFFFF \ 23 | 0x7FFFFFFFFFFFFFFF 0x8000000000000000 0xFFFFFFFFFFFFFFFF; do 24 | scoutfs set-attr-x -p $id "$T_D0/ids" 25 | scoutfs get-attr-x -p "$T_D0/ids" 26 | done 27 | 28 | echo "== created files and dirs inherit project id" 29 | touch "$T_D0/dir/file" 30 | mkdir "$T_D0/dir/sub" 31 | scoutfs get-attr-x -p "$T_D0/dir/file" 32 | scoutfs get-attr-x -p "$T_D0/dir/sub" 33 | 34 | echo "== inheritance continues" 35 | mkdir "$T_D0/dir/sub/more" 36 | scoutfs get-attr-x -p "$T_D0/dir/sub/more" 37 | 38 | # .. just inherits 0 :) 39 | echo "== clearing project id stops inheritance" 40 | scoutfs set-attr-x -p 0 "$T_D0/dir" 41 | touch "$T_D0/dir/another-file" 42 | mkdir "$T_D0/dir/another-sub" 43 | scoutfs get-attr-x -p "$T_D0/dir/another-file" 44 | scoutfs get-attr-x -p "$T_D0/dir/another-sub" 45 | 46 | echo "== o_tmpfile creations inherit dir" 47 | scoutfs set-attr-x -p $ID "$T_D0/dir" 48 | o_tmpfile_linkat "$T_D0/dir" "$T_D0/dir/tmpfile" 49 | scoutfs get-attr-x -p "$T_D0/dir/tmpfile" 50 | 51 | 52 | t_pass 53 | -------------------------------------------------------------------------------- /tests/tests/quorum-heartbeat-timeout.sh: -------------------------------------------------------------------------------- 1 | # 2 | # test that the quorum_heartbeat_time_ms option affects how long it 3 | # takes to recover from a failed mount. 4 | # 5 | 6 | t_require_mounts 2 7 | 8 | time_ms() 9 | { 10 | # time_t in seconds, then trunate nanoseconds to 3 most dig digits 11 | date +%s%3N 12 | } 13 | 14 | set_bad_timeout() { 15 | local to="$1" 16 | t_set_sysfs_mount_option 0 quorum_heartbeat_timeout_ms $to && \ 17 | t_fail "set bad q hb to $to" 18 | } 19 | 20 | set_timeout() 21 | { 22 | local nr="$1" 23 | local how="$2" 24 | local to="$3" 25 | local is 26 | 27 | if [ $how == "sysfs" ]; then 28 | t_set_sysfs_mount_option $nr quorum_heartbeat_timeout_ms $to 29 | fi 30 | if [ $how == "mount" ]; then 31 | t_umount $nr 32 | t_mount_opt $nr "quorum_heartbeat_timeout_ms=$to" 33 | fi 34 | 35 | is=$(t_get_sysfs_mount_option $nr quorum_heartbeat_timeout_ms) 36 | 37 | if [ "$is" != "$to" ]; then 38 | t_fail "tried to set qhbto on $nr via $how to $to but got $is" 39 | fi 40 | } 41 | 42 | test_timeout() 43 | { 44 | local how="$1" 45 | local to="$2" 46 | local start 47 | local nr 48 | local sv 49 | local delay 50 | local low 51 | local high 52 | 53 | # set timeout on non-server quorum mounts 54 | sv=$(t_server_nr) 55 | for nr in $(t_quorum_nrs); do 56 | if [ $nr -ne $sv ]; then 57 | set_timeout $nr $how $to 58 | fi 59 | done 60 | 61 | # give followers time to recv heartbeats and reset timeouts 62 | sleep 1 63 | 64 | # tear down the current server/leader 65 | t_force_umount $sv 66 | 67 | # see how long it takes for the next leader to start 68 | start=$(time_ms) 69 | t_wait_for_leader 70 | delay=$(($(time_ms) - start)) 71 | 72 | # kind of fun to have these logged 73 | echo "to $to delay $delay" >> $T_TMP.delay 74 | 75 | # restore the mount that we tore down 76 | t_mount $sv 77 | 78 | # make sure the new leader delay was reasonable, allowing for some slack 79 | low=$((to - 1000)) 80 | high=$((to + 5000)) 81 | 82 | # make sure the new leader delay was reasonable 83 | test "$delay" -lt "$low" && t_fail "delay $delay < low $low (to $to)" 84 | test "$delay" -gt "$high" && t_fail "delay $delay > high $high (to $to)" 85 | } 86 | 87 | echo "== bad timeout values fail" 88 | set_bad_timeout 0 89 | set_bad_timeout -1 90 | set_bad_timeout 1000000 91 | 92 | echo "== bad mount option fails" 93 | if [ "$(t_server_nr)" == 0 ]; then 94 | nr=1 95 | else 96 | nr=0 97 | fi 98 | t_umount $nr 99 | t_mount_opt $nr "quorum_heartbeat_timeout_ms=1000000" 2>/dev/null && \ 100 | t_fail "bad mount option succeeded" 101 | t_mount $nr 102 | 103 | echo "== mount option" 104 | def=$(t_get_sysfs_mount_option 0 quorum_heartbeat_timeout_ms) 105 | test_timeout mount $def 106 | test_timeout mount 3000 107 | test_timeout mount $((def + 19000)) 108 | 109 | echo "== sysfs" 110 | test_timeout sysfs $def 111 | test_timeout sysfs 3000 112 | test_timeout sysfs $((def + 19000)) 113 | 114 | echo "== reset all options" 115 | t_remount_all 116 | 117 | t_pass 118 | -------------------------------------------------------------------------------- /tests/tests/renameat2-noreplace.sh: -------------------------------------------------------------------------------- 1 | # 2 | # simple renameat2 NOREPLACE unit test 3 | # 4 | 5 | t_require_commands dumb_renameat2 6 | t_require_mounts 2 7 | 8 | echo "=== renameat2 noreplace flag test" 9 | 10 | # give each mount their own dir (lock group) to minimize create contention 11 | mkdir $T_M0/dir0 12 | mkdir $T_M1/dir1 13 | 14 | echo "=== run two asynchronous calls to renameat2 NOREPLACE" 15 | for i in $(seq 0 100); do 16 | # prepare inputs in isolation 17 | touch "$T_M0/dir0/old0" 18 | touch "$T_M1/dir1/old1" 19 | 20 | # race doing noreplace renames, both can't succeed 21 | dumb_renameat2 -n "$T_M0/dir0/old0" "$T_M0/dir0/sharednew" 2> /dev/null & 22 | pid0=$! 23 | dumb_renameat2 -n "$T_M1/dir1/old1" "$T_M1/dir0/sharednew" 2> /dev/null & 24 | pid1=$! 25 | 26 | wait $pid0 27 | rc0=$? 28 | wait $pid1 29 | rc1=$? 30 | 31 | test "$rc0" == 0 -a "$rc1" == 0 && t_fail "both renames succeeded" 32 | 33 | # blow away possible files for either race outcome 34 | rm -f "$T_M0/dir0/old0" "$T_M1/dir1/old1" "$T_M0/dir0/sharednew" "$T_M1/dir1/sharednew" 35 | done 36 | 37 | t_pass 38 | -------------------------------------------------------------------------------- /tests/tests/retention-basic.sh: -------------------------------------------------------------------------------- 1 | t_require_commands scoutfs touch rm setfattr 2 | 3 | touch "$T_D0/file-1" 4 | 5 | echo "== setting retention on dir fails" 6 | scoutfs set-attr-x -t 1 "$T_D0" 2>&1 | t_filter_fs 7 | 8 | echo "== set retention" 9 | scoutfs set-attr-x -t 1 "$T_D0/file-1" 10 | 11 | echo "== get-attr-x shows retention" 12 | scoutfs get-attr-x -t "$T_D0/file-1" 13 | 14 | echo "== unpriv can't clear retention" 15 | setpriv --ruid=12345 --euid=12345 scoutfs set-attr-x -t 0 "$T_D0/file-1" 2>&1 | t_filter_fs 16 | 17 | echo "== can set hidden scoutfs xattr in retention" 18 | setfattr -n scoutfs.hide.srch.retention_test -v val "$T_D0/file-1" 19 | 20 | echo "== setting user. xattr fails in retention" 21 | setfattr -n user.retention_test -v val "$T_D0/file-1" 2>&1 | t_filter_fs 22 | 23 | echo "== file deletion fails in retention" 24 | rm -f "$T_D0/file-1" 2>&1 | t_filter_fs 25 | 26 | echo "== file rename fails in retention" 27 | mv $T_D0/file-1 $T_D0/file-2 2>&1 | t_filter_fs 28 | 29 | echo "== file write fails in retention" 30 | date >> $T_D0/file-1 31 | 32 | echo "== file truncate fails in retention" 33 | truncate -s 0 $T_D0/file-1 2>&1 | t_filter_fs 34 | 35 | echo "== setattr fails in retention" 36 | touch $T_D0/file-1 2>&1 | t_filter_fs 37 | 38 | echo "== clear retention" 39 | scoutfs set-attr-x -t 0 "$T_D0/file-1" 40 | 41 | echo "== file write" 42 | date >> $T_D0/file-1 43 | 44 | echo "== file rename" 45 | mv $T_D0/file-1 $T_D0/file-2 46 | mv $T_D0/file-2 $T_D0/file-1 47 | 48 | echo "== setattr" 49 | touch $T_D0/file-1 2>&1 | t_filter_fs 50 | 51 | echo "== xattr deletion" 52 | setfattr -x scoutfs.hide.srch.retention_test "$T_D0/file-1" 53 | 54 | echo "== cleanup" 55 | rm -f "$T_D0/file-1" 56 | 57 | t_pass 58 | -------------------------------------------------------------------------------- /tests/tests/setattr_more.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Test correctness of the setattr_more ioctl. 3 | # 4 | 5 | t_require_commands scoutfs touch mkdir rm stat mknod 6 | 7 | FILE="$T_D0/file" 8 | 9 | echo "== 0 data_version arg fails" 10 | touch "$FILE" 11 | scoutfs setattr -V 0 -s 1 "$FILE" 2>&1 | t_filter_fs 12 | rm "$FILE" 13 | 14 | echo "== args must specify size and offline" 15 | touch "$FILE" 16 | scoutfs setattr -V 1 -o -s 0 "$FILE" 2>&1 | t_filter_fs 17 | rm "$FILE" 18 | 19 | echo "== only works on regular files" 20 | mkdir "$T_D0/dir" 21 | scoutfs setattr -V 1 -s 1 "$T_D0/dir" 2>&1 | t_filter_fs 22 | rmdir "$T_D0/dir" 23 | mknod "$T_D0/char" c 1 3 24 | scoutfs setattr -V 1 -s 1 "$T_D0/char" 2>&1 | t_filter_fs 25 | rm "$T_D0/char" 26 | 27 | echo "== non-zero file size fails" 28 | echo contents > "$FILE" 29 | scoutfs setattr -V 1 -s 1 "$FILE" 2>&1 | t_filter_fs 30 | rm "$FILE" 31 | 32 | echo "== non-zero file data_version fails" 33 | touch "$FILE" 34 | truncate -s 1M "$FILE" 35 | truncate -s 0 "$FILE" 36 | scoutfs setattr -V 1 -o -s 1 "$FILE" 2>&1 | t_filter_fs 37 | rm "$FILE" 38 | 39 | echo "== large size is set" 40 | touch "$FILE" 41 | scoutfs setattr -V 1 -s 578437695752307201 "$FILE" 2>&1 | t_filter_fs 42 | stat -c "%s" "$FILE" 43 | rm "$FILE" 44 | 45 | echo "== large data_version is set" 46 | touch "$FILE" 47 | scoutfs setattr -V 578437695752307201 -s 1 "$FILE" 2>&1 | t_filter_fs 48 | scoutfs stat -s data_version "$FILE" 49 | rm "$FILE" 50 | 51 | echo "== large ctime is set" 52 | touch "$FILE" 53 | # only doing 32bit sec 'cause stat gets confused 54 | scoutfs setattr -t 67305985.999999999 -V 1 -s 1 "$FILE" 2>&1 | t_filter_fs 55 | TZ=GMT stat -c "%z" "$FILE" 56 | rm "$FILE" 57 | 58 | echo "== large offline extents are created" 59 | touch "$FILE" 60 | scoutfs setattr -V 1 -o -s $((10007 * 4096)) "$FILE" 2>&1 | t_filter_fs 61 | scoutfs get-fiemap "$FILE" 62 | rm "$FILE" 63 | 64 | # had a bug where we were creating extents that were too long 65 | echo "== correct offline extent length" 66 | touch "$FILE" 67 | scoutfs setattr -V 1 -o -s 4000000000 "$FILE" 2>&1 | t_filter_fs 68 | scoutfs stat -s offline_blocks "$FILE" 69 | rm "$FILE" 70 | 71 | # Do not fail if data_version is unset - the unset `0` value should not 72 | # be passed down to attr_x handling code which will -EINVAL on that. 73 | echo "== omitting data_version should not fail" 74 | touch "$FILE" 75 | scoutfs setattr -s 0 -t 1725670311.0 -r 1725670311.0 "$FILE" 76 | rm "$FILE" 77 | 78 | t_pass 79 | -------------------------------------------------------------------------------- /tests/tests/setup-error-teardown.sh: -------------------------------------------------------------------------------- 1 | # 2 | # test setup error teardown 3 | # 4 | # Make sure that we're properly cleaning up partially built up state by 5 | # hitting errors while setting up subsystems during mount. 6 | # 7 | 8 | t_require_commands sleep killall 9 | 10 | echo "== interrupt waiting mount" 11 | t_umount_all 12 | mount -t scoutfs -o metadev_path=$T_MB0 $T_DB0 $T_M0 & 13 | pid="$!" 14 | sleep .1 15 | kill $pid 16 | # silence terminated message 17 | wait "$pid" 2> /dev/null 18 | t_mount_all 19 | 20 | t_pass 21 | -------------------------------------------------------------------------------- /tests/tests/simple-readdir.sh: -------------------------------------------------------------------------------- 1 | # 2 | # verify d_off output of xfs_io is consistent. 3 | # 4 | 5 | t_require_commands xfs_io 6 | 7 | filt() 8 | { 9 | grep d_off | cut -d ' ' -f 1,4- 10 | } 11 | 12 | echo "== create content" 13 | for s in $(seq 1 7 250); do 14 | f=$(printf '%*s' $s | tr ' ' 'a') 15 | touch ${T_D0}/$f 16 | done 17 | 18 | echo "== readdir all" 19 | xfs_io -c "readdir -v" $T_D0 | filt 20 | 21 | echo "== readdir offset" 22 | xfs_io -c "readdir -v -o 20" $T_D0 | filt 23 | 24 | echo "== readdir len (bytes)" 25 | xfs_io -c "readdir -v -l 193" $T_D0 | filt 26 | 27 | echo "== introduce gap" 28 | for s in $(seq 57 7 120); do 29 | f=$(printf '%*s' $s | tr ' ' 'a') 30 | rm -f ${T_D0}/$f 31 | done 32 | xfs_io -c "readdir -v" $T_D0 | filt 33 | 34 | echo "== cleanup" 35 | rm -rf $T_D0 36 | 37 | t_pass 38 | -------------------------------------------------------------------------------- /tests/tests/srch-safe-merge-pos.sh: -------------------------------------------------------------------------------- 1 | # 2 | # There was a bug where srch file compaction could get stuck if a 3 | # partial compaction finished at the specific _SAFE_BYTES offset in a 4 | # block. Resuming from that position would return an error and 5 | # compaction would stop making forward progress. 6 | # 7 | # We use triggers to pad the output of log compaction to end on the safe 8 | # offset and then cause compaction of those padded inputs to stop at the 9 | # safe offset. Continuation will either succeed or return errors. 10 | # 11 | 12 | # forcing rotation, so just a few 13 | NR=10 14 | SEQF="%.20g" 15 | COMPACT_NR=4 16 | 17 | echo "== initialize per-mount values" 18 | declare -a err 19 | declare -a compact_delay 20 | for nr in $(t_fs_nrs); do 21 | err[$nr]=$(t_counter srch_compact_error $nr) 22 | compact_delay[$nr]=$(cat $(t_sysfs_path $nr)/srch/compact_delay_ms) 23 | done 24 | restore_compact_delay() 25 | { 26 | for nr in $(t_fs_nrs); do 27 | echo ${compact_delay[$nr]} > $(t_sysfs_path $nr)/srch/compact_delay_ms 28 | done 29 | } 30 | trap restore_compact_delay EXIT 31 | 32 | echo "== arm compaction triggers" 33 | for nr in $(t_fs_nrs); do 34 | t_trigger_arm srch_compact_logs_pad_safe $nr 35 | t_trigger_arm srch_merge_stop_safe $nr 36 | done 37 | 38 | echo "== compact more often" 39 | for nr in $(t_fs_nrs); do 40 | echo 1000 > $(t_sysfs_path $nr)/srch/compact_delay_ms 41 | done 42 | 43 | echo "== create padded sorted inputs by forcing log rotation" 44 | sv=$(t_server_nr) 45 | for i in $(seq 1 $COMPACT_NR); do 46 | for j in $(seq 1 $COMPACT_NR); do 47 | t_trigger_arm srch_force_log_rotate $sv 48 | 49 | seq -f "f-$i-$j-$SEQF" 1 10 | \ 50 | bulk_create_paths -X "scoutfs.srch.t-srch-safe-merge-pos" -d "$T_D0" > \ 51 | /dev/null 52 | sync 53 | 54 | test "$(t_trigger_get srch_force_log_rotate $sv)" == "0" || \ 55 | t_fail "srch_force_log_rotate didn't trigger" 56 | done 57 | 58 | padded=0 59 | while test $padded == 0 && sleep .5; do 60 | for nr in $(t_fs_nrs); do 61 | if [ "$(t_trigger_get srch_compact_logs_pad_safe $nr)" == "0" ]; then 62 | t_trigger_arm srch_compact_logs_pad_safe $nr 63 | padded=1 64 | break 65 | fi 66 | test "$(t_counter srch_compact_error $nr)" == "${err[$nr]}" || \ 67 | t_fail "srch_compact_error counter increased on mount $nr" 68 | done 69 | done 70 | done 71 | 72 | echo "== compaction of padded should stop at safe" 73 | sleep 2 74 | for nr in $(t_fs_nrs); do 75 | if [ "$(t_trigger_get srch_merge_stop_safe $nr)" == "0" ]; then 76 | break 77 | fi 78 | done 79 | 80 | echo "== verify no compaction errors" 81 | sleep 2 82 | for nr in $(t_fs_nrs); do 83 | test "$(t_counter srch_compact_error $nr)" == "${err[$nr]}" || \ 84 | t_fail "srch_compact_error counter increased on mount $nr" 85 | done 86 | 87 | echo "== cleanup" 88 | find "$T_D0" -type f -name 'f-*' -delete 89 | 90 | t_pass 91 | -------------------------------------------------------------------------------- /tests/tests/stage-multi-part.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Stage a large file in multiple parts and have a reader read it while 3 | # it's being staged. This has found problems with extent access 4 | # locking. 5 | # 6 | 7 | t_require_commands scoutfs perl cmp rm 8 | 9 | FILE_BYTES=$((4 * 1024 * 1024 * 1024)) 10 | FILE_BLOCKS=$((FILE_BYTES / 4096)) 11 | FRAG_BYTES=$((128 * 1024 * 1024)) 12 | FRAG_BLOCKS=$((FRAG_BYTES / 4096)) 13 | NR_FRAGS=$((FILE_BLOCKS / FRAG_BLOCKS)) 14 | 15 | # 16 | # high bandwidth way to generate file contents with predictable 17 | # contents. We use ascii lines with the block identity, padded to 4KB 18 | # with spaces. 19 | # 20 | # $1 is number of 4k blocks to write, and each block gets its block 21 | # number in the line. $2, $3, and $4 are fields that are put in every 22 | # block. 23 | # 24 | gen() { 25 | perl -e 'for (my $i = 0; $i < '$1'; $i++) { printf("mount %020u process %020u file %020u blkno %020u%s\n", '$2', '$3', '$4', $i, " " x 3987); }' 26 | } 27 | 28 | release_file() { 29 | local path="$1" 30 | local vers=$(scoutfs stat -s data_version "$path") 31 | 32 | scoutfs release "$path" -V "$vers" -o 0 -l $FILE_BYTES 33 | } 34 | 35 | stage_file() { 36 | local path="$1" 37 | local vers=$(scoutfs stat -s data_version "$path") 38 | local off=0 39 | 40 | for a in $(seq 1 $NR_FRAGS); do 41 | scoutfs stage <(gen $FRAG_BLOCKS $a $a $a) "$path" -V "$vers" \ 42 | -o $off -l $FRAG_BYTES 43 | ((off+=$FRAG_BYTES)) 44 | done 45 | } 46 | 47 | FILE="$T_D0/file" 48 | 49 | whole_file() { 50 | for a in $(seq 1 $NR_FRAGS); do 51 | gen $FRAG_BLOCKS $a $a $a 52 | done 53 | } 54 | 55 | # 56 | # just one pass through the file. 57 | # 58 | 59 | whole_file > "$FILE" 60 | release_file "$FILE" 61 | 62 | cmp "$FILE" <(whole_file) & 63 | pid=$! 64 | 65 | stage_file "$FILE" 66 | 67 | wait $pid || t_fail "comparison failed" 68 | 69 | t_pass 70 | -------------------------------------------------------------------------------- /tests/tests/stage-release-race-alloc.sh: -------------------------------------------------------------------------------- 1 | # 2 | # concurrent stage and release allocation consistency 3 | # 4 | 5 | t_require_commands rm mkdir dd cp cmp mv find scoutfs 6 | 7 | EACH=4 8 | NR=$((EACH * 4)) 9 | DIR="$T_D0/dir" 10 | BLOCKS=256 11 | BYTES=$(($BLOCKS * 4096)) 12 | 13 | release_file() { 14 | local path="$1" 15 | local vers=$(scoutfs stat -s data_version "$path") 16 | 17 | echo "releasing $path" >> "$T_TMP.log" 18 | scoutfs release "$path" -V "$vers" -o 0 -l $BYTES 19 | echo "released $path" >> "$T_TMP.log" 20 | } 21 | 22 | stage_file() { 23 | local path="$1" 24 | local vers=$(scoutfs stat -s data_version "$path") 25 | 26 | echo "staging $path" >> "$T_TMP.log" 27 | scoutfs stage "$DIR/good/$(basename $path)" "$path" -V "$vers" -o 0 -l $BYTES 28 | 29 | echo "staged $path" >> "$T_TMP.log" 30 | } 31 | 32 | echo "== create initial files" 33 | mkdir -p "$DIR"/{on,off,good} 34 | for i in $(seq 1 $NR); do 35 | dd if=/dev/urandom of="$DIR/good/$i" bs=1MiB count=1 status=none 36 | cp "$DIR/good/$i" "$DIR/on/$i" 37 | done 38 | 39 | echo "== race stage and release" 40 | for r in $(seq 1 1000); do 41 | 42 | on=$(find "$DIR"/on/* 2>/dev/null | shuf | head -$EACH) 43 | off=$(find "$DIR"/off/* 2>/dev/null | shuf | head -$EACH) 44 | echo r $r on $on off $off >> "$T_TMP.log" 45 | 46 | for f in $on; do 47 | release_file $f & 48 | done 49 | for f in $off; do 50 | stage_file $f & 51 | done 52 | wait 53 | 54 | [ -n "$on" ] && mv $on "$DIR/off/" 55 | [ -n "$off" ] && mv $off "$DIR/on/" 56 | 57 | for f in $(find "$DIR"/on/* 2>/dev/null); do 58 | cmp "$f" "$DIR/good/$(basename $f)" 59 | if [ $? != 0 ]; then 60 | t_fail "file $f bad!" 61 | fi 62 | done 63 | done 64 | 65 | echo "== cleanup" 66 | rm -f "$T_TMP.log" 67 | 68 | t_pass 69 | -------------------------------------------------------------------------------- /utils/.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.d 3 | *.swp 4 | src/scoutfs 5 | src/format.h 6 | src/ioctl.h 7 | .sparse* 8 | .mock.build* 9 | cscope.* 10 | scoutfs-utils.spec 11 | scoutfs-utils-*.tar 12 | -------------------------------------------------------------------------------- /utils/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # If we're creating a standalone tarball for distribution we copy the 3 | # headers out of the kmod dir into the tarball. And then when we're 4 | # building in that tarball we use the headers in src/ directly. 5 | # 6 | FMTIOC_H := format.h ioctl.h 7 | FMTIOC_KMOD := $(addprefix ../kmod/src/,$(FMTIOC_H)) 8 | 9 | CFLAGS := -Wall -O2 -Werror -D_FILE_OFFSET_BITS=64 -g -msse4.2 \ 10 | -fno-strict-aliasing \ 11 | -DSCOUTFS_FORMAT_HASH=0x$(SCOUTFS_FORMAT_HASH)LLU 12 | 13 | ifneq ($(wildcard $(firstword $(FMTIOC_KMOD))),) 14 | CFLAGS += -I../kmod/src 15 | endif 16 | 17 | BIN := src/scoutfs 18 | OBJ := $(patsubst %.c,%.o,$(wildcard src/*.c)) 19 | DEPS := $(wildcard */*.d) 20 | 21 | all: $(BIN) 22 | 23 | ifneq ($(DEPS),) 24 | -include $(DEPS) 25 | endif 26 | 27 | ifeq ($(V), ) 28 | QU = @echo 29 | VE = @ 30 | else 31 | QU = @: 32 | VE = 33 | endif 34 | 35 | $(BIN): $(OBJ) 36 | $(QU) [BIN $@] 37 | $(VE)gcc -o $@ $^ -luuid -lm -lcrypto -lblkid 38 | 39 | %.o %.d: %.c Makefile sparse.sh 40 | $(QU) [CC $<] 41 | $(VE)gcc $(CFLAGS) -MD -MP -MF $*.d -c $< -o $*.o 42 | $(QU) [SP $<] 43 | $(VE)./sparse.sh -Wbitwise -D__CHECKER__ $(CFLAGS) $< 44 | 45 | .PHONY: .FORCE 46 | 47 | # - We use the git describe from tags to set up the RPM versioning 48 | RPM_VERSION := $(shell git describe --long --tags | awk -F '-' '{gsub(/^v/,""); print $$1}') 49 | RPM_GITHASH := $(shell git rev-parse --short HEAD) 50 | 51 | %.spec: %.spec.in .FORCE 52 | sed -e 's/@@VERSION@@/$(RPM_VERSION)/g' \ 53 | -e 's/@@GITHASH@@/$(RPM_GITHASH)/g' < $< > $@+ 54 | mv $@+ $@ 55 | 56 | TARFILE = scoutfs-utils-$(RPM_VERSION).tar 57 | 58 | # 59 | # make a stand alone buildable tarball for packaging, arguably this 60 | # shouldn't be included in the dist Makefile :) 61 | # 62 | dist: $(RPM_DIR) scoutfs-utils.spec 63 | git archive --format=tar --prefix scoutfs-utils-$(RPM_VERSION)/ HEAD^{tree} > $(TARFILE) 64 | tar rf $(TARFILE) --transform="s@\(.*\)@scoutfs-utils-$(RPM_VERSION)/\1@" scoutfs-utils.spec 65 | tar rf $(TARFILE) --transform="s@.*\(src/.*\)@scoutfs-utils-$(RPM_VERSION)/\1@" $(FMTIOC_KMOD) 66 | 67 | clean: 68 | @rm -f $(BIN) $(OBJ) $(DEPS) .sparse.* 69 | -------------------------------------------------------------------------------- /utils/fenced/scoutfs-fenced.conf.example: -------------------------------------------------------------------------------- 1 | # delay, in seconds, between each check for pending fence requests. 2 | SCOUTFS_FENCED_DELAY=1 3 | # path to executable to run to service fence request 4 | #SCOUTFS_FENCED_RUN= 5 | # arguments to pass to binary 6 | SCOUTFS_FENCED_RUN_ARGS="" 7 | -------------------------------------------------------------------------------- /utils/fenced/scoutfs-fenced.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=ScoutFS fenced 3 | 4 | [Service] 5 | Restart=on-failure 6 | RestartSec=5s 7 | StartLimitBurst=5 8 | ExecStart=/usr/libexec/scoutfs-fenced/scoutfs-fenced 9 | 10 | [Install] 11 | WantedBy=default.target 12 | -------------------------------------------------------------------------------- /utils/man/scoutfs-fenced.8: -------------------------------------------------------------------------------- 1 | .TH scoutfs-fenced 8 2 | .SH NAME 3 | scoutfs-fenced \- scoutfs fence request monitoring and dispatch daemon 4 | .SH DESCRIPTION 5 | The 6 | .B scoutfs-fenced 7 | daemon runs on hosts with mounts that are configured as quorum members 8 | and could create fence requests. It watches sysfs directories of 9 | mounted scoutfs volumes for the directories store requests 10 | to fence a mount. 11 | 12 | .SH ENVIRONMENT 13 | scoutfs-fenced reads the 14 | .I SCOUTFS_FENCED_CONFIG_FILE 15 | environment variable for the path to the config file that contains its 16 | configuration. The file must be readable and is sourced as a bash 17 | script and is expected to set the following configuration variables. 18 | 19 | .SH CONFIGURATION 20 | 21 | .TP 22 | .B SCOUTFS_FENCED_DELAY 23 | The number of seconds to wait beteween checking for fence request 24 | directories in the sysfs directories of all mounts on the host. 25 | 26 | .TP 27 | .B SCOUTFS_FENCED_RUN 28 | The path to the command to execute for each fence request. The file at 29 | the path must be executable. 30 | 31 | .TP 32 | .B SCOUTFS_FENCED_RUN_ARGS 33 | The arguments that are unconditionally passed through to the run 34 | command. 35 | 36 | .SH DAEMONIZING AND LOGGING 37 | 38 | scoutfs-fenced runs in the foreground and writes to stderr and stdout. 39 | Disconnecting it from parents and redirecting its output are the 40 | responsibility of the host environment. 41 | 42 | .SH RUN COMMAND INTERFACE 43 | 44 | scoutfs-fenced sets enviroment variables for the run command with 45 | information about the mount that must be fenced: 46 | 47 | .TP 48 | .B SCOUTFS_FENCED_REQ_RID 49 | The RID of the mount to be fenced. 50 | .TP 51 | .B SCOUTFS_FENCED_REQ_IP 52 | The dotted quad IPv4 address of the last connection from the mount. 53 | 54 | .RE 55 | The return status of the run command indicates if the mount was 56 | fenced, or not. If the mount was successfully fenced then the command 57 | should return a 0 success status. If the run command returns a non-zero 58 | failure status then the request will be set as errored and the server 59 | will shut down. The next server that starts will create another fence 60 | request for the mount. 61 | 62 | .SH SEE ALSO 63 | .BR scoutfs (5), 64 | 65 | .SH AUTHORS 66 | Zach Brown 67 | -------------------------------------------------------------------------------- /utils/scoutfs-utils.spec.in: -------------------------------------------------------------------------------- 1 | %define pkg_version @@VERSION@@ 2 | %define pkg_git_hash @@GITHASH@@ 3 | %define pkg_date %(date +%%Y%%m%%d) 4 | 5 | %{!?_release: %global _release 0.%{pkg_date}git%{pkg_git_hash}} 6 | 7 | Name: scoutfs-utils 8 | Summary: scoutfs user space utilities 9 | Version: %{pkg_version} 10 | Release: %{_release}%{?dist} 11 | License: GPLv2 12 | Group: System Environment/Base 13 | URL: http://scoutfs.org/ 14 | 15 | BuildRequires: git 16 | BuildRequires: gzip 17 | BuildRequires: libuuid-devel 18 | BuildRequires: openssl-devel 19 | BuildRequires: libblkid-devel 20 | 21 | #Requires: kmod-scoutfs = %{version} 22 | 23 | Source: scoutfs-utils-%{pkg_version}.tar 24 | 25 | # Disable the building of the debug package(s). 26 | %define debug_package %{nil} 27 | 28 | %description 29 | scoutfs - user space utilities 30 | 31 | %package -n scoutfs-devel 32 | Summary: scoutfs devel headers 33 | Version: %{pkg_version} 34 | Release: %{_release}%{?dist} 35 | License: GPLv2 36 | Group: Development/Libraries 37 | URL: http://scoutfs.org/ 38 | 39 | %description -n scoutfs-devel 40 | scoutfs - development headers 41 | 42 | %prep 43 | %setup -q -n scoutfs-utils-%{pkg_version} 44 | 45 | %build 46 | make 47 | gzip man/*.? 48 | 49 | %install 50 | mkdir -p $RPM_BUILD_ROOT%{_mandir}/man{5,7,8} 51 | cp man/*.5.gz $RPM_BUILD_ROOT%{_mandir}/man5/. 52 | cp man/*.7.gz $RPM_BUILD_ROOT%{_mandir}/man7/. 53 | cp man/*.8.gz $RPM_BUILD_ROOT%{_mandir}/man8/. 54 | install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs 55 | install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h 56 | install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h 57 | install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced 58 | install -m 644 -D fenced/scoutfs-fenced.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-fenced.service 59 | install -m 644 -D fenced/scoutfs-fenced.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-fenced.conf.example 60 | 61 | %files 62 | %defattr(644,root,root,755) 63 | %{_mandir}/man*/scoutfs*.gz 64 | /%{_unitdir}/scoutfs-fenced.service 65 | %{_sysconfdir}/scoutfs 66 | %defattr(755,root,root,755) 67 | %{_sbindir}/scoutfs 68 | %{_libexecdir}/scoutfs-fenced 69 | 70 | %files -n scoutfs-devel 71 | %defattr(644,root,root,755) 72 | %{_includedir}/scoutfs 73 | 74 | %clean 75 | rm -rf %{buildroot} 76 | 77 | -------------------------------------------------------------------------------- /utils/sparse.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # can we find sparse? If not, we're done. 4 | which sparse > /dev/null 2>&1 || exit 0 5 | 6 | # 7 | # one of the problems with using sparse in userspace is that it picks up 8 | # things in system headers that we don't care about. We're willing to 9 | # take on the burden of filtering them out so that we can have it tell 10 | # us about problems in our code. 11 | # 12 | # system headers using __transparent_union__ 13 | RE="^/.*error: ignoring attribute __transparent_union__" 14 | 15 | # we don't care if system headers have gcc attributes sparse doesn't 16 | # know about 17 | RE="$RE|error: attribute '__leaf__': unknown attribute" 18 | 19 | # yes, sparse, that's the size of memseting a 4 meg buffer all right 20 | RE="$RE|warning: memset with byte count of 4194304" 21 | 22 | # some sparse versions don't know about some builtins 23 | RE="$RE|error: undefined identifier '__builtin_fpclassify'" 24 | 25 | # 26 | # don't filter out 'too many errors' here, it can signify that 27 | # sparse doesn't understand something and is throwing a *ton* 28 | # of useless errors before giving up and existing. Check 29 | # unfiltered sparse output. 30 | # 31 | 32 | # 33 | # I'm not sure this is needed. 34 | # 35 | search=$(gcc -print-search-dirs | awk '($1 == "install:"){print "-I" $2}') 36 | 37 | # 38 | # We're trying to use sparse against glibc headers which go wild trying to 39 | # use internal compiler macros to test features. We copy gcc's and give 40 | # them to sparse, but not the ones that sparse already has. 41 | # 42 | defines=".sparse.gcc-defines.$$.h" 43 | awk ' 44 | # save defines from gcc 45 | ( FNR == NR ) { lines[$2]=$0 } 46 | 47 | # delete defines that sparse also has 48 | ( FNR < NR ) { delete lines[$2] } 49 | 50 | # dump remaining lines unique to gcc 51 | END { 52 | for (a in lines) { 53 | print lines[a] 54 | } 55 | } 56 | ' <(gcc -dM -E -x c - < /dev/null) <(sparse -dM -E -x c - < /dev/null) > $defines 57 | include="-include $defines" 58 | 59 | # 60 | # sparse doesn't seem to notice when it's on a 64bit host. It warns that 61 | # 64bit values don't fit in 'unsigned long' without this. 62 | # 63 | if grep -q "__LP64__ 1" $defines; then 64 | m64="-m64" 65 | else 66 | m64="" 67 | fi 68 | 69 | sparse $m64 $include $search/include "$@" 2>&1 | egrep -v "($RE)" | tee .sparse.output 70 | 71 | rm -f $defines 72 | 73 | if [ -s .sparse.output ]; then 74 | exit 1 75 | else 76 | exit 0 77 | fi 78 | -------------------------------------------------------------------------------- /utils/src/avl.c: -------------------------------------------------------------------------------- 1 | #include "sparse.h" 2 | #include "util.h" 3 | #include "format.h" 4 | #include "avl.h" 5 | 6 | static struct scoutfs_avl_node *node_ptr(struct scoutfs_avl_root *root, 7 | 8 | __le16 off) 9 | { 10 | return off ? (void *)root + le16_to_cpu(off) : NULL; 11 | } 12 | 13 | __le16 avl_node_off(struct scoutfs_avl_root *root, 14 | struct scoutfs_avl_node *node) 15 | { 16 | if (!node) 17 | return 0; 18 | return cpu_to_le16((void *)node - (void *)root); 19 | } 20 | 21 | struct scoutfs_avl_node *avl_first(struct scoutfs_avl_root *root) 22 | { 23 | struct scoutfs_avl_node *node = node_ptr(root, root->node); 24 | 25 | while (node && node->left) 26 | node = node_ptr(root, node->left); 27 | 28 | return node; 29 | } 30 | 31 | struct scoutfs_avl_node *avl_next(struct scoutfs_avl_root *root, 32 | struct scoutfs_avl_node *node) 33 | { 34 | struct scoutfs_avl_node *parent; 35 | 36 | if (node->right) { 37 | node = node_ptr(root, node->right); 38 | while (node->left) 39 | node = node_ptr(root, node->left); 40 | return node; 41 | } 42 | 43 | while ((parent = node_ptr(root, node->parent)) && 44 | node == node_ptr(root, parent->right)) 45 | node = parent; 46 | 47 | return parent; 48 | } 49 | -------------------------------------------------------------------------------- /utils/src/avl.h: -------------------------------------------------------------------------------- 1 | #ifndef _AVL_H_ 2 | #define _AVL_H_ 3 | 4 | __le16 avl_node_off(struct scoutfs_avl_root *root, 5 | struct scoutfs_avl_node *node); 6 | struct scoutfs_avl_node *avl_first(struct scoutfs_avl_root *root); 7 | struct scoutfs_avl_node *avl_next(struct scoutfs_avl_root *root, 8 | struct scoutfs_avl_node *node); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /utils/src/bitmap.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | 5 | #include "sparse.h" 6 | #include "util.h" 7 | #include "bitmap.h" 8 | 9 | /* 10 | * Just a quick simple native bitmap. 11 | */ 12 | 13 | int test_bit(unsigned long *bits, u64 nr) 14 | { 15 | return !!(bits[nr / BITS_PER_LONG] & (1UL << (nr & (BITS_PER_LONG - 1)))); 16 | } 17 | 18 | void set_bit(unsigned long *bits, u64 nr) 19 | { 20 | bits[nr / BITS_PER_LONG] |= 1UL << (nr & (BITS_PER_LONG - 1)); 21 | } 22 | 23 | void clear_bit(unsigned long *bits, u64 nr) 24 | { 25 | bits[nr / BITS_PER_LONG] &= ~(1UL << (nr & (BITS_PER_LONG - 1))); 26 | } 27 | 28 | u64 find_next_set_bit(unsigned long *map, u64 from, u64 total) 29 | { 30 | unsigned long bits; 31 | u64 base; 32 | u64 nr; 33 | int bit; 34 | 35 | base = from & ~((unsigned long)BITS_PER_LONG - 1); 36 | map += from / BITS_PER_LONG; 37 | 38 | while (base < total) { 39 | bits = *map; 40 | 41 | while (bits) { 42 | bit = ffsl(bits) - 1; 43 | nr = base + bit; 44 | 45 | if (nr >= from) 46 | return min(nr, total); 47 | 48 | bits &= ~(1UL << bit); 49 | } 50 | 51 | base += BITS_PER_LONG; 52 | map++; 53 | } 54 | 55 | return total; 56 | } 57 | 58 | unsigned long *alloc_bits(u64 max) 59 | { 60 | return calloc(DIV_ROUND_UP(max, BITS_PER_LONG), sizeof(unsigned long)); 61 | } 62 | 63 | -------------------------------------------------------------------------------- /utils/src/bitmap.h: -------------------------------------------------------------------------------- 1 | #ifndef _BITMAP_H_ 2 | #define _BITMAP_H_ 3 | 4 | int test_bit(unsigned long *bits, u64 nr); 5 | void set_bit(unsigned long *bits, u64 nr); 6 | void clear_bit(unsigned long *bits, u64 nr); 7 | u64 find_next_set_bit(unsigned long *start, u64 from, u64 total); 8 | unsigned long *alloc_bits(u64 max); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /utils/src/bitops.h: -------------------------------------------------------------------------------- 1 | #ifndef _BITOPS_H_ 2 | #define _BITOPS_H_ 3 | 4 | #include "sparse.h" 5 | 6 | /* 7 | * Implement little endian bitmaps in terms of native longs. __packed 8 | * is used to avoid unaligned accesses. These are neither atomic nor 9 | * particularly efficient. 10 | */ 11 | 12 | #define BITS_PER_LONG (sizeof(long) * 8) 13 | #if __BYTE_ORDER == __LITTLE_ENDIAN 14 | #define BITOP_LE_SWIZZLE 0 15 | #else 16 | #define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7) 17 | #endif 18 | 19 | static inline unsigned long get_nr_word(int nr, void *addr) 20 | { 21 | unsigned long *longs = addr; 22 | unsigned long ind = nr / BITS_PER_LONG; 23 | unsigned long val; 24 | 25 | memcpy(&val, &longs[ind], sizeof(val)); 26 | 27 | return val; 28 | } 29 | 30 | static inline void put_nr_word(int nr, void *addr, unsigned long val) 31 | { 32 | unsigned long *longs = addr; 33 | unsigned long ind = nr / BITS_PER_LONG; 34 | 35 | memcpy(&longs[ind], &val, sizeof(val)); 36 | } 37 | 38 | static inline unsigned long nr_mask(int nr) 39 | { 40 | return 1UL << (nr % BITS_PER_LONG); 41 | } 42 | 43 | static inline int test_bit(int nr, void *addr) 44 | { 45 | unsigned long val = get_nr_word(nr, addr); 46 | 47 | return !!(val & nr_mask(nr)); 48 | } 49 | 50 | static inline void set_bit(int nr, void *addr) 51 | { 52 | unsigned long val = get_nr_word(nr, addr); 53 | 54 | val |= nr_mask(nr); 55 | put_nr_word(nr, addr, val); 56 | } 57 | 58 | static inline void clear_bit(int nr, void *addr) 59 | { 60 | unsigned long val = get_nr_word(nr, addr); 61 | 62 | val &= ~nr_mask(nr); 63 | put_nr_word(nr, addr, val); 64 | } 65 | 66 | static inline int test_bit_le(int nr, void *addr) 67 | { 68 | return test_bit(nr ^ BITOP_LE_SWIZZLE, addr); 69 | } 70 | 71 | static inline int test_and_set_bit_le(int nr, void *addr) 72 | { 73 | int ret; 74 | 75 | nr ^= BITOP_LE_SWIZZLE; 76 | ret = test_bit(nr, addr); 77 | set_bit(nr, addr); 78 | return ret; 79 | } 80 | 81 | static inline void set_bit_le(int nr, void *addr) 82 | { 83 | set_bit(nr ^ BITOP_LE_SWIZZLE, addr); 84 | } 85 | 86 | static inline void clear_bit_le(int nr, void *addr) 87 | { 88 | clear_bit(nr ^ BITOP_LE_SWIZZLE, addr); 89 | } 90 | 91 | static inline int test_and_clear_bit_le(int nr, void *addr) 92 | { 93 | int ret; 94 | 95 | nr ^= BITOP_LE_SWIZZLE; 96 | ret = test_bit(nr, addr); 97 | clear_bit(nr, addr); 98 | return ret; 99 | } 100 | 101 | #endif 102 | -------------------------------------------------------------------------------- /utils/src/blkid.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include "util.h" 13 | #include "format.h" 14 | #include "blkid.h" 15 | 16 | static int check_bdev_blkid(int fd, char *devname, char *usage) 17 | { 18 | blkid_probe pr; 19 | int ret = 0; 20 | 21 | pr = blkid_new_probe_from_filename(devname); 22 | if (!pr) { 23 | fprintf(stderr, "%s: failed to create a new libblkid probe\n", devname); 24 | goto out; 25 | } 26 | 27 | /* enable partitions probing (superblocks are enabled by default) */ 28 | ret = blkid_probe_enable_partitions(pr, true); 29 | if (ret == -1) { 30 | fprintf(stderr, "%s: blkid_probe_enable_partitions() failed\n", devname); 31 | goto out; 32 | } 33 | 34 | ret = blkid_do_fullprobe(pr); 35 | if (ret == -1) { 36 | fprintf(stderr, "%s: blkid_do_fullprobe() failed", devname); 37 | goto out; 38 | } else if (ret == 0) { 39 | const char *type; 40 | 41 | if (!blkid_probe_lookup_value(pr, "TYPE", &type, NULL)) { 42 | fprintf(stderr, "%s: appears to contain an existing " 43 | "%s superblock\n", devname, type); 44 | ret = -1; 45 | goto out; 46 | } 47 | 48 | if (!blkid_probe_lookup_value(pr, "PTTYPE", &type, NULL)) { 49 | fprintf(stderr, "%s: appears to contain a partition " 50 | "table (%s)\n", devname, type); 51 | ret = -1; 52 | goto out; 53 | } 54 | } else { 55 | /* return 0 if ok */ 56 | ret = 0; 57 | } 58 | 59 | out: 60 | blkid_free_probe(pr); 61 | 62 | return ret; 63 | } 64 | 65 | static int check_bdev_scoutfs(int fd, char *devname, char *usage) 66 | { 67 | struct scoutfs_super_block *super = NULL; 68 | int ret; 69 | 70 | ret = read_block(fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT, (void **)&super); 71 | if (ret) 72 | return ret; 73 | 74 | if (le32_to_cpu(super->hdr.magic) == SCOUTFS_BLOCK_MAGIC_SUPER) { 75 | fprintf(stderr, "%s: appears to contain an existing " 76 | "ScoutFS superblock\n", devname); 77 | ret = -EINVAL; 78 | } 79 | 80 | free(super); 81 | 82 | return ret; 83 | } 84 | 85 | 86 | /* 87 | * Returns -1 on error, 0 otherwise. 88 | */ 89 | int check_bdev(int fd, char *devname, char *usage) 90 | { 91 | return check_bdev_blkid(fd, devname, usage) ?: 92 | /* Our sig is not in blkid (yet) so check explicitly for us. */ 93 | check_bdev_scoutfs(fd, devname, usage); 94 | } 95 | -------------------------------------------------------------------------------- /utils/src/blkid.h: -------------------------------------------------------------------------------- 1 | #ifndef _BLKID_H_ 2 | #define _BLKID_H_ 3 | 4 | int check_bdev(int fd, char *path, char *usage); 5 | 6 | #endif 7 | -------------------------------------------------------------------------------- /utils/src/bloom.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "sparse.h" 4 | #include "util.h" 5 | #include "format.h" 6 | #include "hash.h" 7 | #include "bloom.h" 8 | 9 | void calc_bloom_nrs(struct scoutfs_key *key, unsigned int *nrs) 10 | { 11 | u64 hash; 12 | int i; 13 | 14 | hash = scoutfs_hash64(key, sizeof(struct scoutfs_key)); 15 | 16 | for (i = 0; i < SCOUTFS_FOREST_BLOOM_NRS; i++) { 17 | nrs[i] = (u32)hash % SCOUTFS_FOREST_BLOOM_BITS; 18 | hash >>= SCOUTFS_FOREST_BLOOM_FUNC_BITS; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /utils/src/bloom.h: -------------------------------------------------------------------------------- 1 | #ifndef _BLOOM_H_ 2 | #define _BLOOM_H_ 3 | 4 | void calc_bloom_nrs(struct scoutfs_key *key, unsigned int *nrs); 5 | 6 | #endif 7 | -------------------------------------------------------------------------------- /utils/src/btree.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "sparse.h" 4 | #include "util.h" 5 | #include "format.h" 6 | #include "key.h" 7 | #include "avl.h" 8 | #include "leaf_item_hash.h" 9 | #include "btree.h" 10 | 11 | void btree_init_block(struct scoutfs_btree_block *bt, int level) 12 | { 13 | int free; 14 | 15 | free = SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block); 16 | if (level == 0) 17 | free -= SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; 18 | 19 | bt->level = level; 20 | bt->mid_free_len = cpu_to_le16(free); 21 | } 22 | 23 | /* 24 | * Point the root at the single leaf block that makes up a btree. 25 | */ 26 | void btree_init_root_single(struct scoutfs_btree_root *root, 27 | struct scoutfs_btree_block *bt, 28 | u64 seq, u64 blkno) 29 | { 30 | root->ref.blkno = cpu_to_le64(blkno); 31 | root->ref.seq = cpu_to_le64(seq); 32 | root->height = 1; 33 | 34 | memset(bt, 0, SCOUTFS_BLOCK_LG_SIZE); 35 | 36 | btree_init_block(bt, 0); 37 | } 38 | 39 | static void *alloc_val(struct scoutfs_btree_block *bt, int len) 40 | { 41 | le16_add_cpu(&bt->mid_free_len, -len); 42 | le16_add_cpu(&bt->total_item_bytes, len); 43 | return (void *)&bt->items[le16_to_cpu(bt->nr_items)] + le16_to_cpu(bt->mid_free_len); 44 | } 45 | 46 | /* 47 | * Add a sorted item after all the items in the block. 48 | * 49 | * We simply implement the special case of a wildly imbalanced avl tree. 50 | * Mkfs only ever inserts a handful of items and they'll be rebalanced 51 | * over time. 52 | */ 53 | void btree_append_item(struct scoutfs_btree_block *bt, 54 | struct scoutfs_key *key, void *val, int val_len) 55 | { 56 | struct scoutfs_btree_item *item; 57 | struct scoutfs_avl_node *prev; 58 | void *val_buf; 59 | 60 | item = &bt->items[le16_to_cpu(bt->nr_items)]; 61 | 62 | if (bt->nr_items) { 63 | assert(scoutfs_key_compare(key, &(item - 1)->key) > 0); 64 | prev = &(item - 1)->node; 65 | 66 | item->node.height = prev->height++; 67 | item->node.left = avl_node_off(&bt->item_root, prev); 68 | prev->parent = avl_node_off(&bt->item_root, &item->node); 69 | } 70 | 71 | bt->item_root.node = avl_node_off(&bt->item_root, &item->node); 72 | le16_add_cpu(&bt->nr_items, 1); 73 | le16_add_cpu(&bt->mid_free_len, 74 | -(u16)sizeof(struct scoutfs_btree_item)); 75 | le16_add_cpu(&bt->total_item_bytes, sizeof(struct scoutfs_btree_item)); 76 | 77 | item->key = *key; 78 | item->seq = cpu_to_le64(1); 79 | item->flags = 0; 80 | 81 | leaf_item_hash_insert(bt, &item->key, 82 | cpu_to_le16((void *)item - (void *)bt)); 83 | if (val_len == 0) 84 | return; 85 | 86 | val_buf = alloc_val(bt, val_len); 87 | item->val_off = cpu_to_le16((void *)val_buf - (void *)bt); 88 | item->val_len = cpu_to_le16(val_len); 89 | memcpy(val_buf, val, val_len); 90 | } 91 | -------------------------------------------------------------------------------- /utils/src/btree.h: -------------------------------------------------------------------------------- 1 | #ifndef _BTREE_H_ 2 | #define _BTREE_H_ 3 | 4 | void btree_init_block(struct scoutfs_btree_block *bt, int level); 5 | void btree_init_root_single(struct scoutfs_btree_root *root, 6 | struct scoutfs_btree_block *bt, 7 | u64 seq, u64 blkno); 8 | 9 | void btree_append_item(struct scoutfs_btree_block *bt, 10 | struct scoutfs_key *key, void *val, int val_len); 11 | 12 | #endif 13 | -------------------------------------------------------------------------------- /utils/src/cmd.h: -------------------------------------------------------------------------------- 1 | #ifndef _CMD_H_ 2 | #define _CMD_H_ 3 | 4 | #define GROUP_CORE 0 5 | #define GROUP_INFO 1 6 | #define GROUP_SEARCH 2 7 | #define GROUP_AGENT 3 8 | #define GROUP_DEBUG 4 9 | 10 | void cmd_register_argp(char *name, struct argp *argp, int group, 11 | int (*func)(int argc, char **argv)); 12 | 13 | char cmd_execute(int argc, char **argv); 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /utils/src/cmp.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_CMP_H_ 2 | #define _SCOUTFS_CMP_H_ 3 | 4 | /* 5 | * A generic ternary comparison macro with strict type checking. 6 | */ 7 | #define scoutfs_cmp(a, b) \ 8 | ({ \ 9 | __typeof__(a) _a = (a); \ 10 | __typeof__(b) _b = (b); \ 11 | int _ret; \ 12 | \ 13 | (void) (&_a == &_b); \ 14 | _ret = _a < _b ? -1 : _a > _b ? 1 : 0; \ 15 | _ret; \ 16 | }) 17 | 18 | static inline int scoutfs_cmp_u64s(u64 a, u64 b) 19 | { 20 | return a < b ? -1 : a > b ? 1 : 0; 21 | } 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /utils/src/crc.c: -------------------------------------------------------------------------------- 1 | #include "crc.h" 2 | #include "util.h" 3 | #include "format.h" 4 | 5 | u32 crc32c(u32 crc, const void *data, unsigned int len) 6 | { 7 | while (len >= 8) { 8 | crc = __builtin_ia32_crc32di(crc, *(u64 *)data); 9 | len -= 8; 10 | data += 8; 11 | } 12 | if (len & 4) { 13 | crc = __builtin_ia32_crc32si(crc, *(u32 *)data); 14 | data += 4; 15 | } 16 | if (len & 2) { 17 | crc = __builtin_ia32_crc32hi(crc, *(u16 *)data); 18 | data += 2; 19 | } 20 | if (len & 1) 21 | crc = __builtin_ia32_crc32qi(crc, *(u8 *)data); 22 | 23 | return crc; 24 | } 25 | 26 | /* A simple hack to get reasonably solid 64bit hash values */ 27 | u64 crc32c_64(u32 crc, const void *data, unsigned int len) 28 | { 29 | unsigned int half = (len + 1) / 2; 30 | 31 | return ((u64)crc32c(crc, data, half) << 32) | 32 | crc32c(~crc, data + len - half, half); 33 | } 34 | 35 | u32 crc_block(struct scoutfs_block_header *hdr, u32 size) 36 | { 37 | return crc32c(~0, (char *)hdr + sizeof(hdr->crc), 38 | size - sizeof(hdr->crc)); 39 | } 40 | -------------------------------------------------------------------------------- /utils/src/crc.h: -------------------------------------------------------------------------------- 1 | #ifndef _CRC_H_ 2 | #define _CRC_H_ 3 | 4 | #include "sparse.h" 5 | #include "util.h" 6 | #include "format.h" 7 | 8 | u32 crc32c(u32 crc, const void *data, unsigned int len); 9 | u64 crc32c_64(u32 crc, const void *data, unsigned int len); 10 | u32 crc_block(struct scoutfs_block_header *hdr, u32 size); 11 | 12 | #endif 13 | -------------------------------------------------------------------------------- /utils/src/dev.h: -------------------------------------------------------------------------------- 1 | #ifndef _DEV_H_ 2 | #define _DEV_H_ 3 | 4 | #include 5 | 6 | #define BASE_SIZE_FMT "%.2f%s" 7 | #define BASE_SIZE_ARGS(sz) size_flt(sz, 1), size_str(sz, 1) 8 | 9 | #define SIZE_FMT "%llu (%.2f %s)" 10 | #define SIZE_ARGS(nr, sz) (nr), size_flt(nr, sz), size_str(nr, sz) 11 | 12 | int get_device_size(char *path, int fd, u64 *size_ret); 13 | int limit_device_size(char *path, int fd, u64 min_size, u64 max_size, bool allow_small_size, 14 | char *use_type, u64 *size_ret); 15 | float size_flt(u64 nr, unsigned size); 16 | char *size_str(u64 nr, unsigned size); 17 | int flush_device(int fd); 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /utils/src/endian_swap.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_ENDIAN_SWAP_H_ 2 | #define _SCOUTFS_ENDIAN_SWAP_H_ 3 | 4 | #define le64_to_be64(x) cpu_to_be64(le64_to_cpu(x)) 5 | #define le32_to_be32(x) cpu_to_be32(le32_to_cpu(x)) 6 | #define le16_to_be16(x) cpu_to_be16(le16_to_cpu(x)) 7 | 8 | #define be64_to_le64(x) cpu_to_le64(be64_to_cpu(x)) 9 | #define be32_to_le32(x) cpu_to_le32(be32_to_cpu(x)) 10 | #define be16_to_le16(x) cpu_to_le16(be16_to_cpu(x)) 11 | 12 | #define le16_to_le64(x) cpu_to_le64(le16_to_cpu(x)) 13 | #define le32_to_le64(x) cpu_to_le64(le32_to_cpu(x)) 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /utils/src/hash.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_HASH_H_ 2 | #define _SCOUTFS_HASH_H_ 3 | 4 | /* 5 | * We're using FNV1a for now. It's fine. Ish. 6 | * 7 | * The longer term plan is xxh3 but it looks like it'll take just a bit 8 | * more time to be declared stable and then it needs to be ported to the 9 | * kernel. 10 | * 11 | * - https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html 12 | * - https://github.com/Cyan4973/xxHash/releases/tag/v0.7.4 13 | */ 14 | 15 | static inline u32 fnv1a32(const void *data, unsigned int len) 16 | { 17 | u32 hash = 0x811c9dc5; 18 | 19 | while (len--) { 20 | hash ^= *(u8 *)(data++); 21 | hash *= 0x01000193; 22 | } 23 | 24 | return hash; 25 | } 26 | 27 | static inline u64 fnv1a64(const void *data, unsigned int len) 28 | { 29 | u64 hash = 0xcbf29ce484222325ULL; 30 | 31 | while (len--) { 32 | hash ^= *(u8 *)(data++); 33 | hash *= 0x100000001b3ULL; 34 | } 35 | 36 | return hash; 37 | } 38 | 39 | static inline u32 scoutfs_hash32(const void *data, unsigned int len) 40 | { 41 | return fnv1a32(data, len); 42 | } 43 | 44 | static inline u64 scoutfs_hash64(const void *data, unsigned int len) 45 | { 46 | return fnv1a64(data, len); 47 | } 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /utils/src/leaf_item_hash.c: -------------------------------------------------------------------------------- 1 | #include "sparse.h" 2 | #include "util.h" 3 | #include "format.h" 4 | #include "hash.h" 5 | #include "leaf_item_hash.h" 6 | 7 | /* 8 | * A minimal extraction of the leaf item hash from the kernel's btree. 9 | */ 10 | 11 | int leaf_item_hash_ind(struct scoutfs_key *key) 12 | { 13 | return scoutfs_hash32(key, sizeof(struct scoutfs_key)) % 14 | SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; 15 | } 16 | 17 | __le16 *leaf_item_hash_buckets(struct scoutfs_btree_block *bt) 18 | { 19 | return (void *)bt + SCOUTFS_BLOCK_LG_SIZE - 20 | SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; 21 | } 22 | 23 | void leaf_item_hash_insert(struct scoutfs_btree_block *bt, 24 | struct scoutfs_key *key, __le16 off) 25 | { 26 | __le16 *buckets = leaf_item_hash_buckets(bt); 27 | int i; 28 | 29 | if (bt->level > 0) 30 | return; 31 | 32 | for (i = leaf_item_hash_ind(key); 33 | i < SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; i++) { 34 | if (buckets[i] == 0) { 35 | buckets[i] = off; 36 | return; 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /utils/src/leaf_item_hash.h: -------------------------------------------------------------------------------- 1 | #ifndef _LEAF_ITEM_HASH_H_ 2 | #define _LEAF_ITEM_HASH_H_ 3 | 4 | int leaf_item_hash_ind(struct scoutfs_key *key); 5 | __le16 *leaf_item_hash_buckets(struct scoutfs_btree_block *bt); 6 | void leaf_item_hash_insert(struct scoutfs_btree_block *bt, 7 | struct scoutfs_key *key, __le16 off); 8 | 9 | #endif 10 | -------------------------------------------------------------------------------- /utils/src/lk_rbtree_wrapper.h: -------------------------------------------------------------------------------- 1 | #ifndef _LK_RBTREE_WRAPPER_H_ 2 | #define _LK_RBTREE_WRAPPER_H_ 3 | 4 | /* 5 | * We're using this lame hack to build and use the kernel's rbtree in 6 | * userspace. We drop the kernel's rbtree*[ch] implementation in and 7 | * use them with this wrapper. We only have to remove the kernel 8 | * includes from the imported files. 9 | */ 10 | 11 | #include 12 | #include "util.h" 13 | 14 | #define rcu_assign_pointer(a, b) do { a = b; } while (0) 15 | #define READ_ONCE(a) ({ a; }) 16 | #define WRITE_ONCE(a, b) do { a = b; } while (0) 17 | #define unlikely(a) ({ a; }) 18 | #define EXPORT_SYMBOL(a) /* nop */ 19 | 20 | #include "rbtree_types.h" 21 | #include "rbtree.h" 22 | #include "rbtree_augmented.h" 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /utils/src/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "cmd.h" 10 | #include "util.h" 11 | 12 | /* 13 | * Ensure no compiler-added padding sneaks into structs defined in these 14 | * headers. 15 | */ 16 | #pragma GCC diagnostic error "-Wpadded" 17 | #include "format.h" 18 | #include "ioctl.h" 19 | #pragma GCC diagnostic pop 20 | 21 | int main(int argc, char **argv) 22 | { 23 | /* 24 | * XXX parse global options, env, configs, etc. 25 | */ 26 | 27 | return cmd_execute(argc, argv); 28 | } 29 | -------------------------------------------------------------------------------- /utils/src/mode_types.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "sparse.h" 5 | #include "util.h" 6 | #include "format.h" 7 | #include "mode_types.h" 8 | 9 | unsigned int mode_to_type(mode_t mode) 10 | { 11 | #define S_SHIFT 12 12 | static unsigned char mode_types[S_IFMT >> S_SHIFT] = { 13 | [S_IFIFO >> S_SHIFT] = SCOUTFS_DT_FIFO, 14 | [S_IFCHR >> S_SHIFT] = SCOUTFS_DT_CHR, 15 | [S_IFDIR >> S_SHIFT] = SCOUTFS_DT_DIR, 16 | [S_IFBLK >> S_SHIFT] = SCOUTFS_DT_BLK, 17 | [S_IFREG >> S_SHIFT] = SCOUTFS_DT_REG, 18 | [S_IFLNK >> S_SHIFT] = SCOUTFS_DT_LNK, 19 | [S_IFSOCK >> S_SHIFT] = SCOUTFS_DT_SOCK, 20 | }; 21 | 22 | return mode_types[(mode & S_IFMT) >> S_SHIFT]; 23 | #undef S_SHIFT 24 | } 25 | -------------------------------------------------------------------------------- /utils/src/mode_types.h: -------------------------------------------------------------------------------- 1 | #ifndef _MODE_TYPES_H_ 2 | #define _MODE_TYPES_H_ 3 | 4 | unsigned int mode_to_type(mode_t mode); 5 | 6 | #endif 7 | -------------------------------------------------------------------------------- /utils/src/name_hash.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCOUTFS_NAME_HASH_H_ 2 | #define _SCOUTFS_NAME_HASH_H_ 3 | 4 | #include "hash.h" 5 | 6 | /* 7 | * Test a bit number as though an array of bytes is a large len-bit 8 | * big-endian value. nr 0 is the LSB of the final byte, nr (len - 1) is 9 | * the MSB of the first byte. 10 | */ 11 | static int test_be_bytes_bit(int nr, const char *bytes, int len) 12 | { 13 | return bytes[(len - 1 - nr) >> 3] & (1 << (nr & 7)); 14 | } 15 | 16 | /* 17 | * Generate a 32bit "fingerprint" of the name by extracting 32 evenly 18 | * distributed bits from the name. The intent is to have the sort order 19 | * of the fingerprints reflect the memcmp() sort order of the names 20 | * while mapping large names down to small fs keys. 21 | * 22 | * Names that are smaller than 32bits are biased towards the high bits 23 | * of the fingerprint so that most significant bits of the fingerprints 24 | * consistently reflect the initial characters of the names. 25 | */ 26 | static inline u32 dirent_name_fingerprint(const char *name, unsigned int name_len) 27 | { 28 | int name_bits = name_len * 8; 29 | int skip = max(name_bits / 32, 1); 30 | u32 fp = 0; 31 | int f; 32 | int n; 33 | 34 | for (f = 31, n = name_bits - 1; f >= 0 && n >= 0; f--, n -= skip) 35 | fp |= !!test_be_bytes_bit(n, name, name_bits) << f; 36 | 37 | return fp; 38 | } 39 | 40 | static inline u64 dirent_name_hash(const char *name, unsigned int name_len) 41 | { 42 | return scoutfs_hash32(name, name_len) | 43 | ((u64)dirent_name_fingerprint(name, name_len) << 32); 44 | } 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /utils/src/parse.h: -------------------------------------------------------------------------------- 1 | #ifndef _PARSE_H_ 2 | #define _PARSE_H_ 3 | 4 | #include 5 | #include 6 | 7 | struct scoutfs_quorum_slot; 8 | 9 | int parse_human(char* str, u64 *val_ret); 10 | int parse_u64(char *str, u64 *val_ret); 11 | int parse_s64(char *str, s64 *val_ret); 12 | int parse_u32(char *str, u32 *val_ret); 13 | int parse_timespec(char *str, struct timespec *ts); 14 | int parse_quorum_slot(struct scoutfs_quorum_slot *slot, char *arg); 15 | 16 | static inline char* strdup_or_error(const struct argp_state *state, char *str) 17 | { 18 | char *new = strdup(str); 19 | if (!new) 20 | argp_error(state, "memory allocation failed"); 21 | 22 | return new; 23 | } 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /utils/src/quorum.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "sparse.h" 6 | #include "util.h" 7 | #include "format.h" 8 | 9 | #include "quorum.h" 10 | 11 | bool quorum_slot_present(struct scoutfs_super_block *super, int i) 12 | { 13 | return super->qconf.slots[i].addr.v4.family == cpu_to_le16(SCOUTFS_AF_IPV4); 14 | } 15 | 16 | bool valid_quorum_slots(struct scoutfs_quorum_slot *slots) 17 | { 18 | struct in_addr in; 19 | bool valid = true; 20 | char *addr; 21 | int i; 22 | int j; 23 | 24 | for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { 25 | if (slots[i].addr.v4.family == cpu_to_le16(SCOUTFS_AF_NONE)) 26 | continue; 27 | 28 | if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4)) { 29 | fprintf(stderr, "quorum slot nr %u has invalid family %u\n", 30 | i, le16_to_cpu(slots[i].addr.v4.family)); 31 | valid = false; 32 | } 33 | 34 | for (j = i + 1; j < SCOUTFS_QUORUM_MAX_SLOTS; j++) { 35 | if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4)) 36 | continue; 37 | 38 | if (slots[i].addr.v4.addr == slots[j].addr.v4.addr && 39 | slots[i].addr.v4.port == slots[j].addr.v4.port) { 40 | 41 | in.s_addr = 42 | htonl(le32_to_cpu(slots[i].addr.v4.addr)); 43 | addr = inet_ntoa(in); 44 | fprintf(stderr, "quorum slot nr %u and %u have the same address %s:%u\n", 45 | i, j, addr, 46 | le16_to_cpu(slots[i].addr.v4.port)); 47 | valid = false; 48 | } 49 | } 50 | } 51 | 52 | return valid; 53 | } 54 | 55 | /* 56 | * Print quorum slots to stdout, a line at a time. The first line is 57 | * not indented and the rest of the lines use the indent string from the 58 | * caller. 59 | */ 60 | void print_quorum_slots(struct scoutfs_quorum_slot *slots, int nr, char *indent) 61 | { 62 | struct scoutfs_quorum_slot *sl; 63 | struct in_addr in; 64 | bool first = true; 65 | int i; 66 | 67 | for (i = 0, sl = slots; i < SCOUTFS_QUORUM_MAX_SLOTS; i++, sl++) { 68 | 69 | if (sl->addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4)) 70 | continue; 71 | 72 | in.s_addr = htonl(le32_to_cpu(sl->addr.v4.addr)); 73 | printf("%s%u: %s:%u\n", first ? "" : indent, 74 | i, inet_ntoa(in), le16_to_cpu(sl->addr.v4.port)); 75 | 76 | first = false; 77 | } 78 | } 79 | 80 | -------------------------------------------------------------------------------- /utils/src/quorum.h: -------------------------------------------------------------------------------- 1 | #ifndef _QUORUM_H_ 2 | #define _QUORUM_H_ 3 | 4 | #include 5 | 6 | bool quorum_slot_present(struct scoutfs_super_block *super, int i); 7 | bool valid_quorum_slots(struct scoutfs_quorum_slot *slots); 8 | void print_quorum_slots(struct scoutfs_quorum_slot *slots, int nr, char *indent); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /utils/src/rand.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "rand.h" 4 | #include "sparse.h" 5 | #include "util.h" 6 | 7 | #include 8 | 9 | void pseudo_random_bytes(void *data, unsigned int len) 10 | { 11 | RAND_bytes(data, len); 12 | } 13 | -------------------------------------------------------------------------------- /utils/src/rand.h: -------------------------------------------------------------------------------- 1 | #ifndef _RAND_H_ 2 | #define _RAND_H_ 3 | 4 | /* 5 | * We could play around a bit with some macros to get aligned constant 6 | * word sized buffers filled by single instructions. 7 | */ 8 | void pseudo_random_bytes(void *data, unsigned int len); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /utils/src/rbtree_types.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 | #ifndef _LINUX_RBTREE_TYPES_H 3 | #define _LINUX_RBTREE_TYPES_H 4 | 5 | struct rb_node { 6 | unsigned long __rb_parent_color; 7 | struct rb_node *rb_right; 8 | struct rb_node *rb_left; 9 | } __attribute__((aligned(sizeof(long)))); 10 | /* The alignment might seem pointless, but allegedly CRIS needs it */ 11 | 12 | struct rb_root { 13 | struct rb_node *rb_node; 14 | }; 15 | 16 | /* 17 | * Leftmost-cached rbtrees. 18 | * 19 | * We do not cache the rightmost node based on footprint 20 | * size vs number of potential users that could benefit 21 | * from O(1) rb_last(). Just not worth it, users that want 22 | * this feature can always implement the logic explicitly. 23 | * Furthermore, users that want to cache both pointers may 24 | * find it a bit asymmetric, but that's ok. 25 | */ 26 | struct rb_root_cached { 27 | struct rb_root rb_root; 28 | struct rb_node *rb_leftmost; 29 | }; 30 | 31 | #define RB_ROOT (struct rb_root) { NULL, } 32 | #define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL } 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /utils/src/read_xattr_totals.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "sparse.h" 13 | #include "parse.h" 14 | #include "util.h" 15 | #include "format.h" 16 | #include "ioctl.h" 17 | #include "cmd.h" 18 | 19 | struct xattr_args { 20 | char *path; 21 | }; 22 | 23 | static int do_read_xattr_totals(struct xattr_args *args) 24 | { 25 | struct scoutfs_ioctl_read_xattr_totals rxt; 26 | struct scoutfs_ioctl_xattr_total *xts = NULL; 27 | struct scoutfs_ioctl_xattr_total *xt; 28 | u64 bytes = 1024 * 1024; 29 | int fd = -1; 30 | int ret; 31 | int i; 32 | 33 | xts = malloc(bytes); 34 | if (!xts) { 35 | fprintf(stderr, "xattr total mem alloc failed\n"); 36 | ret = -ENOMEM; 37 | goto out; 38 | } 39 | 40 | fd = get_path(args->path, O_RDONLY); 41 | if (fd < 0) 42 | return fd; 43 | 44 | memset(&rxt, 0, sizeof(rxt)); 45 | rxt.totals_ptr = (unsigned long)xts; 46 | rxt.totals_bytes = bytes; 47 | 48 | for (;;) { 49 | ret = ioctl(fd, SCOUTFS_IOC_READ_XATTR_TOTALS, &rxt); 50 | if (ret == 0) 51 | break; 52 | if (ret < 0) { 53 | ret = -errno; 54 | fprintf(stderr, "read_xattr_totals ioctl failed: " 55 | "%s (%d)\n", strerror(errno), errno); 56 | goto out; 57 | } 58 | 59 | for (i = 0, xt = xts; i < ret; i++, xt++) 60 | printf("%llu.%llu.%llu = %lld, %lld\n", 61 | xt->name[0], xt->name[1], xt->name[2], xt->total, xt->count); 62 | 63 | memcpy(&rxt.pos_name, &xts[ret - 1].name, sizeof(rxt.pos_name)); 64 | if (++rxt.pos_name[2] == 0 && ++rxt.pos_name[1] == 0 && ++rxt.pos_name[0] == 0) 65 | break; 66 | } 67 | 68 | ret = 0; 69 | out: 70 | if (fd >= 0) 71 | close(fd); 72 | free(xts); 73 | 74 | return ret; 75 | }; 76 | 77 | static int parse_opt(int key, char *arg, struct argp_state *state) 78 | { 79 | struct xattr_args *args = state->input; 80 | 81 | switch (key) { 82 | case 'p': 83 | args->path = strdup_or_error(state, arg); 84 | break; 85 | default: 86 | break; 87 | } 88 | 89 | return 0; 90 | } 91 | 92 | static struct argp_option options[] = { 93 | { "path", 'p', "PATH", 0, "Path to ScoutFS filesystem"}, 94 | { NULL } 95 | }; 96 | 97 | static struct argp argp = { 98 | options, 99 | parse_opt, 100 | "", 101 | "Print global value totals of .totl. xattrs" 102 | }; 103 | 104 | static int read_xattr_totals_cmd(int argc, char **argv) 105 | { 106 | 107 | struct xattr_args xattr_args = {NULL}; 108 | int ret; 109 | 110 | ret = argp_parse(&argp, argc, argv, 0, NULL, &xattr_args); 111 | if (ret) 112 | return ret; 113 | 114 | return do_read_xattr_totals(&xattr_args); 115 | } 116 | 117 | static void __attribute__((constructor)) read_xattr_totals_ctor(void) 118 | { 119 | cmd_register_argp("read-xattr-totals", &argp, GROUP_INFO, read_xattr_totals_cmd); 120 | } 121 | -------------------------------------------------------------------------------- /utils/src/srch.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "sparse.h" 4 | #include "util.h" 5 | #include "format.h" 6 | #include "srch.h" 7 | 8 | /* shifting by width is undefined :/ */ 9 | #define BYTE_MASK(b) ((1ULL << (b << 3)) - 1) 10 | static u64 byte_masks[] = { 11 | 0, BYTE_MASK(1), BYTE_MASK(2), BYTE_MASK(3), 12 | BYTE_MASK(4), BYTE_MASK(5), BYTE_MASK(6), BYTE_MASK(7), U64_MAX, 13 | }; 14 | 15 | static u64 decode_u64(void *buf, int bytes) 16 | { 17 | u64 val = get_unaligned_le64(buf) & byte_masks[bytes]; 18 | 19 | return (val >> 1) ^ (-(val & 1)); 20 | } 21 | 22 | int srch_decode_entry(void *buf, struct scoutfs_srch_entry *sre, 23 | struct scoutfs_srch_entry *prev) 24 | { 25 | u64 diffs[3]; 26 | u16 lengths; 27 | int bytes; 28 | int tot; 29 | int i; 30 | 31 | lengths = get_unaligned_le16(buf); 32 | tot = 2; 33 | 34 | for (i = 0; i < array_size(diffs); i++) { 35 | bytes = min(8, lengths & 15); 36 | diffs[i] = decode_u64(buf + tot, bytes); 37 | tot += bytes; 38 | lengths >>= 4; 39 | } 40 | 41 | sre->hash = cpu_to_le64(le64_to_cpu(prev->hash) + diffs[0]); 42 | sre->ino = cpu_to_le64(le64_to_cpu(prev->ino) + diffs[1]); 43 | sre->id = cpu_to_le64(le64_to_cpu(prev->id) + diffs[2]); 44 | 45 | return tot; 46 | } 47 | 48 | static int encode_u64(__le64 *buf, u64 val) 49 | { 50 | int bytes; 51 | 52 | val = (val << 1) ^ ((s64)val >> 63); /* shift sign extend */ 53 | bytes = (fls64(val) + 7) >> 3; 54 | 55 | put_unaligned_le64(val, buf); 56 | return bytes; 57 | } 58 | 59 | int srch_encode_entry(void *buf, struct scoutfs_srch_entry *sre, struct scoutfs_srch_entry *prev) 60 | { 61 | u64 diffs[] = { 62 | le64_to_cpu(sre->hash) - le64_to_cpu(prev->hash), 63 | le64_to_cpu(sre->ino) - le64_to_cpu(prev->ino), 64 | le64_to_cpu(sre->id) - le64_to_cpu(prev->id), 65 | }; 66 | u16 lengths = 0; 67 | int bytes; 68 | int tot = 2; 69 | int i; 70 | 71 | for (i = 0; i < array_size(diffs); i++) { 72 | bytes = encode_u64(buf + tot, diffs[i]); 73 | lengths |= bytes << (i << 2); 74 | tot += bytes; 75 | } 76 | 77 | put_unaligned_le16(lengths, buf); 78 | 79 | return tot; 80 | } 81 | -------------------------------------------------------------------------------- /utils/src/srch.h: -------------------------------------------------------------------------------- 1 | #ifndef _SRCH_H_ 2 | #define _SRCH_H_ 3 | 4 | int srch_decode_entry(void *buf, struct scoutfs_srch_entry *sre, 5 | struct scoutfs_srch_entry *prev); 6 | int srch_encode_entry(void *buf, struct scoutfs_srch_entry *sre, struct scoutfs_srch_entry *prev); 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /utils/tex/.gitignore: -------------------------------------------------------------------------------- 1 | missfont.log 2 | *.fls 3 | *.aux 4 | *.d 5 | *.d 6 | *.fdb_latexmk 7 | *.log 8 | *.pdf 9 | -------------------------------------------------------------------------------- /utils/tex/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # # dnf install latexmk texlive 3 | # # make 4 | # 5 | # Tools 6 | LATEXMK = latexmk 7 | RM = rm -f 8 | 9 | # Project-specific settings 10 | DOCNAME = scoutfs 11 | 12 | # Targets 13 | all: doc 14 | doc: pdf 15 | pdf: $(DOCNAME).pdf 16 | 17 | # Rules 18 | %.pdf: %.tex 19 | $(LATEXMK) -pdf -M -MP -MF $*.d $* 20 | 21 | mostlyclean: 22 | $(LATEXMK) -silent -c 23 | $(RM) *.bbl 24 | 25 | clean: mostlyclean 26 | $(LATEXMK) -silent -C 27 | $(RM) *.run.xml *.synctex.gz 28 | $(RM) *.d 29 | 30 | .PHONY: all clean doc mostlyclean pdf 31 | 32 | # Include auto-generated dependencies 33 | -include *.d 34 | --------------------------------------------------------------------------------