├── .github ├── FUNDING.yml └── workflows │ └── build.yml ├── .gitignore ├── .gitmodules ├── COPYING ├── GNUmakefile ├── README.md ├── docs └── kant_handle_my_swag.jpeg ├── harvest ├── hvst ├── LICENSE ├── README.md └── hvst ├── lua ├── Makefile ├── README.md └── src │ ├── Makefile │ ├── align.lua │ ├── cliargs.lua │ ├── cliargs │ ├── config_loader.lua │ ├── constants.lua │ ├── core.lua │ ├── parser.lua │ ├── printer.lua │ └── utils │ │ ├── disect.lua │ │ ├── disect_argument.lua │ │ ├── filter.lua │ │ ├── lookup.lua │ │ ├── shallow_copy.lua │ │ ├── split.lua │ │ ├── trim.lua │ │ └── wordwrap.lua │ ├── extensions.json │ ├── harvest.lua │ ├── import_file_extension_list.lua │ ├── lfs.c │ └── lfs.h └── test ├── GNUmakefile ├── bats ├── bin │ └── bats ├── lib │ └── bats-core │ │ ├── common.bash │ │ ├── formatter.bash │ │ ├── preprocessing.bash │ │ ├── semaphore.bash │ │ ├── test_functions.bash │ │ ├── tracing.bash │ │ ├── validator.bash │ │ └── warnings.bash └── libexec │ └── bats-core │ ├── bats │ ├── bats-exec-file │ ├── bats-exec-suite │ ├── bats-exec-test │ ├── bats-format-cat │ ├── bats-format-junit │ ├── bats-format-pretty │ ├── bats-format-tap │ ├── bats-format-tap13 │ └── bats-preprocess ├── bats_setup ├── setup_suite.bash └── test.bats /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: dyne 2 | patreon: dyneorg 3 | open_collective: dyne 4 | 5 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Test harvest 2 | on: 3 | push: 4 | paths-ignore: 5 | - 'docs/**' 6 | - 'examples/**' 7 | - '*.md' 8 | branches: 9 | - main 10 | pull_request: 11 | branches: 12 | - main 13 | 14 | jobs: 15 | shared: 16 | strategy: 17 | matrix: 18 | os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04] 19 | version: [11] 20 | runs-on: '${{ matrix.os }}' 21 | name: '🐧 Linux: ${{ matrix.os }} / ${{ matrix.version }}' 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v2 25 | - name: Install dependencies 26 | run: | 27 | sudo apt-get update 28 | sudo apt-get install --force-yes zsh 29 | - name: Run tests 30 | run: | 31 | make -C test 32 | 33 | mac-osx: 34 | runs-on: macos-12 35 | name: 🍎 MacOSX 36 | steps: 37 | - name: Checkout 38 | uses: actions/checkout@v2 39 | - name: Install dependencies 40 | run: | 41 | brew install coreutils 42 | - name: Run tests 43 | run: | 44 | make -C test 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **.a 2 | **.o -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "zuper"] 2 | path = shell/zuper 3 | url = https://github.com/dyne/zuper 4 | [submodule "file-extension-list"] 5 | path = file-extension-list 6 | url = https://github.com/dyne/file-extension-list 7 | -------------------------------------------------------------------------------- /GNUmakefile: -------------------------------------------------------------------------------- 1 | 2 | PREFIX ?= /usr/local 3 | BINDIR ?= ${DESTDIR}${PREFIX}/bin 4 | 5 | install: 6 | $(info Installing harvest in ${BINDIR}) 7 | install -p harvest ${BINDIR}/harvest 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Harvest - manage large collections of files and dirs 2 | 3 | Harvest makes it easy to list files and folders by type and copy or 4 | move them around. 5 | 6 | ![Kant handle my swag](docs/kant_handle_my_swag.jpeg) 7 | 8 | Harvest is a compact and portable script to scan files and folders and 9 | recognise their typology. Scanning is based on [file 10 | extensions](https://github.com/dyne/file-extension-list) and a simple 11 | fuzzy logic analysis of **folder contents** (not just files) to 12 | recognise if they are related to video, audio or text materials, etc. 13 | 14 | Harvest is **fast**: it can read approximately 1GB of stored filenames 15 | per second and is operated from the console terminal. It never 16 | modifies the filesystem: that is done explicitly by the user piping 17 | shell commands. 18 | 19 | [![Software by Dyne.org ](https://files.dyne.org/software_by_dyne.png)](https://dyne.org) 20 | 21 | Harvest operates on folders containing files without exploding the 22 | files around: it assesses the typology of a folder from the files 23 | contained, but does not promote move the files outside of that folder. For 24 | instance it works very well to move around large collections of 25 | downloaded torrent folders. 26 | 27 | ## :floppy_disk: Installation 28 | 29 | Harvest is a Zsh script and works on any POSIX platform where it can be installed including GNU/Linux, Apple/OSX and MS/Windows. 30 | 31 | Install the latest harvest with: 32 | ``` 33 | curl https://raw.githubusercontent.com/dyne/harvest/main/harvest | sudo tee /usr/local/bin/harvest 34 | ``` 35 | 36 | Dependencies: `zsh` 37 | 38 | Optional: 39 | - `fuse tmsu` for tagged filesystem 40 | - `setfattr` for setting file attributes 41 | 42 | ## :video_game: Usage 43 | 44 | Scan a folder /PATH/ to show and save results 45 | ``` 46 | harvest scan [PATH] 47 | ``` 48 | 49 | List of supported category types: 50 | ``` 51 | code image video book text font web archiv sheet exec slide audio 52 | ``` 53 | 54 | Move all scanned text files in /PATH/ to /DEST/ 55 | ``` 56 | harvest scan [PATH] | grep ';text;' | xargs -rn1 -I% mv % [DEST] 57 | ``` 58 | 59 | Tag all file attributes in /PATH/ with `harvest.type` categories 60 | ``` 61 | harvest attr [PATH] 62 | ``` 63 | 64 | Tag all files for use with TMSU (See section below about TMSU) 65 | ``` 66 | harvest tmsu [PATH] 67 | ``` 68 | 69 | 70 | ## TMSU 71 | 72 | This implementation supports tagged filesystems using [TMSU](https://github.com/oniony/TMSU). 73 | 74 | To allow the navigation of files in the style of a [Semantic Filesystem](https://en.wikipedia.org/wiki/Semantic_file_system), Harvest supports [TMSU](https://tmsu.org/), an small utility to maintain a database of tags inside an hidden directory `.tmsu` in each harvested folder. 75 | 76 | To initialise a `tmsu` database bootstrapped with harvest's tags in the currently harvested folder, do: 77 | ``` 78 | harvest tmsu 79 | ``` 80 | Directories indexed this way can then be "mounted" (using fuse) and navigated: 81 | ``` 82 | harvest mount 83 | ``` 84 | Inside the `$harvest` hidden subfolder (pointing to `.mnt` inside the folder) tags will become folders containing symbolic links to the actual tagged files. Any filemananger following symbolic links can be used to navigate tags, also tags will be set as bookmarks in graphical filemanagers (GTK3 supported). 85 | 86 | In addition to the tags view, there is also a queries folder in which you can run view queries by listing or creating new folders: 87 | ``` 88 | ls -l "$harvest/queries/text and 2018" 89 | ``` 90 | This automatic creation of the query folders makes it possible to use new file queries within the file chooser of a graphical program simply by typing the query in. Unwanted query folders can be safely removed. 91 | 92 | Limited tag management is also possible via the virtual filesystem. For example one can remove specific tags from a file by deleting the symbolic link in the tag folder, or delete a tag by performing a recursive delete. 93 | 94 | To unmount all TMSU semantic filesystems currently mounted, just do: 95 | ``` 96 | harvest umount 97 | ``` 98 | Further TMSU operations are possible operating directly from inside the directories that have been indexed using `harvest tmsu`, for more information see `tmsu help`. For instance, TMSU also detects duplicate files using `tmsu dupes`. 99 | -------------------------------------------------------------------------------- /docs/kant_handle_my_swag.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyne/harvest/b28df3aca349a9a35aee4a8a05f6a8e64a83e6a5/docs/kant_handle_my_swag.jpeg -------------------------------------------------------------------------------- /hvst/LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. 4 | 5 | In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and 6 | successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 9 | 10 | For more information, please refer to 11 | -------------------------------------------------------------------------------- /hvst/README.md: -------------------------------------------------------------------------------- 1 | # hvst 2 | 3 | A wrapper around [dyne/harvest](https://github.com/dyne/harvest) that fills in [missing functionality](https://github.com/dyne/harvest/issues/5) 4 | 5 | At its heart [dyne/harvest](https://github.com/dyne/harvest) is a file categorization tool. 6 | `hvst` takes these categorized files and lets you copy them or move them in bulk. 7 | 8 | ## Installation 9 | 10 | ```sh 11 | cp hvst ~/.local/bin 12 | ``` 13 | 14 | Dependencies: harvest, perl, sed, mkdir, cp, mv 15 | 16 | Optional Dependency: tmsu-fs-mv 17 | 18 | ## Usage 19 | 20 | ``` 21 | hvst 22 | hvst help 23 | hvst ls [TYPE] [OPTION] 24 | hvst types 25 | hvst cp [OPTION]... 26 | hvst mv [OPTION]... 27 | hvst tmv [OPTION]... 28 | ``` 29 | 30 | ## Examples 31 | 32 | ```sh 33 | # Display help message. 34 | hvst help 35 | 36 | # List files+metadata in current directory. 37 | hvst 38 | 39 | # Also list files+metadata in current directory. 40 | hvst ls 41 | 42 | # List only audio files+metadata. 43 | hvst ls audio 44 | 45 | # List only book files WITHOUT metadata. (filename only) 46 | hvst ls book -1 47 | 48 | # List types of files that exist in this directory and their counts. 49 | hvst types 50 | 51 | # Copy all images to ~/Pictures. 52 | hvst cp image ~/Pictures 53 | 54 | # Copy all images to ~/Pictures and be verbose. 55 | hvst cp image ~/Pictures --verbose 56 | 57 | # Show what would happen if we tried to move all images to ~/Pictures/YYYY/MM. 58 | hvst mv image @'"~/Pictures/$year/$month"' --recon 59 | 60 | # Move all videos to ~/Videos/YYYY/MM. 61 | hvst mv video @'"~/Videos/$year/$month"' 62 | 63 | # Move all books to ~/Dropbox/books using tmsu-fs-mv. 64 | hvst tmv book ~/Dropbox/books 65 | 66 | # The cp, mv, and tmv commands take --verbose and --recon 67 | # (or -v and -r for short). 68 | # --verbose means print the shell command. 69 | # --recon means print the shell command but don't execute it. 70 | ``` 71 | 72 | ### Destination Expressions 73 | 74 | The `` for for `cp`, `mv`, and `tmv` can be a Perl expression. 75 | A Leading "@" tells `hvst` to evaluate the string as Perl code. 76 | The following variables will be available. 77 | 78 | * $i - numeric index 79 | * $nt - node type (file or dir) 80 | * $t - file type 81 | * $d - date in YYYY-MM-DD format 82 | * $mt - modified time in seconds 83 | * $year - year 84 | * $month - month (zero padded) 85 | * $day - day (zero padded) 86 | * $s - size in bytes 87 | * $n - name 88 | 89 | The `$_` variable is also available and is a hashref containing all of the above values. 90 | 91 | ## My System 92 | 93 | To keep my `~/Downloads` tidy, I use these aliases and run them periodically. 94 | I don't run this automatically, because I often delete files I don't want before 95 | running `hvst`. 96 | 97 | ```bash 98 | alias hmvi="hvst mv image @'\"~/Pictures/\$year/\$month\"'" 99 | alias hmvv="hvst mv video @'\"~/Videos/\$year/\$month\"'" 100 | ``` 101 | 102 | To help me find them later, I tag them with [tmsu](https://github.com/oniony/TMSU). 103 | -------------------------------------------------------------------------------- /hvst/hvst: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | use warnings; 4 | use feature 'say'; 5 | use Getopt::Long; 6 | 7 | # run harvest and parse its output 8 | sub harvest { 9 | my @lines = `harvest --output=csv --file 2> /dev/null | sed 1d`; 10 | my @files; 11 | my $i = 1; 12 | for (@lines) { 13 | /^(\w+),(\w+),(\d+),(\d+),(.*)$/; 14 | my @lt = localtime($3); 15 | my $file = { 16 | i => $i, 17 | nt => $1, 18 | t => $2, 19 | mt => $3, 20 | s => $4, 21 | n => $5 22 | }; 23 | $file->{n} =~ s/\s*$//; 24 | $file->{year} = $lt[5]+1900; 25 | $file->{month} = sprintf('%02d', $lt[4]+1); 26 | $file->{day} = sprintf('%02d', $lt[3]); 27 | $file->{d} = "$file->{year}-$file->{month}-$file->{day}"; 28 | push @files, $file; 29 | $i++; 30 | } 31 | return @files; 32 | } 33 | 34 | # evaluate a destination expression 35 | sub expr { 36 | my $file = shift; 37 | my $expr = shift; 38 | if ($expr =~ /^@/) { 39 | # leading '@' means eval as code 40 | $expr =~ s/^@//; 41 | $_ = $file; 42 | # add variables for ease of use in expression 43 | my $i = $_->{i}; 44 | my $nt = $_->{nt}; 45 | my $t = $_->{t}; 46 | my $d = $_->{d}; 47 | my $mt = $_->{mt}; 48 | my $s = $_->{s}; 49 | my $n = $_->{n}; 50 | my $year = $_->{year}; 51 | my $month = $_->{month}; 52 | my $day = $_->{day}; 53 | my $dest = eval($expr); 54 | $dest =~ s/^~/\$HOME/; # do tilde expansion manually 55 | return $dest; 56 | } else { 57 | # just a string 58 | return $expr 59 | } 60 | } 61 | 62 | # list files by type 63 | sub ls { 64 | my $type = shift; 65 | return grep { $_->{t} eq $type } harvest(); 66 | } 67 | 68 | # list types of files and their count 69 | sub types { 70 | my %count; 71 | for (harvest()) { 72 | if (exists $count{$_->{t}}) { 73 | $count{$_->{t}}++; 74 | } else { 75 | $count{$_->{t}} = 1; 76 | } 77 | } 78 | my @pairs; 79 | while (my @p = each %count) { 80 | push @pairs, \@p; 81 | } 82 | return \@pairs; 83 | } 84 | 85 | # A function to generate functions for filesystem operations 86 | sub filesystem_fn { 87 | my $op = shift; 88 | return sub { 89 | my $type = shift; 90 | my $dest_expr = shift; 91 | my $verbose = shift; 92 | my $recon = shift; 93 | my @files = ls $type; 94 | for (@files) { 95 | my $dest = expr($_, $dest_expr); 96 | my $mkdir = qq[mkdir -p "$dest"]; 97 | say $mkdir if ($verbose || $recon); 98 | system $mkdir unless $recon; 99 | my $command = qq[$op "$_->{n}" "$dest"]; 100 | say $command if ($verbose || $recon); 101 | system $command unless $recon; 102 | } 103 | } 104 | } 105 | 106 | # Define filesystem functions. 107 | { 108 | no strict 'refs'; 109 | my @defs = ( 110 | [ 'cp', 'cp' ], 111 | [ 'mv', 'mv' ], 112 | [ 'tmv', 'tmsu-fs-mv' ] 113 | ); 114 | for my $d (@defs) { 115 | my $function_name = $d->[0]; 116 | *$function_name = filesystem_fn($d->[1]); 117 | } 118 | } 119 | 120 | # print file metadata like harvest does 121 | sub print_file { 122 | my $file = shift; 123 | printf("%-7d%-7s%-7s%-11s%-6s %s\n", 124 | $file->{i}, 125 | $file->{nt}, 126 | $file->{t}, 127 | $file->{d}, 128 | $file->{s}, 129 | $file->{n} 130 | ); 131 | } 132 | 133 | my $help = q{A wrapper around dyne/harvest to fill in missing functionality 134 | 135 | Usage: 136 | 137 | hvst 138 | hvst help 139 | hvst ls [TYPE] [OPTION] 140 | hvst types 141 | hvst cp [OPTION]... 142 | hvst mv [OPTION]... 143 | hvst tmv [OPTION]... 144 | 145 | Examples: 146 | 147 | # Display help message. 148 | hvst help 149 | 150 | # List files in current directory. 151 | hvst 152 | 153 | # Also list files in current directory. 154 | hvst ls 155 | 156 | # List only audio files. 157 | hvst ls audio 158 | 159 | # List only book files WITHOUT metadata. (filename only) 160 | hvst ls book -1 161 | 162 | # List types of files that exist in this directory and their counts. 163 | hvst types 164 | 165 | # Copy all images to ~/Pictures. 166 | hvst cp image ~/Pictures 167 | 168 | # Copy all images to ~/Pictures and be verbose. 169 | hvst cp image ~/Pictures --verbose 170 | 171 | # Show what would happen if we tried to move all images to ~/Pictures/YYYY/MM. 172 | hvst mv image @'"~/Pictures/$year/$month"' --recon 173 | 174 | # Move all videos to ~/Videos/YYYY/MM. 175 | hvst mv video @'"~/Videos/$year/$month"' 176 | 177 | # Move all books to ~/Dropbox/books using tmsu-fs-mv. 178 | hvst tmv book ~/Dropbox/books 179 | 180 | # The cp, mv, and tmv commands take --verbose and --recon 181 | # (or -v and -r for short). 182 | # --verbose means print the shell command. 183 | # --recon means print the shell command but don't execute it. 184 | }; 185 | 186 | sub main { 187 | if (@ARGV) { 188 | my $command = shift @ARGV; 189 | $_{verbose} = ''; 190 | $_{recon} = ''; 191 | if ($command eq 'cp') { 192 | GetOptions(\%_, "verbose|v", "recon|r"); 193 | my ($type, $dest) = @ARGV; 194 | cp($type, $dest, $_{verbose}, $_{recon}); 195 | } elsif ($command eq 'mv') { 196 | GetOptions(\%_, "verbose|v", "recon|r"); 197 | my ($type, $dest) = @ARGV; 198 | mv($type, $dest, $_{verbose}, $_{recon}); 199 | } elsif ($command eq 'tmv') { 200 | GetOptions(\%_, "verbose|v", "recon|r"); 201 | my ($type, $dest) = @ARGV; 202 | tmv($type, $dest, $_{verbose}, $_{recon}); 203 | } elsif ($command eq 'ls') { 204 | GetOptions(\%_, "1"); 205 | my ($type) = @ARGV; 206 | my @files; 207 | if ($type) { 208 | @files = ls($type); 209 | } else { 210 | @files = harvest(); 211 | } 212 | for (@files) { 213 | if ($_{1}) { 214 | say $_->{n}; 215 | } else { 216 | print_file($_); 217 | } 218 | } 219 | } elsif ($command eq 'types') { 220 | my @types = sort { $b->[1] <=> $a->[1] } @{types()}; 221 | for (@types) { 222 | printf(qq{%-8s%8d\n}, $_->[0], $_->[1]); 223 | } 224 | } elsif ($command eq "help" || $command eq "-h" || $command eq "--help") { 225 | print $help; 226 | } else { 227 | warn("Command '$command' not recognized."); 228 | return 1; 229 | } 230 | } else { 231 | my @files = harvest(); 232 | for (@files) { 233 | print_file($_); 234 | } 235 | } 236 | return 0; 237 | } 238 | 239 | # https://stackoverflow.com/questions/707022/is-there-a-perl-equivalent-to-pythons-if-name-main 240 | unless (caller) { 241 | exit main(); 242 | } 243 | 244 | 1; 245 | -------------------------------------------------------------------------------- /lua/Makefile: -------------------------------------------------------------------------------- 1 | PREFIX ?= /usr/local 2 | ARCH=$(shell uname -m) 3 | 4 | CC ?= gcc 5 | AR ?= ar 6 | CFLAGS ?= -O3 --fast-math 7 | 8 | LUA_VERSION ?= luajit-2.1 9 | INCLUDE ?= -I /usr/include/$(LUA_VERSION) 10 | LDADD ?= /usr/lib/x86_64-linux-gnu/libluajit-5.1.a -lm -ldl 11 | 12 | MUSL_LDADD ?= /usr/lib/${ARCH}-linux-musl/libc.a 13 | 14 | all: shared 15 | 16 | build-deps: 17 | apt install pkg-config luarocks libluajit-5.1-dev 18 | sudo luarocks install luastatic 19 | 20 | shared: 21 | CC=$(CC) AR=$(AR) INCLUDE="$(INCLUDE)" LDADD="$(LDADD)" CFLAGS="$(CFLAGS)" make -C src 22 | $(CC) -o harvest $(CFLAGS) $(INCLUDE) src/harvest.luastatic.c src/lfs.a $(LDADD) 23 | 24 | static: LDADD = /usr/lib/${ARCH}-linux-gnu/libluajit-5.1.a /usr/lib/${ARCH}-linux-musl/libc.a 25 | static: CC = musl-gcc 26 | static: CFLAGS = -O3 --fast-math -static 27 | static: 28 | CC=$(CC) AR=$(AR) INCLUDE="$(INCLUDE)" LDADD="$(LDADD)" CFLAGS="$(CFLAGS)" make -C src static 29 | $(CC) -static -o harvest $(CFLAGS) $(INCLUDE) src/harvest.luastatic.c src/lfs.a $(LDADD) -lm 30 | 31 | luajit-win64: CC=x86_64-w64-mingw32-gcc 32 | luajit-win64: AR=x86_64-w64-mingw32-ar 33 | luajit-win64: 34 | if ! [ -r luajit.tar.gz ]; then curl -L https://luajit.org/download/LuaJIT-2.1.0-beta3.tar.gz > luajit.tar.gz; fi 35 | if ! [ -d luajit ]; then mkdir -p luajit && tar -C luajit -xf luajit.tar.gz ; fi 36 | mkdir -p luajit/include && cp -ra luajit/*/src/*.h luajit/include/ 37 | make -C luajit/Lua* HOST_CC=gcc CC=$(CC) CFLAGS="$(CFLAGS)" TARGET_SYS=Windows BUILDMODE=static 38 | cp luajit/Lua*/src/libluajit.a luajit/ 39 | 40 | win64: CC=x86_64-w64-mingw32-gcc 41 | win64: AR=x86_64-w64-mingw32-ar 42 | win64: INCLUDE = -I luajit/include 43 | win64: LDADD = luajit/libluajit.a 44 | win64: LDLIBS=-lm 45 | win64: luajit-win64 46 | CC=$(CC) AR=$(AR) INCLUDE="$(INCLUDE)" LDADD="$(LDADD)" CFLAGS="$(CFLAGS)" make -C src static 47 | $(CC) -static -o harvest.exe $(CFLAGS) $(INCLUDE) src/harvest.luastatic.c src/lfs.a $(LDADD) 48 | 49 | win32: mingw32-luajit-static 50 | 51 | install: 52 | install -p harvest $(DESTDIR)$(PREFIX)/bin/harvest 53 | 54 | clean-luajit: 55 | make -C src/luajit clean 56 | 57 | clean: 58 | rm -f src/*.o src/*.a 59 | rm -f src/harvest.luastatic.c 60 | 61 | # install: 62 | # install -d $(PREFIX)/share/harvest/file-extension-list/render/ 63 | # install -p file-extension-list/render/file-extension-parser.zsh \ 64 | # $(PREFIX)/share/harvest/file-extension-list/render/file-extension-parser.zsh 65 | # install -d $(PREFIX)/share/harvest/zuper/ 66 | # install -p zuper/zuper $(PREFIX)/share/harvest/zuper/zuper 67 | # install -p zuper/zuper.init $(PREFIX)/share/harvest/zuper/zuper.init 68 | # install -p harvest $(PREFIX)/bin/harvest 69 | 70 | -------------------------------------------------------------------------------- /lua/README.md: -------------------------------------------------------------------------------- 1 | # Harvest - manage large collections of files and dirs 2 | 3 | Harvest makes it easy to list files and folders by type and copy or 4 | move them around. 5 | 6 | ![Kant handle my swag](https://repository-images.githubusercontent.com/77449851/9d50d480-9766-11ea-98a2-c5aa84501c6e) 7 | 8 | It is compact and portable software that can scan files and folders to 9 | recognise their typology. Scanning is based on [file 10 | extensions](https://github.com/dyne/file-extension-list) and a simple 11 | fuzzy logic analysis of **folder contents** (not just files) to 12 | recognise if they are related to video, audio or text materials, etc. 13 | 14 | It is **fast**: it can process approximately 1GB of stored files per 15 | second and is operated from the console terminal. 16 | 17 | Harvest operates on folders containing files without exploding the 18 | files around: it assesses the typology of a folder from the files 19 | contained, but does not move the files outside of that folder. For 20 | instance it works very well to move around large collections of 21 | downloaded torrent folders. 22 | 23 | ## :floppy_disk: Installation 24 | 25 | Harvest works on all desktop platforms supported by it (GNU/Linux, 26 | Apple/OSX and MS/Windows). 27 | 28 | To be built from source, Harvest requires the following packages to be installed in your system: 29 | - pkg-config 30 | - luarocks 31 | - libluajit-5.1-dev 32 | 33 | Then inside the luarocks package manager it should be installed luastatic and inspect: 34 | ``` 35 | sudo luarocks install luastatic 36 | sudo luarocks install inspect 37 | ``` 38 | 39 | From inside the source, just type: 40 | 41 | Just type 42 | ```bash 43 | git submodule update --init --recursive 44 | make 45 | sudo make install 46 | ``` 47 | to install into `/usr/local/bin/harvest`. 48 | 49 | ## :video_game: Usage 50 | 51 | ``` 52 | Usage: harvest [OPTIONS] 53 | 54 | OPTIONS: 55 | -p, --path=PATH (default is current position) 56 | -t, --type=TYPE text, audio, video, code, etc. 57 | -o, --output=FORMAT csv, json (default: human) 58 | --dir select only directories 59 | --file select only files 60 | -d run in DEBUG mode 61 | -v, --version print the version and exits 62 | ``` 63 | 64 | 65 | To list all image files found in Downloads: 66 | ``` 67 | harvest -p ~/Downloads -t image --file 68 | ``` 69 | 70 | To list all video directories at current filesystem position: 71 | ``` 72 | harvest -t video --dir 73 | ``` 74 | 75 | To list all files and dirs containing reading materials: 76 | ``` 77 | harvest -t text 78 | ``` 79 | 80 | To have a list of supported types use `harvest -t list` at any moment 81 | ``` 82 | Supported types: 83 | code image video book text font web archiv sheet exec slide audio 84 | ``` 85 | For more information about types recognized see the catalogue of file 86 | types we maintain in the [file-extension-list 87 | project](https://github.com/dyne/file-extension-list). 88 | 89 | ### Copy or move 90 | 91 | So far we have seen how to run non-destructive operations, now we come 92 | to **apply actual changes to the filesystem**. 93 | 94 | #### Using shell scripts 95 | 96 | Another simplier solution to move or copy files around is to use shell scripting on the command-line or inside your own scripts. 97 | 98 | For example, here is a short concatenation of commands that will copy all harvested image files and directories to /tmp/images/ 99 | ```bash 100 | harvest -t image -o csv | cut -d, -f5 | xargs -I{} cp -v {} /tmp/images 101 | ``` 102 | 103 | The comma separated list (CSV) output of harvest is organized like this: 104 | ``` 105 | FILE | DIR, TYPE, TIMESTAMP, SIZE, FILENAME 106 | ``` 107 | 108 | #### Using hvst 109 | 110 | One solution is to use a practical wrapper called 111 | [hvst](https://git.coom.tech/gg1234/hvst) which supports distributing 112 | files to destination folders named after Perl expressions based on 113 | file attributes, for instance date. 114 | 115 | For more info about this solution see the [hvst readme documentation](https://git.coom.tech/gg1234/hvst). 116 | 117 | #### Using TMSU 118 | 119 | Support of tagged filesystems is an old feature present in the 120 | [harvest shell implementation](https://github.com/dyne/harvest/tree/master/shell) and 121 | it is easy to bring back. 122 | 123 | If anyone wants it back just say, for more information see the [TMSU project](https://github.com/oniony/TMSU). 124 | 125 | ## :heart_eyes: Acknowledgements 126 | 127 | [![software by Dyne.org](https://files.dyne.org/software_by_dyne.png)](http://www.dyne.org) 128 | 129 | Harvest is Copyright (C) 2014-2022 by the Dyne.org Foundation 130 | 131 | Harvest is designed, written and maintained by Denis "Jaromil" Roio 132 | with contributions by Puria Nafisi Azizi and G Gundam. 133 | 134 | This source code is free software; you can redistribute it and/or 135 | modify it under the terms of the GNU Public License as published by 136 | the Free Software Foundation; either version 3 of the License, or 137 | (at your option) any later version. 138 | 139 | This source code is distributed in the hope that it will be useful, 140 | but WITHOUT ANY WARRANTY; without even the implied warranty of 141 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. Please refer 142 | to the GNU Public License for more details. 143 | 144 | You should have received a copy of the GNU Public License along with 145 | this source code; if not, write to: Free Software Foundation, Inc., 146 | 675 Mass Ave, Cambridge, MA 02139, USA. 147 | -------------------------------------------------------------------------------- /lua/src/Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS += $(shell pkg-config luajit --cflags) 2 | 3 | SOURCES = harvest.lua \ 4 | cliargs.lua cliargs/*.lua cliargs/utils/*.lua \ 5 | align.lua 6 | 7 | all: 8 | $(CC) -c $(INCLUDE) $(CFLAGS) lfs.c -o lfs.o 9 | $(AR) rcs lfs.a lfs.o 10 | CC="" luastatic $(SOURCES) lfs.a 11 | $(CC) -c $(INCLUDE) $(CFLAGS) harvest.luastatic.c 12 | 13 | 14 | static: 15 | $(CC) -c $(INCLUDE) $(CFLAGS) lfs.c -o lfs.o 16 | $(AR) rcs lfs.a lfs.o 17 | -CC="" luastatic $(SOURCES) lfs.a -static $(INCLUDES) 18 | $(CC) -c $(INCLUDE) $(CFLAGS) harvest.luastatic.c 19 | 20 | import-extensions: 21 | lua import_file_extension_list.lua 22 | 23 | -------------------------------------------------------------------------------- /lua/src/align.lua: -------------------------------------------------------------------------------- 1 | -- This code was ripped from Rosetta's stone 2 | 3 | local tWord = {} -- word table 4 | local tColLen = {} -- maximum word length in a column 5 | local rowCount = 0 -- row counter 6 | --store maximum column lengths at 'tColLen'; save words into 'tWord' table 7 | local function readInput(pStr) 8 | for line in pStr:gmatch("([^\n]+)[\n]-") do -- read until '\n' character 9 | rowCount = rowCount + 1 10 | tWord[rowCount] = {} -- create new row 11 | local colCount = 0 12 | for word in line:gmatch("[^,]+") do -- read non '$' character 13 | colCount = colCount + 1 14 | -- 6 is the minimum column width for padding 15 | tColLen[colCount] = math.max((tColLen[colCount] or 6), #word) -- store column length 16 | tWord[rowCount][colCount] = word -- store words 17 | end--for word 18 | end--for line 19 | end--readInput 20 | --repeat space to align the words in the same column 21 | local align = { 22 | ["left"] = function (pWord, pColLen) 23 | local n = (pColLen or 0) - #pWord + 1 24 | return pWord .. (" "):rep(n) 25 | end;--["left"] 26 | ["right"] = function (pWord, pColLen) 27 | local n = (pColLen or 0) - #pWord + 1 28 | return (" "):rep(n) .. pWord 29 | end;--["right"] 30 | ["center"] = function (pWord, pColLen) 31 | local n = (pColLen or 0) - #pWord + 1 32 | local n1 = math.floor(n/2) 33 | return (" "):rep(n1) .. pWord .. (" "):rep(n-n1) 34 | end;--["center"] 35 | } 36 | --word table padder 37 | local function padWordTable(pAlignment) 38 | local alignFunc = align[pAlignment] -- selecting the spacer function 39 | for rowCount, tRow in ipairs(tWord) do 40 | for colCount, word in ipairs(tRow) do 41 | tRow[colCount] = alignFunc(word, tColLen[colCount]) -- save the padded words into the word table 42 | end--for colCount, word 43 | end--for rowCount, tRow 44 | end--padWordTable 45 | --main interface 46 | --------------------------------------------------[] 47 | function alignColumn(pStr, pAlignment) 48 | --------------------------------------------------[] 49 | readInput(pStr) -- store column lengths and words 50 | padWordTable(pAlignment or "left") -- pad the stored words 51 | local output = "" 52 | for rowCount, tRow in ipairs(tWord) do 53 | -- local line = table.concat(tRow) -- concatenate words in one row 54 | -- print(line) -- print the line 55 | -- output = output .. line .. "\n" -- concatenate the line for output, add line break 56 | output = table.concat(tRow) 57 | end--for rowCount, tRow 58 | return output 59 | end--alignColumn 60 | 61 | return alignColumn 62 | -------------------------------------------------------------------------------- /lua/src/cliargs.lua: -------------------------------------------------------------------------------- 1 | -- luacheck: ignore 212 2 | 3 | local core = require('cliargs.core')() 4 | local unpack = _G.unpack or table.unpack -- luacheck: compat 5 | 6 | local cli = setmetatable({},{ __index = core }) 7 | 8 | function cli:parse(arguments, no_cleanup) 9 | if not no_cleanup then 10 | cli:cleanup() 11 | end 12 | 13 | local out = { core.parse(self, arguments) } 14 | 15 | return unpack(out) 16 | end 17 | 18 | -- Clean up the entire module (unload the scripts) as it's expected to be 19 | -- discarded after use. 20 | function cli:cleanup() 21 | for k, v in pairs(package.loaded) do 22 | if (v == cli) or (k:match('cliargs')) then 23 | package.loaded[k] = nil 24 | end 25 | end 26 | 27 | cli = nil 28 | end 29 | 30 | cli.VERSION = "3.0-2" 31 | 32 | return cli -------------------------------------------------------------------------------- /lua/src/cliargs/config_loader.lua: -------------------------------------------------------------------------------- 1 | local trim = require 'cliargs.utils.trim' 2 | 3 | local function read_file(filepath) 4 | local f, err = io.open(filepath, "r") 5 | 6 | if not f then 7 | return nil, err 8 | end 9 | 10 | local contents = f:read('*all') 11 | 12 | f:close() 13 | 14 | return contents 15 | end 16 | 17 | return { 18 | FORMAT_LOADERS = { 19 | ["lua"] = "from_lua", 20 | ["json"] = "from_json", 21 | ["yaml"] = "from_yaml", 22 | ["yml"] = "from_yaml", 23 | ["ini"] = "from_ini", 24 | }, 25 | 26 | --- Load configuration from a Lua file that exports a table. 27 | from_lua = function(filepath) 28 | local file, err = loadfile(filepath) 29 | 30 | if not file and err then 31 | return nil, err 32 | end 33 | 34 | return file() 35 | end, 36 | 37 | --- Load configuration from a JSON file. 38 | --- 39 | --- Requires the "dkjson"[1] module to be present on the system. Get it with: 40 | --- 41 | --- luarocks install dkjson 42 | --- 43 | --- [1] http://dkolf.de/src/dkjson-lua.fsl/home 44 | from_json = function(filepath) 45 | local src, config, _, err 46 | local json = require 'dkjson' 47 | 48 | src, err = read_file(filepath) 49 | 50 | if not src and err then 51 | return nil, err 52 | end 53 | 54 | config, _, err = json.decode(src) 55 | 56 | if err then 57 | return nil, err 58 | end 59 | 60 | return config 61 | end, 62 | 63 | --- Load configuration from an INI file. 64 | --- 65 | --- Requires the "inifile"[1] module to be present on the system. Get it with: 66 | --- 67 | --- luarocks install inifile 68 | --- 69 | --- The INI file must contain a group that lists the default values. For 70 | --- example: 71 | --- 72 | --- [cli] 73 | --- quiet = true 74 | --- compress = lzma 75 | --- 76 | --- The routine will automatically cast boolean values ("true" and "false") 77 | --- into Lua booleans. You may opt out of this behavior by passing `false` 78 | --- to `no_cast`. 79 | --- 80 | --- [1] http://docs.bartbes.com/inifile 81 | from_ini = function(filepath, group, no_cast) 82 | local inifile = require 'inifile' 83 | local config, err 84 | 85 | group = group or 'cli' 86 | 87 | assert(type(group) == 'string', 88 | 'You must provide an INI group to read from.' 89 | ) 90 | 91 | config, err = inifile.parse(filepath) 92 | 93 | if not config and err then 94 | return nil, err 95 | end 96 | 97 | if not no_cast then 98 | for k, src_value in pairs(config[group]) do 99 | local v = trim(src_value) 100 | 101 | if v == 'true' then 102 | v = true 103 | elseif v == 'false' then 104 | v = false 105 | end 106 | 107 | config[group][k] = v 108 | end 109 | end 110 | 111 | return config[group] 112 | end, 113 | 114 | --- Load configuration from a YAML file. 115 | --- 116 | --- Requires the "yaml"[1] module to be present on the system. Get it with: 117 | --- 118 | --- luarocks install yaml 119 | --- 120 | --- [1] http://doc.lubyk.org/yaml.html 121 | from_yaml = function(filepath) 122 | local src, config, err 123 | local yaml = require 'yaml' 124 | 125 | src, err = read_file(filepath) 126 | 127 | if not src and err then 128 | return nil, err 129 | end 130 | 131 | config, err = yaml.load(src) 132 | 133 | if not config and err then 134 | return nil, err 135 | end 136 | 137 | return config 138 | end 139 | } 140 | -------------------------------------------------------------------------------- /lua/src/cliargs/constants.lua: -------------------------------------------------------------------------------- 1 | return { 2 | TYPE_COMMAND = 'command', 3 | TYPE_ARGUMENT = 'argument', 4 | TYPE_SPLAT = 'splat', 5 | TYPE_OPTION = 'option', 6 | } 7 | -------------------------------------------------------------------------------- /lua/src/cliargs/core.lua: -------------------------------------------------------------------------------- 1 | -- luacheck: ignore 212 2 | 3 | local _ 4 | local disect = require('cliargs.utils.disect') 5 | local lookup = require('cliargs.utils.lookup') 6 | local filter = require('cliargs.utils.filter') 7 | local shallow_copy = require('cliargs.utils.shallow_copy') 8 | local create_printer = require('cliargs.printer') 9 | local config_loader = require('cliargs.config_loader') 10 | local parser = require('cliargs.parser') 11 | local K = require 'cliargs.constants' 12 | 13 | local function is_callable(fn) 14 | return type(fn) == "function" or (getmetatable(fn) or {}).__call 15 | end 16 | 17 | local function cast_to_boolean(v) 18 | if v == nil then 19 | return v 20 | else 21 | return v and true or false 22 | end 23 | end 24 | 25 | -- -------- -- 26 | -- CLI Main -- 27 | -- -------- -- 28 | local function create_core() 29 | --- @module 30 | --- 31 | --- The primary export you receive when you require the library. For example: 32 | --- 33 | --- local cli = require 'cliargs' 34 | local cli = {} 35 | local colsz = { 0, 0 } -- column width, help text. Set to 0 for auto detect 36 | local options = {} 37 | 38 | cli.name = "" 39 | cli.description = "" 40 | 41 | cli.printer = create_printer(function() 42 | return { 43 | name = cli.name, 44 | description = cli.description, 45 | options = options, 46 | colsz = colsz 47 | } 48 | end) 49 | 50 | -- Used internally to add an option 51 | local function define_option(k, ek, v, label, desc, default, callback) 52 | local flag = (v == nil) -- no value, so it's a flag 53 | local negatable = flag and (ek and ek:find('^%[no%-]') ~= nil) 54 | 55 | if negatable then 56 | ek = ek:sub(6) 57 | end 58 | 59 | -- guard against duplicates 60 | if lookup(k, ek, options) then 61 | error("Duplicate option: " .. (k or ek) .. ", please rename one of them.") 62 | end 63 | 64 | if negatable and lookup(nil, "no-"..ek, options) then 65 | error("Duplicate option: " .. ("no-"..ek) .. ", please rename one of them.") 66 | end 67 | 68 | -- below description of full entry record, nils included for reference 69 | local entry = { 70 | type = K.TYPE_OPTION, 71 | key = k, 72 | expanded_key = ek, 73 | desc = desc, 74 | default = default, 75 | label = label, 76 | flag = flag, 77 | negatable = negatable, 78 | callback = callback 79 | } 80 | 81 | table.insert(options, entry) 82 | end 83 | 84 | local function define_command_option(key) 85 | --- @module 86 | --- 87 | --- This is a special instance of the [cli]() module that you receive when 88 | --- you define a new command using [cli#command](). 89 | local cmd = create_core() 90 | 91 | cmd.__key__ = key 92 | cmd.type = K.TYPE_COMMAND 93 | 94 | --- Specify a file that the command should run. The rest of the arguments 95 | --- are forward to that file to process, which is free to use or not use 96 | --- lua_cliargs in turn. 97 | --- 98 | --- @param {string} file_path 99 | --- Absolute file-path to a lua script to execute. 100 | function cmd:file(file_path) 101 | cmd.__file__ = file_path 102 | return cmd 103 | end 104 | 105 | --- Define a command handler. This callback will be invoked if the command 106 | --- argument was supplied by the user at runtime. What you return from this 107 | --- callback will be returned to the parent CLI library's parse routine and 108 | --- it will return that in turn! 109 | --- 110 | --- @param {function} callback 111 | function cmd:action(callback) 112 | cmd.__action__ = callback 113 | return cmd 114 | end 115 | 116 | return cmd 117 | end 118 | 119 | -- ------------------------------------------------------------------------ -- 120 | -- PUBLIC API 121 | -- ------------------------------------------------------------------------ -- 122 | 123 | --- CONFIG 124 | 125 | --- Assigns the name of the program which will be used for logging. 126 | function cli:set_name(in_name) 127 | cli.name = in_name 128 | 129 | return self 130 | end 131 | 132 | --- Write down a brief, 1-liner description of what the program does. 133 | function cli:set_description(in_description) 134 | cli.description = in_description 135 | 136 | return self 137 | end 138 | 139 | --- Sets the amount of space allocated to the argument keys and descriptions 140 | --- in the help listing. 141 | --- 142 | --- The sizes are used for wrapping long argument keys and descriptions. 143 | --- 144 | --- @param {number} [key_cols=0] 145 | --- The number of columns assigned to the argument keys, set to 0 to 146 | --- auto detect. 147 | --- 148 | --- @param {number} [desc_cols=0] 149 | --- The number of columns assigned to the argument descriptions, set to 150 | --- 0 to auto set the total width to 72. 151 | function cli:set_colsz(key_cols, desc_cols) 152 | colsz = { key_cols or colsz[1], desc_cols or colsz[2] } 153 | end 154 | 155 | function cli:redefine_default(key, new_default) 156 | local entry = lookup(key, key, options) 157 | 158 | if not entry then 159 | return nil 160 | end 161 | 162 | if entry.flag then 163 | new_default = cast_to_boolean(new_default) 164 | end 165 | 166 | entry.default = shallow_copy(new_default) 167 | 168 | return true 169 | end 170 | 171 | --- Load default values from a table. 172 | --- 173 | --- @param {table} config 174 | --- Your new set of defaults. The keys could either point to the short 175 | --- or expanded option keys, and their values are the new defaults. 176 | --- 177 | --- @param {boolean} [strict=false] 178 | --- Turn this on to return nil and an error message if a key in the 179 | --- config table could not be mapped to any CLI option. 180 | --- 181 | --- @return {true} 182 | --- When the new defaults were loaded successfully, or strict was not 183 | --- set. 184 | --- 185 | --- @return {union} 186 | --- When strict was set and there was an error. 187 | function cli:load_defaults(config, strict) 188 | for k, v in pairs(config) do 189 | local success = self:redefine_default(k, v) 190 | 191 | if strict and not success then 192 | return nil, "Unrecognized option with the key '" .. k .. "'" 193 | end 194 | end 195 | 196 | return true 197 | end 198 | 199 | --- Read config values from a configuration file. 200 | --- 201 | --- @param {string} path 202 | --- Absolute file path. 203 | --- 204 | --- @param {string} [format=nil] 205 | --- The config file format, which has to be one of: 206 | --- "lua", "json", "ini", or "yaml". 207 | --- When this is left blank, we try to auto-detect the format from the 208 | --- file extension. 209 | --- 210 | --- @param {boolean} [strict=false] 211 | --- Forwarded to [#load_defaults](). See that method for the parameter 212 | --- description. 213 | --- 214 | --- @return {true|union} 215 | --- Returns true on successful load. Otherwise, nil and an error 216 | --- message are returned instead. 217 | function cli:read_defaults(path, format) 218 | if not format then 219 | format = path:match('%.([^%.]+)$') 220 | end 221 | 222 | local loader = config_loader.FORMAT_LOADERS[format] 223 | 224 | if not loader then 225 | return nil, 'Unsupported file format "' .. format .. '"' 226 | end 227 | 228 | return config_loader[loader](path) 229 | end 230 | 231 | --- Define a required argument. 232 | --- 233 | --- 234 | --- Required arguments do not take a symbol like `-` or `--`, may not have a 235 | --- default value, and are parsed in the order they are defined. 236 | --- 237 | --- 238 | --- For example: 239 | --- 240 | --- ```lua 241 | --- cli:argument('INPUT', 'path to the input file') 242 | --- cli:argument('OUTPUT', 'path to the output file') 243 | --- ``` 244 | --- 245 | --- At run-time, the arguments have to be specified using the following 246 | --- notation: 247 | --- 248 | --- ```bash 249 | --- $ ./script.lua ./main.c ./a.out 250 | --- ``` 251 | --- 252 | --- If the user does not pass a value to _every_ argument, the parser will 253 | --- raise an error. 254 | --- 255 | --- @param {string} key 256 | --- 257 | --- The argument identifier that will be displayed to the user and 258 | --- be used to reference the run-time value. 259 | --- 260 | --- @param {string} desc 261 | --- 262 | --- A description for this argument to display in usage help. 263 | --- 264 | --- @param {function} [callback] 265 | --- Callback to invoke when this argument is parsed. 266 | function cli:argument(key, desc, callback) 267 | assert(type(key) == "string" and type(desc) == "string", 268 | "Key and description are mandatory arguments (Strings)" 269 | ) 270 | 271 | assert(callback == nil or is_callable(callback), 272 | "Callback argument must be a function" 273 | ) 274 | 275 | if lookup(key, key, options) then 276 | error("Duplicate argument: " .. key .. ", please rename one of them.") 277 | end 278 | 279 | table.insert(options, { 280 | type = K.TYPE_ARGUMENT, 281 | key = key, 282 | desc = desc, 283 | callback = callback 284 | }) 285 | 286 | return self 287 | end 288 | 289 | --- Defines a "splat" (or catch-all) argument. 290 | --- 291 | --- This is a special kind of argument that may be specified 0 or more times, 292 | --- the values being appended to a list. 293 | --- 294 | --- For example, let's assume our program takes a single output file and works 295 | --- on multiple source files: 296 | --- 297 | --- ```lua 298 | --- cli:argument('OUTPUT', 'path to the output file') 299 | --- cli:splat('INPUTS', 'the sources to compile', nil, 10) -- up to 10 source files 300 | --- ``` 301 | --- 302 | --- At run-time, it could be invoked as such: 303 | --- 304 | --- ```bash 305 | --- $ ./script.lua ./a.out file1.c file2.c main.c 306 | --- ``` 307 | --- 308 | --- If you want to make the output optional, you could do something like this: 309 | --- 310 | --- ```lua 311 | --- cli:option('-o, --output=FILE', 'path to the output file', './a.out') 312 | --- cli:splat('INPUTS', 'the sources to compile', nil, 10) 313 | --- ``` 314 | --- 315 | --- And now we may omit the output file path: 316 | --- 317 | --- ```bash 318 | --- $ ./script.lua file1.c file2.c main.c 319 | --- ``` 320 | --- 321 | --- @param {string} key 322 | --- The argument's "name" that will be displayed to the user. 323 | --- 324 | --- @param {string} desc 325 | --- A description of the argument. 326 | --- 327 | --- @param {*} [default=nil] 328 | --- A default value. 329 | --- 330 | --- @param {number} [maxcount=1] 331 | --- The maximum number of occurences allowed. 332 | --- 333 | --- @param {function} [callback] 334 | --- A function to call **everytime** a value for this argument is 335 | --- parsed. 336 | --- 337 | function cli:splat(key, desc, default, maxcount, callback) 338 | assert(#filter(options, 'type', K.TYPE_SPLAT) == 0, 339 | "Only one splat argument may be defined." 340 | ) 341 | 342 | assert(type(key) == "string" and type(desc) == "string", 343 | "Key and description are mandatory arguments (Strings)" 344 | ) 345 | 346 | assert(type(default) == "string" or default == nil, 347 | "Default value must either be omitted or be a string" 348 | ) 349 | 350 | maxcount = tonumber(maxcount or 1) 351 | 352 | assert(maxcount > 0 and maxcount < 1000, 353 | "Maxcount must be a number from 1 to 999" 354 | ) 355 | 356 | assert(is_callable(callback) or callback == nil, 357 | "Callback argument: expected a function or nil" 358 | ) 359 | 360 | local typed_default = default or {} 361 | 362 | if type(typed_default) ~= 'table' then 363 | typed_default = { typed_default } 364 | end 365 | 366 | table.insert(options, { 367 | type = K.TYPE_SPLAT, 368 | key = key, 369 | desc = desc, 370 | default = typed_default, 371 | maxcount = maxcount, 372 | callback = callback 373 | }) 374 | 375 | return self 376 | end 377 | 378 | --- Defines an optional argument. 379 | --- 380 | --- Optional arguments can use 3 different notations, and can accept a value. 381 | --- 382 | --- @param {string} key 383 | --- 384 | --- The argument identifier. This can either be `-key`, or 385 | --- `-key, --expanded-key`. 386 | --- Values can be specified either by appending a space after the 387 | --- identifier (e.g. `-key value` or `--expanded-key value`) or by 388 | --- separating them with a `=` (e.g. `-key=value` or 389 | --- `--expanded-key=value`). 390 | --- 391 | --- @param {string} desc 392 | --- 393 | --- A description for the argument to be shown in --help. 394 | --- 395 | --- @param {bool} [default=nil] 396 | --- 397 | --- A default value to use in case the option was not specified at 398 | --- run-time (the default value is nil if you leave this blank.) 399 | --- 400 | --- @param {function} [callback] 401 | --- 402 | --- A callback to invoke when this option is parsed. 403 | --- 404 | --- @example 405 | --- 406 | --- The following option will be stored in `args["i"]` and `args["input"]` 407 | --- with a default value of `file.txt`: 408 | --- 409 | --- cli:option("-i, --input=FILE", "path to the input file", "file.txt") 410 | function cli:option(key, desc, default, callback) 411 | assert(type(key) == "string" and type(desc) == "string", 412 | "Key and description are mandatory arguments (Strings)" 413 | ) 414 | 415 | assert(is_callable(callback) or callback == nil, 416 | "Callback argument: expected a function or nil" 417 | ) 418 | 419 | local k, ek, v = disect(key) 420 | 421 | -- if there's no VALUE indicator anywhere, what they want really is a flag. 422 | -- e.g: 423 | -- 424 | -- cli:option('-q, --quiet', '...') 425 | if v == nil then 426 | return self:flag(key, desc, default, callback) 427 | end 428 | 429 | define_option(k, ek, v, key, desc, default, callback) 430 | 431 | return self 432 | end 433 | 434 | --- Define an optional "flag" argument. 435 | --- 436 | --- Flags are a special subset of options that can either be `true` or `false`. 437 | --- 438 | --- For example: 439 | --- ```lua 440 | --- cli:flag('-q, --quiet', 'Suppress output.', true) 441 | --- ``` 442 | --- 443 | --- At run-time: 444 | --- 445 | --- ```bash 446 | --- $ ./script.lua --quiet 447 | --- $ ./script.lua -q 448 | --- ``` 449 | --- 450 | --- Passing a value to a flag raises an error: 451 | --- 452 | --- ```bash 453 | --- $ ./script.lua --quiet=foo 454 | --- $ echo $? # => 1 455 | --- ``` 456 | --- 457 | --- Flags may be _negatable_ by prepending `[no-]` to their key: 458 | --- 459 | --- ```lua 460 | --- cli:flag('-c, --[no-]compress', 'whether to compress or not', true) 461 | --- ``` 462 | --- 463 | --- Now the user gets to pass `--no-compress` if they want to skip 464 | --- compression, or either specify `--compress` explicitly or leave it 465 | --- unspecified to use compression. 466 | --- 467 | --- @param {string} key 468 | --- @param {string} desc 469 | --- @param {*} default 470 | --- @param {function} callback 471 | function cli:flag(key, desc, default, callback) 472 | if type(default) == "function" then 473 | callback = default 474 | default = nil 475 | end 476 | 477 | assert(type(key) == "string" and type(desc) == "string", 478 | "Key and description are mandatory arguments (Strings)" 479 | ) 480 | 481 | local k, ek, v = disect(key) 482 | 483 | if v ~= nil then 484 | error("A flag type option cannot have a value set: " .. key) 485 | end 486 | 487 | define_option(k, ek, nil, key, desc, cast_to_boolean(default), callback) 488 | 489 | return self 490 | end 491 | 492 | --- Define a command argument. 493 | --- 494 | --- @param {string} name 495 | --- The name of the command and the argument that the user has to 496 | --- supply to invoke it. 497 | --- 498 | --- @param {string} [desc] 499 | --- An optional string to show in the help listing which should 500 | --- describe what the command does. It will be displayed if --help 501 | --- was run on the main program. 502 | --- 503 | --- 504 | --- @return {cmd} 505 | --- Another instance of the CLI library which is scoped to that 506 | --- command. 507 | function cli:command(name, desc) 508 | local cmd = define_command_option(name) 509 | 510 | cmd:set_name(cli.name .. ' ' .. name) 511 | cmd:set_description(desc) 512 | 513 | table.insert(options, cmd) 514 | 515 | return cmd 516 | end 517 | 518 | --- Parse the process arguments table. 519 | --- 520 | --- @param {table} [arguments=_G.arg] 521 | --- The list of arguments to parse. Defaults to the global `arg` table 522 | --- which contains the arguments the process was started with. 523 | --- 524 | --- @return {table} 525 | --- A table containing all the arguments, options, flags, 526 | --- and splat arguments that were specified or had a default 527 | --- (where applicable). 528 | --- 529 | --- @return {array} 530 | --- If a parsing error has occured, note that the --help option is 531 | --- also considered an error. 532 | function cli:parse(arguments) 533 | return parser(arguments, options, cli.printer) 534 | end 535 | 536 | --- Prints the USAGE message. 537 | --- 538 | --- @return {string} 539 | --- The USAGE message. 540 | function cli:print_usage() 541 | cli.printer.print(cli:get_usage_message()) 542 | end 543 | 544 | function cli:get_usage_message() 545 | return cli.printer.generate_usage() 546 | end 547 | 548 | --- Prints the HELP information. 549 | --- 550 | --- @return {string} 551 | --- The HELP message. 552 | function cli:print_help() 553 | cli.printer.print(cli.printer.generate_help_and_usage()) 554 | end 555 | 556 | return cli 557 | end 558 | 559 | return create_core -------------------------------------------------------------------------------- /lua/src/cliargs/parser.lua: -------------------------------------------------------------------------------- 1 | local K = require 'cliargs.constants' 2 | 3 | ------------------------------------------------------------------------------- 4 | -- UTILS 5 | ------------------------------------------------------------------------------- 6 | local shallow_copy = require 'cliargs.utils.shallow_copy' 7 | local filter = require 'cliargs.utils.filter' 8 | local disect_argument = require 'cliargs.utils.disect_argument' 9 | local lookup = require 'cliargs.utils.lookup' 10 | 11 | local function clone_table_shift(t) 12 | local clone = shallow_copy(t) 13 | table.remove(clone, 1) 14 | return clone 15 | end 16 | 17 | local function clone_table_remove(t, index) 18 | local clone = shallow_copy(t) 19 | table.remove(clone, index) 20 | return clone 21 | end 22 | 23 | ------------------------------------------------------------------------------- 24 | -- PARSE ROUTINES 25 | ------------------------------------------------------------------------------- 26 | local p = {} 27 | function p.invoke_command(args, options, done) 28 | local commands = filter(options, 'type', K.TYPE_COMMAND) 29 | 30 | for index, opt in ipairs(args) do 31 | local command = filter(commands, '__key__', opt)[1] 32 | 33 | if command then 34 | local command_args = clone_table_remove(args, index) 35 | 36 | if command.__action__ then 37 | local parsed_command_args, err = command:parse(command_args) 38 | 39 | if err then 40 | return nil, err 41 | end 42 | 43 | return command.__action__(parsed_command_args) 44 | elseif command.__file__ then 45 | local filename = command.__file__ 46 | 47 | if type(filename) == 'function' then 48 | filename = filename() 49 | end 50 | 51 | local run_command_file = function() 52 | _G.arg = command_args 53 | 54 | local res, err = assert(loadfile(filename))() 55 | 56 | _G.arg = args 57 | 58 | return res, err 59 | end 60 | 61 | return run_command_file() 62 | end 63 | end 64 | end 65 | 66 | return done() 67 | end 68 | 69 | function p.print_help(args, printer, done) 70 | -- has --help or -h ? display the help listing and abort! 71 | for _, v in pairs(args) do 72 | if v == "--help" or v == "-h" then 73 | return nil, printer.generate_help_and_usage() 74 | end 75 | end 76 | 77 | return done() 78 | end 79 | 80 | function p.track_dump_request(args, done) 81 | -- starts with --__DUMP__; set dump to true to dump the parsed arguments 82 | if args[1] == "--__DUMP__" then 83 | return done(true, clone_table_shift(args)) 84 | else 85 | return done(false, args) 86 | end 87 | end 88 | 89 | function p.process_arguments(args, options, done) 90 | local values = {} 91 | local cursor = 0 92 | local argument_cursor = 1 93 | local argument_delimiter_found = false 94 | local function consume() 95 | cursor = cursor + 1 96 | 97 | return args[cursor] 98 | end 99 | 100 | local required = filter(options, 'type', K.TYPE_ARGUMENT) 101 | 102 | while cursor < #args do 103 | local curr_opt = consume() 104 | local symbol, key, value, flag_negated = disect_argument(curr_opt) 105 | 106 | -- end-of-options indicator: 107 | if curr_opt == "--" then 108 | argument_delimiter_found = true 109 | 110 | -- an option: 111 | elseif not argument_delimiter_found and symbol then 112 | local entry = lookup(key, key, options) 113 | 114 | if not key or not entry then 115 | local option_type = value and "option" or "flag" 116 | 117 | return nil, "unknown/bad " .. option_type .. ": " .. curr_opt 118 | end 119 | 120 | if flag_negated and not entry.negatable then 121 | return nil, "flag '" .. curr_opt .. "' may not be negated using --no-" 122 | end 123 | 124 | -- a flag and a value specified? that's an error 125 | if entry.flag and value then 126 | return nil, "flag " .. curr_opt .. " does not take a value" 127 | elseif entry.flag then 128 | value = not flag_negated 129 | -- an option: 130 | else 131 | -- the value might be in the next argument, e.g: 132 | -- 133 | -- --compress lzma 134 | if not value then 135 | -- if the option contained a = and there's no value, it means they 136 | -- want to nullify an option's default value. eg: 137 | -- 138 | -- --compress= 139 | if curr_opt:find('=') then 140 | value = '__CLIARGS_NULL__' 141 | else 142 | -- NOTE: this has the potential to be buggy and swallow the next 143 | -- entry as this entry's value even though that entry may be an 144 | -- actual argument/option 145 | -- 146 | -- this would be a user error and there is no determinate way to 147 | -- figure it out because if there's no leading symbol (- or --) 148 | -- in that entry it can be an actual argument. :shrug: 149 | value = consume() 150 | 151 | if not value then 152 | return nil, "option " .. curr_opt .. " requires a value to be set" 153 | end 154 | end 155 | end 156 | end 157 | 158 | table.insert(values, { entry = entry, value = value }) 159 | 160 | if entry.callback then 161 | local altkey = entry.key 162 | local status, err 163 | 164 | if key == entry.key then 165 | altkey = entry.expanded_key 166 | else 167 | key = entry.expanded_key 168 | end 169 | 170 | status, err = entry.callback(key, value, altkey, curr_opt) 171 | 172 | if status == nil and err then 173 | return nil, err 174 | end 175 | end 176 | 177 | -- a regular argument: 178 | elseif argument_cursor <= #required then 179 | local entry = required[argument_cursor] 180 | 181 | table.insert(values, { entry = entry, value = curr_opt }) 182 | 183 | if entry.callback then 184 | local status, err = entry.callback(entry.key, curr_opt) 185 | 186 | if status == nil and err then 187 | return nil, err 188 | end 189 | end 190 | 191 | argument_cursor = argument_cursor + 1 192 | 193 | -- a splat argument: 194 | else 195 | local entry = filter(options, 'type', K.TYPE_SPLAT)[1] 196 | 197 | if entry then 198 | table.insert(values, { entry = entry, value = curr_opt }) 199 | 200 | if entry.callback then 201 | local status, err = entry.callback(entry.key, curr_opt) 202 | 203 | if status == nil and err then 204 | return nil, err 205 | end 206 | end 207 | end 208 | 209 | argument_cursor = argument_cursor + 1 210 | end 211 | end 212 | 213 | return done(values, argument_cursor - 1) 214 | end 215 | 216 | function p.validate(options, arg_count, done) 217 | local required = filter(options, 'type', K.TYPE_ARGUMENT) 218 | local splatarg = filter(options, 'type', K.TYPE_SPLAT)[1] or { maxcount = 0 } 219 | 220 | local min_arg_count = #required 221 | local max_arg_count = #required + splatarg.maxcount 222 | 223 | -- missing any required arguments, or too many? 224 | if arg_count < min_arg_count or arg_count > max_arg_count then 225 | if splatarg.maxcount > 0 then 226 | return nil, ( 227 | "bad number of arguments: " .. 228 | min_arg_count .. "-" .. max_arg_count .. 229 | " argument(s) must be specified, not " .. arg_count 230 | ) 231 | else 232 | return nil, ( 233 | "bad number of arguments: " .. 234 | min_arg_count .. " argument(s) must be specified, not " .. arg_count 235 | ) 236 | end 237 | end 238 | 239 | return done() 240 | end 241 | 242 | function p.collect_results(cli_values, options) 243 | local results = {} 244 | local function collect_with_default(entry) 245 | local entry_values = {} 246 | local _ 247 | 248 | for _, item in ipairs(cli_values) do 249 | if item.entry == entry then 250 | table.insert(entry_values, item.value) 251 | end 252 | end 253 | 254 | if #entry_values == 0 then 255 | return type(entry.default) == 'table' and entry.default or { entry.default } 256 | else 257 | return entry_values 258 | end 259 | end 260 | 261 | local function write(entry, value) 262 | if entry.key then results[entry.key] = value end 263 | if entry.expanded_key then results[entry.expanded_key] = value end 264 | end 265 | 266 | for _, entry in pairs(options) do 267 | local entry_cli_values = collect_with_default(entry) 268 | local maxcount = entry.maxcount 269 | 270 | if maxcount == nil then 271 | maxcount = type(entry.default) == 'table' and 999 or 1 272 | end 273 | 274 | local entry_value = entry_cli_values 275 | 276 | if maxcount == 1 and type(entry_cli_values) == 'table' then 277 | -- take the last value 278 | entry_value = entry_cli_values[#entry_cli_values] 279 | 280 | if entry_value == '__CLIARGS_NULL__' then 281 | entry_value = nil 282 | end 283 | end 284 | 285 | write(entry, entry_value) 286 | end 287 | 288 | return results 289 | end 290 | 291 | 292 | return function(arguments, options, printer) 293 | assert(arguments == nil or type(arguments) == "table", 294 | "expected an argument table to be passed in, " .. 295 | "got something of type " .. type(arguments) 296 | ) 297 | 298 | local args = arguments or _G.arg or {} 299 | 300 | -- the spiral of DOOM: 301 | return p.invoke_command(args, options, function() 302 | return p.track_dump_request(args, function(dump, args_without_dump) 303 | return p.print_help(args_without_dump, printer, function() 304 | return p.process_arguments(args_without_dump, options, function(values, arg_count) 305 | return p.validate(options, arg_count, function() 306 | if dump then 307 | return nil, printer.dump_internal_state(values) 308 | else 309 | return p.collect_results(values, options) 310 | end 311 | end) 312 | end) 313 | end) 314 | end) 315 | end) 316 | end 317 | -------------------------------------------------------------------------------- /lua/src/cliargs/printer.lua: -------------------------------------------------------------------------------- 1 | local wordwrap = require('cliargs.utils.wordwrap') 2 | local filter = require('cliargs.utils.filter') 3 | local K = require('cliargs.constants') 4 | local MAX_COLS = 72 5 | local _ 6 | 7 | local function create_printer(get_parser_state) 8 | local printer = {} 9 | 10 | function printer.print(msg) 11 | return _G.print(msg) 12 | end 13 | 14 | local function get_max_label_length() 15 | local maxsz = 0 16 | local state = get_parser_state() 17 | local optargument = filter(state.options, 'type', K.TYPE_SPLAT)[1] 18 | local commands = filter(state.options, 'type', K.TYPE_COMMAND) 19 | 20 | for _, entry in ipairs(commands) do 21 | if #entry.__key__ > maxsz then 22 | maxsz = #entry.__key__ 23 | end 24 | end 25 | 26 | for _,table_name in ipairs({"options"}) do 27 | for _, entry in ipairs(state[table_name]) do 28 | local key = entry.label or entry.key or entry.__key__ 29 | 30 | if #key > maxsz then 31 | maxsz = #key 32 | end 33 | end 34 | end 35 | 36 | if optargument and #optargument.key > maxsz then 37 | maxsz = #optargument.key 38 | end 39 | 40 | return maxsz 41 | end 42 | 43 | -- Generate the USAGE heading message. 44 | function printer.generate_usage() 45 | local state = get_parser_state() 46 | local msg = "Usage:" 47 | 48 | local required = filter(state.options, 'type', K.TYPE_ARGUMENT) 49 | local optional = filter(state.options, 'type', K.TYPE_OPTION) 50 | local optargument = filter(state.options, 'type', K.TYPE_SPLAT)[1] 51 | 52 | if #state.name > 0 then 53 | msg = msg .. ' ' .. tostring(state.name) 54 | end 55 | 56 | if #optional > 0 then 57 | msg = msg .. " [OPTIONS]" 58 | end 59 | 60 | if #required > 0 or optargument then 61 | msg = msg .. " [--]" 62 | end 63 | 64 | if #required > 0 then 65 | for _,entry in ipairs(required) do 66 | msg = msg .. " " .. entry.key 67 | end 68 | end 69 | 70 | if optargument then 71 | if optargument.maxcount == 1 then 72 | msg = msg .. " [" .. optargument.key .. "]" 73 | elseif optargument.maxcount == 2 then 74 | msg = msg .. " [" .. optargument.key .. "-1 [" .. optargument.key .. "-2]]" 75 | elseif optargument.maxcount > 2 then 76 | msg = msg .. " [" .. optargument.key .. "-1 [" .. optargument.key .. "-2 [...]]]" 77 | end 78 | end 79 | 80 | return msg 81 | end 82 | 83 | function printer.generate_help() 84 | local msg = '' 85 | local state = get_parser_state() 86 | local col1 = state.colsz[1] 87 | local col2 = state.colsz[2] 88 | local required = filter(state.options, 'type', K.TYPE_ARGUMENT) 89 | local optional = filter(state.options, 'type', K.TYPE_OPTION) 90 | local commands = filter(state.options, 'type', K.TYPE_COMMAND) 91 | local optargument = filter(state.options, 'type', K.TYPE_SPLAT)[1] 92 | 93 | local function append(label, desc) 94 | label = " " .. label .. string.rep(" ", col1 - (#label + 2)) 95 | desc = table.concat(wordwrap(desc, col2), "\n") -- word-wrap 96 | desc = desc:gsub("\n", "\n" .. string.rep(" ", col1)) -- add padding 97 | 98 | msg = msg .. label .. desc .. "\n" 99 | end 100 | 101 | if col1 == 0 then 102 | col1 = get_max_label_length(state) 103 | end 104 | 105 | -- add margins 106 | col1 = col1 + 3 107 | 108 | if col2 == 0 then 109 | col2 = MAX_COLS - col1 110 | end 111 | 112 | if col2 < 10 then 113 | col2 = 10 114 | end 115 | 116 | if #commands > 0 then 117 | msg = msg .. "\nCOMMANDS: \n" 118 | 119 | for _, entry in ipairs(commands) do 120 | append(entry.__key__, entry.description or '') 121 | end 122 | end 123 | 124 | if required[1] or optargument then 125 | msg = msg .. "\nARGUMENTS: \n" 126 | 127 | for _,entry in ipairs(required) do 128 | append(entry.key, entry.desc .. " (required)") 129 | end 130 | end 131 | 132 | if optargument then 133 | local optarg_desc = ' ' .. optargument.desc 134 | local default_value = optargument.maxcount > 1 and 135 | optargument.default[1] or 136 | optargument.default 137 | 138 | if #optargument.default > 0 then 139 | optarg_desc = optarg_desc .. " (optional, default: " .. tostring(default_value[1]) .. ")" 140 | else 141 | optarg_desc = optarg_desc .. " (optional)" 142 | end 143 | 144 | append(optargument.key, optarg_desc) 145 | end 146 | 147 | if #optional > 0 then 148 | msg = msg .. "\nOPTIONS: \n" 149 | 150 | for _,entry in ipairs(optional) do 151 | local desc = entry.desc 152 | if not entry.flag and entry.default and #tostring(entry.default) > 0 then 153 | local readable_default = type(entry.default) == "table" and "[]" or tostring(entry.default) 154 | desc = desc .. " (default: " .. readable_default .. ")" 155 | elseif entry.flag and entry.negatable then 156 | local readable_default = entry.default and 'on' or 'off' 157 | desc = desc .. " (default: " .. readable_default .. ")" 158 | end 159 | append(entry.label, desc) 160 | end 161 | end 162 | 163 | return msg 164 | end 165 | 166 | function printer.dump_internal_state(values) 167 | local state = get_parser_state() 168 | local required = filter(state.options, 'type', K.TYPE_ARGUMENT) 169 | local optional = filter(state.options, 'type', K.TYPE_OPTION) 170 | local optargument = filter(state.options, 'type', K.TYPE_SPLAT)[1] 171 | local maxlabel = get_max_label_length() 172 | local msg = '' 173 | 174 | local function print(fragment) 175 | msg = msg .. fragment .. '\n' 176 | end 177 | 178 | print("\n======= Provided command line =============") 179 | print("\nNumber of arguments: ", #arg) 180 | 181 | for i,v in ipairs(arg) do -- use gloabl 'arg' not the modified local 'args' 182 | print(string.format("%3i = '%s'", i, v)) 183 | end 184 | 185 | print("\n======= Parsed command line ===============") 186 | if #required > 0 then print("\nArguments:") end 187 | for _, entry in ipairs(required) do 188 | print( 189 | " " .. 190 | entry.key .. string.rep(" ", maxlabel + 2 - #entry.key) .. 191 | " => '" .. 192 | tostring(values[entry]) .. 193 | "'" 194 | ) 195 | end 196 | 197 | if optargument then 198 | print( 199 | "\nOptional arguments:" .. 200 | optargument.key .. 201 | "; allowed are " .. 202 | tostring(optargument.maxcount) .. 203 | " arguments" 204 | ) 205 | 206 | if optargument.maxcount == 1 then 207 | print( 208 | " " .. optargument.key .. 209 | string.rep(" ", maxlabel + 2 - #optargument.key) .. 210 | " => '" .. 211 | optargument.key .. 212 | "'" 213 | ) 214 | else 215 | for i = 1, optargument.maxcount do 216 | if values[optargument] and values[optargument][i] then 217 | print( 218 | " " .. tostring(i) .. 219 | string.rep(" ", maxlabel + 2 - #tostring(i)) .. 220 | " => '" .. 221 | tostring(values[optargument][i]) .. 222 | "'" 223 | ) 224 | end 225 | end 226 | end 227 | end 228 | 229 | if #optional > 0 then print("\nOptional parameters:") end 230 | local doubles = {} 231 | for _, entry in pairs(optional) do 232 | if not doubles[entry] then 233 | local value = values[entry] 234 | 235 | if type(value) == "string" then 236 | value = "'"..value.."'" 237 | else 238 | value = tostring(value) .." (" .. type(value) .. ")" 239 | end 240 | 241 | print(" " .. entry.label .. string.rep(" ", maxlabel + 2 - #entry.label) .. " => " .. value) 242 | 243 | doubles[entry] = entry 244 | end 245 | end 246 | 247 | print("\n===========================================\n\n") 248 | 249 | return msg 250 | end 251 | 252 | function printer.generate_help_and_usage() 253 | local msg = '' 254 | 255 | msg = msg .. printer.generate_usage() .. '\n' 256 | msg = msg .. printer.generate_help() 257 | 258 | return msg 259 | end 260 | 261 | return printer 262 | end 263 | 264 | return create_printer -------------------------------------------------------------------------------- /lua/src/cliargs/utils/disect.lua: -------------------------------------------------------------------------------- 1 | local split = require('cliargs.utils.split') 2 | 3 | local RE_ADD_COMMA = "^%-([%a%d]+)[%s]%-%-" 4 | local RE_ADJUST_DELIMITER = "(%-%-?)([%a%d]+)[%s]" 5 | 6 | -- parameterize the key if needed, possible variations: 7 | -- 8 | -- -key 9 | -- -key VALUE 10 | -- -key=VALUE 11 | -- 12 | -- -key, --expanded 13 | -- -key, --expanded VALUE 14 | -- -key, --expanded=VALUE 15 | -- 16 | -- -key --expanded 17 | -- -key --expanded VALUE 18 | -- -key --expanded=VALUE 19 | -- 20 | -- --expanded 21 | -- --expanded VALUE 22 | -- --expanded=VALUE 23 | local function disect(key) 24 | -- characters allowed are a-z, A-Z, 0-9 25 | -- extended + values also allow; # @ _ + - 26 | local k, ek, v, _ 27 | local dummy 28 | 29 | -- leading "-" or "--" 30 | local prefix 31 | 32 | -- if there is no comma, between short and extended, add one 33 | _, _, dummy = key:find(RE_ADD_COMMA) 34 | if dummy then 35 | key = key:gsub(RE_ADD_COMMA, "-" .. dummy .. ", --", 1) 36 | end 37 | 38 | -- replace space delimiting the value indicator by "=" 39 | -- 40 | -- -key VALUE => -key=VALUE 41 | -- --expanded-key VALUE => --expanded-key=VALUE 42 | _, _, prefix, dummy = key:find(RE_ADJUST_DELIMITER) 43 | if prefix and dummy then 44 | key = key:gsub(RE_ADJUST_DELIMITER, prefix .. dummy .. "=", 1) 45 | end 46 | 47 | -- if there is no "=", then append one 48 | if not key:find("=") then 49 | key = key .. "=" 50 | end 51 | 52 | -- get value 53 | _, _, v = key:find(".-%=(.+)") 54 | 55 | -- get key(s), remove spaces 56 | key = split(key, "=")[1]:gsub(" ", "") 57 | 58 | -- get short key & extended key 59 | _, _, k = key:find("^%-([^-][^%s,]*)") 60 | _, _, ek = key:find("%-%-(.+)$") 61 | 62 | if v == "" then 63 | v = nil 64 | end 65 | 66 | return k,ek,v 67 | end 68 | 69 | return disect -------------------------------------------------------------------------------- /lua/src/cliargs/utils/disect_argument.lua: -------------------------------------------------------------------------------- 1 | local function disect_argument(str) 2 | local _, symbol, key, value 3 | local negated = false 4 | 5 | _, _, symbol, key = str:find("^([%-]*)(.*)") 6 | 7 | if key then 8 | local actual_key 9 | 10 | -- split value and key 11 | _, _, actual_key, value = key:find("([^%=]+)[%=]?(.*)") 12 | 13 | if value then 14 | key = actual_key 15 | end 16 | 17 | if key:sub(1,3) == "no-" then 18 | key = key:sub(4,-1) 19 | negated = true 20 | end 21 | end 22 | 23 | -- no leading symbol means the sole fragment is the value. 24 | if #symbol == 0 then 25 | value = str 26 | key = nil 27 | end 28 | 29 | return 30 | #symbol > 0 and symbol or nil, 31 | key and #key > 0 and key or nil, 32 | value and #value > 0 and value or nil, 33 | negated and true or false 34 | end 35 | 36 | return disect_argument -------------------------------------------------------------------------------- /lua/src/cliargs/utils/filter.lua: -------------------------------------------------------------------------------- 1 | return function(t, k, v) 2 | local out = {} 3 | 4 | for _, item in ipairs(t) do 5 | if item[k] == v then 6 | table.insert(out, item) 7 | end 8 | end 9 | 10 | return out 11 | end -------------------------------------------------------------------------------- /lua/src/cliargs/utils/lookup.lua: -------------------------------------------------------------------------------- 1 | 2 | -- Used internally to lookup an entry using either its short or expanded keys 3 | local function lookup(k, ek, ...) 4 | local _ 5 | 6 | for _, t in ipairs({...}) do 7 | for _, entry in ipairs(t) do 8 | if k and entry.key == k then 9 | return entry 10 | end 11 | 12 | if ek and entry.expanded_key == ek then 13 | return entry 14 | end 15 | 16 | if entry.negatable then 17 | if ek and ("no-"..entry.expanded_key) == ek then return entry end 18 | end 19 | end 20 | end 21 | 22 | return nil 23 | end 24 | 25 | return lookup -------------------------------------------------------------------------------- /lua/src/cliargs/utils/shallow_copy.lua: -------------------------------------------------------------------------------- 1 | -- courtesy of http://lua-users.org/wiki/CopyTable 2 | local function shallow_copy(orig) 3 | if type(orig) == 'table' then 4 | local copy = {} 5 | 6 | for orig_key, orig_value in pairs(orig) do 7 | copy[orig_key] = orig_value 8 | end 9 | 10 | return copy 11 | else -- number, string, boolean, etc 12 | return orig 13 | end 14 | end 15 | 16 | return shallow_copy -------------------------------------------------------------------------------- /lua/src/cliargs/utils/split.lua: -------------------------------------------------------------------------------- 1 | local function split(str, pat) 2 | local t = {} 3 | local fpat = "(.-)" .. pat 4 | local last_end = 1 5 | local s, e, cap = str:find(fpat, 1) 6 | 7 | while s do 8 | if s ~= 1 or cap ~= "" then 9 | table.insert(t,cap) 10 | end 11 | 12 | last_end = e + 1 13 | s, e, cap = str:find(fpat, last_end) 14 | end 15 | 16 | if last_end <= #str then 17 | cap = str:sub(last_end) 18 | table.insert(t, cap) 19 | end 20 | 21 | return t 22 | end 23 | 24 | return split -------------------------------------------------------------------------------- /lua/src/cliargs/utils/trim.lua: -------------------------------------------------------------------------------- 1 | -- courtesy of the jungle: http://lua-users.org/wiki/StringTrim 2 | return function(str) 3 | return str:match "^%s*(.-)%s*$" 4 | end -------------------------------------------------------------------------------- /lua/src/cliargs/utils/wordwrap.lua: -------------------------------------------------------------------------------- 1 | local split = require('cliargs.utils.split') 2 | 3 | local function buildline(words, size, overflow) 4 | -- if overflow is set, a word longer than size, will overflow the size 5 | -- otherwise it will be chopped in line-length pieces 6 | local line = {} 7 | if #words[1] > size then 8 | -- word longer than line 9 | if overflow then 10 | line[1] = words[1] 11 | table.remove(words, 1) 12 | else 13 | line[1] = words[1]:sub(1, size) 14 | words[1] = words[1]:sub(size + 1, -1) 15 | end 16 | else 17 | local len = 0 18 | while words[1] and (len + #words[1] + 1 <= size) or (len == 0 and #words[1] == size) do 19 | line[#line+1] = words[1] 20 | len = len + #words[1] + 1 21 | table.remove(words, 1) 22 | end 23 | end 24 | return table.concat(line, " "), words 25 | end 26 | 27 | local function wordwrap(str, size, overflow) 28 | -- if overflow is set, then words longer than a line will overflow 29 | -- otherwise, they'll be chopped in pieces 30 | local out, words = {}, split(str, ' ') 31 | while words[1] do 32 | out[#out+1], words = buildline(words, size, overflow) 33 | end 34 | return out 35 | end 36 | 37 | return wordwrap -------------------------------------------------------------------------------- /lua/src/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "1.ada": [ 3 | "code" 4 | ], 5 | "2.ada": [ 6 | "code" 7 | ], 8 | "3dm": [ 9 | "image" 10 | ], 11 | "3ds": [ 12 | "image" 13 | ], 14 | "3g2": [ 15 | "video" 16 | ], 17 | "3gp": [ 18 | "video" 19 | ], 20 | "7z": [ 21 | "archive" 22 | ], 23 | "a": [ 24 | "archive" 25 | ], 26 | "aac": [ 27 | "audio" 28 | ], 29 | "aaf": [ 30 | "video" 31 | ], 32 | "ada": [ 33 | "code" 34 | ], 35 | "adb": [ 36 | "code" 37 | ], 38 | "ads": [ 39 | "code" 40 | ], 41 | "ai": [ 42 | "image" 43 | ], 44 | "aiff": [ 45 | "audio" 46 | ], 47 | "ape": [ 48 | "audio" 49 | ], 50 | "apk": [ 51 | "archive" 52 | ], 53 | "ar": [ 54 | "archive" 55 | ], 56 | "asf": [ 57 | "video" 58 | ], 59 | "asm": [ 60 | "code" 61 | ], 62 | "au": [ 63 | "audio" 64 | ], 65 | "avchd": [ 66 | "video" 67 | ], 68 | "avi": [ 69 | "video" 70 | ], 71 | "azw": [ 72 | "book" 73 | ], 74 | "azw1": [ 75 | "book" 76 | ], 77 | "azw3": [ 78 | "book" 79 | ], 80 | "azw4": [ 81 | "book" 82 | ], 83 | "azw6": [ 84 | "book" 85 | ], 86 | "bas": [ 87 | "code" 88 | ], 89 | "bash": [ 90 | "code", 91 | "exec" 92 | ], 93 | "bat": [ 94 | "code", 95 | "exec" 96 | ], 97 | "bin": [ 98 | "exec" 99 | ], 100 | "bmp": [ 101 | "image" 102 | ], 103 | "bz2": [ 104 | "archive" 105 | ], 106 | "c": [ 107 | "code" 108 | ], 109 | "c++": [ 110 | "code" 111 | ], 112 | "cab": [ 113 | "archive" 114 | ], 115 | "cbl": [ 116 | "code" 117 | ], 118 | "cbr": [ 119 | "book" 120 | ], 121 | "cbz": [ 122 | "book" 123 | ], 124 | "cc": [ 125 | "code" 126 | ], 127 | "class": [ 128 | "code" 129 | ], 130 | "clj": [ 131 | "code" 132 | ], 133 | "cob": [ 134 | "code" 135 | ], 136 | "command": [ 137 | "exec" 138 | ], 139 | "cpio": [ 140 | "archive" 141 | ], 142 | "cpp": [ 143 | "code" 144 | ], 145 | "crx": [ 146 | "exec" 147 | ], 148 | "cs": [ 149 | "code" 150 | ], 151 | "csh": [ 152 | "code", 153 | "exec" 154 | ], 155 | "css": [ 156 | "web" 157 | ], 158 | "csv": [ 159 | "sheet" 160 | ], 161 | "cxx": [ 162 | "code" 163 | ], 164 | "d": [ 165 | "code" 166 | ], 167 | "dds": [ 168 | "image" 169 | ], 170 | "deb": [ 171 | "archive" 172 | ], 173 | "diff": [ 174 | "code" 175 | ], 176 | "dmg": [ 177 | "archive" 178 | ], 179 | "doc": [ 180 | "text" 181 | ], 182 | "docx": [ 183 | "text" 184 | ], 185 | "drc": [ 186 | "video" 187 | ], 188 | "dwg": [ 189 | "image" 190 | ], 191 | "dxf": [ 192 | "image" 193 | ], 194 | "e": [ 195 | "code" 196 | ], 197 | "ebook": [ 198 | "text" 199 | ], 200 | "egg": [ 201 | "archive" 202 | ], 203 | "el": [ 204 | "code" 205 | ], 206 | "eot": [ 207 | "font" 208 | ], 209 | "eps": [ 210 | "image" 211 | ], 212 | "epub": [ 213 | "book" 214 | ], 215 | "exe": [ 216 | "exec" 217 | ], 218 | "f": [ 219 | "code" 220 | ], 221 | "f77": [ 222 | "code" 223 | ], 224 | "f90": [ 225 | "code" 226 | ], 227 | "fish": [ 228 | "code", 229 | "exec" 230 | ], 231 | "flac": [ 232 | "audio" 233 | ], 234 | "flv": [ 235 | "video" 236 | ], 237 | "for": [ 238 | "code" 239 | ], 240 | "fth": [ 241 | "code" 242 | ], 243 | "ftn": [ 244 | "code" 245 | ], 246 | "gif": [ 247 | "image" 248 | ], 249 | "go": [ 250 | "code" 251 | ], 252 | "gpx": [ 253 | "image" 254 | ], 255 | "groovy": [ 256 | "code" 257 | ], 258 | "gsm": [ 259 | "audio" 260 | ], 261 | "gz": [ 262 | "archive" 263 | ], 264 | "h": [ 265 | "code" 266 | ], 267 | "hh": [ 268 | "code" 269 | ], 270 | "hpp": [ 271 | "code" 272 | ], 273 | "hs": [ 274 | "code" 275 | ], 276 | "htm": [ 277 | "code", 278 | "web" 279 | ], 280 | "html": [ 281 | "code", 282 | "web" 283 | ], 284 | "hxx": [ 285 | "code" 286 | ], 287 | "ics": [ 288 | "sheet" 289 | ], 290 | "iso": [ 291 | "archive" 292 | ], 293 | "it": [ 294 | "audio" 295 | ], 296 | "jar": [ 297 | "archive" 298 | ], 299 | "java": [ 300 | "code" 301 | ], 302 | "jpeg": [ 303 | "image" 304 | ], 305 | "jpg": [ 306 | "image" 307 | ], 308 | "js": [ 309 | "code", 310 | "web" 311 | ], 312 | "jsp": [ 313 | "code" 314 | ], 315 | "jsx": [ 316 | "code", 317 | "web" 318 | ], 319 | "kml": [ 320 | "image" 321 | ], 322 | "kmz": [ 323 | "image" 324 | ], 325 | "ksh": [ 326 | "code", 327 | "exec" 328 | ], 329 | "kt": [ 330 | "code" 331 | ], 332 | "less": [ 333 | "web" 334 | ], 335 | "lha": [ 336 | "archive" 337 | ], 338 | "lhs": [ 339 | "code" 340 | ], 341 | "lisp": [ 342 | "code" 343 | ], 344 | "log": [ 345 | "text" 346 | ], 347 | "lua": [ 348 | "code" 349 | ], 350 | "m": [ 351 | "code" 352 | ], 353 | "m2v": [ 354 | "video" 355 | ], 356 | "m3u": [ 357 | "audio" 358 | ], 359 | "m4": [ 360 | "code" 361 | ], 362 | "m4a": [ 363 | "audio" 364 | ], 365 | "m4p": [ 366 | "video" 367 | ], 368 | "m4v": [ 369 | "video" 370 | ], 371 | "mar": [ 372 | "archive" 373 | ], 374 | "max": [ 375 | "image" 376 | ], 377 | "md": [ 378 | "text" 379 | ], 380 | "mid": [ 381 | "audio" 382 | ], 383 | "mkv": [ 384 | "video" 385 | ], 386 | "mng": [ 387 | "video" 388 | ], 389 | "mobi": [ 390 | "book" 391 | ], 392 | "mod": [ 393 | "audio" 394 | ], 395 | "mov": [ 396 | "video" 397 | ], 398 | "mp2": [ 399 | "video" 400 | ], 401 | "mp3": [ 402 | "audio" 403 | ], 404 | "mp4": [ 405 | "video" 406 | ], 407 | "mpa": [ 408 | "audio" 409 | ], 410 | "mpe": [ 411 | "video" 412 | ], 413 | "mpeg": [ 414 | "video" 415 | ], 416 | "mpg": [ 417 | "video" 418 | ], 419 | "mpv": [ 420 | "video" 421 | ], 422 | "msg": [ 423 | "text" 424 | ], 425 | "msi": [ 426 | "exec" 427 | ], 428 | "mxf": [ 429 | "video" 430 | ], 431 | "nim": [ 432 | "code" 433 | ], 434 | "nsv": [ 435 | "video" 436 | ], 437 | "odp": [ 438 | "slide" 439 | ], 440 | "ods": [ 441 | "sheet" 442 | ], 443 | "odt": [ 444 | "text" 445 | ], 446 | "ogg": [ 447 | "video" 448 | ], 449 | "ogm": [ 450 | "video" 451 | ], 452 | "ogv": [ 453 | "video" 454 | ], 455 | "org": [ 456 | "text" 457 | ], 458 | "otf": [ 459 | "font" 460 | ], 461 | "pages": [ 462 | "text" 463 | ], 464 | "pak": [ 465 | "archive" 466 | ], 467 | "patch": [ 468 | "code" 469 | ], 470 | "pdf": [ 471 | "text" 472 | ], 473 | "pea": [ 474 | "archive" 475 | ], 476 | "php": [ 477 | "code", 478 | "web" 479 | ], 480 | "pl": [ 481 | "code" 482 | ], 483 | "pls": [ 484 | "audio" 485 | ], 486 | "png": [ 487 | "image" 488 | ], 489 | "po": [ 490 | "code" 491 | ], 492 | "pp": [ 493 | "code" 494 | ], 495 | "ppt": [ 496 | "slide" 497 | ], 498 | "ps": [ 499 | "image" 500 | ], 501 | "psd": [ 502 | "image" 503 | ], 504 | "py": [ 505 | "code" 506 | ], 507 | "qt": [ 508 | "video" 509 | ], 510 | "r": [ 511 | "code" 512 | ], 513 | "ra": [ 514 | "audio" 515 | ], 516 | "rar": [ 517 | "archive" 518 | ], 519 | "rb": [ 520 | "code" 521 | ], 522 | "rm": [ 523 | "video" 524 | ], 525 | "rmvb": [ 526 | "video" 527 | ], 528 | "roq": [ 529 | "video" 530 | ], 531 | "rpm": [ 532 | "archive" 533 | ], 534 | "rs": [ 535 | "code" 536 | ], 537 | "rst": [ 538 | "text" 539 | ], 540 | "rtf": [ 541 | "text" 542 | ], 543 | "s": [ 544 | "code" 545 | ], 546 | "s3m": [ 547 | "audio" 548 | ], 549 | "s7z": [ 550 | "archive" 551 | ], 552 | "scala": [ 553 | "code" 554 | ], 555 | "scss": [ 556 | "web" 557 | ], 558 | "sh": [ 559 | "code", 560 | "exec" 561 | ], 562 | "shar": [ 563 | "archive" 564 | ], 565 | "sid": [ 566 | "audio" 567 | ], 568 | "srt": [ 569 | "video" 570 | ], 571 | "svg": [ 572 | "image" 573 | ], 574 | "svi": [ 575 | "video" 576 | ], 577 | "swg": [ 578 | "code" 579 | ], 580 | "swift": [ 581 | "code" 582 | ], 583 | "tar": [ 584 | "archive" 585 | ], 586 | "tbz2": [ 587 | "archive" 588 | ], 589 | "tex": [ 590 | "text" 591 | ], 592 | "tga": [ 593 | "image" 594 | ], 595 | "tgz": [ 596 | "archive" 597 | ], 598 | "thm": [ 599 | "image" 600 | ], 601 | "tif": [ 602 | "image" 603 | ], 604 | "tiff": [ 605 | "image" 606 | ], 607 | "tlz": [ 608 | "archive" 609 | ], 610 | "ttf": [ 611 | "font" 612 | ], 613 | "txt": [ 614 | "text" 615 | ], 616 | "v": [ 617 | "code" 618 | ], 619 | "vb": [ 620 | "code" 621 | ], 622 | "vcf": [ 623 | "sheet" 624 | ], 625 | "vcxproj": [ 626 | "code" 627 | ], 628 | "vob": [ 629 | "video" 630 | ], 631 | "war": [ 632 | "archive" 633 | ], 634 | "wasm": [ 635 | "web" 636 | ], 637 | "wav": [ 638 | "audio" 639 | ], 640 | "webm": [ 641 | "video" 642 | ], 643 | "webp": [ 644 | "image" 645 | ], 646 | "whl": [ 647 | "archive" 648 | ], 649 | "wma": [ 650 | "audio" 651 | ], 652 | "wmv": [ 653 | "video" 654 | ], 655 | "woff": [ 656 | "font" 657 | ], 658 | "woff2": [ 659 | "font" 660 | ], 661 | "wpd": [ 662 | "text" 663 | ], 664 | "wps": [ 665 | "text" 666 | ], 667 | "xcf": [ 668 | "image" 669 | ], 670 | "xcodeproj": [ 671 | "code" 672 | ], 673 | "xls": [ 674 | "sheet" 675 | ], 676 | "xlsx": [ 677 | "sheet" 678 | ], 679 | "xm": [ 680 | "audio" 681 | ], 682 | "xml": [ 683 | "code" 684 | ], 685 | "xpi": [ 686 | "archive" 687 | ], 688 | "xz": [ 689 | "archive" 690 | ], 691 | "yuv": [ 692 | "image", 693 | "video" 694 | ], 695 | "zip": [ 696 | "archive" 697 | ], 698 | "zipx": [ 699 | "archive" 700 | ], 701 | "zsh": [ 702 | "code", 703 | "exec" 704 | ] 705 | } 706 | -------------------------------------------------------------------------------- /lua/src/harvest.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env luajit 2 | 3 | -- Copyright (C) 2014-2022 Dyne.org Foundation 4 | 5 | -- Harvest is designed, written and maintained by Denis "Jaromil" Roio 6 | 7 | -- This source code is free software; you can redistribute it and/or 8 | -- modify it under the terms of the GNU Public License as published by 9 | -- the Free Software Foundation; either version 3 of the License, or 10 | -- (at your option) any later version. 11 | -- 12 | -- This source code is distributed in the hope that it will be useful, 13 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. Please refer 15 | -- to the GNU Public License for more details. 16 | -- 17 | -- You should have received a copy of the GNU Public License along with 18 | -- this source code; if not, write to: Free Software Foundation, Inc., 19 | -- 675 Mass Ave, Cambridge, MA 02139, USA. 20 | 21 | local lfs = require'lfs' 22 | 23 | -- require local lua libs from src 24 | package.path = package.path ..";"..lfs.currentdir().."/src/?.lua" 25 | 26 | local cli = require'cliargs' 27 | 28 | local align = require'align' 29 | 30 | local function stderr(_msg) io.stderr:write(_msg..'\n') end 31 | local function stdout(_msg) io.stdout:write(_msg..'\n') end 32 | 33 | -- # fuzzy thresholds 34 | -- # 35 | -- # this is the most important section to tune the selection: the higher 36 | -- # the values the more file of that type need to be present in a 37 | -- # directory to classify it with their own type. In other words a lower 38 | -- # number makes the type "dominant". 39 | local fuzzy = { 40 | video=1, -- minimum video files to increase the video factor 41 | audio=3, -- minimum audio files to increase the audio factor 42 | text=10, -- minimum text files to increase the text factor 43 | image=10, -- minimum image files to increase the image factor 44 | other=25, -- minimum other files to increase the other factor 45 | code=5, -- minimum code files to increase the code factor 46 | web=10, 47 | slide=2, 48 | sheet=3, 49 | archiv=10 50 | } 51 | 52 | -- https://github.com/dyne/file-extension-list 53 | local file_extension_list = { ["ogm"] = "video",["doc"] = "text",["hs"] = "code",["scala"] = "code",["js"] = "code",["swift"] = "code",["cc"] = "code",["jsp"] = "code",["tga"] = "image",["ape"] = "audio",["woff2"] = "font",["cab"] = "archive",["whl"] = "archive",["mpe"] = "video",["rmvb"] = "video",["srt"] = "video",["csh"] = "code",["tex"] = "text",["cs"] = "code",["exe"] = "exec",["m4a"] = "audio",["zsh"] = "code",["crx"] = "exec",["vob"] = "video",["xm"] = "audio",["gz"] = "archive",["org"] = "text",["ada"] = "code",["lhs"] = "code",["azw"] = "book",["for"] = "code",["gif"] = "image",["rb"] = "code",["3g2"] = "video",["cob"] = "code",["ar"] = "archive",["vb"] = "code",["sid"] = "audio",["ai"] = "image",["wma"] = "audio",["pea"] = "archive",["lisp"] = "code",["bmp"] = "image",["py"] = "code",["2.ada"] = "code",["mp4"] = "video",["m4p"] = "video",["aaf"] = "video",["jpeg"] = "image",["3dm"] = "image",["command"] = "exec",["go"] = "code",["azw4"] = "book",["otf"] = "font",["ebook"] = "text",["eps"] = "image",["rtf"] = "text",["cbz"] = "book",["ttf"] = "font",["1.ada"] = "code",["bat"] = "code",["mobi"] = "book",["diff"] = "code",["ra"] = "audio",["cpio"] = "archive",["xz"] = "archive",["php"] = "code",["s"] = "code",["dmg"] = "archive",["flv"] = "video",["asf"] = "video",["css"] = "web",["zipx"] = "archive",["mpg"] = "video",["xls"] = "sheet",["cpp"] = "code",["jpg"] = "image",["mkv"] = "video",["nsv"] = "video",["jsx"] = "code",["mp3"] = "audio",["adb"] = "code",["h"] = "code",["m4"] = "code",["java"] = "code",["cbl"] = "code",["hpp"] = "code",["class"] = "code",["lua"] = "code",["m2v"] = "video",["fth"] = "code",["deb"] = "archive",["rst"] = "text",["csv"] = "sheet",["hh"] = "code",["hxx"] = "code",["c"] = "code",["m4v"] = "video",["pls"] = "audio",["pak"] = "archive",["tbz2"] = "archive",["aiff"] = "audio",["egg"] = "archive",["log"] = "text",["swg"] = "code",["gpx"] = "image",["e"] = "code",["d"] = "code",["bz2"] = "archive",["f"] = "code",["fish"] = "code",["iso"] = "archive",["apk"] = "archive",["it"] = "audio",["webm"] = "video",["3ds"] = "image",["au"] = "audio",["patch"] = "code",["rs"] = "code",["kml"] = "image",["woff"] = "font",["r"] = "code",["max"] = "image",["3gp"] = "video",["po"] = "code",["v"] = "code",["mng"] = "video",["rpm"] = "archive",["a"] = "archive",["htm"] = "code",["s7z"] = "archive",["ics"] = "sheet",["bash"] = "code",["f90"] = "code",["flac"] = "audio",["azw3"] = "book",["mp2"] = "video",["asm"] = "code",["xml"] = "code",["ksh"] = "code",["epub"] = "book",["bas"] = "code",["svg"] = "image",["tgz"] = "archive",["mpa"] = "audio",["wmv"] = "video",["vcxproj"] = "code",["mpeg"] = "video",["mpv"] = "video",["less"] = "web",["f77"] = "code",["c++"] = "code",["m3u"] = "audio",["dwg"] = "image",["odt"] = "text",["msg"] = "text",["ads"] = "code",["msi"] = "exec",["png"] = "image",["gsm"] = "audio",["ogg"] = "video",["cbr"] = "book",["azw1"] = "book",["pages"] = "text",["dds"] = "image",["docx"] = "text",["azw6"] = "book",["mid"] = "audio",["ftn"] = "code",["odp"] = "slide",["aac"] = "audio",["s3m"] = "audio",["avi"] = "video",["ogv"] = "video",["ods"] = "sheet",["groovy"] = "code",["eot"] = "font",["dxf"] = "image",["nim"] = "code",["html"] = "code",["wpd"] = "text",["bin"] = "exec",["txt"] = "text",["pp"] = "code",["rm"] = "video",["m"] = "code",["ps"] = "image",["psd"] = "image",["ppt"] = "slide",["clj"] = "code",["roq"] = "video",["mod"] = "audio",["tiff"] = "image",["lha"] = "archive",["mxf"] = "video",["7z"] = "archive",["drc"] = "video",["yuv"] = "image",["wps"] = "text",["sh"] = "code",["mar"] = "archive",["vcf"] = "sheet",["shar"] = "archive",["xcf"] = "image",["tlz"] = "archive",["jar"] = "archive",["qt"] = "video",["tar"] = "archive",["xpi"] = "archive",["zip"] = "archive",["xcodeproj"] = "code",["cxx"] = "code",["kt"] = "code",["rar"] = "archive",["md"] = "text",["scss"] = "web",["pdf"] = "text",["webp"] = "image",["war"] = "archive",["pl"] = "code",["xlsx"] = "sheet",["svi"] = "video",["thm"] = "image",["avchd"] = "video",["tif"] = "image",["mov"] = "video",["kmz"] = "image",["wasm"] = "web",["el"] = "code",["wav"] = "audio",} 54 | 55 | local function extparser(arg) 56 | local curr = 0 57 | repeat 58 | local n = arg:find('.',curr+1, true) 59 | if n then curr = n end 60 | until (not n) 61 | if (curr == 0) then return nil end 62 | return(arg:sub( curr + 1 )) 63 | end 64 | 65 | -- recurse into directories 66 | local function analyse_path(args, pathname, level) 67 | local target = pathname or args.path 68 | local curlev = tonumber(level or 1) 69 | if curlev > tonumber(args.maxdepth) then return end 70 | local scores = { other = { } } 71 | local path 72 | for path in lfs.dir(target) do 73 | if not (path == '.' or path == '..') then 74 | local tarpath = target..'/'..path 75 | if lfs.attributes(tarpath,"mode") == "directory" then 76 | analyse_path(args, tarpath, curlev+1) 77 | 78 | else -- file in subdir 79 | local ftype = file_extension_list[ extparser(tarpath) ] 80 | if ftype then 81 | if not scores[ftype] then scores[ftype] = { } end 82 | table.insert(scores[ftype], tarpath) 83 | else 84 | table.insert(scores['other'], tarpath) 85 | end 86 | end 87 | end 88 | end 89 | return scores 90 | end 91 | 92 | local function fuzzyguess(scores) 93 | -- compute a very, very simple linear fuzzy logic for each 94 | local res = { guess = 'other', 95 | totals = { } } 96 | if scores then 97 | for k,v in pairs(scores) do 98 | res.totals[k] = #v / (fuzzy[k] or fuzzy['other']) 99 | end 100 | end 101 | local max = 0 102 | for k,v in pairs(res.totals) do 103 | if v > max then 104 | max = v 105 | res.guess = k 106 | end 107 | end 108 | return res 109 | end 110 | 111 | -- checks that the file attributes match the selection arguments 112 | local function filter_selection(args, attr) 113 | if args.type and (args.type ~= attr.guess) then return false end 114 | if args.file and attr.mode == 'directory' then return false end 115 | if args.dir and attr.mode ~= 'directory' then return false end 116 | return true 117 | end 118 | 119 | local function show_selection(args, selection) 120 | if args.output == 'human' then 121 | stderr(align('LINE,MODE,TYPE,YYYY-MM-DD,SIZE,NAME')) 122 | stderr(align('----,----,----,----------,----,----')) 123 | end 124 | -- for k,v in pairs(selection) do 125 | for k=1, #selection do 126 | local v = selection[k] 127 | if args.output == 'csv' then 128 | stdout(v.type..","..v.guess..","..v.modification.."," 129 | ..v.size..","..v.name) 130 | -- human friendly formatting 131 | else 132 | local size = v.size 133 | local guess = v.guess 134 | if v.type == 'dir' then size = '/' end 135 | if v.guess == 'other' then guess = '? ? ?' end 136 | if v.guess == 'archiv' then guess = 'archv' end 137 | stdout(align(k..","..v.type..","..guess.."," 138 | ..os.date('%Y-%m-%d',v.modification).."," 139 | ..size..","..v.name)) 140 | end 141 | end 142 | end 143 | 144 | -- CLI: command line argument parsing 145 | 146 | cli:set_name("harvest") 147 | cli:set_description('manage large collections of files and directories') 148 | 149 | cli:option("-p, --path=PATH", "", lfs.currentdir()) 150 | cli:option("-t, --type=TYPE", "text, audio, video, etc. (-t list)") 151 | cli:option("-o, --output=FORMAT", "csv", 'human') 152 | cli:option("-m, --maxdepth=NUM", "max levels of recursion inside dirs", 3) 153 | cli:flag("--dir", "select only directories") 154 | cli:flag("--file", "select only files") 155 | cli:flag("-d", "run in DEBUG mode", function() DEBUG=1 end) 156 | cli:flag("-v, --version", "print the version and exit", function() 157 | print("Harvest version 0.8") os.exit(0) end) 158 | 159 | local args, err = cli:parse(arg) 160 | local selection = { } 161 | 162 | -- MAIN() 163 | if not args and err then 164 | -- print(cli.name .. ': command not recognized') 165 | stderr(err) 166 | os.exit(1) 167 | elseif args then -- default command is scan 168 | stderr("Harvest "..args.path) 169 | if args.type == 'list' then 170 | local list = { } 171 | for k,v in pairs(file_extension_list) do 172 | table.insert(list, v) 173 | end 174 | print'Supported types:' 175 | local hash = { } 176 | for _,v in ipairs(list) do 177 | if not hash[v] then 178 | io.stdout:write(' '..v) 179 | hash[v] = true 180 | end 181 | end 182 | io.stdout:write('\n') 183 | os.exit(0) 184 | end 185 | if args.type then stderr("type: "..args.type) end 186 | 187 | -- recursive 188 | local fattr = lfs.attributes 189 | for file in lfs.dir(args.path) do 190 | local filepath = args.path.."/"..file 191 | if not (file == '.' or file == '..') then 192 | local attr = fattr(filepath) 193 | attr.name = file 194 | -- I(attr) 195 | if type(attr) == 'table' then -- safety to os stat 196 | if attr.mode == "directory" then 197 | attr.type = 'dir' 198 | attr.guess = fuzzyguess( 199 | analyse_path(args, filepath) ).guess 200 | collectgarbage'collect' -- recursion costs memory 201 | if filter_selection(args,attr) then 202 | table.insert(selection, attr) 203 | end 204 | else 205 | attr.type = 'file' 206 | attr.guess = 207 | file_extension_list[ extparser(filepath) ] 208 | or 'other' 209 | if filter_selection(args,attr) then 210 | table.insert(selection, attr) 211 | end 212 | end 213 | end 214 | end 215 | end 216 | end 217 | 218 | -- print to screen 219 | show_selection(args,selection) 220 | -------------------------------------------------------------------------------- /lua/src/import_file_extension_list.lua: -------------------------------------------------------------------------------- 1 | JSON = require'json' 2 | 3 | local function stderr(_msg) io.stderr:write(_msg..'\n') end 4 | 5 | -- see if the file exists 6 | function file_exists(file) 7 | local f = io.open(file, "rb") 8 | if f then f:close() end 9 | return f ~= nil 10 | end 11 | 12 | -- get all lines from a file, returns an empty 13 | -- list/table if the file does not exist 14 | function read_from(file) 15 | if not file_exists(file) then return {} end 16 | local buf = "" 17 | for line in io.lines(file) do 18 | buf = buf .. line 19 | end 20 | return buf 21 | end 22 | 23 | 24 | function dump(o) 25 | if type(o) == 'table' then 26 | local s = '{ ' 27 | for k,v in pairs(o) do 28 | if type(k) ~= 'number' then k = '"'..k..'"' end 29 | s = s .. '['..k..'] = "' .. dump(v) .. '",' 30 | end 31 | return s .. '} ' 32 | else 33 | return tostring(o) 34 | end 35 | end 36 | 37 | local file = 'extensions.json' 38 | local buf 39 | if not file_exists(file) then 40 | stderr("file not found: ".. file) 41 | os.exit(1) 42 | end 43 | if file_exists(file) then 44 | buf = read_from(file) 45 | end 46 | -- print(buf) 47 | local extensions = JSON.decode(buf) 48 | -- remove nested array 49 | local res = { } 50 | for k,v in pairs(extensions) do 51 | res[k] = v[1] 52 | end 53 | print(dump(res)) 54 | -------------------------------------------------------------------------------- /lua/src/lfs.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** LuaFileSystem 3 | ** Copyright Kepler Project 2003 - 2020 4 | ** (http://keplerproject.github.io/luafilesystem) 5 | */ 6 | 7 | /* Define 'chdir' for systems that do not implement it */ 8 | #ifdef NO_CHDIR 9 | #define chdir(p) (-1) 10 | #define chdir_error "Function 'chdir' not provided by system" 11 | #else 12 | #define chdir_error strerror(errno) 13 | #endif 14 | 15 | #ifdef _WIN32 16 | #define chdir(p) (_chdir(p)) 17 | #define getcwd(d, s) (_getcwd(d, s)) 18 | #define rmdir(p) (_rmdir(p)) 19 | #define LFS_EXPORT __declspec (dllexport) 20 | #ifndef fileno 21 | #define fileno(f) (_fileno(f)) 22 | #endif 23 | #else 24 | #define LFS_EXPORT 25 | #endif 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | LFS_EXPORT int luaopen_lfs(lua_State * L); 32 | 33 | #ifdef __cplusplus 34 | } 35 | #endif 36 | -------------------------------------------------------------------------------- /test/GNUmakefile: -------------------------------------------------------------------------------- 1 | 2 | test: 3 | $(info Running harvest tests) 4 | ./bats/bin/bats . 5 | 6 | clean: 7 | $(info Cleaning up test files) 8 | @rm -rf harvest harvested 9 | @rm -rf /tmp/harvest_test 10 | @rm -rf bob_dylan bob_marley herzog nice_books zenroom 11 | -------------------------------------------------------------------------------- /test/bats/bin/bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | if command -v greadlink >/dev/null; then 6 | bats_readlinkf() { 7 | greadlink -f "$1" 8 | } 9 | else 10 | bats_readlinkf() { 11 | readlink -f "$1" 12 | } 13 | fi 14 | 15 | fallback_to_readlinkf_posix() { 16 | bats_readlinkf() { 17 | [ "${1:-}" ] || return 1 18 | max_symlinks=40 19 | CDPATH='' # to avoid changing to an unexpected directory 20 | 21 | target=$1 22 | [ -e "${target%/}" ] || target=${1%"${1##*[!/]}"} # trim trailing slashes 23 | [ -d "${target:-/}" ] && target="$target/" 24 | 25 | cd -P . 2>/dev/null || return 1 26 | while [ "$max_symlinks" -ge 0 ] && max_symlinks=$((max_symlinks - 1)); do 27 | if [ ! "$target" = "${target%/*}" ]; then 28 | case $target in 29 | /*) cd -P "${target%/*}/" 2>/dev/null || break ;; 30 | *) cd -P "./${target%/*}" 2>/dev/null || break ;; 31 | esac 32 | target=${target##*/} 33 | fi 34 | 35 | if [ ! -L "$target" ]; then 36 | target="${PWD%/}${target:+/}${target}" 37 | printf '%s\n' "${target:-/}" 38 | return 0 39 | fi 40 | 41 | # `ls -dl` format: "%s %u %s %s %u %s %s -> %s\n", 42 | # , , , , 43 | # , , , 44 | # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/ls.html 45 | link=$(ls -dl -- "$target" 2>/dev/null) || break 46 | target=${link#*" $target -> "} 47 | done 48 | return 1 49 | } 50 | } 51 | 52 | if ! BATS_PATH=$(bats_readlinkf "${BASH_SOURCE[0]}" 2>/dev/null); then 53 | fallback_to_readlinkf_posix 54 | BATS_PATH=$(bats_readlinkf "${BASH_SOURCE[0]}") 55 | fi 56 | 57 | export BATS_ROOT=${BATS_PATH%/*/*} 58 | export -f bats_readlinkf 59 | exec env BATS_ROOT="$BATS_ROOT" "$BATS_ROOT/libexec/bats-core/bats" "$@" 60 | -------------------------------------------------------------------------------- /test/bats/lib/bats-core/common.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | bats_prefix_lines_for_tap_output() { 4 | while IFS= read -r line; do 5 | printf '# %s\n' "$line" || break # avoid feedback loop when errors are redirected into BATS_OUT (see #353) 6 | done 7 | if [[ -n "$line" ]]; then 8 | printf '# %s\n' "$line" 9 | fi 10 | } 11 | 12 | function bats_replace_filename() { 13 | local line 14 | while read -r line; do 15 | printf "%s\n" "${line//$BATS_TEST_SOURCE/$BATS_TEST_FILENAME}" 16 | done 17 | if [[ -n "$line" ]]; then 18 | printf "%s\n" "${line//$BATS_TEST_SOURCE/$BATS_TEST_FILENAME}" 19 | fi 20 | } 21 | 22 | bats_quote_code() { # 23 | printf -v "$1" -- "%s%s%s" "$BATS_BEGIN_CODE_QUOTE" "$2" "$BATS_END_CODE_QUOTE" 24 | } 25 | 26 | bats_check_valid_version() { 27 | if [[ ! $1 =~ [0-9]+.[0-9]+.[0-9]+ ]]; then 28 | printf "ERROR: version '%s' must be of format ..!\n" "$1" >&2 29 | exit 1 30 | fi 31 | } 32 | 33 | # compares two versions. Return 0 when version1 < version2 34 | bats_version_lt() { # 35 | bats_check_valid_version "$1" 36 | bats_check_valid_version "$2" 37 | 38 | local -a version1_parts version2_parts 39 | IFS=. read -ra version1_parts <<< "$1" 40 | IFS=. read -ra version2_parts <<< "$2" 41 | 42 | for i in {0..2}; do 43 | if (( version1_parts[i] < version2_parts[i] )); then 44 | return 0 45 | elif (( version1_parts[i] > version2_parts[i] )); then 46 | return 1 47 | fi 48 | done 49 | # if we made it this far, they are equal -> also not less then 50 | return 2 # use other failing return code to distinguish equal from gt 51 | } 52 | 53 | # ensure a minimum version of bats is running or exit with failure 54 | bats_require_minimum_version() { # 55 | local required_minimum_version=$1 56 | 57 | if bats_version_lt "$BATS_VERSION" "$required_minimum_version"; then 58 | printf "BATS_VERSION=%s does not meet required minimum %s\n" "$BATS_VERSION" "$required_minimum_version" 59 | exit 1 60 | fi 61 | 62 | if bats_version_lt "$BATS_GUARANTEED_MINIMUM_VERSION" "$required_minimum_version"; then 63 | BATS_GUARANTEED_MINIMUM_VERSION="$required_minimum_version" 64 | fi 65 | } 66 | 67 | bats_binary_search() { # 68 | if [[ $# -ne 2 ]]; then 69 | printf "ERROR: bats_binary_search requires exactly 2 arguments: \n" >&2 70 | return 2 71 | fi 72 | 73 | local -r search_value=$1 array_name=$2 74 | 75 | # we'd like to test if array is set but we cannot distinguish unset from empty arrays, so we need to skip that 76 | 77 | local start=0 mid end mid_value 78 | # start is inclusive, end is exclusive ... 79 | eval "end=\${#${array_name}[@]}" 80 | 81 | # so start == end means empty search space 82 | while (( start < end )); do 83 | mid=$(( (start + end) / 2 )) 84 | eval "mid_value=\${${array_name}[$mid]}" 85 | if [[ "$mid_value" == "$search_value" ]]; then 86 | return 0 87 | elif [[ "$mid_value" < "$search_value" ]]; then 88 | # This branch excludes equality -> +1 to skip the mid element. 89 | # This +1 also avoids endless recursion on odd sized search ranges. 90 | start=$((mid + 1)) 91 | else 92 | end=$mid 93 | fi 94 | done 95 | 96 | # did not find it -> its not there 97 | return 1 98 | } -------------------------------------------------------------------------------- /test/bats/lib/bats-core/formatter.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # reads (extended) bats tap streams from stdin and calls callback functions for each line 4 | # bats_tap_stream_plan -> when the test plan is encountered 5 | # bats_tap_stream_begin -> when a new test is begun WARNING: extended only 6 | # bats_tap_stream_ok [--duration -> when a test was successful 7 | # bats_tap_stream_not_ok [--duration ] -> when a test has failed 8 | # bats_tap_stream_skipped -> when a test was skipped 9 | # bats_tap_stream_comment -> when a comment line was encountered, 10 | # scope tells the last encountered of plan, begin, ok, not_ok, skipped, suite 11 | # bats_tap_stream_suite -> when a new file is begun WARNING: extended only 12 | # bats_tap_stream_unknown -> when a line is encountered that does not match the previous entries, 13 | # scope @see bats_tap_stream_comment 14 | # forwards all input as is, when there is no TAP test plan header 15 | function bats_parse_internal_extended_tap() { 16 | local header_pattern='[0-9]+\.\.[0-9]+' 17 | IFS= read -r header 18 | 19 | if [[ "$header" =~ $header_pattern ]]; then 20 | bats_tap_stream_plan "${header:3}" 21 | else 22 | # If the first line isn't a TAP plan, print it and pass the rest through 23 | printf '%s\n' "$header" 24 | exec cat 25 | fi 26 | 27 | ok_line_regexpr="ok ([0-9]+) (.*)" 28 | skip_line_regexpr="ok ([0-9]+) (.*) # skip( (.*))?$" 29 | not_ok_line_regexpr="not ok ([0-9]+) (.*)" 30 | 31 | timing_expr="in ([0-9]+)ms$" 32 | local test_name begin_index ok_index not_ok_index index scope 33 | begin_index=0 34 | index=0 35 | scope=plan 36 | while IFS= read -r line; do 37 | case "$line" in 38 | 'begin '*) # this might only be called in extended tap output 39 | ((++begin_index)) 40 | scope=begin 41 | test_name="${line#* "$begin_index" }" 42 | bats_tap_stream_begin "$begin_index" "$test_name" 43 | ;; 44 | 'ok '*) 45 | ((++index)) 46 | if [[ "$line" =~ $ok_line_regexpr ]]; then 47 | ok_index="${BASH_REMATCH[1]}" 48 | test_name="${BASH_REMATCH[2]}" 49 | if [[ "$line" =~ $skip_line_regexpr ]]; then 50 | scope=skipped 51 | test_name="${BASH_REMATCH[2]}" # cut off name before "# skip" 52 | local skip_reason="${BASH_REMATCH[4]}" 53 | bats_tap_stream_skipped "$ok_index" "$test_name" "$skip_reason" 54 | else 55 | scope=ok 56 | if [[ "$line" =~ $timing_expr ]]; then 57 | bats_tap_stream_ok --duration "${BASH_REMATCH[1]}" "$ok_index" "$test_name" 58 | else 59 | bats_tap_stream_ok "$ok_index" "$test_name" 60 | fi 61 | fi 62 | else 63 | printf "ERROR: could not match ok line: %s" "$line" >&2 64 | exit 1 65 | fi 66 | ;; 67 | 'not ok '*) 68 | ((++index)) 69 | scope=not_ok 70 | if [[ "$line" =~ $not_ok_line_regexpr ]]; then 71 | not_ok_index="${BASH_REMATCH[1]}" 72 | test_name="${BASH_REMATCH[2]}" 73 | if [[ "$line" =~ $timing_expr ]]; then 74 | bats_tap_stream_not_ok --duration "${BASH_REMATCH[1]}" "$not_ok_index" "$test_name" 75 | else 76 | bats_tap_stream_not_ok "$not_ok_index" "$test_name" 77 | fi 78 | else 79 | printf "ERROR: could not match not ok line: %s" "$line" >&2 80 | exit 1 81 | fi 82 | ;; 83 | '# '*) 84 | bats_tap_stream_comment "${line:2}" "$scope" 85 | ;; 86 | '#') 87 | bats_tap_stream_comment "" "$scope" 88 | ;; 89 | 'suite '*) 90 | scope=suite 91 | # pass on the 92 | bats_tap_stream_suite "${line:6}" 93 | ;; 94 | *) 95 | bats_tap_stream_unknown "$line" "$scope" 96 | ;; 97 | esac 98 | done 99 | } 100 | 101 | normalize_base_path() { # 102 | # the relative path root to use for reporting filenames 103 | # this is mainly intended for suite mode, where this will be the suite root folder 104 | local base_path="$2" 105 | # use the containing directory when --base-path is a file 106 | if [[ ! -d "$base_path" ]]; then 107 | base_path="$(dirname "$base_path")" 108 | fi 109 | # get the absolute path 110 | base_path="$(cd "$base_path" && pwd)" 111 | # ensure the path ends with / to strip that later on 112 | if [[ "${base_path}" != *"/" ]]; then 113 | base_path="$base_path/" 114 | fi 115 | printf -v "$1" "%s" "$base_path" 116 | } 117 | -------------------------------------------------------------------------------- /test/bats/lib/bats-core/preprocessing.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BATS_TMPNAME="$BATS_RUN_TMPDIR/bats.$$" 4 | BATS_PARENT_TMPNAME="$BATS_RUN_TMPDIR/bats.$PPID" 5 | # shellcheck disable=SC2034 6 | BATS_OUT="${BATS_TMPNAME}.out" # used in bats-exec-file 7 | 8 | bats_preprocess_source() { 9 | # export to make it visible to bats_evaluate_preprocessed_source 10 | # since the latter runs in bats-exec-test's bash while this runs in bats-exec-file's 11 | export BATS_TEST_SOURCE="${BATS_TMPNAME}.src" 12 | bats-preprocess "$BATS_TEST_FILENAME" >"$BATS_TEST_SOURCE" 13 | } 14 | 15 | bats_evaluate_preprocessed_source() { 16 | if [[ -z "${BATS_TEST_SOURCE:-}" ]]; then 17 | BATS_TEST_SOURCE="${BATS_PARENT_TMPNAME}.src" 18 | fi 19 | # Dynamically loaded user files provided outside of Bats. 20 | # shellcheck disable=SC1090 21 | source "$BATS_TEST_SOURCE" 22 | } 23 | -------------------------------------------------------------------------------- /test/bats/lib/bats-core/semaphore.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # setup the semaphore environment for the loading file 4 | bats_semaphore_setup() { 5 | export -f bats_semaphore_get_free_slot_count 6 | export -f bats_semaphore_acquire_while_locked 7 | export BATS_SEMAPHORE_DIR="$BATS_RUN_TMPDIR/semaphores" 8 | 9 | if command -v flock >/dev/null; then 10 | bats_run_under_lock() { 11 | flock "$BATS_SEMAPHORE_DIR" "$@" 12 | } 13 | elif command -v shlock >/dev/null; then 14 | bats_run_under_lock() { 15 | local lockfile="$BATS_SEMAPHORE_DIR/shlock.lock" 16 | while ! shlock -p $$ -f "$lockfile"; do 17 | sleep 1 18 | done 19 | # we got the lock now, execute the command 20 | "$@" 21 | local status=$? 22 | # free the lock 23 | rm -f "$lockfile" 24 | return $status 25 | } 26 | else 27 | printf "ERROR: flock/shlock is required for parallelization within files!\n" >&2 28 | exit 1 29 | fi 30 | } 31 | 32 | # $1 - output directory for stdout/stderr 33 | # $@ - command to run 34 | # run the given command in a semaphore 35 | # block when there is no free slot for the semaphore 36 | # when there is a free slot, run the command in background 37 | # gather the output of the command in files in the given directory 38 | bats_semaphore_run() { 39 | local output_dir=$1 40 | shift 41 | local semaphore_slot 42 | semaphore_slot=$(bats_semaphore_acquire_slot) 43 | bats_semaphore_release_wrapper "$output_dir" "$semaphore_slot" "$@" & 44 | printf "%d\n" "$!" 45 | } 46 | 47 | # $1 - output directory for stdout/stderr 48 | # $@ - command to run 49 | # this wraps the actual function call to install some traps on exiting 50 | bats_semaphore_release_wrapper() { 51 | local output_dir="$1" 52 | local semaphore_name="$2" 53 | shift 2 # all other parameters will be use for the command to execute 54 | 55 | # shellcheck disable=SC2064 # we want to expand the semaphore_name right now! 56 | trap "status=$?; bats_semaphore_release_slot '$semaphore_name'; exit $status" EXIT 57 | 58 | mkdir -p "$output_dir" 59 | "$@" 2>"$output_dir/stderr" >"$output_dir/stdout" 60 | local status=$? 61 | 62 | # bash bug: the exit trap is not called for the background process 63 | bats_semaphore_release_slot "$semaphore_name" 64 | trap - EXIT # avoid calling release twice 65 | return $status 66 | } 67 | 68 | bats_semaphore_acquire_while_locked() { 69 | if [[ $(bats_semaphore_get_free_slot_count) -gt 0 ]]; then 70 | local slot=0 71 | while [[ -e "$BATS_SEMAPHORE_DIR/slot-$slot" ]]; do 72 | (( ++slot )) 73 | done 74 | if [[ $slot -lt $BATS_SEMAPHORE_NUMBER_OF_SLOTS ]]; then 75 | touch "$BATS_SEMAPHORE_DIR/slot-$slot" && printf "%d\n" "$slot" && return 0 76 | fi 77 | fi 78 | return 1 79 | } 80 | 81 | # block until a semaphore slot becomes free 82 | # prints the number of the slot that it received 83 | bats_semaphore_acquire_slot() { 84 | mkdir -p "$BATS_SEMAPHORE_DIR" 85 | # wait for a slot to become free 86 | # TODO: avoid busy waiting by using signals -> this opens op prioritizing possibilities as well 87 | while true; do 88 | # don't lock for reading, we are fine with spuriously getting no free slot 89 | if [[ $(bats_semaphore_get_free_slot_count) -gt 0 ]]; then 90 | bats_run_under_lock bash -c bats_semaphore_acquire_while_locked && break 91 | fi 92 | sleep 1 93 | done 94 | } 95 | 96 | bats_semaphore_release_slot() { 97 | # we don't need to lock this, since only our process owns this file 98 | # and freeing a semaphore cannot lead to conflicts with others 99 | rm "$BATS_SEMAPHORE_DIR/slot-$1" # this will fail if we had not acquired a semaphore! 100 | } 101 | 102 | bats_semaphore_get_free_slot_count() { 103 | # find might error out without returning something useful when a file is deleted, 104 | # while the directory is traversed -> only continue when there was no error 105 | until used_slots=$(find "$BATS_SEMAPHORE_DIR" -name 'slot-*' 2>/dev/null | wc -l); do :; done 106 | echo $(( BATS_SEMAPHORE_NUMBER_OF_SLOTS - used_slots )) 107 | } 108 | -------------------------------------------------------------------------------- /test/bats/lib/bats-core/test_functions.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BATS_TEST_DIRNAME="${BATS_TEST_FILENAME%/*}" 4 | BATS_TEST_NAMES=() 5 | 6 | # shellcheck source=lib/bats-core/warnings.bash 7 | source "$BATS_ROOT/lib/bats-core/warnings.bash" 8 | 9 | # find_in_bats_lib_path echoes the first recognized load path to 10 | # a library in BATS_LIB_PATH or relative to BATS_TEST_DIRNAME. 11 | # 12 | # Libraries relative to BATS_TEST_DIRNAME take precedence over 13 | # BATS_LIB_PATH. 14 | # 15 | # Library load paths are recognized using find_library_load_path. 16 | # 17 | # If no library is found find_in_bats_lib_path returns 1. 18 | find_in_bats_lib_path() { # 19 | local return_var="${1:?}" 20 | local library_name="${2:?}" 21 | 22 | local -a bats_lib_paths 23 | IFS=: read -ra bats_lib_paths <<< "$BATS_LIB_PATH" 24 | 25 | for path in "${bats_lib_paths[@]}"; do 26 | if [[ -f "$path/$library_name" ]]; then 27 | printf -v "$return_var" "%s" "$path/$library_name" 28 | # A library load path was found, return 29 | return 0 30 | elif [[ -f "$path/$library_name/load.bash" ]]; then 31 | printf -v "$return_var" "%s" "$path/$library_name/load.bash" 32 | # A library load path was found, return 33 | return 0 34 | fi 35 | done 36 | 37 | return 1 38 | } 39 | 40 | # bats_internal_load expects an absolute path that is a library load path. 41 | # 42 | # If the library load path points to a file (a library loader) it is 43 | # sourced. 44 | # 45 | # If it points to a directory all files ending in .bash inside of the 46 | # directory are sourced. 47 | # 48 | # If the sourcing of the library loader or of a file in a library 49 | # directory fails bats_internal_load prints an error message and returns 1. 50 | # 51 | # If the passed library load path is not absolute or is not a valid file 52 | # or directory bats_internal_load prints an error message and returns 1. 53 | bats_internal_load() { 54 | local library_load_path="${1:?}" 55 | 56 | if [[ "${library_load_path:0:1}" != / ]]; then 57 | printf "Passed library load path is not an absolute path: %s\n" "$library_load_path" >&2 58 | return 1 59 | fi 60 | 61 | # library_load_path is a library loader 62 | if [[ -f "$library_load_path" ]]; then 63 | # shellcheck disable=SC1090 64 | if ! source "$library_load_path"; then 65 | printf "Error while sourcing library loader at '%s'\n" "$library_load_path" >&2 66 | return 1 67 | fi 68 | return 0 69 | fi 70 | 71 | printf "Passed library load path is neither a library loader nor library directory: %s\n" "$library_load_path" >&2 72 | return 1 73 | } 74 | 75 | # bats_load_safe accepts an argument called 'slug' and attempts to find and 76 | # source a library based on the slug. 77 | # 78 | # A slug can be an absolute path, a library name or a relative path. 79 | # 80 | # If the slug is an absolute path bats_load_safe attempts to find the library 81 | # load path using find_library_load_path. 82 | # What is considered a library load path is documented in the 83 | # documentation for find_library_load_path. 84 | # 85 | # If the slug is not an absolute path it is considered a library name or 86 | # relative path. bats_load_safe attempts to find the library load path using 87 | # find_in_bats_lib_path. 88 | # 89 | # If bats_load_safe can find a library load path it is passed to bats_internal_load. 90 | # If bats_internal_load fails bats_load_safe returns 1. 91 | # 92 | # If no library load path can be found bats_load_safe prints an error message 93 | # and returns 1. 94 | bats_load_safe() { 95 | local slug="${1:?}" 96 | if [[ ${slug:0:1} != / ]]; then # relative paths are relative to BATS_TEST_DIRNAME 97 | slug="$BATS_TEST_DIRNAME/$slug" 98 | fi 99 | 100 | if [[ -f "$slug.bash" ]]; then 101 | bats_internal_load "$slug.bash" 102 | return $? 103 | elif [[ -f "$slug" ]]; then 104 | bats_internal_load "$slug" 105 | return $? 106 | fi 107 | 108 | # loading from PATH (retained for backwards compatibility) 109 | if [[ ! -f "$1" ]] && type -P "$1" >/dev/null; then 110 | # shellcheck disable=SC1090 111 | source "$1" 112 | return $? 113 | fi 114 | 115 | # No library load path can be found 116 | printf "bats_load_safe: Could not find '%s'[.bash]\n" "$slug" >&2 117 | return 1 118 | } 119 | 120 | bats_require_lib_path() { 121 | if [[ -z "${BATS_LIB_PATH:-}" ]]; then 122 | printf "%s: requires BATS_LIB_PATH to be set!\n" "${FUNCNAME[1]}" >&2 123 | exit 1 124 | fi 125 | } 126 | 127 | bats_load_library_safe() { # 128 | local slug="${1:?}" library_path 129 | 130 | bats_require_lib_path 131 | 132 | # Check for library load paths in BATS_TEST_DIRNAME and BATS_LIB_PATH 133 | if [[ ${slug:0:1} != / ]]; then 134 | find_in_bats_lib_path library_path "$slug" 135 | if [[ -z "$library_path" ]]; then 136 | printf "Could not find library '%s' relative to test file or in BATS_LIB_PATH\n" "$slug" >&2 137 | return 1 138 | fi 139 | else 140 | # absolute paths are taken as is 141 | library_path="$slug" 142 | if [[ ! -f "$library_path" ]]; then 143 | printf "Could not find library on absolute path '%s'\n" "$library_path" >&2 144 | return 1 145 | fi 146 | fi 147 | 148 | bats_internal_load "$library_path" 149 | return $? 150 | } 151 | 152 | # immediately exit on error, use bats_load_library_safe to catch and handle errors 153 | bats_load_library() { # 154 | bats_require_lib_path 155 | if ! bats_load_library_safe "$@"; then 156 | exit 1 157 | fi 158 | } 159 | 160 | # load acts like bats_load_safe but exits the shell instead of returning 1. 161 | load() { 162 | if ! bats_load_safe "$@"; then 163 | echo "${FUNCNAME[0]} $LINENO" >&3 164 | exit 1 165 | fi 166 | } 167 | 168 | bats_redirect_stderr_into_file() { 169 | "$@" 2>>"$bats_run_separate_stderr_file" # use >> to see collisions' content 170 | } 171 | 172 | bats_merge_stdout_and_stderr() { 173 | "$@" 2>&1 174 | } 175 | 176 | # write separate lines from into 177 | bats_separate_lines() { # 178 | local output_array_name="$1" 179 | local input_var_name="$2" 180 | if [[ $keep_empty_lines ]]; then 181 | local bats_separate_lines_lines=() 182 | if [[ -n "${!input_var_name}" ]]; then # avoid getting an empty line for empty input 183 | while IFS= read -r line; do 184 | bats_separate_lines_lines+=("$line") 185 | done <<<"${!input_var_name}" 186 | fi 187 | eval "${output_array_name}=(\"\${bats_separate_lines_lines[@]}\")" 188 | else 189 | # shellcheck disable=SC2034,SC2206 190 | IFS=$'\n' read -d '' -r -a "$output_array_name" <<<"${!input_var_name}" || true # don't fail due to EOF 191 | fi 192 | } 193 | 194 | run() { # [!|-N] [--keep-empty-lines] [--separate-stderr] [--] 195 | # This has to be restored on exit from this function to avoid leaking our trap INT into surrounding code. 196 | # Non zero exits won't restore under the assumption that they will fail the test before it can be aborted, 197 | # which allows us to avoid duplicating the restore code on every exit path 198 | trap bats_interrupt_trap_in_run INT 199 | local expected_rc= 200 | local keep_empty_lines= 201 | local output_case=merged 202 | local has_flags= 203 | # parse options starting with - 204 | while [[ $# -gt 0 ]] && [[ $1 == -* || $1 == '!' ]]; do 205 | has_flags=1 206 | case "$1" in 207 | '!') 208 | expected_rc=-1 209 | ;; 210 | -[0-9]*) 211 | expected_rc=${1#-} 212 | if [[ $expected_rc =~ [^0-9] ]]; then 213 | printf "Usage error: run: '-NNN' requires numeric NNN (got: %s)\n" "$expected_rc" >&2 214 | return 1 215 | elif [[ $expected_rc -gt 255 ]]; then 216 | printf "Usage error: run: '-NNN': NNN must be <= 255 (got: %d)\n" "$expected_rc" >&2 217 | return 1 218 | fi 219 | ;; 220 | --keep-empty-lines) 221 | keep_empty_lines=1 222 | ;; 223 | --separate-stderr) 224 | output_case="separate" 225 | ;; 226 | --) 227 | shift # eat the -- before breaking away 228 | break 229 | ;; 230 | *) 231 | printf "Usage error: unknown flag '%s'" "$1" >&2 232 | return 1 233 | ;; 234 | esac 235 | shift 236 | done 237 | 238 | if [[ -n $has_flags ]]; then 239 | bats_warn_minimum_guaranteed_version "Using flags on \`run\`" 1.5.0 240 | fi 241 | 242 | local pre_command= 243 | 244 | case "$output_case" in 245 | merged) # redirects stderr into stdout and fills only $output/$lines 246 | pre_command=bats_merge_stdout_and_stderr 247 | ;; 248 | separate) # splits stderr into own file and fills $stderr/$stderr_lines too 249 | local bats_run_separate_stderr_file 250 | bats_run_separate_stderr_file="$(mktemp "${BATS_TEST_TMPDIR}/separate-stderr-XXXXXX")" 251 | pre_command=bats_redirect_stderr_into_file 252 | ;; 253 | esac 254 | 255 | local origFlags="$-" 256 | set +eET 257 | local origIFS="$IFS" 258 | if [[ $keep_empty_lines ]]; then 259 | # 'output', 'status', 'lines' are global variables available to tests. 260 | # preserve trailing newlines by appending . and removing it later 261 | # shellcheck disable=SC2034 262 | output="$($pre_command "$@"; status=$?; printf .; exit $status)" && status=0 || status=$? 263 | output="${output%.}" 264 | else 265 | # 'output', 'status', 'lines' are global variables available to tests. 266 | # shellcheck disable=SC2034 267 | output="$($pre_command "$@")" && status=0 || status=$? 268 | fi 269 | 270 | bats_separate_lines lines output 271 | 272 | if [[ "$output_case" == separate ]]; then 273 | # shellcheck disable=SC2034 274 | read -d '' -r stderr < "$bats_run_separate_stderr_file" 275 | bats_separate_lines stderr_lines stderr 276 | fi 277 | 278 | # shellcheck disable=SC2034 279 | BATS_RUN_COMMAND="${*}" 280 | IFS="$origIFS" 281 | set "-$origFlags" 282 | 283 | if [[ ${BATS_VERBOSE_RUN:-} ]]; then 284 | printf "%s\n" "$output" 285 | fi 286 | 287 | 288 | if [[ -n "$expected_rc" ]]; then 289 | if [[ "$expected_rc" = "-1" ]]; then 290 | if [[ "$status" -eq 0 ]]; then 291 | BATS_ERROR_SUFFIX=", expected nonzero exit code!" 292 | return 1 293 | fi 294 | elif [ "$status" -ne "$expected_rc" ]; then 295 | # shellcheck disable=SC2034 296 | BATS_ERROR_SUFFIX=", expected exit code $expected_rc, got $status" 297 | return 1 298 | fi 299 | elif [[ "$status" -eq 127 ]]; then # "command not found" 300 | bats_generate_warning 1 "$BATS_RUN_COMMAND" 301 | fi 302 | # don't leak our trap into surrounding code 303 | trap bats_interrupt_trap INT 304 | } 305 | 306 | setup() { 307 | return 0 308 | } 309 | 310 | teardown() { 311 | return 0 312 | } 313 | 314 | skip() { 315 | # if this is a skip in teardown ... 316 | if [[ -n "${BATS_TEARDOWN_STARTED-}" ]]; then 317 | # ... we want to skip the rest of teardown. 318 | # communicate to bats_exit_trap that the teardown was completed without error 319 | # shellcheck disable=SC2034 320 | BATS_TEARDOWN_COMPLETED=1 321 | # if we are already in the exit trap (e.g. due to previous skip) ... 322 | if [[ "$BATS_TEARDOWN_STARTED" == as-exit-trap ]]; then 323 | # ... we need to do the rest of the tear_down_trap that would otherwise be skipped after the next call to exit 324 | bats_exit_trap 325 | # and then do the exit (at the end of this function) 326 | fi 327 | # if we aren't in exit trap, the normal exit handling should suffice 328 | else 329 | # ... this is either skip in test or skip in setup. 330 | # Following variables are used in bats-exec-test which sources this file 331 | # shellcheck disable=SC2034 332 | BATS_TEST_SKIPPED="${1:-1}" 333 | # shellcheck disable=SC2034 334 | BATS_TEST_COMPLETED=1 335 | fi 336 | exit 0 337 | } 338 | 339 | bats_test_begin() { 340 | BATS_TEST_DESCRIPTION="$1" 341 | if [[ -n "$BATS_EXTENDED_SYNTAX" ]]; then 342 | printf 'begin %d %s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}$BATS_TEST_DESCRIPTION" >&3 343 | fi 344 | setup 345 | } 346 | 347 | bats_test_function() { 348 | local test_name="$1" 349 | BATS_TEST_NAMES+=("$test_name") 350 | } 351 | 352 | # decides whether a failed test should be run again 353 | bats_should_retry_test() { 354 | # test try number starts at 1 355 | # 0 retries means run only first try 356 | (( BATS_TEST_TRY_NUMBER <= BATS_TEST_RETRIES )) 357 | } 358 | -------------------------------------------------------------------------------- /test/bats/lib/bats-core/tracing.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # shellcheck source=lib/bats-core/common.bash 4 | source "$BATS_ROOT/lib/bats-core/common.bash" 5 | 6 | bats_capture_stack_trace() { 7 | local test_file 8 | local funcname 9 | local i 10 | 11 | BATS_DEBUG_LAST_STACK_TRACE=() 12 | 13 | for ((i = 2; i != ${#FUNCNAME[@]}; ++i)); do 14 | # Use BATS_TEST_SOURCE if necessary to work around Bash < 4.4 bug whereby 15 | # calling an exported function erases the test file's BASH_SOURCE entry. 16 | test_file="${BASH_SOURCE[$i]:-$BATS_TEST_SOURCE}" 17 | funcname="${FUNCNAME[$i]}" 18 | BATS_DEBUG_LAST_STACK_TRACE+=("${BASH_LINENO[$((i-1))]} $funcname $test_file") 19 | case "$funcname" in 20 | "$BATS_TEST_NAME" | setup | teardown | setup_file | teardown_file | setup_suite | teardown_suite) 21 | break 22 | ;; 23 | esac 24 | if [[ "${BASH_SOURCE[$i + 1]:-}" == *"bats-exec-file" ]] && [[ "$funcname" == 'source' ]]; then 25 | break 26 | fi 27 | done 28 | } 29 | 30 | bats_get_failure_stack_trace() { 31 | local stack_trace_var 32 | # See bats_debug_trap for details. 33 | if [[ -n "${BATS_DEBUG_LAST_STACK_TRACE_IS_VALID:-}" ]]; then 34 | stack_trace_var=BATS_DEBUG_LAST_STACK_TRACE 35 | else 36 | stack_trace_var=BATS_DEBUG_LASTLAST_STACK_TRACE 37 | fi 38 | # shellcheck disable=SC2016 39 | eval "$(printf \ 40 | '%s=(${%s[@]+"${%s[@]}"})' \ 41 | "${1}" \ 42 | "${stack_trace_var}" \ 43 | "${stack_trace_var}")" 44 | } 45 | 46 | bats_print_stack_trace() { 47 | local frame 48 | local index=1 49 | local count="${#@}" 50 | local filename 51 | local lineno 52 | 53 | for frame in "$@"; do 54 | bats_frame_filename "$frame" 'filename' 55 | bats_trim_filename "$filename" 'filename' 56 | bats_frame_lineno "$frame" 'lineno' 57 | 58 | printf '%s' "${BATS_STACK_TRACE_PREFIX-# }" 59 | if [[ $index -eq 1 ]]; then 60 | printf '(' 61 | else 62 | printf ' ' 63 | fi 64 | 65 | local fn 66 | bats_frame_function "$frame" 'fn' 67 | if [[ "$fn" != "$BATS_TEST_NAME" ]] && 68 | # don't print "from function `source'"", 69 | # when failing in free code during `source $test_file` from bats-exec-file 70 | ! [[ "$fn" == 'source' && $index -eq $count ]]; then 71 | local quoted_fn 72 | bats_quote_code quoted_fn "$fn" 73 | printf "from function %s " "$quoted_fn" 74 | fi 75 | 76 | if [[ $index -eq $count ]]; then 77 | printf 'in test file %s, line %d)\n' "$filename" "$lineno" 78 | else 79 | printf 'in file %s, line %d,\n' "$filename" "$lineno" 80 | fi 81 | 82 | ((++index)) 83 | done 84 | } 85 | 86 | bats_print_failed_command() { 87 | local stack_trace=("${@}") 88 | if [[ ${#stack_trace[@]} -eq 0 ]]; then 89 | return 90 | fi 91 | local frame="${stack_trace[${#stack_trace[@]} - 1]}" 92 | local filename 93 | local lineno 94 | local failed_line 95 | local failed_command 96 | 97 | bats_frame_filename "$frame" 'filename' 98 | bats_frame_lineno "$frame" 'lineno' 99 | bats_extract_line "$filename" "$lineno" 'failed_line' 100 | bats_strip_string "$failed_line" 'failed_command' 101 | local quoted_failed_command 102 | bats_quote_code quoted_failed_command "$failed_command" 103 | printf '# %s ' "${quoted_failed_command}" 104 | 105 | if [[ "$BATS_ERROR_STATUS" -eq 1 ]]; then 106 | printf 'failed%s\n' "$BATS_ERROR_SUFFIX" 107 | else 108 | printf 'failed with status %d%s\n' "$BATS_ERROR_STATUS" "$BATS_ERROR_SUFFIX" 109 | fi 110 | } 111 | 112 | bats_frame_lineno() { 113 | printf -v "$2" '%s' "${1%% *}" 114 | } 115 | 116 | bats_frame_function() { 117 | local __bff_function="${1#* }" 118 | printf -v "$2" '%s' "${__bff_function%% *}" 119 | } 120 | 121 | bats_frame_filename() { 122 | local __bff_filename="${1#* }" 123 | __bff_filename="${__bff_filename#* }" 124 | 125 | if [[ "$__bff_filename" == "$BATS_TEST_SOURCE" ]]; then 126 | __bff_filename="$BATS_TEST_FILENAME" 127 | fi 128 | printf -v "$2" '%s' "$__bff_filename" 129 | } 130 | 131 | bats_extract_line() { 132 | local __bats_extract_line_line 133 | local __bats_extract_line_index=0 134 | 135 | while IFS= read -r __bats_extract_line_line; do 136 | if [[ "$((++__bats_extract_line_index))" -eq "$2" ]]; then 137 | printf -v "$3" '%s' "${__bats_extract_line_line%$'\r'}" 138 | break 139 | fi 140 | done <"$1" 141 | } 142 | 143 | bats_strip_string() { 144 | [[ "$1" =~ ^[[:space:]]*(.*)[[:space:]]*$ ]] 145 | printf -v "$2" '%s' "${BASH_REMATCH[1]}" 146 | } 147 | 148 | bats_trim_filename() { 149 | printf -v "$2" '%s' "${1#"$BATS_CWD"/}" 150 | } 151 | 152 | # normalize a windows path from e.g. C:/directory to /c/directory 153 | # The path must point to an existing/accessable directory, not a file! 154 | bats_normalize_windows_dir_path() { # 155 | local output_var="$1" path="$2" 156 | if [[ "$output_var" != NORMALIZED_INPUT ]]; then 157 | local NORMALIZED_INPUT 158 | fi 159 | if [[ $path == ?:* ]]; then 160 | NORMALIZED_INPUT="$(cd "$path" || exit 1; pwd)" 161 | else 162 | NORMALIZED_INPUT="$path" 163 | fi 164 | printf -v "$output_var" "%s" "$NORMALIZED_INPUT" 165 | } 166 | 167 | bats_emit_trace() { 168 | if [[ $BATS_TRACE_LEVEL -gt 0 ]]; then 169 | local line=${BASH_LINENO[1]} 170 | # shellcheck disable=SC2016 171 | if [[ $BASH_COMMAND != '"$BATS_TEST_NAME" >> "$BATS_OUT" 2>&1 4>&1' && $BASH_COMMAND != "bats_test_begin "* ]] && # don't emit these internal calls 172 | [[ $BASH_COMMAND != "$BATS_LAST_BASH_COMMAND" || $line != "$BATS_LAST_BASH_LINENO" ]] && 173 | # avoid printing a function twice (at call site and at definition site) 174 | [[ $BASH_COMMAND != "$BATS_LAST_BASH_COMMAND" || ${BASH_LINENO[2]} != "$BATS_LAST_BASH_LINENO" || ${BASH_SOURCE[3]} != "$BATS_LAST_BASH_SOURCE" ]]; then 175 | local file="${BASH_SOURCE[2]}" # index 2: skip over bats_emit_trace and bats_debug_trap 176 | if [[ $file == "${BATS_TEST_SOURCE}" ]]; then 177 | file="$BATS_TEST_FILENAME" 178 | fi 179 | local padding='$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$' 180 | if (( BATS_LAST_STACK_DEPTH != ${#BASH_LINENO[@]} )); then 181 | printf '%s [%s:%d]\n' "${padding::${#BASH_LINENO[@]}-4}" "${file##*/}" "$line" >&4 182 | fi 183 | printf '%s %s\n' "${padding::${#BASH_LINENO[@]}-4}" "$BASH_COMMAND" >&4 184 | BATS_LAST_BASH_COMMAND="$BASH_COMMAND" 185 | BATS_LAST_BASH_LINENO="$line" 186 | BATS_LAST_BASH_SOURCE="${BASH_SOURCE[2]}" 187 | BATS_LAST_STACK_DEPTH="${#BASH_LINENO[@]}" 188 | fi 189 | fi 190 | } 191 | 192 | # bats_debug_trap tracks the last line of code executed within a test. This is 193 | # necessary because $BASH_LINENO is often incorrect inside of ERR and EXIT 194 | # trap handlers. 195 | # 196 | # Below are tables describing different command failure scenarios and the 197 | # reliability of $BASH_LINENO within different the executed DEBUG, ERR, and EXIT 198 | # trap handlers. Naturally, the behaviors change between versions of Bash. 199 | # 200 | # Table rows should be read left to right. For example, on bash version 201 | # 4.0.44(2)-release, if a test executes `false` (or any other failing external 202 | # command), bash will do the following in order: 203 | # 1. Call the DEBUG trap handler (bats_debug_trap) with $BASH_LINENO referring 204 | # to the source line containing the `false` command, then 205 | # 2. Call the DEBUG trap handler again, but with an incorrect $BASH_LINENO, then 206 | # 3. Call the ERR trap handler, but with a (possibly-different) incorrect 207 | # $BASH_LINENO, then 208 | # 4. Call the DEBUG trap handler again, but with $BASH_LINENO set to 1, then 209 | # 5. Call the EXIT trap handler, with $BASH_LINENO set to 1. 210 | # 211 | # bash version 4.4.20(1)-release 212 | # command | first DEBUG | second DEBUG | ERR | third DEBUG | EXIT 213 | # -------------+-------------+--------------+---------+-------------+-------- 214 | # false | OK | OK | OK | BAD[1] | BAD[1] 215 | # [[ 1 = 2 ]] | OK | BAD[2] | BAD[2] | BAD[1] | BAD[1] 216 | # (( 1 = 2 )) | OK | BAD[2] | BAD[2] | BAD[1] | BAD[1] 217 | # ! true | OK | --- | BAD[4] | --- | BAD[1] 218 | # $var_dne | OK | --- | --- | BAD[1] | BAD[1] 219 | # source /dne | OK | --- | --- | BAD[1] | BAD[1] 220 | # 221 | # bash version 4.0.44(2)-release 222 | # command | first DEBUG | second DEBUG | ERR | third DEBUG | EXIT 223 | # -------------+-------------+--------------+---------+-------------+-------- 224 | # false | OK | BAD[3] | BAD[3] | BAD[1] | BAD[1] 225 | # [[ 1 = 2 ]] | OK | --- | BAD[3] | --- | BAD[1] 226 | # (( 1 = 2 )) | OK | --- | BAD[3] | --- | BAD[1] 227 | # ! true | OK | --- | BAD[3] | --- | BAD[1] 228 | # $var_dne | OK | --- | --- | BAD[1] | BAD[1] 229 | # source /dne | OK | --- | --- | BAD[1] | BAD[1] 230 | # 231 | # [1] The reported line number is always 1. 232 | # [2] The reported source location is that of the beginning of the function 233 | # calling the command. 234 | # [3] The reported line is that of the last command executed in the DEBUG trap 235 | # handler. 236 | # [4] The reported source location is that of the call to the function calling 237 | # the command. 238 | bats_debug_trap() { 239 | # on windows we sometimes get a mix of paths (when install via nmp install -g) 240 | # which have C:/... or /c/... comparing them is going to be problematic. 241 | # We need to normalize them to a common format! 242 | local NORMALIZED_INPUT 243 | bats_normalize_windows_dir_path NORMALIZED_INPUT "${1%/*}" 244 | local file_excluded='' path 245 | for path in "${BATS_DEBUG_EXCLUDE_PATHS[@]}"; do 246 | if [[ "$NORMALIZED_INPUT" == "$path"* ]]; then 247 | file_excluded=1 248 | break 249 | fi 250 | done 251 | 252 | # don't update the trace within library functions or we get backtraces from inside traps 253 | # also don't record new stack traces while handling interruptions, to avoid overriding the interrupted command 254 | if [[ -z "$file_excluded" && "${BATS_INTERRUPTED-NOTSET}" == NOTSET ]]; then 255 | BATS_DEBUG_LASTLAST_STACK_TRACE=( 256 | ${BATS_DEBUG_LAST_STACK_TRACE[@]+"${BATS_DEBUG_LAST_STACK_TRACE[@]}"} 257 | ) 258 | 259 | BATS_DEBUG_LAST_LINENO=(${BASH_LINENO[@]+"${BASH_LINENO[@]}"}) 260 | BATS_DEBUG_LAST_SOURCE=(${BASH_SOURCE[@]+"${BASH_SOURCE[@]}"}) 261 | bats_capture_stack_trace 262 | bats_emit_trace 263 | fi 264 | } 265 | 266 | # For some versions of Bash, the `ERR` trap may not always fire for every 267 | # command failure, but the `EXIT` trap will. Also, some command failures may not 268 | # set `$?` properly. See #72 and #81 for details. 269 | # 270 | # For this reason, we call `bats_check_status_from_trap` at the very beginning 271 | # of `bats_teardown_trap` and check the value of `$BATS_TEST_COMPLETED` before 272 | # taking other actions. We also adjust the exit status value if needed. 273 | # 274 | # See `bats_exit_trap` for an additional EXIT error handling case when `$?` 275 | # isn't set properly during `teardown()` errors. 276 | bats_check_status_from_trap() { 277 | local status="$?" 278 | if [[ -z "${BATS_TEST_COMPLETED:-}" ]]; then 279 | BATS_ERROR_STATUS="${BATS_ERROR_STATUS:-$status}" 280 | if [[ "$BATS_ERROR_STATUS" -eq 0 ]]; then 281 | BATS_ERROR_STATUS=1 282 | fi 283 | trap - DEBUG 284 | fi 285 | } 286 | 287 | bats_add_debug_exclude_path() { # 288 | if [[ -z "$1" ]]; then # don't exclude everything 289 | printf "bats_add_debug_exclude_path: Exclude path must not be empty!\n" >&2 290 | return 1 291 | fi 292 | if [[ "$OSTYPE" == cygwin || "$OSTYPE" == msys ]]; then 293 | local normalized_dir 294 | bats_normalize_windows_dir_path normalized_dir "$1" 295 | BATS_DEBUG_EXCLUDE_PATHS+=("$normalized_dir") 296 | else 297 | BATS_DEBUG_EXCLUDE_PATHS+=("$1") 298 | fi 299 | } 300 | 301 | bats_setup_tracing() { 302 | # Variables for capturing accurate stack traces. See bats_debug_trap for 303 | # details. 304 | # 305 | # BATS_DEBUG_LAST_LINENO, BATS_DEBUG_LAST_SOURCE, and 306 | # BATS_DEBUG_LAST_STACK_TRACE hold data from the most recent call to 307 | # bats_debug_trap. 308 | # 309 | # BATS_DEBUG_LASTLAST_STACK_TRACE holds data from two bats_debug_trap calls 310 | # ago. 311 | # 312 | # BATS_DEBUG_LAST_STACK_TRACE_IS_VALID indicates that 313 | # BATS_DEBUG_LAST_STACK_TRACE contains the stack trace of the test's error. If 314 | # unset, BATS_DEBUG_LAST_STACK_TRACE is unreliable and 315 | # BATS_DEBUG_LASTLAST_STACK_TRACE should be used instead. 316 | BATS_DEBUG_LASTLAST_STACK_TRACE=() 317 | BATS_DEBUG_LAST_LINENO=() 318 | BATS_DEBUG_LAST_SOURCE=() 319 | BATS_DEBUG_LAST_STACK_TRACE=() 320 | BATS_DEBUG_LAST_STACK_TRACE_IS_VALID= 321 | BATS_ERROR_SUFFIX= 322 | BATS_DEBUG_EXCLUDE_PATHS=() 323 | # exclude some paths by default 324 | bats_add_debug_exclude_path "$BATS_ROOT/lib/" 325 | bats_add_debug_exclude_path "$BATS_ROOT/libexec/" 326 | 327 | 328 | exec 4<&1 # used for tracing 329 | if [[ "${BATS_TRACE_LEVEL:-0}" -gt 0 ]]; then 330 | # avoid undefined variable errors 331 | BATS_LAST_BASH_COMMAND= 332 | BATS_LAST_BASH_LINENO= 333 | BATS_LAST_BASH_SOURCE= 334 | BATS_LAST_STACK_DEPTH= 335 | # try to exclude helper libraries if found, this is only relevant for tracing 336 | while read -r path; do 337 | bats_add_debug_exclude_path "$path" 338 | done < <(find "$PWD" -type d -name bats-assert -o -name bats-support) 339 | fi 340 | 341 | local exclude_paths path 342 | # exclude user defined libraries 343 | IFS=':' read -r exclude_paths <<< "${BATS_DEBUG_EXCLUDE_PATHS:-}" 344 | for path in "${exclude_paths[@]}"; do 345 | if [[ -n "$path" ]]; then 346 | bats_add_debug_exclude_path "$path" 347 | fi 348 | done 349 | 350 | # turn on traps after setting excludes to avoid tracing the exclude setup 351 | trap 'bats_debug_trap "$BASH_SOURCE"' DEBUG 352 | trap 'bats_error_trap' ERR 353 | } 354 | 355 | bats_error_trap() { 356 | bats_check_status_from_trap 357 | 358 | # If necessary, undo the most recent stack trace captured by bats_debug_trap. 359 | # See bats_debug_trap for details. 360 | if [[ "${BASH_LINENO[*]}" = "${BATS_DEBUG_LAST_LINENO[*]:-}" 361 | && "${BASH_SOURCE[*]}" = "${BATS_DEBUG_LAST_SOURCE[*]:-}" 362 | && -z "$BATS_DEBUG_LAST_STACK_TRACE_IS_VALID" ]]; then 363 | BATS_DEBUG_LAST_STACK_TRACE=( 364 | ${BATS_DEBUG_LASTLAST_STACK_TRACE[@]+"${BATS_DEBUG_LASTLAST_STACK_TRACE[@]}"} 365 | ) 366 | fi 367 | BATS_DEBUG_LAST_STACK_TRACE_IS_VALID=1 368 | } 369 | 370 | bats_interrupt_trap() { 371 | # mark the interruption, to handle during exit 372 | BATS_INTERRUPTED=true 373 | BATS_ERROR_STATUS=130 374 | # debug trap fires before interrupt trap but gets wrong linenumber (line 1) 375 | # -> use last stack trace 376 | exit $BATS_ERROR_STATUS 377 | } 378 | 379 | # this is used inside run() 380 | bats_interrupt_trap_in_run() { 381 | # mark the interruption, to handle during exit 382 | BATS_INTERRUPTED=true 383 | BATS_ERROR_STATUS=130 384 | BATS_DEBUG_LAST_STACK_TRACE_IS_VALID=true 385 | exit $BATS_ERROR_STATUS 386 | } 387 | -------------------------------------------------------------------------------- /test/bats/lib/bats-core/validator.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | bats_test_count_validator() { 4 | trap '' INT # continue forwarding 5 | header_pattern='[0-9]+\.\.[0-9]+' 6 | IFS= read -r header 7 | # repeat the header 8 | printf "%s\n" "$header" 9 | 10 | # if we detect a TAP plan 11 | if [[ "$header" =~ $header_pattern ]]; then 12 | # extract the number of tests ... 13 | local expected_number_of_tests="${header:3}" 14 | # ... count the actual number of [not ] oks... 15 | local actual_number_of_tests=0 16 | while IFS= read -r line; do 17 | # forward line 18 | printf "%s\n" "$line" 19 | case "$line" in 20 | 'ok '*) 21 | (( ++actual_number_of_tests )) 22 | ;; 23 | 'not ok'*) 24 | (( ++actual_number_of_tests )) 25 | ;; 26 | esac 27 | done 28 | # ... and error if they are not the same 29 | if [[ "${actual_number_of_tests}" != "${expected_number_of_tests}" ]]; then 30 | printf '# bats warning: Executed %s instead of expected %s tests\n' "$actual_number_of_tests" "$expected_number_of_tests" 31 | return 1 32 | fi 33 | else 34 | # forward output unchanged 35 | cat 36 | fi 37 | } -------------------------------------------------------------------------------- /test/bats/lib/bats-core/warnings.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # shellcheck source=lib/bats-core/tracing.bash 4 | source "$BATS_ROOT/lib/bats-core/tracing.bash" 5 | 6 | BATS_WARNING_SHORT_DESCS=( 7 | # to start with 1 8 | 'PADDING' 9 | # see issue #578 for context 10 | "\`run\`'s command \`%s\` exited with code 127, indicating 'Command not found'. Use run's return code checks, e.g. \`run -127\`, to fix this message." 11 | "%s requires at least BATS_VERSION=%s. Use \`bats_require_minimum_version %s\` to fix this message." 12 | ) 13 | 14 | # generate a warning report for the parent call's call site 15 | bats_generate_warning() { # [...] 16 | local warning_number="$1" padding="00" 17 | shift 18 | if [[ $warning_number =~ [0-9]+ ]] && ((warning_number < ${#BATS_WARNING_SHORT_DESCS[@]} )); then 19 | { 20 | printf "BW%s: ${BATS_WARNING_SHORT_DESCS[$warning_number]}\n" "${padding:${#warning_number}}${warning_number}" "$@" 21 | bats_capture_stack_trace 22 | BATS_STACK_TRACE_PREFIX=' ' bats_print_stack_trace "${BATS_DEBUG_LAST_STACK_TRACE[@]}" 23 | } >> "$BATS_WARNING_FILE" 2>&3 24 | else 25 | printf "Invalid Bats warning number '%s'. It must be an integer between 1 and %d." "$warning_number" "$((${#BATS_WARNING_SHORT_DESCS[@]} - 1))" >&2 26 | exit 1 27 | fi 28 | } 29 | 30 | # generate a warning if the BATS_GUARANTEED_MINIMUM_VERSION is not high enough 31 | bats_warn_minimum_guaranteed_version() { # 32 | if bats_version_lt "$BATS_GUARANTEED_MINIMUM_VERSION" "$2"; then 33 | bats_generate_warning 2 "$1" "$2" "$2" 34 | fi 35 | } 36 | -------------------------------------------------------------------------------- /test/bats/libexec/bats-core/bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | export BATS_VERSION='1.7.0' 5 | VALID_FORMATTERS="pretty, junit, tap, tap13" 6 | 7 | version() { 8 | printf 'Bats %s\n' "$BATS_VERSION" 9 | } 10 | 11 | abort() { 12 | local print_usage=1 13 | if [[ ${1:-} == --no-print-usage ]]; then 14 | print_usage= 15 | shift 16 | fi 17 | printf 'Error: %s\n' "$1" >&2 18 | if [[ -n $print_usage ]]; then 19 | usage >&2 20 | fi 21 | exit 1 22 | } 23 | 24 | usage() { 25 | local cmd="${0##*/}" 26 | local line 27 | 28 | cat < 30 | ${cmd} [-h | -v] 31 | 32 | HELP_TEXT_HEADER 33 | 34 | cat <<'HELP_TEXT_BODY' 35 | is the path to a Bats test file, or the path to a directory 36 | containing Bats test files (ending with ".bats") 37 | 38 | -c, --count Count test cases without running any tests 39 | --code-quote-style