├── .circleci └── config.yml ├── .drone.yml ├── .github └── workflows │ ├── build.yml │ └── codeql-analysis.yml ├── .gitignore ├── AUTHORS ├── CHECK ├── CHECKSUMS ├── COPYING ├── HISTORY ├── INSTALL ├── INSTALL.windows ├── Makefile.am ├── README ├── TODO ├── acinclude.m4 ├── autogen.sh ├── autover.sh ├── cmdline ├── bw.c ├── bw.h ├── check.c ├── device.c ├── dry.c ├── dup.c ├── elem.c ├── elem.h ├── fnmatch.c ├── fnmatch.h ├── handle.c ├── handle.h ├── import.c ├── import.h ├── io.c ├── io.h ├── list.c ├── metro.c ├── mingw.c ├── mingw.h ├── mkstream.c ├── mktest.c ├── murmur3.c ├── murmur3test.c ├── parity.c ├── parity.h ├── pool.c ├── portable.h ├── rehash.c ├── scan.c ├── scrub.c ├── search.c ├── search.h ├── selftest.c ├── snapraid.c ├── snapraid.h ├── speed.c ├── spooky2.c ├── spooky2test.c ├── state.c ├── state.h ├── status.c ├── stream.c ├── stream.h ├── support.c ├── support.h ├── sync.c ├── touch.c ├── unix.c ├── unix.h ├── util.c └── util.h ├── configure.ac ├── configure.windows-x64 ├── configure.windows-x86 ├── makecov.sh ├── makedist.sh ├── makesan.sh ├── makescan.sh ├── makesum.sh ├── maketest.sh ├── raid ├── COPYING ├── check.c ├── combo.h ├── cpu.h ├── gf.h ├── helper.c ├── helper.h ├── int.c ├── internal.h ├── intz.c ├── memory.c ├── memory.h ├── mktables.c ├── module.c ├── raid.c ├── raid.h ├── tables.c ├── tag.c ├── test.c ├── test.h ├── test │ ├── Makefile │ ├── fulltest.c │ ├── invtest.c │ ├── selftest.c │ └── speedtest.c ├── x86.c └── x86z.c ├── snapraid-rpm.spec ├── snapraid.1 ├── snapraid.conf.example ├── snapraid.conf.example.windows ├── snapraid.d ├── snapraid.txt ├── test ├── test-par1.conf ├── test-par2.conf ├── test-par3.conf ├── test-par4.conf ├── test-par5.conf ├── test-par6-hole.conf ├── test-par6-noaccess.conf ├── test-par6-rename.conf └── test-par6.conf ├── tommyds ├── LICENSE ├── tommy.c ├── tommyarray.c ├── tommyarray.h ├── tommyarrayblkof.c ├── tommyarrayblkof.h ├── tommychain.h ├── tommyhash.c ├── tommyhash.h ├── tommyhashdyn.c ├── tommyhashdyn.h ├── tommylist.c ├── tommylist.h ├── tommytree.c ├── tommytree.h └── tommytypes.h └── valgrind.supp /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Build configuration for https://circleci.com/ 2 | 3 | version: 2.1 4 | 5 | orbs: 6 | win: circleci/windows@2.2.0 7 | 8 | workflows: 9 | main: 10 | jobs: 11 | - build_ubuntu 12 | - build_docker 13 | 14 | jobs: 15 | build_docker: 16 | docker: 17 | - image: circleci/buildpack-deps:stable 18 | steps: 19 | - checkout 20 | - run: autoreconf -i && ./configure --enable-warning-as-error && make all dist 21 | build_ubuntu: 22 | machine: 23 | image: ubuntu-2004:202101-01 24 | steps: 25 | - checkout 26 | - run: autoreconf -i && ./configure --enable-warning-as-error && make all dist 27 | build_win: 28 | executor: 29 | name: win/default 30 | steps: 31 | - run: 32 | name: Installing MSYS2 33 | shell: powershell.exe 34 | command: 'choco install msys2' 35 | - run: 36 | name: Installing tools 37 | shell: powershell.exe 38 | command: 'C:\tools\msys64\usr\bin\bash.exe -l -c "pacman --needed --noconfirm -S autoreconf automake mingw-w64-x86_64-toolchain"' 39 | - checkout 40 | - run: 41 | name: Autoreconf 42 | shell: C:\\tools\\msys64\\usr\\bin\\bash.exe -l 43 | command: 'cd /c/Users/circleci/project && autoreconf -i' 44 | - run: 45 | name: Configure 46 | shell: C:\\tools\\msys64\\usr\\bin\\bash.exe -l 47 | command: 'cd /c/Users/circleci/project && ./configure --enable-warning-as-error' 48 | - run: 49 | name: Make 50 | shell: C:\\tools\\msys64\\usr\\bin\\bash.exe -l 51 | command: 'cd /c/Users/circleci/project && make' 52 | 53 | -------------------------------------------------------------------------------- /.drone.yml: -------------------------------------------------------------------------------- 1 | # Build configuration for https://www.tea-ci.org 2 | 3 | build: 4 | image: teaci/msys$$arch 5 | shell: mingw$$arch 6 | pull: true 7 | commands: 8 | - autoreconf -i 9 | - ./configure --enable-warning-as-error 10 | - make 11 | - make check 12 | 13 | matrix: 14 | arch: 15 | - 64 16 | - 32 17 | 18 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build-mac: 11 | runs-on: macos-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - run: brew install automake 15 | - run: autoreconf -i 16 | - run: ./configure --enable-warning-as-error 17 | - run: make all 18 | - run: make distcheck 19 | build-linux: 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@v2 23 | - run: autoreconf -i 24 | - run: ./configure --enable-warning-as-error 25 | - run: make all 26 | - run: make distcheck 27 | build-win: 28 | runs-on: windows-latest 29 | defaults: 30 | run: 31 | shell: msys2 {0} 32 | steps: 33 | - uses: actions/checkout@v2 34 | - uses: msys2/setup-msys2@v2 35 | with: 36 | msystem: MINGW64 37 | update: true 38 | install: mingw-w64-x86_64-gcc autoconf automake make 39 | - run: autoreconf -i 40 | - run: ./configure --enable-warning-as-error 41 | - run: make all 42 | - run: make distcheck 43 | 44 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '20 6 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'cpp' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@v4 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v3 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v3 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v3 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # archives 2 | *.zip 3 | *.tar.gz 4 | *.tgz 5 | 6 | # backups 7 | *~ 8 | 9 | # logs 10 | *.log 11 | stream*.bin 12 | 13 | # upload 14 | makepush.sh 15 | makemount.sh 16 | makeumount.sh 17 | 18 | # autotools 19 | Makefile 20 | Makefile.in 21 | compile 22 | aclocal.m4 23 | autom4te.cache/ 24 | config.guess 25 | config.h 26 | config.h.in 27 | config.log 28 | config.status 29 | config.sub 30 | configure 31 | install-sh 32 | missing 33 | stamp-h1 34 | .dirstamp 35 | 36 | # objects 37 | snapraid 38 | mktest 39 | mkstream 40 | *.exe 41 | *.o 42 | *.gcda 43 | *.gcno 44 | *.gcov 45 | 46 | # projects 47 | *.dst 48 | *.epr 49 | *.geany 50 | 51 | # AFL 52 | makeafl.sh 53 | afl/ 54 | afl_corpus/ 55 | 56 | # specific 57 | cov/ 58 | contrib/ 59 | paper/ 60 | kernel/ 61 | obj/ 62 | bench/ 63 | support/ 64 | archive/ 65 | www/ 66 | wiki/ 67 | beta/ 68 | ftp/ 69 | 70 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | SnapRAID AUTHORS 2 | ================ 3 | 4 | The author of SnapRAID is Andrea Mazzoleni. 5 | 6 | You can contact me sending an email at: 7 | 8 | amadvance@gmail.com 9 | 10 | Please don't send support requests at this address, but use the 11 | SnapRAID Forum. 12 | 13 | 14 | ACKNOWLEDGMENTS 15 | =============== 16 | 17 | Special thanks to Leifi, the king of the Forum! 18 | 19 | Thanks for the testing, suggestions and bug reports to klbl, 20 | jwill42, tholiin, uhclem, reardonia, Jens, rubylaser and the 21 | whole SnapRAID Forum. 22 | 23 | Thanks to Maxim Tikhonov for making the Ubuntu packages. 24 | 25 | Also thanks for the support to Ben_in_COSprings, Darin, micksh, RamanRB 26 | and the whole AVS Forum. 27 | 28 | -------------------------------------------------------------------------------- /CHECK: -------------------------------------------------------------------------------- 1 | SnapRAID CHECK 2 | ============== 3 | 4 | The regression test of SnapRAID is run using the command: 5 | 6 | make check 7 | 8 | You can also run the regression test in valgrind with: 9 | 10 | ./configure --enable-valgrind 11 | make check 12 | 13 | To run a coverage test you should use: 14 | 15 | ./configure --enable-coverage 16 | make lcov_reset 17 | make check 18 | make lcov_capture 19 | make lcov_html 20 | 21 | and open the file ./cov/index.html in your browser to see the results. 22 | 23 | Please note that in the coverage analysis we exclude the handling of all 24 | the error conditions that result in an immediate termination of the program. 25 | You can recognize this excluded code because it's enclosed between 26 | the LCOV_EXCL_START and LCOV_EXCL_STOP keywords. 27 | 28 | To test with the clang static analyzer use: 29 | 30 | scan-build ./configure 31 | scan-build make 32 | 33 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | SnapRAID INSTALL 2 | ================ 3 | 4 | To build and install SnapRAID you need to download the source 5 | code from http://www.snapraid.it and unpack it with: 6 | 7 | tar xf snapraid-*.tar.gz 8 | cd snapraid-* 9 | 10 | To configure and build, run: 11 | 12 | ./configure 13 | make 14 | 15 | To run the correctness tests for the application: 16 | 17 | make check 18 | 19 | If this terminates with "Success", you can install the application and 20 | the documentation by running: 21 | 22 | sudo make install 23 | 24 | (The snapraid application itself does not require root privileges.) 25 | 26 | To start using SnapRAID, change the example configuration 27 | file snapraid.conf.example to fit your needs and copy it to /etc/snapraid.conf 28 | 29 | To get more help, see the "Getting Started" section in the snapraid manpage 30 | by typing: 31 | 32 | man snapraid 33 | 34 | -------------------------------------------------------------------------------- /INSTALL.windows: -------------------------------------------------------------------------------- 1 | SnapRAID INSTALL for Windows 2 | ============================ 3 | 4 | To start using SnapRAID you have to change the example configuration 5 | file snapraid.conf.example to fit your needs and copy it with the 6 | name snapraid.conf in the directory where you run snapraid.exe. 7 | 8 | To get more help, see the "Getting Started" section in snapraid.txt. 9 | 10 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | SnapRAID 2 | ======== 3 | 4 | SnapRAID is a backup program designed for disk arrays, storing 5 | parity information for data recovery in the event of up to six 6 | disk failures. 7 | 8 | Primarily intended for home media centers with large, 9 | infrequently changing files, SnapRAID offers several features: 10 | 11 | * You can utilize disks already filled with files without the 12 | need to reformat them, accessing them as usual. 13 | * All your data is hashed to ensure data integrity and prevent 14 | silent corruption. 15 | * When the number of failed disks exceeds the parity count, 16 | data loss is confined to the affected disks; data on 17 | other disks remains accessible. 18 | * If you accidentally delete files on a disk, recovery is 19 | possible. 20 | * Disks can have different sizes. 21 | * You can add disks at any time. 22 | * SnapRAID doesn't lock in your data; you can stop using it 23 | anytime without reformatting or moving data. 24 | * To access a file, only a single disk needs to spin, saving 25 | power and reducing noise. 26 | 27 | For more information, please visit the official SnapRAID site: 28 | 29 | :http://www.snapraid.it/ 30 | -------------------------------------------------------------------------------- /acinclude.m4: -------------------------------------------------------------------------------- 1 | dnl @synopsis AC_CHECK_CC_OPT(flag, ifyes, ifno) 2 | dnl 3 | dnl Shows a message as like "checking whether gcc accepts flag ... no" 4 | dnl and executes ifyes or ifno. 5 | 6 | AC_DEFUN([AC_CHECK_CC_OPT], 7 | [ 8 | AC_MSG_CHECKING([whether ${CC-cc} accepts $1]) 9 | echo 'void f(){}' > conftest.c 10 | if test -z "`${CC-cc} -c $1 conftest.c 2>&1`"; then 11 | AC_MSG_RESULT([yes]) 12 | $2 13 | else 14 | AC_MSG_RESULT([no]) 15 | $3 16 | fi 17 | rm -f conftest* 18 | ]) 19 | 20 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | echo "Generating build information using autoreconf" 4 | 5 | # All is done by autoreconf 6 | autoreconf -f -i 7 | 8 | # Run configure for this platform 9 | echo "Now you are ready to run ./configure" 10 | 11 | -------------------------------------------------------------------------------- /autover.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | 4 | if [ -d .git ]; then 5 | # Get version from git tags, removing the 'v' prefix 6 | VERSION=$(git describe --match 'v*' 2>/dev/null | sed 's/^v//') 7 | fi 8 | 9 | if [ -f .version ]; then 10 | # Get version from the .version file 11 | VERSION=$(cat .version) 12 | fi 13 | 14 | if [ -z "$VERSION" ] && [ -d .git ]; then 15 | # Fall back to short commit hash 16 | VERSION=0-$(git rev-parse --short HEAD 2>/dev/null) 17 | fi 18 | 19 | if [ -z $VERSION ]; then 20 | # No version, but still use a number 21 | VERSION="0" 22 | fi 23 | 24 | printf '%s' "$VERSION" 25 | -------------------------------------------------------------------------------- /cmdline/bw.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #include "portable.h" 19 | 20 | #include "bw.h" 21 | 22 | void bw_init(struct snapraid_bw* bw, uint64_t limit) 23 | { 24 | bw->limit = limit; 25 | bw->total = 0; 26 | bw->start = tick_ms(); 27 | } 28 | 29 | void bw_limit(struct snapraid_bw* bw, uint64_t bytes) 30 | { 31 | if (!bw || bw->limit == 0) 32 | return; 33 | 34 | uint64_t elapsed = tick_ms() - bw->start; 35 | uint64_t done; 36 | uint64_t eta; 37 | 38 | done = __atomic_fetch_add(&bw->total, bytes, __ATOMIC_SEQ_CST); 39 | 40 | eta = done * 1000 / bw->limit; 41 | 42 | if (eta > elapsed) { 43 | eta -= elapsed; 44 | usleep(eta * 1000); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /cmdline/bw.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #ifndef __BW_H 19 | #define __BW_H 20 | 21 | #include "state.h" 22 | #include "support.h" 23 | 24 | /** 25 | * Bandwidth limiting 26 | */ 27 | struct snapraid_bw { 28 | uint64_t limit; /**< Bandwidth limit in bytes per second */ 29 | uint64_t total; /**< Remaining bytes allowed in current second */ 30 | uint64_t start; /**< Time when to reset the bandwidth counter */ 31 | }; 32 | 33 | /** 34 | * Initialize the bandwidth limit 35 | */ 36 | void bw_init(struct snapraid_bw* bw, uint64_t limit); 37 | 38 | /** 39 | * Limit IO bandwidth to stay within the configured limit. 40 | * If no limit is set, returns immediately. 41 | * Otherwise sleeps as needed to maintain the rate limit. 42 | */ 43 | void bw_limit(struct snapraid_bw* bw, uint64_t bytes); 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /cmdline/dup.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2011 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #include "portable.h" 19 | 20 | #include "support.h" 21 | #include "util.h" 22 | #include "elem.h" 23 | #include "state.h" 24 | #include "parity.h" 25 | #include "handle.h" 26 | 27 | /****************************************************************************/ 28 | /* dup */ 29 | 30 | struct snapraid_hash { 31 | struct snapraid_disk* disk; /**< Disk. */ 32 | struct snapraid_file* file; /**< File. */ 33 | unsigned char hash[HASH_MAX]; /**< Hash of the whole file. */ 34 | 35 | /* nodes for data structures */ 36 | tommy_hashdyn_node node; 37 | }; 38 | 39 | struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) 40 | { 41 | struct snapraid_hash* hash; 42 | block_off_t i; 43 | unsigned char* buf; 44 | size_t hash_size = BLOCK_HASH_SIZE; 45 | 46 | hash = malloc_nofail(sizeof(struct snapraid_hash)); 47 | hash->disk = disk; 48 | hash->file = file; 49 | 50 | buf = malloc_nofail(file->blockmax * hash_size); 51 | 52 | /* set the back pointer */ 53 | for (i = 0; i < file->blockmax; ++i) { 54 | struct snapraid_block* block = fs_file2block_get(file, i); 55 | 56 | memcpy(buf + i * hash_size, block->hash, hash_size); 57 | 58 | if (!block_has_updated_hash(block)) { 59 | free(buf); 60 | free(hash); 61 | return 0; 62 | } 63 | } 64 | 65 | memhash(state->besthash, state->hashseed, hash->hash, buf, file->blockmax * hash_size); 66 | 67 | free(buf); 68 | 69 | return hash; 70 | } 71 | 72 | static inline tommy_uint32_t hash_hash(struct snapraid_hash* hash) 73 | { 74 | return tommy_hash_u32(0, hash->hash, HASH_MAX); 75 | } 76 | 77 | void hash_free(struct snapraid_hash* hash) 78 | { 79 | free(hash); 80 | } 81 | 82 | int hash_compare(const void* void_arg, const void* void_data) 83 | { 84 | const char* arg = void_arg; 85 | const struct snapraid_hash* hash = void_data; 86 | 87 | return memcmp(arg, hash->hash, HASH_MAX); 88 | } 89 | 90 | void state_dup(struct snapraid_state* state) 91 | { 92 | tommy_hashdyn hashset; 93 | tommy_node* i; 94 | unsigned count; 95 | data_off_t size; 96 | char esc_buffer[ESC_MAX]; 97 | char esc_buffer_alt[ESC_MAX]; 98 | 99 | tommy_hashdyn_init(&hashset); 100 | 101 | count = 0; 102 | size = 0; 103 | 104 | msg_progress("Comparing...\n"); 105 | 106 | /* for each disk */ 107 | for (i = state->disklist; i != 0; i = i->next) { 108 | tommy_node* j; 109 | struct snapraid_disk* disk = i->data; 110 | 111 | /* for each file */ 112 | for (j = disk->filelist; j != 0; j = j->next) { 113 | struct snapraid_file* file = j->data; 114 | struct snapraid_hash* hash; 115 | tommy_hash_t hash32; 116 | 117 | /* if empty, skip it */ 118 | if (file->size == 0) 119 | continue; 120 | 121 | hash = hash_alloc(state, disk, file); 122 | 123 | /* if no hash, skip it */ 124 | if (!hash) 125 | continue; 126 | 127 | hash32 = hash_hash(hash); 128 | 129 | struct snapraid_hash* found = tommy_hashdyn_search(&hashset, hash_compare, hash->hash, hash32); 130 | if (found) { 131 | ++count; 132 | size += found->file->size; 133 | log_tag("dup:%s:%s:%s:%s:%" PRIu64 ": dup\n", disk->name, esc_tag(file->sub, esc_buffer), found->disk->name, esc_tag(found->file->sub, esc_buffer_alt), found->file->size); 134 | printf("%12" PRIu64 " %s = %s\n", file->size, fmt_term(disk, file->sub, esc_buffer), fmt_term(found->disk, found->file->sub, esc_buffer_alt)); 135 | hash_free(hash); 136 | } else { 137 | tommy_hashdyn_insert(&hashset, &hash->node, hash, hash32); 138 | } 139 | } 140 | } 141 | 142 | tommy_hashdyn_foreach(&hashset, (tommy_foreach_func*)hash_free); 143 | tommy_hashdyn_done(&hashset); 144 | 145 | msg_status("\n"); 146 | msg_status("%8u duplicates, for %" PRIu64 " GB\n", count, size / GIGA); 147 | if (count) 148 | msg_status("There are duplicates!\n"); 149 | else 150 | msg_status("No duplicates\n"); 151 | 152 | log_tag("summary:dup_count:%u\n", count); 153 | log_tag("summary:dup_size:%" PRIu64 "\n", size); 154 | if (count == 0) { 155 | log_tag("summary:exit:unique\n"); 156 | } else { 157 | log_tag("summary:exit:dup\n"); 158 | } 159 | log_flush(); 160 | } 161 | 162 | -------------------------------------------------------------------------------- /cmdline/fnmatch.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 1991, 92, 93, 96, 97, 98, 99 Free Software Foundation, Inc. 2 | This file is part of the GNU C Library. 3 | 4 | The GNU C Library is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU Library General Public License as 6 | published by the Free Software Foundation; either version 2 of the 7 | License, or (at your option) any later version. 8 | 9 | The GNU C Library is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | Library General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . */ 16 | 17 | #ifndef _FNMATCH_H 18 | #define _FNMATCH_H 1 19 | 20 | #ifdef __cplusplus 21 | extern "C" { 22 | #endif 23 | 24 | #if defined __cplusplus || (defined __STDC__ && __STDC__) || defined WINDOWS32 25 | # if !defined __GLIBC__ || !defined __P 26 | # undef __P 27 | # define __P(protos) protos 28 | # endif 29 | #else /* Not C++ or ANSI C. */ 30 | # undef __P 31 | # define __P(protos) () 32 | /* We can get away without defining `const' here only because in this file 33 | it is used only inside the prototype for `fnmatch', which is elided in 34 | non-ANSI C where `const' is problematical. */ 35 | #endif /* C++ or ANSI C. */ 36 | 37 | #ifndef const 38 | # if (defined __STDC__ && __STDC__) || defined __cplusplus 39 | # define __const const 40 | # else 41 | # define __const 42 | # endif 43 | #endif 44 | 45 | /* We #undef these before defining them because some losing systems 46 | (HP-UX A.08.07 for example) define these in . */ 47 | #undef FNM_PATHNAME 48 | #undef FNM_NOESCAPE 49 | #undef FNM_PERIOD 50 | #undef FNM_LEADING_DIR 51 | #undef FNM_CASEFOLD 52 | 53 | /* Bits set in the FLAGS argument to `fnmatch'. */ 54 | #define FNM_PATHNAME (1 << 0) /* No wildcard can ever match `/'. */ 55 | #define FNM_NOESCAPE (1 << 1) /* Backslashes don't quote special chars. */ 56 | #define FNM_PERIOD (1 << 2) /* Leading `.' is matched only explicitly. */ 57 | 58 | #if !defined _POSIX_C_SOURCE || _POSIX_C_SOURCE < 2 || defined _GNU_SOURCE 59 | # define FNM_FILE_NAME FNM_PATHNAME /* Preferred GNU name. */ 60 | # define FNM_LEADING_DIR (1 << 3) /* Ignore `/...' after a match. */ 61 | # define FNM_CASEFOLD (1 << 4) /* Compare without regard to case. */ 62 | #endif 63 | 64 | /* Value returned by `fnmatch' if STRING does not match PATTERN. */ 65 | #define FNM_NOMATCH 1 66 | 67 | /* This value is returned if the implementation does not support 68 | `fnmatch'. Since this is not the case here it will never be 69 | returned but the conformance test suites still require the symbol 70 | to be defined. */ 71 | #ifdef _XOPEN_SOURCE 72 | # define FNM_NOSYS (-1) 73 | #endif 74 | 75 | /* Match NAME against the filename pattern PATTERN, 76 | returning zero if it matches, FNM_NOMATCH if not. */ 77 | extern int fnmatch __P ((__const char *__pattern, __const char *__name, 78 | int __flags)); 79 | 80 | #ifdef __cplusplus 81 | } 82 | #endif 83 | 84 | #endif /* fnmatch.h */ 85 | -------------------------------------------------------------------------------- /cmdline/handle.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2011 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #ifndef __HANDLE_H 19 | #define __HANDLE_H 20 | 21 | #include "state.h" 22 | #include "support.h" 23 | #include "bw.h" 24 | 25 | /****************************************************************************/ 26 | /* handle */ 27 | 28 | struct snapraid_handle { 29 | char path[PATH_MAX]; /**< Path of the file. */ 30 | struct snapraid_disk* disk; /**< Disk of the file. */ 31 | struct snapraid_file* file; /**< File opened. When the file is closed, it's set to 0. */ 32 | int f; /**< Handle of the file. */ 33 | struct stat st; /**< Stat info of the opened file. */ 34 | struct advise_struct advise; /**< Advise information. */ 35 | data_off_t valid_size; /**< Size of the valid data. */ 36 | int created; /**< If the file was created, otherwise it was already existing. */ 37 | struct snapraid_bw* bw; /**< Context for bandwidth limiting. */ 38 | }; 39 | 40 | /** 41 | * Create a file. 42 | * The file is created if missing, and opened with write access. 43 | * If the file is created, the handle->created is set. 44 | * The initial size of the file is stored in the file->st struct. 45 | * If the file cannot be opened for write access, it's opened with read-only access. 46 | * The read-only access works only if the file has already the correct size and doesn't need to be modified. 47 | */ 48 | int handle_create(struct snapraid_handle* handle, struct snapraid_file* file, int mode); 49 | 50 | /** 51 | * Truncate a file if required. 52 | */ 53 | int handle_truncate(struct snapraid_handle* handle, struct snapraid_file* file); 54 | 55 | /** 56 | * Open a file. 57 | * The file is opened for reading. 58 | */ 59 | int handle_open(struct snapraid_handle* handle, struct snapraid_file* file, int mode, fptr* out, fptr* out_missing); 60 | 61 | /** 62 | * Close a file. 63 | */ 64 | int handle_close(struct snapraid_handle* handle); 65 | 66 | /** 67 | * Read a block from a file. 68 | * If the read block is shorter, it's padded with 0. 69 | */ 70 | int handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size, fptr* out, fptr* out_missing); 71 | 72 | /** 73 | * Write a block to a file. 74 | */ 75 | int handle_write(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size); 76 | 77 | /** 78 | * Change the modification time of the file to the saved value. 79 | */ 80 | int handle_utime(struct snapraid_handle* handle); 81 | 82 | /** 83 | * Map the unsorted list of disk to an ordered vector. 84 | * \param diskmax The size of the vector. 85 | * \return The allocated vector of pointers. 86 | */ 87 | struct snapraid_handle* handle_mapping(struct snapraid_state* state, unsigned* diskmax); 88 | 89 | #endif 90 | 91 | -------------------------------------------------------------------------------- /cmdline/import.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #ifndef __IMPORT_H 19 | #define __IMPORT_H 20 | 21 | #include "elem.h" 22 | #include "state.h" 23 | 24 | /****************************************************************************/ 25 | /* import */ 26 | 27 | /** 28 | * Import block. 29 | * Block used to import data external when recovering by hash. 30 | */ 31 | struct snapraid_import_block { 32 | struct snapraid_import_file* file; /**< Back pointer to the file owning this block. */ 33 | unsigned size; /**< Size of the block. */ 34 | data_off_t offset; /**< Position of the block in the file. */ 35 | unsigned char hash[HASH_MAX]; /**< Hash of the block. */ 36 | unsigned char prevhash[HASH_MAX]; /**< Previous hash of the block. Valid only if we are in rehash state. */ 37 | 38 | /* nodes for data structures */ 39 | tommy_hashdyn_node nodeset; 40 | tommy_hashdyn_node prevnodeset; 41 | }; 42 | 43 | /** 44 | * Import file. 45 | * File used to import data external when recovering by hash. 46 | */ 47 | struct snapraid_import_file { 48 | data_off_t size; /**< Size of the file. */ 49 | struct snapraid_import_block* blockimp; /**< All the blocks of the file. */ 50 | block_off_t blockmax; /**< Number of blocks. */ 51 | char* path; /**< Full path of the file. */ 52 | 53 | /* nodes for data structures */ 54 | tommy_node nodelist; 55 | }; 56 | 57 | /** 58 | * Deallocate an import file. 59 | */ 60 | void import_file_free(struct snapraid_import_file* file); 61 | 62 | /** 63 | * Fetch a block from the specified hash. 64 | * Return ==0 if the block is found, and copied into buffer. 65 | */ 66 | int state_import_fetch(struct snapraid_state* state, int prevhash, struct snapraid_block* missing_block, unsigned char* buffer); 67 | 68 | /** 69 | * Import files from the specified directory. 70 | */ 71 | void state_import(struct snapraid_state* state, const char* dir); 72 | 73 | #endif 74 | 75 | -------------------------------------------------------------------------------- /cmdline/list.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #include "portable.h" 19 | 20 | #include "util.h" 21 | #include "elem.h" 22 | #include "state.h" 23 | #include "parity.h" 24 | #include "handle.h" 25 | 26 | /****************************************************************************/ 27 | /* list */ 28 | 29 | void state_list(struct snapraid_state* state) 30 | { 31 | tommy_node* i; 32 | unsigned file_count; 33 | data_off_t file_size; 34 | unsigned link_count; 35 | char esc_buffer[ESC_MAX]; 36 | char esc_buffer_alt[ESC_MAX]; 37 | 38 | file_count = 0; 39 | file_size = 0; 40 | link_count = 0; 41 | 42 | msg_progress("Listing...\n"); 43 | 44 | /* for each disk */ 45 | for (i = state->disklist; i != 0; i = i->next) { 46 | tommy_node* j; 47 | struct snapraid_disk* disk = i->data; 48 | 49 | /* sort by name */ 50 | tommy_list_sort(&disk->filelist, file_path_compare); 51 | 52 | /* for each file */ 53 | for (j = disk->filelist; j != 0; j = j->next) { 54 | struct snapraid_file* file = j->data; 55 | #if HAVE_LOCALTIME_R 56 | struct tm tm_res; 57 | #endif 58 | struct tm* tm; 59 | time_t t; 60 | 61 | ++file_count; 62 | file_size += file->size; 63 | 64 | log_tag("file:%s:%s:%" PRIu64 ":%" PRIi64 ":%u:%" PRIi64 "\n", disk->name, esc_tag(file->sub, esc_buffer), file->size, file->mtime_sec, file->mtime_nsec, file->inode); 65 | 66 | t = file->mtime_sec; 67 | #if HAVE_LOCALTIME_R 68 | tm = localtime_r(&t, &tm_res); 69 | #else 70 | tm = localtime(&t); 71 | #endif 72 | 73 | printf("%12" PRIu64 " ", file->size); 74 | if (tm) { 75 | printf("%04u/%02u/%02u %02u:%02u", tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min); 76 | if (msg_level >= MSG_VERBOSE) 77 | printf(":%02u.%09u", tm->tm_sec, file->mtime_nsec); 78 | printf(" "); 79 | } 80 | printf("%s\n", fmt_term(disk, file->sub, esc_buffer)); 81 | } 82 | 83 | /* sort by name */ 84 | tommy_list_sort(&disk->linklist, link_alpha_compare); 85 | 86 | /* for each link */ 87 | for (j = disk->linklist; j != 0; j = j->next) { 88 | struct snapraid_link* slink = j->data; 89 | const char* type; 90 | 91 | switch (slink->flag & FILE_IS_LINK_MASK) { 92 | case FILE_IS_HARDLINK : type = "hardlink"; break; 93 | case FILE_IS_SYMLINK : type = "symlink"; break; 94 | case FILE_IS_SYMDIR : type = "symdir"; break; 95 | case FILE_IS_JUNCTION : type = "junction"; break; 96 | /* LCOV_EXCL_START */ 97 | default : type = "unknown"; break; 98 | /* LCOV_EXCL_STOP */ 99 | } 100 | 101 | ++link_count; 102 | 103 | log_tag("link_%s:%s:%s:%s\n", type, disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); 104 | 105 | printf("%12s ", type); 106 | printf(" "); 107 | if (msg_level >= MSG_VERBOSE) 108 | printf(" "); 109 | printf("%s -> %s\n", fmt_term(disk, slink->sub, esc_buffer), fmt_term(disk, slink->linkto, esc_buffer_alt)); 110 | } 111 | } 112 | 113 | msg_status("\n"); 114 | msg_status("%8u files, for %" PRIu64 " GB\n", file_count, file_size / GIGA); 115 | msg_status("%8u links\n", link_count); 116 | 117 | log_tag("summary:file_count:%u\n", file_count); 118 | log_tag("summary:file_size:%" PRIu64 "\n", file_size); 119 | log_tag("summary:link_count:%u\n", link_count); 120 | log_tag("summary:exit:ok\n"); 121 | log_flush(); 122 | } 123 | 124 | -------------------------------------------------------------------------------- /cmdline/metro.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | /* 19 | * Derivative work from metrohash128.cpp 20 | * 21 | * metrohash128.cpp 22 | * 23 | * Copyright 2015-2018 J. Andrew Rogers 24 | * 25 | * Licensed under the Apache License, Version 2.0 (the "License"); 26 | * you may not use this file except in compliance with the License. 27 | * You may obtain a copy of the License at 28 | * 29 | * http://www.apache.org/licenses/LICENSE-2.0 30 | * 31 | * Unless required by applicable law or agreed to in writing, software 32 | * distributed under the License is distributed on an "AS IS" BASIS, 33 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 34 | * See the License for the specific language governing permissions and 35 | * limitations under the License. 36 | */ 37 | 38 | static const uint64_t k0 = 0xC83A91E1; 39 | static const uint64_t k1 = 0x8648DBDB; 40 | static const uint64_t k2 = 0x7BDEC03B; 41 | static const uint64_t k3 = 0x2F5870A5; 42 | 43 | void MetroHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest) 44 | { 45 | const uint8_t* ptr = data; 46 | uint64_t v[4]; 47 | 48 | v[0] = (util_read64(seed) - k0) * k3; 49 | v[1] = (util_read64(seed + 8) + k1) * k2; 50 | 51 | if (size >= 32) { 52 | v[2] = (util_read64(seed) + k0) * k2; 53 | v[3] = (util_read64(seed + 8) - k1) * k3; 54 | 55 | do { 56 | v[0] += util_read64(ptr) * k0; ptr += 8; v[0] = util_rotr64(v[0], 29) + v[2]; 57 | v[1] += util_read64(ptr) * k1; ptr += 8; v[1] = util_rotr64(v[1], 29) + v[3]; 58 | v[2] += util_read64(ptr) * k2; ptr += 8; v[2] = util_rotr64(v[2], 29) + v[0]; 59 | v[3] += util_read64(ptr) * k3; ptr += 8; v[3] = util_rotr64(v[3], 29) + v[1]; 60 | size -= 32; 61 | } while (size >= 32); 62 | 63 | v[2] ^= util_rotr64(((v[0] + v[3]) * k0) + v[1], 21) * k1; 64 | v[3] ^= util_rotr64(((v[1] + v[2]) * k1) + v[0], 21) * k0; 65 | v[0] ^= util_rotr64(((v[0] + v[2]) * k0) + v[3], 21) * k1; 66 | v[1] ^= util_rotr64(((v[1] + v[3]) * k1) + v[2], 21) * k0; 67 | } 68 | 69 | if (size >= 16) { 70 | v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3; 71 | v[1] += util_read64(ptr) * k2; ptr += 8; v[1] = util_rotr64(v[1], 33) * k3; 72 | v[0] ^= util_rotr64((v[0] * k2) + v[1], 45) * k1; 73 | v[1] ^= util_rotr64((v[1] * k3) + v[0], 45) * k0; 74 | size -= 16; 75 | } 76 | 77 | if (size >= 8) { 78 | v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3; 79 | v[0] ^= util_rotr64((v[0] * k2) + v[1], 27) * k1; 80 | size -= 8; 81 | } 82 | 83 | if (size >= 4) { 84 | v[1] += util_read32(ptr) * k2; ptr += 4; v[1] = util_rotr64(v[1], 33) * k3; 85 | v[1] ^= util_rotr64((v[1] * k3) + v[0], 46) * k0; 86 | size -= 4; 87 | } 88 | 89 | if (size >= 2) { 90 | v[0] += util_read16(ptr) * k2; ptr += 2; v[0] = util_rotr64(v[0], 33) * k3; 91 | v[0] ^= util_rotr64((v[0] * k2) + v[1], 22) * k1; 92 | size -= 2; 93 | } 94 | 95 | if (size >= 1) { 96 | v[1] += util_read8(ptr) * k2; v[1] = util_rotr64(v[1], 33) * k3; 97 | v[1] ^= util_rotr64((v[1] * k3) + v[0], 58) * k0; 98 | } 99 | 100 | v[0] += util_rotr64((v[0] * k0) + v[1], 13); 101 | v[1] += util_rotr64((v[1] * k1) + v[0], 37); 102 | v[0] += util_rotr64((v[0] * k2) + v[1], 13); 103 | v[1] += util_rotr64((v[1] * k3) + v[0], 37); 104 | 105 | util_write64(digest, v[0]); 106 | util_write64(digest + 8, v[0]); 107 | } 108 | 109 | -------------------------------------------------------------------------------- /cmdline/mkstream.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #include "portable.h" 19 | 20 | #include "stream.h" 21 | #include "support.h" 22 | 23 | #define STREAM_MAX 8 24 | #define BUFFER_MAX 64 25 | #define STR_MAX 128 26 | 27 | void test(void) 28 | { 29 | struct stream* s; 30 | char file[32]; 31 | unsigned char buffer[BUFFER_MAX]; 32 | char str[STR_MAX]; 33 | unsigned i, j; 34 | uint32_t u32 = -1L; 35 | uint64_t u64 = -1LL; 36 | uint32_t put_crc_stored; 37 | uint32_t put_crc_computed; 38 | 39 | crc32c_init(); 40 | 41 | s = sopen_multi_write(STREAM_MAX); 42 | for (i = 0; i < STREAM_MAX; ++i) { 43 | snprintf(file, sizeof(file), "stream%u.bin", i); 44 | remove(file); 45 | if (sopen_multi_file(s, i, file) != 0) { 46 | /* LCOV_EXCL_START */ 47 | exit(EXIT_FAILURE); 48 | /* LCOV_EXCL_STOP */ 49 | } 50 | } 51 | 52 | for (j = 0; j < 256; ++j) { 53 | if (sputc(j, s) != 0) { 54 | /* LCOV_EXCL_START */ 55 | exit(EXIT_FAILURE); 56 | /* LCOV_EXCL_STOP */ 57 | } 58 | } 59 | 60 | for (j = 0; j < 32; ++j) { 61 | if (sputb32(u32 >> j, s) != 0) { 62 | /* LCOV_EXCL_START */ 63 | exit(EXIT_FAILURE); 64 | /* LCOV_EXCL_STOP */ 65 | } 66 | } 67 | 68 | for (j = 0; j < 64; ++j) { 69 | if (sputb64(u64 >> j, s) != 0) { 70 | /* LCOV_EXCL_START */ 71 | exit(EXIT_FAILURE); 72 | /* LCOV_EXCL_STOP */ 73 | } 74 | } 75 | 76 | for (j = 0; j < BUFFER_MAX; ++j) { 77 | memset(buffer, j, j); 78 | if (swrite(buffer, j, s) != 0) { 79 | /* LCOV_EXCL_START */ 80 | exit(EXIT_FAILURE); 81 | /* LCOV_EXCL_STOP */ 82 | } 83 | } 84 | 85 | for (j = 1; j < STR_MAX; ++j) { 86 | memset(str, ' ' + j, j - 1); 87 | str[j - 1] = 0; 88 | if (sputbs(str, s) != 0) { 89 | /* LCOV_EXCL_START */ 90 | exit(EXIT_FAILURE); 91 | /* LCOV_EXCL_STOP */ 92 | } 93 | } 94 | 95 | put_crc_stored = scrc(s); 96 | put_crc_computed = scrc_stream(s); 97 | 98 | if (put_crc_stored != put_crc_computed) { 99 | /* LCOV_EXCL_START */ 100 | exit(EXIT_FAILURE); 101 | /* LCOV_EXCL_STOP */ 102 | } 103 | 104 | if (sputble32(put_crc_stored, s) != 0) { 105 | /* LCOV_EXCL_START */ 106 | exit(EXIT_FAILURE); 107 | /* LCOV_EXCL_STOP */ 108 | } 109 | 110 | if (sclose(s) != 0) { 111 | /* LCOV_EXCL_START */ 112 | exit(EXIT_FAILURE); 113 | /* LCOV_EXCL_STOP */ 114 | } 115 | 116 | for (i = 0; i < STREAM_MAX; ++i) { 117 | uint32_t get_crc_stored; 118 | uint32_t get_crc_computed; 119 | snprintf(file, sizeof(file), "stream%u.bin", i); 120 | 121 | s = sopen_read(file); 122 | if (s == 0) { 123 | /* LCOV_EXCL_START */ 124 | exit(EXIT_FAILURE); 125 | /* LCOV_EXCL_STOP */ 126 | } 127 | 128 | for (j = 0; j < 256; ++j) { 129 | int c = sgetc(s); 130 | if (c == EOF || (unsigned char)c != j) { 131 | /* LCOV_EXCL_START */ 132 | exit(EXIT_FAILURE); 133 | /* LCOV_EXCL_STOP */ 134 | } 135 | } 136 | 137 | for (j = 0; j < 32; ++j) { 138 | uint32_t v32; 139 | if (sgetb32(s, &v32) != 0 || v32 != (u32 >> j)) { 140 | /* LCOV_EXCL_START */ 141 | exit(EXIT_FAILURE); 142 | /* LCOV_EXCL_STOP */ 143 | } 144 | } 145 | 146 | for (j = 0; j < 64; ++j) { 147 | uint64_t v64; 148 | if (sgetb64(s, &v64) != 0 || v64 != (u64 >> j)) { 149 | /* LCOV_EXCL_START */ 150 | exit(EXIT_FAILURE); 151 | /* LCOV_EXCL_STOP */ 152 | } 153 | } 154 | 155 | for (j = 1; j < BUFFER_MAX; ++j) { 156 | char copy[BUFFER_MAX]; 157 | memset(buffer, j, j); 158 | if (sread(s, copy, j) != 0 || memcmp(copy, buffer, j) != 0) { 159 | /* LCOV_EXCL_START */ 160 | exit(EXIT_FAILURE); 161 | /* LCOV_EXCL_STOP */ 162 | } 163 | } 164 | 165 | for (j = 1; j < STR_MAX; ++j) { 166 | char copy[STR_MAX]; 167 | memset(str, ' ' + j, j - 1); 168 | str[j - 1] = 0; 169 | if (sgetbs(s, copy, sizeof(copy)) != 0 || strcmp(copy, str) != 0) { 170 | /* LCOV_EXCL_START */ 171 | exit(EXIT_FAILURE); 172 | /* LCOV_EXCL_STOP */ 173 | } 174 | } 175 | 176 | /* get the computed CRC *before* reading the stored one */ 177 | get_crc_computed = scrc(s); 178 | 179 | if (sgetble32(s, &get_crc_stored) != 0) { 180 | /* LCOV_EXCL_START */ 181 | exit(EXIT_FAILURE); 182 | /* LCOV_EXCL_STOP */ 183 | } 184 | 185 | if (get_crc_stored != put_crc_stored) { 186 | /* LCOV_EXCL_START */ 187 | exit(EXIT_FAILURE); 188 | /* LCOV_EXCL_STOP */ 189 | } 190 | 191 | if (get_crc_stored != get_crc_computed) { 192 | /* LCOV_EXCL_START */ 193 | exit(EXIT_FAILURE); 194 | /* LCOV_EXCL_STOP */ 195 | } 196 | 197 | if (sclose(s) != 0) { 198 | /* LCOV_EXCL_START */ 199 | exit(EXIT_FAILURE); 200 | /* LCOV_EXCL_STOP */ 201 | } 202 | } 203 | 204 | for (i = 0; i < STREAM_MAX; ++i) { 205 | uint32_t get_crc_stored; 206 | uint32_t get_crc_computed; 207 | unsigned char buf[4]; 208 | snprintf(file, sizeof(file), "stream%u.bin", i); 209 | 210 | s = sopen_read(file); 211 | if (s == 0) { 212 | /* LCOV_EXCL_START */ 213 | exit(EXIT_FAILURE); 214 | /* LCOV_EXCL_STOP */ 215 | } 216 | 217 | if (sdeplete(s, buf) != 0) { 218 | /* LCOV_EXCL_START */ 219 | exit(EXIT_FAILURE); 220 | /* LCOV_EXCL_STOP */ 221 | } 222 | 223 | /* get the stored crc from the last four bytes */ 224 | get_crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24; 225 | 226 | if (get_crc_stored != put_crc_stored) { 227 | /* LCOV_EXCL_START */ 228 | exit(EXIT_FAILURE); 229 | /* LCOV_EXCL_STOP */ 230 | } 231 | 232 | /* get the computed CRC *after* reading the stored one */ 233 | get_crc_computed = scrc(s); 234 | 235 | /* adjust the stored crc to include itself */ 236 | get_crc_stored = crc32c(get_crc_stored, buf, 4); 237 | 238 | if (get_crc_stored != get_crc_computed) { 239 | /* LCOV_EXCL_START */ 240 | exit(EXIT_FAILURE); 241 | /* LCOV_EXCL_STOP */ 242 | } 243 | 244 | if (sclose(s) != 0) { 245 | /* LCOV_EXCL_START */ 246 | exit(EXIT_FAILURE); 247 | /* LCOV_EXCL_STOP */ 248 | } 249 | } 250 | } 251 | 252 | int main(void) 253 | { 254 | unsigned i; 255 | 256 | lock_init(); 257 | 258 | for (i = 1; i <= 16; ++i) { 259 | 260 | /* test with different stream buffer size */ 261 | STREAM_SIZE = i; 262 | 263 | printf("Test stream buffer size %u\n", i); 264 | 265 | test(); 266 | } 267 | 268 | return 0; 269 | } 270 | 271 | -------------------------------------------------------------------------------- /cmdline/murmur3.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2011 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | /* 19 | * Derivative work from MurmorHash3.cpp revision r136 20 | * 21 | * SMHasher & MurmurHash 22 | * http://code.google.com/p/smhasher/ 23 | * 24 | * Exact source used as reference: 25 | * http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp?spec=svn136&r=136 26 | */ 27 | 28 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 29 | // domain. The author hereby disclaims copyright to this source code. 30 | 31 | /* Finalization mix - force all bits of a hash block to avalanche */ 32 | static inline uint32_t fmix32(uint32_t h) 33 | { 34 | h ^= h >> 16; 35 | h *= 0x85ebca6b; 36 | h ^= h >> 13; 37 | h *= 0xc2b2ae35; 38 | h ^= h >> 16; 39 | return h; 40 | } 41 | 42 | /* 43 | * Warning! 44 | * Don't declare these variables static, otherwise the gcc optimizer 45 | * may generate very slow code for multiplication with these constants, 46 | * like: 47 | 48 | -> .cpp 49 | k1 *= c1; 50 | -> .asm 51 | 152: 8d 14 80 lea (%eax,%eax,4),%edx 52 | 155: 8d 14 90 lea (%eax,%edx,4),%edx 53 | 158: c1 e2 03 shl $0x3,%edx 54 | 15b: 29 c2 sub %eax,%edx 55 | 15d: 8d 14 d2 lea (%edx,%edx,8),%edx 56 | 160: 8d 14 90 lea (%eax,%edx,4),%edx 57 | 163: 8d 14 d0 lea (%eax,%edx,8),%edx 58 | 166: 8d 14 90 lea (%eax,%edx,4),%edx 59 | 169: 8d 14 50 lea (%eax,%edx,2),%edx 60 | 16c: 8d 14 90 lea (%eax,%edx,4),%edx 61 | 16f: 8d 14 92 lea (%edx,%edx,4),%edx 62 | 172: 8d 14 50 lea (%eax,%edx,2),%edx 63 | 175: 8d 04 d0 lea (%eax,%edx,8),%eax 64 | 178: 8d 14 c5 00 00 00 00 lea 0x0(,%eax,8),%edx 65 | 17f: 29 d0 sub %edx,%eax 66 | 67 | * resulting in speeds of 500 MB/s instead of 3000 MB/s. 68 | * 69 | * Verified with gcc 4.4.4 compiling with : 70 | * 71 | * g++ -g -c -O2 MurmurHash3.cpp -o MurmurHash3.o 72 | */ 73 | uint32_t c1 = 0x239b961b; 74 | uint32_t c2 = 0xab0e9789; 75 | uint32_t c3 = 0x38b34ae5; 76 | uint32_t c4 = 0xa1e38b93; 77 | 78 | void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* seed, void* digest) 79 | { 80 | size_t nblocks; 81 | const uint32_t* blocks; 82 | const uint32_t* end; 83 | size_t size_remainder; 84 | uint32_t h1, h2, h3, h4; 85 | 86 | h1 = util_read32(seed + 0); 87 | h2 = util_read32(seed + 4); 88 | h3 = util_read32(seed + 8); 89 | h4 = util_read32(seed + 12); 90 | 91 | nblocks = size / 16; 92 | blocks = data; 93 | end = blocks + nblocks * 4; 94 | 95 | /* body */ 96 | while (blocks < end) { 97 | uint32_t k1 = blocks[0]; 98 | uint32_t k2 = blocks[1]; 99 | uint32_t k3 = blocks[2]; 100 | uint32_t k4 = blocks[3]; 101 | 102 | #if WORDS_BIGENDIAN 103 | k1 = util_swap32(k1); 104 | k2 = util_swap32(k2); 105 | k3 = util_swap32(k3); 106 | k4 = util_swap32(k4); 107 | #endif 108 | 109 | k1 *= c1; k1 = util_rotl32(k1, 15); k1 *= c2; h1 ^= k1; 110 | 111 | h1 = util_rotl32(h1, 19); h1 += h2; h1 = h1 * 5 + 0x561ccd1b; 112 | 113 | k2 *= c2; k2 = util_rotl32(k2, 16); k2 *= c3; h2 ^= k2; 114 | 115 | h2 = util_rotl32(h2, 17); h2 += h3; h2 = h2 * 5 + 0x0bcaa747; 116 | 117 | k3 *= c3; k3 = util_rotl32(k3, 17); k3 *= c4; h3 ^= k3; 118 | 119 | h3 = util_rotl32(h3, 15); h3 += h4; h3 = h3 * 5 + 0x96cd1c35; 120 | 121 | k4 *= c4; k4 = util_rotl32(k4, 18); k4 *= c1; h4 ^= k4; 122 | 123 | h4 = util_rotl32(h4, 13); h4 += h1; h4 = h4 * 5 + 0x32ac3b17; 124 | 125 | blocks += 4; 126 | } 127 | 128 | /* tail */ 129 | size_remainder = size & 15; 130 | if (size_remainder != 0) { 131 | const uint8_t* tail = (const uint8_t*)blocks; 132 | 133 | uint32_t k1 = 0; 134 | uint32_t k2 = 0; 135 | uint32_t k3 = 0; 136 | uint32_t k4 = 0; 137 | 138 | switch (size_remainder) { 139 | case 15 : k4 ^= (uint32_t)tail[14] << 16; /* fallthrough */ 140 | case 14 : k4 ^= (uint32_t)tail[13] << 8; /* fallthrough */ 141 | case 13 : k4 ^= (uint32_t)tail[12] << 0; /* fallthrough */ 142 | k4 *= c4; k4 = util_rotl32(k4, 18); k4 *= c1; h4 ^= k4; 143 | /* fallthrough */ 144 | case 12 : k3 ^= (uint32_t)tail[11] << 24; /* fallthrough */ 145 | case 11 : k3 ^= (uint32_t)tail[10] << 16; /* fallthrough */ 146 | case 10 : k3 ^= (uint32_t)tail[ 9] << 8; /* fallthrough */ 147 | case 9 : k3 ^= (uint32_t)tail[ 8] << 0; /* fallthrough */ 148 | k3 *= c3; k3 = util_rotl32(k3, 17); k3 *= c4; h3 ^= k3; 149 | /* fallthrough */ 150 | case 8 : k2 ^= (uint32_t)tail[ 7] << 24; /* fallthrough */ 151 | case 7 : k2 ^= (uint32_t)tail[ 6] << 16; /* fallthrough */ 152 | case 6 : k2 ^= (uint32_t)tail[ 5] << 8; /* fallthrough */ 153 | case 5 : k2 ^= (uint32_t)tail[ 4] << 0; /* fallthrough */ 154 | k2 *= c2; k2 = util_rotl32(k2, 16); k2 *= c3; h2 ^= k2; 155 | /* fallthrough */ 156 | case 4 : k1 ^= (uint32_t)tail[ 3] << 24; /* fallthrough */ 157 | case 3 : k1 ^= (uint32_t)tail[ 2] << 16; /* fallthrough */ 158 | case 2 : k1 ^= (uint32_t)tail[ 1] << 8; /* fallthrough */ 159 | case 1 : k1 ^= (uint32_t)tail[ 0] << 0; /* fallthrough */ 160 | k1 *= c1; k1 = util_rotl32(k1, 15); k1 *= c2; h1 ^= k1; 161 | /* fallthrough */ 162 | } 163 | } 164 | 165 | /* finalization */ 166 | h1 ^= size; h2 ^= size; h3 ^= size; h4 ^= size; 167 | 168 | h1 += h2; h1 += h3; h1 += h4; 169 | h2 += h1; h3 += h1; h4 += h1; 170 | 171 | h1 = fmix32(h1); 172 | h2 = fmix32(h2); 173 | h3 = fmix32(h3); 174 | h4 = fmix32(h4); 175 | 176 | h1 += h2; h1 += h3; h1 += h4; 177 | h2 += h1; h3 += h1; h4 += h1; 178 | 179 | util_write32(digest + 0, h1); 180 | util_write32(digest + 4, h2); 181 | util_write32(digest + 8, h3); 182 | util_write32(digest + 12, h4); 183 | } 184 | 185 | -------------------------------------------------------------------------------- /cmdline/parity.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2011 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #ifndef __PARITY_H 19 | #define __PARITY_H 20 | 21 | #include "support.h" 22 | #include "bw.h" 23 | 24 | /****************************************************************************/ 25 | /* parity */ 26 | 27 | struct snapraid_split_handle { 28 | char path[PATH_MAX]; /**< Path of the file. */ 29 | int f; /**< Handle of the files. */ 30 | struct stat st; /**< Stat info of the opened file. */ 31 | struct advise_struct advise; /**< Advise information. */ 32 | 33 | /** 34 | * Size of the parity split. 35 | * Only the latest not zero size is allowed to grow. 36 | * Note that this value CANNOT be PARITY_SIZE_INVALID. 37 | */ 38 | data_off_t size; 39 | 40 | /** 41 | * Valid size of the parity split. 42 | * This is the size effectively written, and not the result of a chsize operation. 43 | * It's used to make read operations failing if read over that size. 44 | * 45 | * Parity is also truncated to that size when fixing it, in case of a Break (Ctrl+C) 46 | * of the program. 47 | */ 48 | data_off_t valid_size; 49 | 50 | /** 51 | * Artificial size limit for testing. 52 | * 0 means unlimited. 53 | */ 54 | data_off_t limit_size; 55 | }; 56 | 57 | struct snapraid_parity_handle { 58 | struct snapraid_split_handle split_map[SPLIT_MAX]; 59 | unsigned split_mac; /**< Number of parity splits. */ 60 | unsigned level; /**< Level of the parity. */ 61 | struct snapraid_bw* bw; /**< Context for bandwidth limiting. */ 62 | }; 63 | 64 | /** 65 | * Compute the size of the allocated parity data in number of blocks. 66 | * 67 | * This includes parity blocks not yet written and still invalid. 68 | */ 69 | block_off_t parity_allocated_size(struct snapraid_state* state); 70 | 71 | /** 72 | * Compute the size of the used parity data in number of blocks. 73 | * 74 | * This includes only parity blocks used for files, not counting 75 | * potential invalid parity at the end. 76 | * 77 | * If the array is fully synced there is no difference between 78 | * parity_allocate_size() and parity_used_size(). 79 | * But if the sync is interrupted, the parity_used_size() returns 80 | * the position of the latest BLK block, ignoring CHG, REL and DELETED ones, 81 | * because their parity may be still not even written in the parity file. 82 | */ 83 | block_off_t parity_used_size(struct snapraid_state* state); 84 | 85 | /** 86 | * Check if the parity needs to be updated with a "sync". 87 | * 88 | * This is the same logic used in "status" to detect an incomplete "sync", 89 | * that ignores invalid block, if they are not used by a file in any disk. 90 | * This means that DELETED blocks won't necessarily imply an invalid parity. 91 | */ 92 | int parity_is_invalid(struct snapraid_state* state); 93 | 94 | /** 95 | * Report all the files outside the specified parity size. 96 | */ 97 | void parity_overflow(struct snapraid_state* state, data_off_t size); 98 | 99 | /** 100 | * Create the parity file. 101 | * \param out_size Return the size of the parity file. 102 | */ 103 | int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode, uint32_t block_size, data_off_t limit_size); 104 | 105 | /** 106 | * Change the parity size. 107 | * \param out_size Return the size of the parity file. The out_size is set also on error to reflect a partial resize. 108 | */ 109 | int parity_chsize(struct snapraid_parity_handle* handle, struct snapraid_parity* parity, int* is_modified, data_off_t size, uint32_t block_size, int skip_fallocate, int skip_space_holder); 110 | 111 | /** 112 | * Get the size of the parity. 113 | * 114 | * This returns the cached/expected version of the split sizes, and not the real file size. 115 | */ 116 | void parity_size(struct snapraid_parity_handle* handle, data_off_t* out_size); 117 | 118 | /** 119 | * Open an already existing parity file. 120 | */ 121 | int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode, uint32_t block_size, data_off_t limit_size); 122 | 123 | /** 124 | * Flush the parity file in the disk. 125 | */ 126 | int parity_sync(struct snapraid_parity_handle* handle); 127 | 128 | /** 129 | * Truncate the parity file to the valid size. 130 | */ 131 | int parity_truncate(struct snapraid_parity_handle* handle); 132 | 133 | /** 134 | * Close the parity file. 135 | */ 136 | int parity_close(struct snapraid_parity_handle* handle); 137 | 138 | /** 139 | * Read a block from the parity file. 140 | */ 141 | int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size, fptr* out); 142 | 143 | /** 144 | * Write a block in the parity file. 145 | */ 146 | int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size); 147 | 148 | #endif 149 | 150 | -------------------------------------------------------------------------------- /cmdline/rehash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #include "portable.h" 19 | 20 | #include "util.h" 21 | #include "elem.h" 22 | #include "import.h" 23 | #include "state.h" 24 | #include "parity.h" 25 | #include "handle.h" 26 | #include "raid/raid.h" 27 | 28 | /****************************************************************************/ 29 | /* rehash */ 30 | 31 | void state_rehash(struct snapraid_state* state) 32 | { 33 | block_off_t blockmax; 34 | block_off_t i; 35 | 36 | blockmax = parity_allocated_size(state); 37 | 38 | /* check if a rehash is already in progress */ 39 | if (state->prevhash != HASH_UNDEFINED) { 40 | /* LCOV_EXCL_START */ 41 | log_fatal("You already have a rehash in progress.\n"); 42 | exit(EXIT_FAILURE); 43 | /* LCOV_EXCL_STOP */ 44 | } 45 | 46 | if (state->hash == state->besthash) { 47 | /* LCOV_EXCL_START */ 48 | log_fatal("You are already using the best hash for your platform.\n"); 49 | exit(EXIT_FAILURE); 50 | /* LCOV_EXCL_STOP */ 51 | } 52 | 53 | /* copy the present hash as previous one */ 54 | state->prevhash = state->hash; 55 | memcpy(state->prevhashseed, state->hashseed, HASH_MAX); 56 | 57 | /* set the new hash and seed */ 58 | state->hash = state->besthash; 59 | if (randomize(state->hashseed, HASH_MAX) != 0) { 60 | /* LCOV_EXCL_START */ 61 | log_fatal("Failed to get random values.\n"); 62 | exit(EXIT_FAILURE); 63 | /* LCOV_EXCL_STOP */ 64 | } 65 | 66 | /* mark all the block for rehashing */ 67 | for (i = 0; i < blockmax; ++i) { 68 | snapraid_info info; 69 | 70 | /* if it's unused */ 71 | info = info_get(&state->infoarr, i); 72 | if (info == 0) { 73 | /* skip it */ 74 | continue; 75 | } 76 | 77 | if (info_get_rehash(info)) { 78 | /* LCOV_EXCL_START */ 79 | log_fatal("Internal inconsistency for a rehash already in progress\n"); 80 | os_abort(); 81 | /* LCOV_EXCL_STOP */ 82 | } 83 | 84 | /* enable the rehash */ 85 | info = info_set_rehash(info); 86 | 87 | /* save it */ 88 | info_set(&state->infoarr, i, info); 89 | } 90 | 91 | /* save the new content file */ 92 | state->need_write = 1; 93 | 94 | msg_status("A rehash is now scheduled. It will take place progressively in the next\n"); 95 | msg_status("'sync' and 'scrub' commands. You can check the rehash progress using the\n"); 96 | msg_status("'status' command.\n"); 97 | } 98 | 99 | -------------------------------------------------------------------------------- /cmdline/search.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2014 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #ifndef __SEARCH_H 19 | #define __SEARCH_H 20 | 21 | #include "elem.h" 22 | #include "state.h" 23 | 24 | /****************************************************************************/ 25 | /* search */ 26 | 27 | /** 28 | * Search file. 29 | * File used to search for moved data. 30 | */ 31 | struct snapraid_search_file { 32 | char* path; /**< Full path of the file. */ 33 | char* name; /**< Pointer of the name inside the path. */ 34 | data_off_t size; 35 | int64_t mtime_sec; 36 | int mtime_nsec; 37 | 38 | /* nodes for data structures */ 39 | tommy_node node; 40 | }; 41 | 42 | /** 43 | * Deallocate a search file. 44 | */ 45 | void search_file_free(struct snapraid_search_file* file); 46 | 47 | /** 48 | * Fetch a file from the size, timestamp and name. 49 | * Return ==0 if the block is found, and copied into buffer. 50 | */ 51 | int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_file* missing_file, block_off_t missing_file_pos, struct snapraid_block* missing_block, unsigned char* buffer); 52 | 53 | /** 54 | * Import files from the specified directory. 55 | */ 56 | void state_search(struct snapraid_state* state, const char* dir); 57 | 58 | /** 59 | * Import files from all the data disks. 60 | */ 61 | void state_search_array(struct snapraid_state* state); 62 | 63 | #endif 64 | 65 | -------------------------------------------------------------------------------- /cmdline/snapraid.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2011 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #ifndef __SNAPRAID_H 19 | #define __SNAPRAID_H 20 | 21 | /****************************************************************************/ 22 | /* snapraid */ 23 | 24 | void speed(int period); 25 | void selftest(void); 26 | 27 | #endif 28 | 29 | -------------------------------------------------------------------------------- /cmdline/spooky2.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | /* 19 | * Derivative work from SpookyV2.cpp/h 20 | * 21 | * WARNING!!!! Note that this implementation doesn't use the short hash optimization 22 | * resulting in different hashes for any length shorter than 192 bytes 23 | * 24 | * SpookyHash 25 | * http://burtleburtle.net/bob/hash/spooky.html 26 | * 27 | * Exact source used as reference: 28 | * http://burtleburtle.net/bob/c/SpookyV2.h 29 | * http://burtleburtle.net/bob/c/SpookyV2.cpp 30 | */ 31 | 32 | // Spooky Hash 33 | // A 128-bit noncryptographic hash, for checksums and table lookup 34 | // By Bob Jenkins. Public domain. 35 | // Oct 31 2010: published framework, disclaimer ShortHash isn't right 36 | // Nov 7 2010: disabled ShortHash 37 | // Oct 31 2011: replace End, ShortMix, ShortEnd, enable ShortHash again 38 | // April 10 2012: buffer overflow on platforms without unaligned reads 39 | // July 12 2012: was passing out variables in final to in/out in short 40 | // July 30 2012: I reintroduced the buffer overflow 41 | // August 5 2012: SpookyV2: d = should be d += in short hash, and remove extra mix from long hash 42 | // 43 | // Up to 3 bytes/cycle for long messages. Reasonably fast for short messages. 44 | // All 1 or 2 bit deltas achieve avalanche within 1% bias per output bit. 45 | // 46 | // This was developed for and tested on 64-bit x86-compatible processors. 47 | // It assumes the processor is little-endian. There is a macro 48 | // controlling whether unaligned reads are allowed (by default they are). 49 | // This should be an equally good hash on big-endian machines, but it will 50 | // compute different results on them than on little-endian machines. 51 | // 52 | // Google's CityHash has similar specs to SpookyHash, and CityHash is faster 53 | // on new Intel boxes. MD4 and MD5 also have similar specs, but they are orders 54 | // of magnitude slower. CRCs are two or more times slower, but unlike 55 | // SpookyHash, they have nice math for combining the CRCs of pieces to form 56 | // the CRCs of wholes. There are also cryptographic hashes, but those are even 57 | // slower than MD5. 58 | // 59 | 60 | #define Mix(data, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11) \ 61 | s0 += data[0]; s2 ^= s10; s11 ^= s0; s0 = util_rotl64(s0, 11); s11 += s1; \ 62 | s1 += data[1]; s3 ^= s11; s0 ^= s1; s1 = util_rotl64(s1, 32); s0 += s2; \ 63 | s2 += data[2]; s4 ^= s0; s1 ^= s2; s2 = util_rotl64(s2, 43); s1 += s3; \ 64 | s3 += data[3]; s5 ^= s1; s2 ^= s3; s3 = util_rotl64(s3, 31); s2 += s4; \ 65 | s4 += data[4]; s6 ^= s2; s3 ^= s4; s4 = util_rotl64(s4, 17); s3 += s5; \ 66 | s5 += data[5]; s7 ^= s3; s4 ^= s5; s5 = util_rotl64(s5, 28); s4 += s6; \ 67 | s6 += data[6]; s8 ^= s4; s5 ^= s6; s6 = util_rotl64(s6, 39); s5 += s7; \ 68 | s7 += data[7]; s9 ^= s5; s6 ^= s7; s7 = util_rotl64(s7, 57); s6 += s8; \ 69 | s8 += data[8]; s10 ^= s6; s7 ^= s8; s8 = util_rotl64(s8, 55); s7 += s9; \ 70 | s9 += data[9]; s11 ^= s7; s8 ^= s9; s9 = util_rotl64(s9, 54); s8 += s10; \ 71 | s10 += data[10]; s0 ^= s8; s9 ^= s10; s10 = util_rotl64(s10, 22); s9 += s11; \ 72 | s11 += data[11]; s1 ^= s9; s10 ^= s11; s11 = util_rotl64(s11, 46); s10 += s0; 73 | 74 | #define EndPartial(h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11) \ 75 | h11 += h1; h2 ^= h11; h1 = util_rotl64(h1, 44); \ 76 | h0 += h2; h3 ^= h0; h2 = util_rotl64(h2, 15); \ 77 | h1 += h3; h4 ^= h1; h3 = util_rotl64(h3, 34); \ 78 | h2 += h4; h5 ^= h2; h4 = util_rotl64(h4, 21); \ 79 | h3 += h5; h6 ^= h3; h5 = util_rotl64(h5, 38); \ 80 | h4 += h6; h7 ^= h4; h6 = util_rotl64(h6, 33); \ 81 | h5 += h7; h8 ^= h5; h7 = util_rotl64(h7, 10); \ 82 | h6 += h8; h9 ^= h6; h8 = util_rotl64(h8, 13); \ 83 | h7 += h9; h10 ^= h7; h9 = util_rotl64(h9, 38); \ 84 | h8 += h10; h11 ^= h8; h10 = util_rotl64(h10, 53); \ 85 | h9 += h11; h0 ^= h9; h11 = util_rotl64(h11, 42); \ 86 | h10 += h0; h1 ^= h10; h0 = util_rotl64(h0, 54); 87 | 88 | #define End(data, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11) \ 89 | h0 += data[0]; h1 += data[1]; h2 += data[2]; h3 += data[3]; \ 90 | h4 += data[4]; h5 += data[5]; h6 += data[6]; h7 += data[7]; \ 91 | h8 += data[8]; h9 += data[9]; h10 += data[10]; h11 += data[11]; \ 92 | EndPartial(h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11); \ 93 | EndPartial(h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11); \ 94 | EndPartial(h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11); 95 | 96 | // number of uint64_t's in internal state 97 | #define sc_numVars 12 98 | 99 | // size of the internal state 100 | #define sc_blockSize (sc_numVars * 8) 101 | 102 | // 103 | // sc_const: a constant which: 104 | // * is not zero 105 | // * is odd 106 | // * is a not-very-regular mix of 1's and 0's 107 | // * does not need any other special mathematical properties 108 | // 109 | #define sc_const 0xdeadbeefdeadbeefLL 110 | 111 | void SpookyHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest) 112 | { 113 | uint64_t h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11; 114 | uint64_t buf[sc_numVars]; 115 | size_t nblocks; 116 | const uint64_t* blocks; 117 | const uint64_t* end; 118 | size_t size_remainder; 119 | #if WORDS_BIGENDIAN 120 | unsigned i; 121 | #endif 122 | 123 | h9 = util_read64(seed + 0); 124 | h10 = util_read64(seed + 8); 125 | 126 | h0 = h3 = h6 = h9; 127 | h1 = h4 = h7 = h10; 128 | h2 = h5 = h8 = h11 = sc_const; 129 | 130 | nblocks = size / sc_blockSize; 131 | blocks = data; 132 | end = blocks + nblocks * sc_numVars; 133 | 134 | /* body */ 135 | while (blocks < end) { 136 | #if WORDS_BIGENDIAN 137 | for (i = 0; i < sc_numVars; ++i) 138 | buf[i] = util_swap64(blocks[i]); 139 | Mix(buf, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11); 140 | #else 141 | Mix(blocks, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11); 142 | #endif 143 | blocks += sc_numVars; 144 | } 145 | 146 | /* tail */ 147 | size_remainder = (size - ((const uint8_t*)end - (const uint8_t*)data)); 148 | memcpy(buf, end, size_remainder); 149 | memset(((uint8_t*)buf) + size_remainder, 0, sc_blockSize - size_remainder); 150 | ((uint8_t*)buf)[sc_blockSize - 1] = size_remainder; 151 | 152 | /* finalization */ 153 | #if WORDS_BIGENDIAN 154 | for (i = 0; i < sc_numVars; ++i) 155 | buf[i] = util_swap64(buf[i]); 156 | #endif 157 | End(buf, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11); 158 | 159 | util_write64(digest + 0, h0); 160 | util_write64(digest + 8, h1); 161 | } 162 | 163 | -------------------------------------------------------------------------------- /cmdline/touch.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2014 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #include "portable.h" 19 | 20 | #include "support.h" 21 | #include "elem.h" 22 | #include "state.h" 23 | #include "handle.h" 24 | 25 | void state_touch(struct snapraid_state* state) 26 | { 27 | tommy_node* i; 28 | char esc_buffer[ESC_MAX]; 29 | 30 | msg_progress("Setting sub-second timestamps...\n"); 31 | 32 | /* for all disks */ 33 | for (i = state->disklist; i != 0; i = i->next) { 34 | struct snapraid_disk* disk = i->data; 35 | tommy_node* j; 36 | 37 | /* for all files */ 38 | for (j = disk->filelist; j != 0; j = j->next) { 39 | struct snapraid_file* file = j->data; 40 | 41 | /* if the file has a zero nanosecond timestamp */ 42 | /* note that symbolic links are not in the file list */ 43 | /* and then are not processed */ 44 | if (file->mtime_nsec == 0) { 45 | char path[PATH_MAX]; 46 | struct stat st; 47 | int f; 48 | int ret; 49 | int nsec; 50 | int flags; 51 | 52 | pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub); 53 | 54 | /* set a new nanosecond timestamp different than 0 */ 55 | do { 56 | uint32_t nano; 57 | 58 | /* get a random nanosecond value */ 59 | if (randomize(&nano, sizeof(nano)) != 0) { 60 | /* LCOV_EXCL_START */ 61 | log_fatal("Failed to retrieve random values.\n"); 62 | exit(EXIT_FAILURE); 63 | /* LCOV_EXCL_STOP */ 64 | } 65 | 66 | nsec = nano % 1000000000; 67 | } while (nsec == 0); 68 | 69 | /* O_BINARY: open as binary file (Windows only) */ 70 | /* O_NOFOLLOW: do not follow links to ensure to open the real file */ 71 | flags = O_BINARY | O_NOFOLLOW; 72 | #ifdef _WIN32 73 | /* in Windows we must have write access at the file */ 74 | flags |= O_RDWR; 75 | #else 76 | /* in all others platforms, read access is enough */ 77 | flags |= O_RDONLY; 78 | #endif 79 | 80 | /* open it */ 81 | f = open(path, flags); 82 | if (f == -1) { 83 | /* LCOV_EXCL_START */ 84 | log_fatal("Error opening file '%s'. %s.\n", path, strerror(errno)); 85 | continue; 86 | /* LCOV_EXCL_STOP */ 87 | } 88 | 89 | /* get the present timestamp, that may be different than the one */ 90 | /* in the content file */ 91 | ret = fstat(f, &st); 92 | if (ret == -1) { 93 | /* LCOV_EXCL_START */ 94 | close(f); 95 | log_fatal("Error accessing file '%s'. %s.\n", path, strerror(errno)); 96 | continue; 97 | /* LCOV_EXCL_STOP */ 98 | } 99 | 100 | /* set the tweaked modification time, with new nano seconds */ 101 | ret = fmtime(f, st.st_mtime, nsec); 102 | if (ret != 0) { 103 | /* LCOV_EXCL_START */ 104 | close(f); 105 | log_fatal("Error timing file '%s'. %s.\n", path, strerror(errno)); 106 | continue; 107 | /* LCOV_EXCL_STOP */ 108 | } 109 | 110 | /* uses fstat again to get the present timestamp */ 111 | /* this is needed because the value read */ 112 | /* may be different than the written one */ 113 | ret = fstat(f, &st); 114 | if (ret == -1) { 115 | /* LCOV_EXCL_START */ 116 | close(f); 117 | log_fatal("Error accessing file '%s'. %s.\n", path, strerror(errno)); 118 | continue; 119 | /* LCOV_EXCL_STOP */ 120 | } 121 | 122 | /* close it */ 123 | ret = close(f); 124 | if (ret != 0) { 125 | /* LCOV_EXCL_START */ 126 | log_fatal("Error closing file '%s'. %s.\n", path, strerror(errno)); 127 | continue; 128 | /* LCOV_EXCL_STOP */ 129 | } 130 | 131 | /* set the same nanosecond value in the content file */ 132 | /* note that if the seconds value is already matching */ 133 | /* the file won't be synced because the content file will */ 134 | /* contain the new updated timestamp */ 135 | file->mtime_nsec = STAT_NSEC(&st); 136 | 137 | /* state changed, we need to update it */ 138 | state->need_write = 1; 139 | 140 | log_tag("touch:%s:%s: %" PRIu64 ".%d\n", disk->name, esc_tag(file->sub, esc_buffer), (uint64_t)st.st_mtime, STAT_NSEC(&st)); 141 | msg_info("touch %s\n", fmt_term(disk, file->sub, esc_buffer)); 142 | } 143 | } 144 | } 145 | } 146 | 147 | -------------------------------------------------------------------------------- /cmdline/unix.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2011 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #ifndef __UNIX_H 19 | #define __UNIX_H 20 | 21 | #ifdef __linux__ 22 | #define HAVE_LINUX_DEVICE 1 /**< In Linux enables special device support. */ 23 | #define HAVE_DIRECT_IO 1 /**< Support O_DIRECT in open(). */ 24 | #endif 25 | 26 | #define O_BINARY 0 /**< Not used in Unix. */ 27 | #define O_SEQUENTIAL 0 /**< In Unix posix_fadvise() shall be used. */ 28 | 29 | /** 30 | * If nanoseconds are not supported, we report the special STAT_NSEC_INVALID value, 31 | * to mark that it's undefined. 32 | */ 33 | #define STAT_NSEC_INVALID -1 34 | 35 | /* Check if we have nanoseconds support */ 36 | #if HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC 37 | #define STAT_NSEC(st) ((int)(st)->st_mtim.tv_nsec) /* Linux */ 38 | #elif HAVE_STRUCT_STAT_ST_MTIMENSEC 39 | #define STAT_NSEC(st) ((int)(st)->st_mtimensec) /* NetBSD */ 40 | #elif HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC 41 | #define STAT_NSEC(st) ((int)(st)->st_mtimespec.tv_nsec) /* FreeBSD, Mac OS X */ 42 | #else 43 | #define STAT_NSEC(st) STAT_NSEC_INVALID 44 | #endif 45 | 46 | /** 47 | * Open a file with the O_NOATIME flag to avoid to update the access time. 48 | */ 49 | int open_noatime(const char* file, int flags); 50 | 51 | /** 52 | * Check if the specified file is hidden. 53 | */ 54 | int dirent_hidden(struct dirent* dd); 55 | 56 | /** 57 | * Return a description of the file type. 58 | */ 59 | const char* stat_desc(struct stat* st); 60 | 61 | /** 62 | * Return the alignment requirement for direct IO. 63 | */ 64 | size_t direct_size(void); 65 | 66 | #endif 67 | 68 | -------------------------------------------------------------------------------- /cmdline/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2011 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #ifndef __UTIL_H 19 | #define __UTIL_H 20 | 21 | 22 | /****************************************************************************/ 23 | /* memory */ 24 | 25 | /** 26 | * Safe aligned malloc. 27 | * If no memory is available, it aborts. 28 | */ 29 | void* malloc_nofail_align(size_t size, void** freeptr); 30 | 31 | /** 32 | * Safe aligned malloc. Usable for direct io. 33 | */ 34 | void* malloc_nofail_direct(size_t size, void** freeptr); 35 | 36 | /** 37 | * Safe aligned vector allocation. 38 | * If no memory is available, it aborts. 39 | */ 40 | void** malloc_nofail_vector_align(int nd, int n, size_t size, void** freeptr); 41 | 42 | /** 43 | * Safe page vector allocation. Usable for direct io. 44 | * If no memory is available, it aborts. 45 | */ 46 | void** malloc_nofail_vector_direct(int nd, int n, size_t size, void** freeptr); 47 | 48 | /** 49 | * Safe allocation with memory test. 50 | */ 51 | void* malloc_nofail_test(size_t size); 52 | 53 | /** 54 | * Test the memory vector for RAM problems. 55 | * If a problem is found, it crashes. 56 | */ 57 | void mtest_vector(int n, size_t size, void** vv); 58 | 59 | /****************************************************************************/ 60 | /* crc */ 61 | 62 | /** 63 | * CRC initial value. 64 | * Using a not zero value allows to detect a leading run of zeros. 65 | */ 66 | #define CRC_IV 0xffffffffU 67 | 68 | /** 69 | * CRC-32 (Castagnoli) table. 70 | */ 71 | extern uint32_t CRC32C_0[256]; 72 | extern uint32_t CRC32C_1[256]; 73 | extern uint32_t CRC32C_2[256]; 74 | extern uint32_t CRC32C_3[256]; 75 | 76 | /** 77 | * If the CPU support the CRC instructions. 78 | */ 79 | #if HAVE_SSE42 80 | extern int crc_x86; 81 | #endif 82 | 83 | /** 84 | * Compute CRC-32 (Castagnoli) for a single byte without the IV. 85 | */ 86 | static inline uint32_t crc32c_plain_char(uint32_t crc, unsigned char c) 87 | { 88 | #if HAVE_SSE42 89 | if (tommy_likely(crc_x86)) { 90 | asm ("crc32b %1, %0\n" : "+r" (crc) : "m" (c)); 91 | return crc; 92 | } 93 | #endif 94 | return CRC32C_0[(crc ^ c) & 0xff] ^ (crc >> 8); 95 | } 96 | 97 | /** 98 | * Compute the CRC-32 (Castagnoli) without the IV. 99 | */ 100 | static inline uint32_t crc32c_gen_plain(uint32_t crc, const unsigned char* ptr, unsigned size) 101 | { 102 | while (size >= 4) { 103 | crc ^= ptr[0] | (uint32_t)ptr[1] << 8 | (uint32_t)ptr[2] << 16 | (uint32_t)ptr[3] << 24; 104 | crc = CRC32C_3[crc & 0xff] ^ CRC32C_2[(crc >> 8) & 0xff] ^ CRC32C_1[(crc >> 16) & 0xff] ^ CRC32C_0[crc >> 24]; 105 | ptr += 4; 106 | size -= 4; 107 | } 108 | 109 | while (size) { 110 | crc = CRC32C_0[(crc ^ *ptr) & 0xff] ^ (crc >> 8); 111 | ++ptr; 112 | --size; 113 | } 114 | 115 | return crc; 116 | } 117 | 118 | /** 119 | * Compute the CRC-32 (Castagnoli) without the IV. 120 | */ 121 | #if HAVE_SSE42 122 | static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr, unsigned size) 123 | { 124 | #ifdef CONFIG_X86_64 125 | uint64_t crc64 = crc; 126 | while (size >= 8) { 127 | asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr)); 128 | ptr += 8; 129 | size -= 8; 130 | } 131 | crc = crc64; 132 | #else 133 | while (size >= 4) { 134 | asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr)); 135 | ptr += 4; 136 | size -= 4; 137 | } 138 | #endif 139 | while (size) { 140 | asm ("crc32b %1, %0\n" : "+r" (crc) : "m" (*ptr)); 141 | ++ptr; 142 | --size; 143 | } 144 | 145 | return crc; 146 | } 147 | #endif 148 | 149 | /** 150 | * Compute CRC-32 (Castagnoli) without the IV. 151 | */ 152 | static inline uint32_t crc32c_plain(uint32_t crc, const unsigned char* ptr, unsigned size) 153 | { 154 | #if HAVE_SSE42 155 | if (tommy_likely(crc_x86)) { 156 | return crc32c_x86_plain(crc, ptr, size); 157 | } 158 | #endif 159 | return crc32c_gen_plain(crc, ptr, size); 160 | } 161 | 162 | /** 163 | * Compute the CRC-32 (Castagnoli) 164 | */ 165 | extern uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, unsigned size); 166 | 167 | /** 168 | * Internal entry points for testing. 169 | */ 170 | uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, unsigned size); 171 | uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, unsigned size); 172 | 173 | /** 174 | * Initialize the CRC-32 (Castagnoli) support. 175 | */ 176 | void crc32c_init(void); 177 | 178 | /****************************************************************************/ 179 | /* hash */ 180 | 181 | /** 182 | * Size of the hash. 183 | */ 184 | #define HASH_MAX 16 185 | 186 | /** 187 | * Hash kinds. 188 | */ 189 | #define HASH_UNDEFINED 0 190 | #define HASH_MURMUR3 1 191 | #define HASH_SPOOKY2 2 192 | #define HASH_METRO 3 193 | 194 | /** 195 | * Compute the HASH of a memory block. 196 | * Seed is a 128 bit vector. 197 | */ 198 | void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size); 199 | 200 | /** 201 | * Return the hash name. 202 | */ 203 | const char* hash_config_name(unsigned kind); 204 | 205 | /** 206 | * Count the number of different bits in the two buffers. 207 | */ 208 | unsigned memdiff(const unsigned char* data1, const unsigned char* data2, size_t size); 209 | 210 | /****************************************************************************/ 211 | /* lock */ 212 | 213 | /** 214 | * Create and locks the lock file. 215 | * Return -1 on error, otherwise it's the file handle to pass to lock_unlock(). 216 | */ 217 | int lock_lock(const char* file); 218 | 219 | /** 220 | * Unlock the lock file. 221 | * Return -1 on error. 222 | */ 223 | int lock_unlock(int f); 224 | 225 | /****************************************************************************/ 226 | /* bitvect */ 227 | 228 | typedef unsigned char bit_vect_t; 229 | #define BIT_VECT_SIZE (sizeof(bit_vect_t) * 8) 230 | 231 | static inline size_t bit_vect_size(size_t max) 232 | { 233 | return (max + BIT_VECT_SIZE - 1) / BIT_VECT_SIZE; 234 | } 235 | 236 | static inline void bit_vect_set(bit_vect_t* bit_vect, size_t off) 237 | { 238 | bit_vect_t mask = 1 << (off % BIT_VECT_SIZE); 239 | bit_vect[off / BIT_VECT_SIZE] |= mask; 240 | } 241 | 242 | static inline void bit_vect_clear(bit_vect_t* bit_vect, size_t off) 243 | { 244 | bit_vect_t mask = 1 << (off % BIT_VECT_SIZE); 245 | bit_vect[off / BIT_VECT_SIZE] &= ~mask; 246 | } 247 | 248 | static inline int bit_vect_test(bit_vect_t* bit_vect, size_t off) 249 | { 250 | bit_vect_t mask = 1 << (off % BIT_VECT_SIZE); 251 | return (bit_vect[off / BIT_VECT_SIZE] & mask) != 0; 252 | } 253 | 254 | /****************************************************************************/ 255 | /* muldiv */ 256 | 257 | unsigned muldiv(uint64_t v, uint64_t mul, uint64_t div); 258 | unsigned muldiv_upper(uint64_t v, uint64_t mul, uint64_t div); 259 | 260 | #endif 261 | -------------------------------------------------------------------------------- /configure.windows-x64: -------------------------------------------------------------------------------- 1 | ./configure --host=x86_64-w64-mingw32.static --build=`./config.guess` $@ 2 | 3 | 4 | -------------------------------------------------------------------------------- /configure.windows-x86: -------------------------------------------------------------------------------- 1 | ./configure --host=i686-w64-mingw32.static --build=`./config.guess` $@ 2 | 3 | 4 | -------------------------------------------------------------------------------- /makecov.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Run the Coverage test 4 | # 5 | 6 | make distclean 7 | 8 | if ! ./configure --enable-coverage --enable-sde; then 9 | exit 1 10 | fi 11 | 12 | if ! make lcov_reset check lcov_capture lcov_html; then 13 | exit 1 14 | fi 15 | 16 | 17 | -------------------------------------------------------------------------------- /makedist.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | 4 | CHECK=check 5 | DEBUG= 6 | 7 | if test "x$1" = "x-f"; then 8 | CHECK=all 9 | fi 10 | 11 | if test "x$1" = "x-d"; then 12 | DEBUG=--enable-debug 13 | fi 14 | 15 | make distclean 16 | 17 | # Reconfigure (with force) to get the latest revision from git 18 | autoreconf -f 19 | 20 | if ! ./configure.windows-x86 $DEBUG; then 21 | exit 1 22 | fi 23 | 24 | if ! make distwindows-x86 distclean; then 25 | exit 1 26 | fi 27 | 28 | if ! ./configure.windows-x64 $DEBUG; then 29 | exit 1 30 | fi 31 | 32 | if ! make -j4 $CHECK; then 33 | exit 1 34 | fi 35 | 36 | if ! make distwindows-x64 distclean; then 37 | exit 1 38 | fi 39 | 40 | if ! ./configure ; then 41 | exit 1 42 | fi 43 | 44 | if ! make dist; then 45 | exit 1 46 | fi 47 | 48 | -------------------------------------------------------------------------------- /makesan.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Run all the Sanitizers available 4 | # 5 | 6 | # Compiler to use 7 | COMPILER=clang 8 | 9 | # Options for configure 10 | # --disable-asm 11 | # Inline assembly is not supported by the Sanitizers 12 | # --without-blkid 13 | # External libraries are not supported by the Sanitizers 14 | OPTIONS="--disable-asm --without-blkid" 15 | 16 | # Source directory 17 | SOURCE=`pwd` 18 | 19 | # Dest directory 20 | DEST=`mktemp -d` 21 | 22 | make distclean 23 | 24 | cd $DEST 25 | 26 | # AddressSanitizer 27 | if ! $SOURCE/configure --enable-asan $OPTIONS CC=$COMPILER; then 28 | exit 1 29 | fi 30 | 31 | if ! make check distclean; then 32 | exit 1 33 | fi 34 | 35 | # UndefinedBehaviourSanitizer 36 | if ! $SOURCE/configure --enable-ubsan $OPTIONS CC=$COMPILER; then 37 | exit 1 38 | fi 39 | 40 | if ! make check distclean; then 41 | exit 1 42 | fi 43 | 44 | # MemorySanitizer 45 | if ! $SOURCE/configure --enable-msan $OPTIONS CC=$COMPILER; then 46 | exit 1 47 | fi 48 | 49 | if ! make check distclean; then 50 | exit 1 51 | fi 52 | 53 | # ThreadSanitizer 54 | if ! $SOURCE/configure --enable-tsan $OPTIONS CC=$COMPILER; then 55 | exit 1 56 | fi 57 | 58 | if ! make check distclean; then 59 | exit 1 60 | fi 61 | 62 | cd $SOURCE 63 | 64 | if ! ./configure; then 65 | exit 1 66 | fi 67 | 68 | -------------------------------------------------------------------------------- /makescan.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Run the Coverity Scan static analyzer 4 | # 5 | 6 | rm -r cov-int 7 | 8 | make distclean 9 | 10 | # Reconfigure (with force) to get the latest revision from git 11 | autoreconf -f 12 | 13 | if ! ./configure ; then 14 | exit 1 15 | fi 16 | 17 | export PATH=$PATH:contrib/cov-analysis-linux64-2020.09/bin 18 | 19 | if ! cov-build --dir cov-int make; then 20 | exit 1 21 | fi 22 | 23 | REVISION=`sh autover.sh` 24 | 25 | tar czf snapraid-$REVISION.tgz cov-int 26 | 27 | rm -r cov-int 28 | 29 | echo snapraid-$REVISION.tgz ready to upload to https://scan.coverity.com/projects/1986/builds/new 30 | 31 | -------------------------------------------------------------------------------- /makesum.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo sha256 > CHECKSUMS 4 | cd archive && sha256sum * | sort -k 2 -V >> ../CHECKSUMS 5 | 6 | -------------------------------------------------------------------------------- /maketest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Run all the Coverage and Valgrind tests 4 | # 5 | 6 | # Source directory 7 | SOURCE=`pwd` 8 | 9 | # Dest directory 10 | DEST=`mktemp -d` 11 | 12 | make distclean 13 | 14 | cd $DEST 15 | 16 | # Coverage 17 | if ! $SOURCE/configure --enable-coverage --enable-sde; then 18 | exit 1 19 | fi 20 | 21 | if ! make lcov_reset check lcov_capture lcov_html; then 22 | exit 1 23 | fi 24 | 25 | cp -a cov $SOURCE/cov 26 | 27 | if ! make distclean; then 28 | exit 1 29 | fi 30 | 31 | # Valgrind 32 | if ! $SOURCE/configure --enable-valgrind; then 33 | exit 1 34 | fi 35 | 36 | if ! make check distclean; then 37 | exit 1 38 | fi 39 | 40 | # Helgrind 41 | if ! $SOURCE/configure --enable-helgrind; then 42 | exit 1 43 | fi 44 | 45 | if ! make check distclean; then 46 | exit 1 47 | fi 48 | 49 | # Drd 50 | if ! $SOURCE/configure --enable-drd; then 51 | exit 1 52 | fi 53 | 54 | if ! make check distclean; then 55 | exit 1 56 | fi 57 | 58 | cd $SOURCE 59 | 60 | if ! ./configure; then 61 | exit 1 62 | fi 63 | 64 | -------------------------------------------------------------------------------- /raid/check.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #include "internal.h" 16 | #include "combo.h" 17 | #include "gf.h" 18 | 19 | /** 20 | * Validate the provided failed blocks. 21 | * 22 | * This function checks if the specified failed blocks satisfy the redundancy 23 | * information using the data from the known valid parity blocks. 24 | * 25 | * It's similar at raid_check(), just with a different format for arguments. 26 | * 27 | * The number of failed blocks @nr must be strictly less than the number of 28 | * parities @nv, because you need one more parity to validate the recovering. 29 | * 30 | * No data or parity blocks are modified. 31 | * 32 | * @nr Number of failed data blocks. 33 | * @id[] Vector of @nr indexes of the failed data blocks. 34 | * The indexes start from 0. They must be in order. 35 | * @nv Number of valid parity blocks. 36 | * @ip[] Vector of @nv indexes of the valid parity blocks. 37 | * The indexes start from 0. They must be in order. 38 | * @nd Number of data blocks. 39 | * @size Size of the blocks pointed by @v. It must be a multiplier of 64. 40 | * @v Vector of pointers to the blocks of data and parity. 41 | * It has (@nd + @ip[@nv - 1] + 1) elements. The starting elements are the 42 | * blocks for data, following with the parity blocks. 43 | * Each block has @size bytes. 44 | * @return 0 if the check is satisfied. -1 otherwise. 45 | */ 46 | static int raid_validate(int nr, int *id, int nv, int *ip, int nd, size_t size, void **vv) 47 | { 48 | uint8_t **v = (uint8_t **)vv; 49 | const uint8_t *T[RAID_PARITY_MAX][RAID_PARITY_MAX]; 50 | uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; 51 | uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; 52 | size_t i; 53 | int j, k, l; 54 | 55 | BUG_ON(nr >= nv); 56 | 57 | /* setup the coefficients matrix */ 58 | for (j = 0; j < nr; ++j) 59 | for (k = 0; k < nr; ++k) 60 | G[j * nr + k] = A(ip[j], id[k]); 61 | 62 | /* invert it to solve the system of linear equations */ 63 | raid_invert(G, V, nr); 64 | 65 | /* get multiplication tables */ 66 | for (j = 0; j < nr; ++j) 67 | for (k = 0; k < nr; ++k) 68 | T[j][k] = table(V[j * nr + k]); 69 | 70 | /* check all positions */ 71 | for (i = 0; i < size; ++i) { 72 | uint8_t p[RAID_PARITY_MAX]; 73 | 74 | /* get parity */ 75 | for (j = 0; j < nv; ++j) 76 | p[j] = v[nd + ip[j]][i]; 77 | 78 | /* compute delta parity, skipping broken disks */ 79 | for (j = 0, k = 0; j < nd; ++j) { 80 | uint8_t b; 81 | 82 | /* skip broken disks */ 83 | if (k < nr && id[k] == j) { 84 | ++k; 85 | continue; 86 | } 87 | 88 | b = v[j][i]; 89 | for (l = 0; l < nv; ++l) 90 | p[l] ^= gfmul[b][gfgen[ip[l]][j]]; 91 | } 92 | 93 | /* reconstruct data */ 94 | for (j = 0; j < nr; ++j) { 95 | uint8_t b = 0; 96 | int idj = id[j]; 97 | 98 | /* recompute the data */ 99 | for (k = 0; k < nr; ++k) 100 | b ^= T[j][k][p[k]]; 101 | 102 | /* add the parity contribution of the reconstructed data */ 103 | for (l = nr; l < nv; ++l) 104 | p[l] ^= gfmul[b][gfgen[ip[l]][idj]]; 105 | } 106 | 107 | /* check that the final parity is 0 */ 108 | for (l = nr; l < nv; ++l) 109 | if (p[l] != 0) 110 | return -1; 111 | } 112 | 113 | return 0; 114 | } 115 | 116 | int raid_check(int nr, int *ir, int nd, int np, size_t size, void **v) 117 | { 118 | /* valid parity index */ 119 | int ip[RAID_PARITY_MAX]; 120 | int vp; 121 | int rd; 122 | int i, j; 123 | 124 | /* enforce limit on size */ 125 | BUG_ON(size % 64 != 0); 126 | 127 | /* enforce limit on number of failures */ 128 | BUG_ON(nr >= np); /* >= because we check with extra parity */ 129 | BUG_ON(np > RAID_PARITY_MAX); 130 | 131 | /* enforce order in index vector */ 132 | BUG_ON(nr >= 2 && ir[0] >= ir[1]); 133 | BUG_ON(nr >= 3 && ir[1] >= ir[2]); 134 | BUG_ON(nr >= 4 && ir[2] >= ir[3]); 135 | BUG_ON(nr >= 5 && ir[3] >= ir[4]); 136 | BUG_ON(nr >= 6 && ir[4] >= ir[5]); 137 | 138 | /* enforce limit on index vector */ 139 | BUG_ON(nr > 0 && ir[nr-1] >= nd + np); 140 | 141 | /* count failed data disk */ 142 | rd = 0; 143 | while (rd < nr && ir[rd] < nd) 144 | ++rd; 145 | 146 | /* put valid parities into ip[] */ 147 | vp = 0; 148 | for (i = rd, j = 0; j < np; ++j) { 149 | /* if parity is failed */ 150 | if (i < nr && ir[i] == nd + j) { 151 | /* skip broken parity */ 152 | ++i; 153 | } else { 154 | /* store valid parity */ 155 | ip[vp] = j; 156 | ++vp; 157 | } 158 | } 159 | 160 | return raid_validate(rd, ir, vp, ip, nd, size, v); 161 | } 162 | 163 | int raid_scan(int *ir, int nd, int np, size_t size, void **v) 164 | { 165 | int r; 166 | 167 | /* check the special case of no failure */ 168 | if (np != 0 && raid_check(0, 0, nd, np, size, v) == 0) 169 | return 0; 170 | 171 | /* for each number of possible failures */ 172 | for (r = 1; r < np; ++r) { 173 | /* try all combinations of r failures on n disks */ 174 | combination_first(r, nd + np, ir); 175 | do { 176 | /* verify if the combination is a valid one */ 177 | if (raid_check(r, ir, nd, np, size, v) == 0) 178 | return r; 179 | } while (combination_next(r, nd + np, ir)); 180 | } 181 | 182 | /* no solution found */ 183 | return -1; 184 | } 185 | 186 | -------------------------------------------------------------------------------- /raid/combo.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #ifndef __RAID_COMBO_H 16 | #define __RAID_COMBO_H 17 | 18 | #include 19 | 20 | /** 21 | * Get the first permutation with repetition of r of n elements. 22 | * 23 | * Typical use is with permutation_next() in the form : 24 | * 25 | * int i[R]; 26 | * permutation_first(R, N, i); 27 | * do { 28 | * code using i[0], i[1], ..., i[R-1] 29 | * } while (permutation_next(R, N, i)); 30 | * 31 | * It's equivalent at the code : 32 | * 33 | * for(i[0]=0;i[0]= n) { 65 | 66 | /* if we are at the first level, we have finished */ 67 | if (i == 0) 68 | return 0; 69 | 70 | /* increase the previous position */ 71 | --i; 72 | goto recurse; 73 | } 74 | 75 | ++i; 76 | 77 | /* initialize all the next positions, if any */ 78 | while (i < r) { 79 | c[i] = 0; 80 | ++i; 81 | } 82 | 83 | return 1; 84 | } 85 | 86 | /** 87 | * Get the first combination without repetition of r of n elements. 88 | * 89 | * Typical use is with combination_next() in the form : 90 | * 91 | * int i[R]; 92 | * combination_first(R, N, i); 93 | * do { 94 | * code using i[0], i[1], ..., i[R-1] 95 | * } while (combination_next(R, N, i)); 96 | * 97 | * It's equivalent at the code : 98 | * 99 | * for(i[0]=0;i[0]= h) { 132 | 133 | /* if we are at the first level, we have finished */ 134 | if (i == 0) 135 | return 0; 136 | 137 | /* increase the previous position */ 138 | --i; 139 | --h; 140 | goto recurse; 141 | } 142 | 143 | ++i; 144 | 145 | /* initialize all the next positions, if any */ 146 | while (i < r) { 147 | /* each position start at the next value of the previous one */ 148 | c[i] = c[i - 1] + 1; 149 | ++i; 150 | } 151 | 152 | return 1; 153 | } 154 | #endif 155 | 156 | -------------------------------------------------------------------------------- /raid/cpu.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amadvance/snapraid/09897fcdb35c99b0147415f0118f21d4d90b22ad/raid/cpu.h -------------------------------------------------------------------------------- /raid/gf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #ifndef __RAID_GF_H 16 | #define __RAID_GF_H 17 | 18 | /* 19 | * Galois field operations. 20 | * 21 | * Basic range checks are implemented using BUG_ON(). 22 | */ 23 | 24 | /* 25 | * GF a*b. 26 | */ 27 | static __always_inline uint8_t mul(uint8_t a, uint8_t b) 28 | { 29 | return gfmul[a][b]; 30 | } 31 | 32 | /* 33 | * GF 1/a. 34 | * Not defined for a == 0. 35 | */ 36 | static __always_inline uint8_t inv(uint8_t v) 37 | { 38 | BUG_ON(v == 0); /* division by zero */ 39 | 40 | return gfinv[v]; 41 | } 42 | 43 | /* 44 | * GF 2^a. 45 | */ 46 | static __always_inline uint8_t pow2(int v) 47 | { 48 | BUG_ON(v < 0 || v > 254); /* invalid exponent */ 49 | 50 | return gfexp[v]; 51 | } 52 | 53 | /* 54 | * Gets the multiplication table for a specified value. 55 | */ 56 | static __always_inline const uint8_t *table(uint8_t v) 57 | { 58 | return gfmul[v]; 59 | } 60 | 61 | /* 62 | * Gets the generator matrix coefficient for parity 'p' and disk 'd'. 63 | */ 64 | static __always_inline uint8_t A(int p, int d) 65 | { 66 | return gfgen[p][d]; 67 | } 68 | 69 | /* 70 | * Dereference as uint8_t 71 | */ 72 | #define v_8(p) (*(uint8_t *)&(p)) 73 | 74 | /* 75 | * Dereference as uint32_t 76 | */ 77 | #define v_32(p) (*(uint32_t *)&(p)) 78 | 79 | /* 80 | * Dereference as uint64_t 81 | */ 82 | #define v_64(p) (*(uint64_t *)&(p)) 83 | 84 | /* 85 | * Multiply each byte of a uint32 by 2 in the GF(2^8). 86 | */ 87 | static __always_inline uint32_t x2_32(uint32_t v) 88 | { 89 | uint32_t mask = v & 0x80808080U; 90 | 91 | mask = (mask << 1) - (mask >> 7); 92 | v = (v << 1) & 0xfefefefeU; 93 | v ^= mask & 0x1d1d1d1dU; 94 | return v; 95 | } 96 | 97 | /* 98 | * Multiply each byte of a uint64 by 2 in the GF(2^8). 99 | */ 100 | static __always_inline uint64_t x2_64(uint64_t v) 101 | { 102 | uint64_t mask = v & 0x8080808080808080ULL; 103 | 104 | mask = (mask << 1) - (mask >> 7); 105 | v = (v << 1) & 0xfefefefefefefefeULL; 106 | v ^= mask & 0x1d1d1d1d1d1d1d1dULL; 107 | return v; 108 | } 109 | 110 | /* 111 | * Divide each byte of a uint32 by 2 in the GF(2^8). 112 | */ 113 | static __always_inline uint32_t d2_32(uint32_t v) 114 | { 115 | uint32_t mask = v & 0x01010101U; 116 | 117 | mask = (mask << 8) - mask; 118 | v = (v >> 1) & 0x7f7f7f7fU; 119 | v ^= mask & 0x8e8e8e8eU; 120 | return v; 121 | } 122 | 123 | /* 124 | * Divide each byte of a uint64 by 2 in the GF(2^8). 125 | */ 126 | static __always_inline uint64_t d2_64(uint64_t v) 127 | { 128 | uint64_t mask = v & 0x0101010101010101ULL; 129 | 130 | mask = (mask << 8) - mask; 131 | v = (v >> 1) & 0x7f7f7f7f7f7f7f7fULL; 132 | v ^= mask & 0x8e8e8e8e8e8e8e8eULL; 133 | return v; 134 | } 135 | 136 | #endif 137 | 138 | -------------------------------------------------------------------------------- /raid/helper.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #include "internal.h" 16 | 17 | #define RAID_SWAP(a, b) \ 18 | do { \ 19 | if (v[a] > v[b]) { \ 20 | int t = v[a]; \ 21 | v[a] = v[b]; \ 22 | v[b] = t; \ 23 | } \ 24 | } while (0) 25 | 26 | void raid_sort(int n, int *v) 27 | { 28 | /* sorting networks generated with Batcher's Merge-Exchange */ 29 | switch (n) { 30 | case 2: 31 | RAID_SWAP(0, 1); 32 | break; 33 | case 3: 34 | RAID_SWAP(0, 2); 35 | RAID_SWAP(0, 1); 36 | RAID_SWAP(1, 2); 37 | break; 38 | case 4: 39 | RAID_SWAP(0, 2); 40 | RAID_SWAP(1, 3); 41 | RAID_SWAP(0, 1); 42 | RAID_SWAP(2, 3); 43 | RAID_SWAP(1, 2); 44 | break; 45 | case 5: 46 | RAID_SWAP(0, 4); 47 | RAID_SWAP(0, 2); 48 | RAID_SWAP(1, 3); 49 | RAID_SWAP(2, 4); 50 | RAID_SWAP(0, 1); 51 | RAID_SWAP(2, 3); 52 | RAID_SWAP(1, 4); 53 | RAID_SWAP(1, 2); 54 | RAID_SWAP(3, 4); 55 | break; 56 | case 6: 57 | RAID_SWAP(0, 4); 58 | RAID_SWAP(1, 5); 59 | RAID_SWAP(0, 2); 60 | RAID_SWAP(1, 3); 61 | RAID_SWAP(2, 4); 62 | RAID_SWAP(3, 5); 63 | RAID_SWAP(0, 1); 64 | RAID_SWAP(2, 3); 65 | RAID_SWAP(4, 5); 66 | RAID_SWAP(1, 4); 67 | RAID_SWAP(1, 2); 68 | RAID_SWAP(3, 4); 69 | break; 70 | } 71 | } 72 | 73 | void raid_insert(int n, int *v, int i) 74 | { 75 | /* we don't use binary search because this is intended */ 76 | /* for very small vectors and we want to optimize the case */ 77 | /* of elements inserted already in order */ 78 | 79 | /* insert at the end */ 80 | v[n] = i; 81 | 82 | /* swap until in the correct position */ 83 | while (n > 0 && v[n - 1] > v[n]) { 84 | /* swap */ 85 | int t = v[n - 1]; 86 | 87 | v[n - 1] = v[n]; 88 | v[n] = t; 89 | 90 | /* previous position */ 91 | --n; 92 | } 93 | } 94 | 95 | -------------------------------------------------------------------------------- /raid/helper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #ifndef __RAID_HELPER_H 16 | #define __RAID_HELPER_H 17 | 18 | /** 19 | * Inserts an integer in a sorted vector. 20 | * 21 | * This function can be used to insert indexes in order, ready to be used for 22 | * calling raid_rec(). 23 | * 24 | * @n Number of integers currently in the vector. 25 | * @v Vector of integers already sorted. 26 | * It must have extra space for the new element at the end. 27 | * @i Value to insert. 28 | */ 29 | void raid_insert(int n, int *v, int i); 30 | 31 | /** 32 | * Sorts a small vector of integers. 33 | * 34 | * If you have indexes not in order, you can use this function to sort them 35 | * before calling raid_rec(). 36 | * 37 | * @n Number of integers. No more than RAID_PARITY_MAX. 38 | * @v Vector of integers. 39 | */ 40 | void raid_sort(int n, int *v); 41 | 42 | #endif 43 | 44 | -------------------------------------------------------------------------------- /raid/intz.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #include "internal.h" 16 | #include "gf.h" 17 | 18 | /* 19 | * GENz (triple parity with powers of 2^-1) 32bit C implementation 20 | */ 21 | void raid_genz_int32(int nd, size_t size, void **vv) 22 | { 23 | uint8_t **v = (uint8_t**)vv; 24 | uint8_t *p; 25 | uint8_t *q; 26 | uint8_t *r; 27 | int d, l; 28 | size_t i; 29 | 30 | uint32_t d0, r0, q0, p0; 31 | uint32_t d1, r1, q1, p1; 32 | 33 | l = nd - 1; 34 | p = v[nd]; 35 | q = v[nd + 1]; 36 | r = v[nd + 2]; 37 | 38 | for (i = 0; i < size; i += 8) { 39 | r0 = q0 = p0 = v_32(v[l][i]); 40 | r1 = q1 = p1 = v_32(v[l][i + 4]); 41 | for (d = l - 1; d >= 0; --d) { 42 | d0 = v_32(v[d][i]); 43 | d1 = v_32(v[d][i + 4]); 44 | 45 | p0 ^= d0; 46 | p1 ^= d1; 47 | 48 | q0 = x2_32(q0); 49 | q1 = x2_32(q1); 50 | 51 | q0 ^= d0; 52 | q1 ^= d1; 53 | 54 | r0 = d2_32(r0); 55 | r1 = d2_32(r1); 56 | 57 | r0 ^= d0; 58 | r1 ^= d1; 59 | } 60 | v_32(p[i]) = p0; 61 | v_32(p[i + 4]) = p1; 62 | v_32(q[i]) = q0; 63 | v_32(q[i + 4]) = q1; 64 | v_32(r[i]) = r0; 65 | v_32(r[i + 4]) = r1; 66 | } 67 | } 68 | 69 | /* 70 | * GENz (triple parity with powers of 2^-1) 64bit C implementation 71 | */ 72 | void raid_genz_int64(int nd, size_t size, void **vv) 73 | { 74 | uint8_t **v = (uint8_t**)vv; 75 | uint8_t *p; 76 | uint8_t *q; 77 | uint8_t *r; 78 | int d, l; 79 | size_t i; 80 | 81 | uint64_t d0, r0, q0, p0; 82 | uint64_t d1, r1, q1, p1; 83 | 84 | l = nd - 1; 85 | p = v[nd]; 86 | q = v[nd + 1]; 87 | r = v[nd + 2]; 88 | 89 | for (i = 0; i < size; i += 16) { 90 | r0 = q0 = p0 = v_64(v[l][i]); 91 | r1 = q1 = p1 = v_64(v[l][i + 8]); 92 | for (d = l - 1; d >= 0; --d) { 93 | d0 = v_64(v[d][i]); 94 | d1 = v_64(v[d][i + 8]); 95 | 96 | p0 ^= d0; 97 | p1 ^= d1; 98 | 99 | q0 = x2_64(q0); 100 | q1 = x2_64(q1); 101 | 102 | q0 ^= d0; 103 | q1 ^= d1; 104 | 105 | r0 = d2_64(r0); 106 | r1 = d2_64(r1); 107 | 108 | r0 ^= d0; 109 | r1 ^= d1; 110 | } 111 | v_64(p[i]) = p0; 112 | v_64(p[i + 8]) = p1; 113 | v_64(q[i]) = q0; 114 | v_64(q[i + 8]) = q1; 115 | v_64(r[i]) = r0; 116 | v_64(r[i + 8]) = r1; 117 | } 118 | } 119 | 120 | -------------------------------------------------------------------------------- /raid/memory.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #include "internal.h" 16 | #include "memory.h" 17 | 18 | void *raid_malloc_align(size_t size, size_t align_size, void **freeptr) 19 | { 20 | unsigned char *ptr; 21 | uintptr_t offset; 22 | 23 | ptr = malloc(size + align_size); 24 | if (!ptr) { 25 | /* LCOV_EXCL_START */ 26 | return 0; 27 | /* LCOV_EXCL_STOP */ 28 | } 29 | 30 | *freeptr = ptr; 31 | 32 | offset = ((uintptr_t)ptr) % align_size; 33 | 34 | if (offset != 0) 35 | ptr += align_size - offset; 36 | 37 | return ptr; 38 | } 39 | 40 | void *raid_malloc(size_t size, void **freeptr) 41 | { 42 | return raid_malloc_align(size, RAID_MALLOC_ALIGN, freeptr); 43 | } 44 | 45 | void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, size_t displacement_size, void **freeptr) 46 | { 47 | void **v; 48 | unsigned char *va; 49 | int i; 50 | 51 | BUG_ON(n <= 0 || nd < 0); 52 | 53 | v = malloc(n * sizeof(void *)); 54 | if (!v) { 55 | /* LCOV_EXCL_START */ 56 | return 0; 57 | /* LCOV_EXCL_STOP */ 58 | } 59 | 60 | va = raid_malloc_align(n * (size + displacement_size), align_size, freeptr); 61 | if (!va) { 62 | /* LCOV_EXCL_START */ 63 | free(v); 64 | return 0; 65 | /* LCOV_EXCL_STOP */ 66 | } 67 | 68 | for (i = 0; i < n; ++i) { 69 | v[i] = va; 70 | va += size + displacement_size; 71 | } 72 | 73 | /* reverse order of the data blocks */ 74 | /* because they are usually accessed from the last one */ 75 | for (i = 0; i < nd / 2; ++i) { 76 | void *ptr = v[i]; 77 | 78 | v[i] = v[nd - 1 - i]; 79 | v[nd - 1 - i] = ptr; 80 | } 81 | 82 | return v; 83 | } 84 | 85 | void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr) 86 | { 87 | return raid_malloc_vector_align(nd, n, size, RAID_MALLOC_ALIGN, RAID_MALLOC_DISPLACEMENT, freeptr); 88 | } 89 | 90 | void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv) 91 | { 92 | unsigned char **v = (unsigned char **)vv; 93 | int i; 94 | size_t j; 95 | 96 | for (i = 0; i < n; ++i) 97 | for (j = 0; j < size; ++j) { 98 | /* basic C99/C11 linear congruential generator */ 99 | seed = seed * 1103515245U + 12345U; 100 | 101 | v[i][j] = seed >> 16; 102 | } 103 | } 104 | 105 | int raid_mtest_vector(int n, size_t size, void **vv) 106 | { 107 | unsigned char **v = (unsigned char **)vv; 108 | int i; 109 | size_t j; 110 | unsigned k; 111 | unsigned char d; 112 | unsigned char p; 113 | 114 | /* fill with 0 */ 115 | d = 0; 116 | for (i = 0; i < n; ++i) 117 | for (j = 0; j < size; ++j) 118 | v[i][j] = d; 119 | 120 | /* test with all the byte patterns */ 121 | for (k = 1; k < 256; ++k) { 122 | p = d; 123 | d = k; 124 | 125 | /* forward fill */ 126 | for (i = 0; i < n; ++i) { 127 | for (j = 0; j < size; ++j) { 128 | if (v[i][j] != p) { 129 | /* LCOV_EXCL_START */ 130 | return -1; 131 | /* LCOV_EXCL_STOP */ 132 | } 133 | v[i][j] = d; 134 | } 135 | } 136 | 137 | p = d; 138 | d = ~p; 139 | /* backward fill with complement */ 140 | for (i = 0; i < n; ++i) { 141 | for (j = size; j > 0; --j) { 142 | if (v[i][j - 1] != p) { 143 | /* LCOV_EXCL_START */ 144 | return -1; 145 | /* LCOV_EXCL_STOP */ 146 | } 147 | v[i][j - 1] = d; 148 | } 149 | } 150 | } 151 | 152 | return 0; 153 | } 154 | 155 | -------------------------------------------------------------------------------- /raid/memory.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #ifndef __RAID_MEMORY_H 16 | #define __RAID_MEMORY_H 17 | 18 | /** 19 | * Memory alignment provided by raid_malloc(). 20 | * 21 | * It should guarantee good cache performance everywhere. 22 | */ 23 | #define RAID_MALLOC_ALIGN 256 24 | 25 | /** 26 | * Memory displacement to avoid cache address sharing on contiguous blocks, 27 | * used by raid_malloc_vector(). 28 | * 29 | * When allocating a sequence of blocks with a size of power of 2, 30 | * there is the risk that the addresses of each block are mapped into the 31 | * same cache line and prefetching predictor, resulting in a lot of cache 32 | * sharing if you access all the blocks in parallel, from the start to the 33 | * end. 34 | * 35 | * To avoid this effect, it's better if all the blocks are allocated 36 | * with a fixed displacement trying to reduce the cache addresses sharing. 37 | * 38 | * The selected displacement was chosen empirically with some speed tests 39 | * with 8/12/16/20/24 data buffers of 256 KB. 40 | * 41 | * These are the results in MB/s with no displacement: 42 | * 43 | * sse2 44 | * gen1 15368 [MB/s] 45 | * gen2 6814 [MB/s] 46 | * genz 3033 [MB/s] 47 | * 48 | * These are the results with displacement resulting in improvements 49 | * in the order of 20% or more: 50 | * 51 | * sse2 52 | * gen1 21936 [MB/s] 53 | * gen2 11902 [MB/s] 54 | * genz 5838 [MB/s] 55 | * 56 | */ 57 | #define RAID_MALLOC_DISPLACEMENT (7*256) 58 | 59 | /** 60 | * Aligned malloc. 61 | * Use an alignment suitable for the raid functions. 62 | */ 63 | void *raid_malloc(size_t size, void **freeptr); 64 | 65 | /** 66 | * Arbitrary aligned malloc. 67 | */ 68 | void *raid_malloc_align(size_t size, size_t align_size, void **freeptr); 69 | 70 | /** 71 | * Aligned vector allocation. 72 | * Use an alignment suitable for the raid functions. 73 | * Returns a vector of @n pointers, each one pointing to a block of 74 | * the specified @size. 75 | * The first @nd elements are reversed in order. 76 | */ 77 | void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr); 78 | 79 | /** 80 | * Arbitrary aligned vector allocation. 81 | */ 82 | void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, size_t displacement_size, void **freeptr); 83 | 84 | /** 85 | * Fills the memory vector with pseudo-random data based on the specified seed. 86 | */ 87 | void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv); 88 | 89 | /** 90 | * Tests the memory vector for RAM problems. 91 | * If a problem is found, it crashes. 92 | */ 93 | int raid_mtest_vector(int n, size_t size, void **vv); 94 | 95 | #endif 96 | 97 | -------------------------------------------------------------------------------- /raid/raid.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #ifndef __RAID_H 16 | #define __RAID_H 17 | 18 | /** 19 | * RAID mode supporting up to 6 parities. 20 | * 21 | * It requires SSSE3 to get good performance with triple or more parities. 22 | * 23 | * This is the default mode set after calling raid_init(). 24 | */ 25 | #define RAID_MODE_CAUCHY 0 26 | 27 | /** 28 | * RAID mode supporting up to 3 parities, 29 | * 30 | * It has a fast triple parity implementation without SSSE3, but it cannot 31 | * go beyond triple parity. 32 | * 33 | * This is mostly intended for low end CPUs like ARM and AMD Athlon. 34 | */ 35 | #define RAID_MODE_VANDERMONDE 1 36 | 37 | /** 38 | * Maximum number of parity disks supported. 39 | */ 40 | #define RAID_PARITY_MAX 6 41 | 42 | /** 43 | * Maximum number of data disks supported. 44 | */ 45 | #define RAID_DATA_MAX 251 46 | 47 | /** 48 | * Initializes the RAID system. 49 | * 50 | * You must call this function before any other. 51 | * 52 | * The RAID system is initialized in the RAID_MODE_CAUCHY mode. 53 | */ 54 | void raid_init(void); 55 | 56 | /** 57 | * Runs a basic functionality self test. 58 | * 59 | * The test is immediate, and it's intended to be run at application 60 | * startup to check the integrity of the RAID system. 61 | * 62 | * It returns 0 on success. 63 | */ 64 | int raid_selftest(void); 65 | 66 | /** 67 | * Sets the mode to use. One of RAID_MODE_*. 68 | * 69 | * You can change mode at any time, and it will affect next calls to raid_gen(), 70 | * raid_rec() and raid_data(). 71 | * 72 | * The two modes are compatible for the first two levels of parity. 73 | * The third one is different. 74 | */ 75 | void raid_mode(int mode); 76 | 77 | /** 78 | * Sets the zero buffer to use in recovering. 79 | * 80 | * Before calling raid_rec() and raid_data() you must provide a memory 81 | * buffer filled with zero with the same size of the blocks to recover. 82 | * 83 | * This buffer is only read and never written. 84 | */ 85 | void raid_zero(void *zero); 86 | 87 | /** 88 | * Computes parity blocks. 89 | * 90 | * This function computes the specified number of parity blocks of the 91 | * provided set of data blocks. 92 | * 93 | * Each parity block allows to recover one data block. 94 | * 95 | * @nd Number of data blocks. 96 | * @np Number of parities blocks to compute. 97 | * @size Size of the blocks pointed by @v. It must be a multiplier of 64. 98 | * @v Vector of pointers to the blocks of data and parity. 99 | * It has (@nd + @np) elements. The starting elements are the blocks for 100 | * data, following with the parity blocks. 101 | * Data blocks are only read and not modified. Parity blocks are written. 102 | * Each block has @size bytes. 103 | */ 104 | void raid_gen(int nd, int np, size_t size, void **v); 105 | 106 | /** 107 | * Recovers failures in data and parity blocks. 108 | * 109 | * This function recovers all the data and parity blocks marked as bad 110 | * in the @ir vector. 111 | * 112 | * Ensure to have @nr <= @np, otherwise recovering is not possible. 113 | * 114 | * The parities blocks used for recovering are automatically selected from 115 | * the ones NOT present in the @ir vector. 116 | * 117 | * In case there are more parity blocks than needed, the parities at lower 118 | * indexes are used in the recovering, and the others are ignored. 119 | * 120 | * Note that no internal integrity check is done when recovering. If the 121 | * provided parities are correct, the resulting data will be correct. 122 | * If parities are wrong, the resulting recovered data will be wrong. 123 | * This happens even in the case you have more parities blocks than needed, 124 | * and some form of integrity verification would be possible. 125 | * 126 | * @nr Number of failed data and parity blocks to recover. 127 | * @ir[] Vector of @nr indexes of the failed data and parity blocks. 128 | * The indexes start from 0. They must be in order. 129 | * The first parity is represented with value @nd, the second with value 130 | * @nd + 1, just like positions in the @v vector. 131 | * @nd Number of data blocks. 132 | * @np Number of parity blocks. 133 | * @size Size of the blocks pointed by @v. It must be a multiplier of 64. 134 | * @v Vector of pointers to the blocks of data and parity. 135 | * It has (@nd + @np) elements. The starting elements are the blocks 136 | * for data, following with the parity blocks. 137 | * Each block has @size bytes. 138 | */ 139 | void raid_rec(int nr, int *ir, int nd, int np, size_t size, void **v); 140 | 141 | /** 142 | * Recovers failures in data blocks only. 143 | * 144 | * This function recovers all the data blocks marked as bad in the @id vector. 145 | * The parity blocks are not modified. 146 | * 147 | * @nr Number of failed data blocks to recover. 148 | * @id[] Vector of @nr indexes of the data blocks to recover. 149 | * The indexes start from 0. They must be in order. 150 | * @ip[] Vector of @nr indexes of the parity blocks to use for recovering. 151 | * The indexes start from 0. They must be in order. 152 | * @nd Number of data blocks. 153 | * @size Size of the blocks pointed by @v. It must be a multiplier of 64. 154 | * @v Vector of pointers to the blocks of data and parity. 155 | * It has (@nd + @ip[@nr - 1] + 1) elements. The starting elements are the 156 | * blocks for data, following with the parity blocks. 157 | * Each blocks has @size bytes. 158 | */ 159 | void raid_data(int nr, int *id, int *ip, int nd, size_t size, void **v); 160 | 161 | /** 162 | * Check the provided failed blocks combination. 163 | * 164 | * This function checks if the specified failed blocks combination satisfies 165 | * the redundancy information. A combination is assumed matching, if the 166 | * remaining valid parity is matching the expected value after recovering. 167 | * 168 | * The number of failed blocks @nr must be strictly less than the number of 169 | * parities @np, because you need one more parity to validate the recovering. 170 | * 171 | * No data or parity blocks are modified. 172 | * 173 | * @nr Number of failed data and parity blocks. 174 | * @ir[] Vector of @nr indexes of the failed data and parity blocks. 175 | * The indexes start from 0. They must be in order. 176 | * The first parity is represented with value @nd, the second with value 177 | * @nd + 1, just like positions in the @v vector. 178 | * @nd Number of data blocks. 179 | * @np Number of parity blocks. 180 | * @size Size of the blocks pointed by @v. It must be a multiplier of 64. 181 | * @v Vector of pointers to the blocks of data and parity. 182 | * It has (@nd + @np) elements. The starting elements are the blocks 183 | * for data, following with the parity blocks. 184 | * Each block has @size bytes. 185 | * @return 0 if the check is satisfied. -1 otherwise. 186 | */ 187 | int raid_check(int nr, int *ir, int nd, int np, size_t size, void **v); 188 | 189 | /** 190 | * Scan for failed blocks. 191 | * 192 | * This function identifies the failed data and parity blocks using the 193 | * available redundancy. 194 | * 195 | * It uses a brute force method, and then the call can be expensive. 196 | * The expected execution time is proportional at the binomial coefficient 197 | * @np + @nd choose @np - 1, usually written as: 198 | * 199 | * ( @np + @nd ) 200 | * ( ) 201 | * ( @np - 1 ) 202 | * 203 | * No data or parity blocks are modified. 204 | * 205 | * The failed block indexes are returned in the @ir vector. 206 | * It must have space for at least @np - 1 values. 207 | * 208 | * The returned @ir vector can then be used in a raid_rec() call to recover 209 | * the failed data and parity blocks. 210 | * 211 | * @ir[] Vector filled with the indexes of the failed data and parity blocks. 212 | * The indexes start from 0 and they are in order. 213 | * The first parity is represented with value @nd, the second with value 214 | * @nd + 1, just like positions in the @v vector. 215 | * @nd Number of data blocks. 216 | * @np Number of parity blocks. 217 | * @size Size of the blocks pointed by @v. It must be a multiplier of 64. 218 | * @v Vector of pointers to the blocks of data and parity. 219 | * It has (@nd + @np) elements. The starting elements are the blocks 220 | * for data, following with the parity blocks. 221 | * Each block has @size bytes. 222 | * @return Number of block indexes returned in the @ir vector. 223 | * 0 if no error is detected. 224 | * -1 if it's not possible to identify the failed disks. 225 | */ 226 | int raid_scan(int *ir, int nd, int np, size_t size, void **v); 227 | 228 | #endif 229 | 230 | -------------------------------------------------------------------------------- /raid/tag.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #include "internal.h" 16 | 17 | typedef void (void_f)(void); 18 | 19 | static struct raid_func { 20 | const char *name; 21 | void_f* func; 22 | } RAID_FUNC[] = { 23 | { "int8", (void_f*)raid_gen3_int8 }, 24 | { "int8", (void_f*)raid_gen4_int8 }, 25 | { "int8", (void_f*)raid_gen5_int8 }, 26 | { "int8", (void_f*)raid_gen6_int8 }, 27 | { "int32", (void_f*)raid_gen1_int32 }, 28 | { "int64", (void_f*)raid_gen1_int64 }, 29 | { "int32", (void_f*)raid_gen2_int32 }, 30 | { "int64", (void_f*)raid_gen2_int64 }, 31 | { "int32", (void_f*)raid_genz_int32 }, 32 | { "int64", (void_f*)raid_genz_int64 }, 33 | { "int8", (void_f*)raid_rec1_int8 }, 34 | { "int8", (void_f*)raid_rec2_int8 }, 35 | { "int8", (void_f*)raid_recX_int8 }, 36 | 37 | #ifdef CONFIG_X86 38 | #ifdef CONFIG_SSE2 39 | { "sse2", (void_f*)raid_gen1_sse2 }, 40 | { "sse2", (void_f*)raid_gen2_sse2 }, 41 | { "sse2", (void_f*)raid_genz_sse2 }, 42 | #endif 43 | #ifdef CONFIG_SSSE3 44 | { "ssse3", (void_f*)raid_gen3_ssse3 }, 45 | { "ssse3", (void_f*)raid_gen4_ssse3 }, 46 | { "ssse3", (void_f*)raid_gen5_ssse3 }, 47 | { "ssse3", (void_f*)raid_gen6_ssse3 }, 48 | { "ssse3", (void_f*)raid_rec1_ssse3 }, 49 | { "ssse3", (void_f*)raid_rec2_ssse3 }, 50 | { "ssse3", (void_f*)raid_recX_ssse3 }, 51 | #endif 52 | #ifdef CONFIG_AVX2 53 | { "avx2", (void_f*)raid_gen1_avx2 }, 54 | { "avx2", (void_f*)raid_gen2_avx2 }, 55 | { "avx2", (void_f*)raid_rec1_avx2 }, 56 | { "avx2", (void_f*)raid_rec2_avx2 }, 57 | { "avx2", (void_f*)raid_recX_avx2 }, 58 | #endif 59 | #endif 60 | 61 | #ifdef CONFIG_X86_64 62 | #ifdef CONFIG_SSE2 63 | { "sse2e", (void_f*)raid_gen2_sse2ext }, 64 | { "sse2e", (void_f*)raid_genz_sse2ext }, 65 | #endif 66 | #ifdef CONFIG_SSSE3 67 | { "ssse3e", (void_f*)raid_gen3_ssse3ext }, 68 | { "ssse3e", (void_f*)raid_gen4_ssse3ext }, 69 | { "ssse3e", (void_f*)raid_gen5_ssse3ext }, 70 | { "ssse3e", (void_f*)raid_gen6_ssse3ext }, 71 | #endif 72 | #ifdef CONFIG_AVX2 73 | { "avx2e", (void_f*)raid_gen3_avx2ext }, 74 | { "avx2e", (void_f*)raid_genz_avx2ext }, 75 | { "avx2e", (void_f*)raid_gen4_avx2ext }, 76 | { "avx2e", (void_f*)raid_gen5_avx2ext }, 77 | { "avx2e", (void_f*)raid_gen6_avx2ext }, 78 | #endif 79 | #endif 80 | { 0, 0 } 81 | }; 82 | 83 | static const char *raid_tag(void_f* func) 84 | { 85 | struct raid_func *i = RAID_FUNC; 86 | 87 | while (i->name != 0) { 88 | if (i->func == func) 89 | return i->name; 90 | ++i; 91 | } 92 | 93 | /* LCOV_EXCL_START */ 94 | return "unknown"; 95 | /* LCOV_EXCL_STOP */ 96 | } 97 | 98 | const char *raid_gen1_tag(void) 99 | { 100 | return raid_tag((void_f*)raid_gen_ptr[0]); 101 | } 102 | 103 | const char *raid_gen2_tag(void) 104 | { 105 | return raid_tag((void_f*)raid_gen_ptr[1]); 106 | } 107 | 108 | const char *raid_genz_tag(void) 109 | { 110 | return raid_tag((void_f*)raid_genz_ptr); 111 | } 112 | 113 | const char *raid_gen3_tag(void) 114 | { 115 | return raid_tag((void_f*)raid_gen_ptr[2]); 116 | } 117 | 118 | const char *raid_gen4_tag(void) 119 | { 120 | return raid_tag((void_f*)raid_gen_ptr[3]); 121 | } 122 | 123 | const char *raid_gen5_tag(void) 124 | { 125 | return raid_tag((void_f*)raid_gen_ptr[4]); 126 | } 127 | 128 | const char *raid_gen6_tag(void) 129 | { 130 | return raid_tag((void_f*)raid_gen_ptr[5]); 131 | } 132 | 133 | const char *raid_rec1_tag(void) 134 | { 135 | return raid_tag((void_f*)raid_rec_ptr[0]); 136 | } 137 | 138 | const char *raid_rec2_tag(void) 139 | { 140 | return raid_tag((void_f*)raid_rec_ptr[1]); 141 | } 142 | 143 | const char *raid_recX_tag(void) 144 | { 145 | return raid_tag((void_f*)raid_rec_ptr[2]); 146 | } 147 | 148 | -------------------------------------------------------------------------------- /raid/test.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | #ifndef __RAID_TEST_H 16 | #define __RAID_TEST_H 17 | 18 | /** 19 | * Tests insertion function. 20 | * 21 | * Test raid_insert() with all the possible combinations of elements to insert. 22 | * 23 | * Returns 0 on success. 24 | */ 25 | int raid_test_insert(void); 26 | 27 | /** 28 | * Tests sorting function. 29 | * 30 | * Test raid_sort() with all the possible combinations of elements to sort. 31 | * 32 | * Returns 0 on success. 33 | */ 34 | int raid_test_sort(void); 35 | 36 | /** 37 | * Tests combination functions. 38 | * 39 | * Tests combination_first() and combination_next() for all the parity levels. 40 | * 41 | * Returns 0 on success. 42 | */ 43 | int raid_test_combo(void); 44 | 45 | /** 46 | * Tests recovering functions. 47 | * 48 | * All the recovering functions are tested with all the combinations 49 | * of failing disks and recovering parities. 50 | * 51 | * Take care that the test time grows exponentially with the number of disks. 52 | * 53 | * Returns 0 on success. 54 | */ 55 | int raid_test_rec(unsigned mode, int nd, size_t size); 56 | 57 | /** 58 | * Tests parity generation functions. 59 | * 60 | * All the parity generation functions are tested with the specified 61 | * number of disks. 62 | * 63 | * Returns 0 on success. 64 | */ 65 | int raid_test_par(unsigned mode, int nd, size_t size); 66 | 67 | #endif 68 | 69 | -------------------------------------------------------------------------------- /raid/test/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Test programs for the RAID library 3 | # 4 | # selftest - Runs the same selftest and speedtest executed at the module startup. 5 | # fulltest - Runs a more extensive test that checks all the built-in functions. 6 | # speedtest - Runs a more complete speed test. 7 | # invtest - Runs an extensive matrix inversion test of all the 377.342.351.231 8 | # possible square submatrices of the Cauchy matrix used. 9 | # covtest - Runs a coverage test. 10 | # sdecovtest - Runs a coverage test with the sde emulator. 11 | # 12 | 13 | MACHINE = $(shell uname -m) 14 | ifeq ($(MACHINE),i686) 15 | SDE = sde 16 | else 17 | SDE = sde64 18 | endif 19 | CC = gcc 20 | LD = ld 21 | CFLAGS = -I.. -Wall -Wextra -g 22 | ifeq ($(COVERAGE),) 23 | CFLAGS += -O2 24 | else 25 | CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1 26 | endif 27 | OBJS = raid.o check.o int.o intz.o x86.o x86z.o tables.o memory.o test.o helper.o module.o tag.o 28 | 29 | %.o: ../%.c 30 | $(CC) $(CFLAGS) -c -o $@ $< 31 | 32 | all: fulltest speedtest selftest invtest 33 | 34 | fulltest: $(OBJS) fulltest.o 35 | $(CC) $(CFLAGS) -o fulltest $^ 36 | 37 | speedtest: $(OBJS) speedtest.o 38 | $(CC) $(CFLAGS) -o speedtest $^ 39 | 40 | selftest: $(OBJS) selftest.o 41 | $(CC) $(CFLAGS) -o selftest $^ 42 | 43 | invtest: $(OBJS) invtest.o 44 | $(CC) $(CFLAGS) -o invtest $^ 45 | 46 | mktables: mktables.o 47 | $(CC) $(CFLAGS) -o mktables $^ 48 | 49 | tables.c: mktables 50 | ./mktables > tables.c 51 | 52 | # Use this target to run a coverage test using lcov 53 | covtest: 54 | $(MAKE) clean 55 | $(MAKE) lcov_reset 56 | $(MAKE) COVERAGE=1 all 57 | ./fulltest 58 | ./selftest 59 | ./speedtest 60 | $(MAKE) lcov_capture 61 | $(MAKE) lcov_html 62 | 63 | # Use this target to run a coverage test using lcov and the sde 64 | sdecovtest: 65 | $(MAKE) clean 66 | $(MAKE) lcov_reset 67 | $(MAKE) COVERAGE=1 all 68 | $(SDE) -p4p -- ./fulltest 69 | $(SDE) -mrm -- ./fulltest 70 | $(SDE) -nhm -- ./fulltest 71 | $(SDE) -hsw -- ./fulltest 72 | $(SDE) -p4p -- ./selftest 73 | $(SDE) -mrm -- ./selftest 74 | $(SDE) -nhm -- ./selftest 75 | $(SDE) -hsw -- ./selftest 76 | $(SDE) -hsw -- ./speedtest 77 | $(MAKE) lcov_capture 78 | $(MAKE) lcov_html 79 | 80 | lcov_reset: 81 | lcov --directory . -z 82 | rm -f lcov.info 83 | 84 | lcov_capture: 85 | lcov --directory . --capture --rc lcov_branch_coverage=1 -o lcov.info 86 | 87 | lcov_html: 88 | rm -rf coverage 89 | mkdir coverage 90 | genhtml --branch-coverage -o coverage lcov.info 91 | 92 | clean: 93 | rm -f *.o mktables tables.c 94 | rm -f *.gcda *.gcno lcov.info 95 | rm -rf coverage 96 | 97 | distclean: clean 98 | rm -f fulltest speedtest selftest invtest 99 | 100 | -------------------------------------------------------------------------------- /raid/test/fulltest.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | /* Full sanity test for the RAID library */ 16 | 17 | #include "internal.h" 18 | #include "test.h" 19 | #include "cpu.h" 20 | 21 | #include 22 | #include 23 | 24 | /* 25 | * Size of the blocks to test. 26 | */ 27 | #define TEST_SIZE 256 28 | 29 | /** 30 | * Number of disks in the long parity test. 31 | */ 32 | #ifdef COVERAGE 33 | #define TEST_COUNT 10 34 | #else 35 | #define TEST_COUNT 32 36 | #endif 37 | 38 | int main(void) 39 | { 40 | printf("Full sanity test for the RAID Cauchy library\n\n"); 41 | 42 | raid_init(); 43 | 44 | #ifdef CONFIG_X86 45 | if (raid_cpu_has_sse2()) 46 | printf("Including x86 SSE2 functions\n"); 47 | if (raid_cpu_has_ssse3()) 48 | printf("Including x86 SSSE3 functions\n"); 49 | if (raid_cpu_has_avx2()) 50 | printf("Including x86 AVX2 functions\n"); 51 | #endif 52 | #ifdef CONFIG_X86_64 53 | printf("Including x64 extended SSE register set\n"); 54 | #endif 55 | 56 | printf("\nPlease wait about 60 seconds...\n\n"); 57 | 58 | printf("Test sorting...\n"); 59 | if (raid_test_sort() != 0) { 60 | /* LCOV_EXCL_START */ 61 | goto bail; 62 | /* LCOV_EXCL_STOP */ 63 | } 64 | 65 | printf("Test insertion...\n"); 66 | if (raid_test_insert() != 0) { 67 | /* LCOV_EXCL_START */ 68 | goto bail; 69 | /* LCOV_EXCL_STOP */ 70 | } 71 | 72 | printf("Test combinations/permutations...\n"); 73 | if (raid_test_combo() != 0) { 74 | /* LCOV_EXCL_START */ 75 | goto bail; 76 | /* LCOV_EXCL_STOP */ 77 | } 78 | 79 | printf("Test Cauchy parity generation with %u data disks...\n", RAID_DATA_MAX); 80 | if (raid_test_par(RAID_MODE_CAUCHY, RAID_DATA_MAX, TEST_SIZE) != 0) { 81 | /* LCOV_EXCL_START */ 82 | goto bail; 83 | /* LCOV_EXCL_STOP */ 84 | } 85 | 86 | printf("Test Cauchy parity generation with 1 data disk...\n"); 87 | if (raid_test_par(RAID_MODE_CAUCHY, 1, TEST_SIZE) != 0) { 88 | /* LCOV_EXCL_START */ 89 | goto bail; 90 | /* LCOV_EXCL_STOP */ 91 | } 92 | 93 | printf("Test Cauchy recovering with all combinations of %u data and 6 parity blocks...\n", TEST_COUNT); 94 | if (raid_test_rec(RAID_MODE_CAUCHY, TEST_COUNT, TEST_SIZE) != 0) { 95 | /* LCOV_EXCL_START */ 96 | goto bail; 97 | /* LCOV_EXCL_STOP */ 98 | } 99 | 100 | printf("Test Vandermonde parity generation with %u data disks...\n", RAID_DATA_MAX); 101 | if (raid_test_par(RAID_MODE_VANDERMONDE, RAID_DATA_MAX, TEST_SIZE) != 0) { 102 | /* LCOV_EXCL_START */ 103 | goto bail; 104 | /* LCOV_EXCL_STOP */ 105 | } 106 | 107 | printf("Test Vandermonde recovering with all combinations of %u data and 3 parity blocks...\n", TEST_COUNT); 108 | if (raid_test_rec(RAID_MODE_VANDERMONDE, TEST_COUNT, TEST_SIZE) != 0) { 109 | /* LCOV_EXCL_START */ 110 | goto bail; 111 | /* LCOV_EXCL_STOP */ 112 | } 113 | 114 | 115 | printf("OK\n"); 116 | return 0; 117 | 118 | bail: 119 | /* LCOV_EXCL_START */ 120 | printf("FAILED!\n"); 121 | exit(EXIT_FAILURE); 122 | /* LCOV_EXCL_STOP */ 123 | } 124 | 125 | -------------------------------------------------------------------------------- /raid/test/invtest.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | /* Matrix inversion test for the RAID library */ 16 | 17 | #include "internal.h" 18 | 19 | #include "combo.h" 20 | #include "gf.h" 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | /** 28 | * Like raid_invert() but optimized to only check if the matrix is 29 | * invertible. 30 | */ 31 | static __always_inline int raid_invert_fast(uint8_t *M, int n) 32 | { 33 | int i, j, k; 34 | 35 | /* for each element in the diagonal */ 36 | for (k = 0; k < n; ++k) { 37 | uint8_t f; 38 | 39 | /* the diagonal element cannot be 0 because */ 40 | /* we are inverting matrices with all the square */ 41 | /* submatrices not singular */ 42 | if (M[k * n + k] == 0) 43 | return -1; 44 | 45 | /* make the diagonal element to be 1 */ 46 | f = inv(M[k * n + k]); 47 | for (j = 0; j < n; ++j) 48 | M[k * n + j] = mul(f, M[k * n + j]); 49 | 50 | /* make all the elements over and under the diagonal */ 51 | /* to be zero */ 52 | for (i = 0; i < n; ++i) { 53 | if (i == k) 54 | continue; 55 | f = M[i * n + k]; 56 | for (j = 0; j < n; ++j) 57 | M[i * n + j] ^= mul(f, M[k * n + j]); 58 | } 59 | } 60 | 61 | return 0; 62 | } 63 | 64 | #define TEST_REFRESH (4 * 1024 * 1024) 65 | 66 | /** 67 | * Precomputed number of square submatrices of size nr. 68 | * 69 | * It's bc(np,nr) * bc(nd,nr) 70 | * 71 | * With 1<=nr<=6 and bc(n, r) == binomial coefficient of (n over r). 72 | */ 73 | long long EXPECTED[RAID_PARITY_MAX] = { 74 | 1506LL, 75 | 470625LL, 76 | 52082500LL, 77 | 2421836250LL, 78 | 47855484300LL, 79 | 327012476050LL 80 | }; 81 | 82 | static __always_inline int test_sub_matrix(int nr, long long *total) 83 | { 84 | uint8_t M[RAID_PARITY_MAX * RAID_PARITY_MAX]; 85 | int np = RAID_PARITY_MAX; 86 | int nd = RAID_DATA_MAX; 87 | int ip[RAID_PARITY_MAX]; 88 | int id[RAID_DATA_MAX]; 89 | long long count; 90 | long long expected; 91 | 92 | printf("\n%ux%u\n", nr, nr); 93 | 94 | count = 0; 95 | expected = EXPECTED[nr - 1]; 96 | 97 | /* all combinations (nr of nd) disks */ 98 | combination_first(nr, nd, id); 99 | do { 100 | /* all combinations (nr of np) parities */ 101 | combination_first(nr, np, ip); 102 | do { 103 | int i, j; 104 | 105 | /* setup the submatrix */ 106 | for (i = 0; i < nr; ++i) 107 | for (j = 0; j < nr; ++j) 108 | M[i * nr + j] = gfgen[ip[i]][id[j]]; 109 | 110 | /* invert */ 111 | if (raid_invert_fast(M, nr) != 0) 112 | return -1; 113 | 114 | if (++count % TEST_REFRESH == 0) { 115 | printf("\r%.3f %%", count * (double)100 / expected); 116 | fflush(stdout); 117 | } 118 | } while (combination_next(nr, np, ip)); 119 | } while (combination_next(nr, nd, id)); 120 | 121 | if (count != expected) 122 | return -1; 123 | 124 | printf("\rTested %" PRIi64 " matrix\n", count); 125 | 126 | *total += count; 127 | 128 | return 0; 129 | } 130 | 131 | int test_all_sub_matrix(void) 132 | { 133 | long long total; 134 | 135 | printf("Invert all square submatrices of the %dx%d Cauchy matrix\n", 136 | RAID_PARITY_MAX, RAID_DATA_MAX); 137 | 138 | printf("\nPlease wait about 2 days...\n"); 139 | 140 | total = 0; 141 | 142 | /* force inlining of everything */ 143 | if (test_sub_matrix(1, &total) != 0) 144 | return -1; 145 | if (test_sub_matrix(2, &total) != 0) 146 | return -1; 147 | if (test_sub_matrix(3, &total) != 0) 148 | return -1; 149 | if (test_sub_matrix(4, &total) != 0) 150 | return -1; 151 | if (test_sub_matrix(5, &total) != 0) 152 | return -1; 153 | if (test_sub_matrix(6, &total) != 0) 154 | return -1; 155 | 156 | printf("\nTested in total %" PRIi64 " matrix\n", total); 157 | 158 | return 0; 159 | } 160 | 161 | int main(void) 162 | { 163 | printf("Matrix inversion test for the RAID Cauchy library\n\n"); 164 | 165 | /* required to set the gfgen table */ 166 | raid_init(); 167 | 168 | if (test_all_sub_matrix() != 0) { 169 | printf("FAILED!\n"); 170 | exit(EXIT_FAILURE); 171 | } 172 | printf("OK\n"); 173 | 174 | return 0; 175 | } 176 | 177 | -------------------------------------------------------------------------------- /raid/test/selftest.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Andrea Mazzoleni 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | */ 14 | 15 | /* Self sanity test for the RAID library */ 16 | 17 | #include "internal.h" 18 | #include "cpu.h" 19 | 20 | #include 21 | #include 22 | 23 | int main(void) 24 | { 25 | printf("Self sanity test for the RAID Cauchy library\n\n"); 26 | 27 | raid_init(); 28 | 29 | printf("Self test...\n"); 30 | if (raid_selftest() != 0) { 31 | /* LCOV_EXCL_START */ 32 | printf("FAILED!\n"); 33 | exit(EXIT_FAILURE); 34 | /* LCOV_EXCL_STOP */ 35 | } 36 | printf("OK\n\n"); 37 | 38 | return 0; 39 | } 40 | 41 | -------------------------------------------------------------------------------- /snapraid-rpm.spec: -------------------------------------------------------------------------------- 1 | Name: snapraid 2 | Summary: Disk array backup for many large rarely-changed files 3 | Version: 12.4 4 | Release: 3%{?dist} 5 | License: GPLv3+ 6 | Group: Applications/System 7 | URL: http://www.snapraid.it/ 8 | Source: https://github.com/amadvance/snapraid/releases/download/v%{version}/snapraid-%{version}.tar.gz 9 | BuildRequires: gcc 10 | 11 | %description 12 | SnapRAID is a backup program for disk arrays. It stores parity 13 | information of your data and it's able to recover from up to six disk 14 | failures. SnapRAID is mainly targeted for a home media center, with a 15 | lot of big files that rarely change. 16 | 17 | %prep 18 | %setup -q 19 | 20 | %build 21 | %configure 22 | make %{?_smp_mflags} 23 | 24 | %check 25 | make check 26 | 27 | %install 28 | make install DESTDIR=%{buildroot} 29 | 30 | %files 31 | %doc COPYING AUTHORS HISTORY README 32 | %{_bindir}/snapraid 33 | %{_mandir}/man1/snapraid.1* 34 | 35 | %changelog 36 | 37 | -------------------------------------------------------------------------------- /snapraid.conf.example: -------------------------------------------------------------------------------- 1 | # Example configuration for snapraid 2 | 3 | # Defines the file to use as parity storage 4 | # It must NOT be in a data disk 5 | # Format: "parity FILE [,FILE] ..." 6 | parity /mnt/diskp/snapraid.parity 7 | 8 | # Defines the files to use as additional parity storage. 9 | # If specified, they enable the multiple failures protection 10 | # from two to six level of parity. 11 | # To enable, uncomment one parity file for each level of extra 12 | # protection required. Start from 2-parity, and follow in order. 13 | # It must NOT be in a data disk 14 | # Format: "X-parity FILE [,FILE] ..." 15 | #2-parity /mnt/diskq/snapraid.2-parity 16 | #3-parity /mnt/diskr/snapraid.3-parity 17 | #4-parity /mnt/disks/snapraid.4-parity 18 | #5-parity /mnt/diskt/snapraid.5-parity 19 | #6-parity /mnt/disku/snapraid.6-parity 20 | 21 | # Defines the files to use as content list 22 | # You can use multiple specification to store more copies 23 | # You must have least one copy for each parity file plus one. Some more don't hurt 24 | # They can be in the disks used for data, parity or boot, 25 | # but each file must be in a different disk 26 | # Format: "content FILE" 27 | content /var/snapraid.content 28 | content /mnt/disk1/snapraid.content 29 | content /mnt/disk2/snapraid.content 30 | 31 | # Defines the data disks to use 32 | # The name and mount point association is relevant for parity, do not change it 33 | # WARNING: Adding here your /home, /var or /tmp disks is NOT a good idea! 34 | # SnapRAID is better suited for files that rarely changes! 35 | # Format: "data DISK_NAME DISK_MOUNT_POINT" 36 | data d1 /mnt/disk1/ 37 | data d2 /mnt/disk2/ 38 | data d3 /mnt/disk3/ 39 | 40 | # Excludes hidden files and directories (uncomment to enable). 41 | #nohidden 42 | 43 | # Defines files and directories to exclude 44 | # Remember that all the paths are relative at the mount points 45 | # Format: "exclude FILE" 46 | # Format: "exclude DIR/" 47 | # Format: "exclude /PATH/FILE" 48 | # Format: "exclude /PATH/DIR/" 49 | exclude *.unrecoverable 50 | exclude /tmp/ 51 | exclude /lost+found/ 52 | 53 | # Defines the block size in kibi bytes (1024 bytes) (uncomment to enable). 54 | # WARNING: Changing this value is for experts only! 55 | # Default value is 256 -> 256 kibi bytes -> 262144 bytes 56 | # Format: "blocksize SIZE_IN_KiB" 57 | #blocksize 256 58 | 59 | # Defines the hash size in bytes (uncomment to enable). 60 | # WARNING: Changing this value is for experts only! 61 | # Default value is 16 -> 128 bits 62 | # Format: "hashsize SIZE_IN_BYTES" 63 | #hashsize 16 64 | 65 | # Automatically save the state when syncing after the specified amount 66 | # of GB processed (uncomment to enable). 67 | # This option is useful to avoid to restart from scratch long 'sync' 68 | # commands interrupted by a machine crash. 69 | # It also improves the recovering if a disk break during a 'sync'. 70 | # Default value is 0, meaning disabled. 71 | # Format: "autosave SIZE_IN_GB" 72 | #autosave 500 73 | 74 | # Defines the pooling directory where the virtual view of the disk 75 | # array is created using the "pool" command (uncomment to enable). 76 | # The files are not really copied here, but just linked using 77 | # symbolic links. 78 | # This directory must be outside the array. 79 | # Format: "pool DIR" 80 | #pool /pool 81 | 82 | # Defines a custom smartctl command to obtain the SMART attributes 83 | # for each disk. This may be required for RAID controllers and for 84 | # some USB disk that cannot be autodetected. 85 | # In the specified options, the "%s" string is replaced by the device name. 86 | # Refers at the smartmontools documentation about the possible options: 87 | # RAID -> https://www.smartmontools.org/wiki/Supported_RAID-Controllers 88 | # USB -> https://www.smartmontools.org/wiki/Supported_USB-Devices 89 | #smartctl d1 -d sat %s 90 | #smartctl d2 -d usbjmicron %s 91 | #smartctl parity -d areca,1/1 /dev/sg0 92 | #smartctl 2-parity -d areca,2/1 /dev/sg0 93 | 94 | -------------------------------------------------------------------------------- /snapraid.conf.example.windows: -------------------------------------------------------------------------------- 1 | # Example configuration for snapraid for Windows 2 | 3 | # Defines the file to use as parity storage 4 | # It must NOT be in a data disk 5 | # Format: "parity FILE [,FILE] ..." 6 | parity E:\snapraid.parity 7 | 8 | # Defines the files to use as additional parity storage. 9 | # If specified, they enable the multiple failures protection 10 | # from two to six level of parity. 11 | # To enable, uncomment one parity file for each level of extra 12 | # protection required. Start from 2-parity, and follow in order. 13 | # It must NOT be in a data disk 14 | # Format: "X-parity FILE [,FILE] ..." 15 | #2-parity F:\snapraid.2-parity 16 | #3-parity G:\snapraid.3-parity 17 | #4-parity H:\snapraid.4-parity 18 | #5-parity I:\snapraid.5-parity 19 | #6-parity J:\snapraid.6-parity 20 | 21 | # Defines the files to use as content list 22 | # You can use multiple specification to store more copies 23 | # You must have least one copy for each parity file plus one. Some more don't hurt 24 | # They can be in the disks used for data, parity or boot, 25 | # but each file must be in a different disk 26 | # Format: "content FILE" 27 | content C:\snapraid\snapraid.content 28 | content K:\array\snapraid.content 29 | content L:\array\snapraid.content 30 | 31 | # Defines the data disks to use 32 | # The name and mount point association is relevant for parity, do not change it 33 | # WARNING: Adding here your boot C:\ disk is NOT a good idea! 34 | # SnapRAID is better suited for files that rarely changes! 35 | # Format: "data DISK_NAME DISK_MOUNT_POINT" 36 | data d1 K:\array\ 37 | data d2 L:\array\ 38 | data d3 M:\array\ 39 | 40 | # Excludes hidden files and directories (uncomment to enable). 41 | #nohidden 42 | 43 | # Defines files and directories to exclude 44 | # Remember that all the paths are relative at the mount points 45 | # Format: "exclude FILE" 46 | # Format: "exclude DIR\" 47 | # Format: "exclude \PATH\FILE" 48 | # Format: "exclude \PATH\DIR\" 49 | exclude *.unrecoverable 50 | exclude Thumbs.db 51 | exclude \$RECYCLE.BIN 52 | exclude \System Volume Information 53 | exclude \Program Files\ 54 | exclude \Program Files (x86)\ 55 | exclude \Windows\ 56 | 57 | # Defines the block size in kibi bytes (1024 bytes) (uncomment to enable). 58 | # WARNING: Changing this value is for experts only! 59 | # Default value is 256 -> 256 kibi bytes -> 262144 bytes 60 | # Format: "blocksize SIZE_IN_KiB" 61 | #blocksize 256 62 | 63 | # Defines the hash size in bytes (uncomment to enable). 64 | # WARNING: Changing this value is for experts only! 65 | # Default value is 16 -> 128 bits 66 | # Format: "hashsize SIZE_IN_BYTES" 67 | #hashsize 16 68 | 69 | # Automatically save the state when syncing after the specified amount 70 | # of GB processed (uncomment to enable). 71 | # This option is useful to avoid to restart from scratch long 'sync' 72 | # commands interrupted by a machine crash. 73 | # It also improves the recovering if a disk break during a 'sync'. 74 | # Default value is 0, meaning disabled. 75 | # Format: "autosave SIZE_IN_GB" 76 | #autosave 500 77 | 78 | # Defines the pooling directory where the virtual view of the disk 79 | # array is created using the "pool" command (uncomment to enable). 80 | # The files are not really copied here, but just linked using 81 | # symbolic links. 82 | # This directory must be outside the array. 83 | # Format: "pool DIR" 84 | #pool C:\pool 85 | 86 | # Defines the Windows UNC path required to access disks from the pooling 87 | # directory when shared in the network. 88 | # If present (uncomment to enable), the symbolic links created in the 89 | # pool virtual view, instead of using local paths, are created using the 90 | # specified UNC path, adding the disk names and file path. 91 | # This allows to share the pool directory in the network. 92 | # See the manual page for more details. 93 | # 94 | # Format: "share UNC_DIR" 95 | #share \\server 96 | 97 | # Defines a custom smartctl command to obtain the SMART attributes 98 | # for each disk. This may be required for RAID controllers and for 99 | # some USB disk that cannot be autodetected. 100 | # In the specified options, the "%s" string is replaced by the device name. 101 | # Refers at the smartmontools documentation about the possible options: 102 | # RAID -> https://www.smartmontools.org/wiki/Supported_RAID-Controllers 103 | # USB -> https://www.smartmontools.org/wiki/Supported_USB-Devices 104 | #smartctl d1 -d sat %s 105 | #smartctl d2 -d usbjmicron %s 106 | #smartctl parity -d areca,1/1 /dev/arcmsr0 107 | #smartctl 2-parity -d areca,2/1 /dev/arcmsr0 108 | 109 | -------------------------------------------------------------------------------- /test/test-par1.conf: -------------------------------------------------------------------------------- 1 | # Test configuration file 2 | blocksize 1 3 | parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 4 | content bench/content 5 | content bench/1-content 6 | disk disk1 bench/disk1/ 7 | disk disk2 bench/disk2/ 8 | disk disk3 bench/disk3/ 9 | disk disk4 bench/disk4/ 10 | disk disk5 bench/disk5/ 11 | disk disk6 bench/disk6/ 12 | include *.hidden 13 | exclude *.unrecoverable 14 | pool bench/pool 15 | share \\server\jbod 16 | autosave 1 17 | 18 | -------------------------------------------------------------------------------- /test/test-par2.conf: -------------------------------------------------------------------------------- 1 | blocksize 1 2 | parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 3 | 2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 4 | content bench/content 5 | content bench/1-content 6 | content bench/2-content 7 | disk disk1 bench/disk1/ 8 | disk disk2 bench/disk2/ 9 | disk disk3 bench/disk3/ 10 | disk disk4 bench/disk4/ 11 | disk disk5 bench/disk5/ 12 | disk disk6 bench/disk6/ 13 | include *.hidden 14 | exclude *.unrecoverable 15 | 16 | -------------------------------------------------------------------------------- /test/test-par3.conf: -------------------------------------------------------------------------------- 1 | blocksize 1 2 | parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 3 | 2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 4 | 3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 5 | content bench/content 6 | content bench/1-content 7 | content bench/2-content 8 | content bench/3-content 9 | disk disk1 bench/disk1/ 10 | disk disk2 bench/disk2/ 11 | disk disk3 bench/disk3/ 12 | disk disk4 bench/disk4/ 13 | disk disk5 bench/disk5/ 14 | disk disk6 bench/disk6/ 15 | include *.hidden 16 | exclude *.unrecoverable 17 | 18 | -------------------------------------------------------------------------------- /test/test-par4.conf: -------------------------------------------------------------------------------- 1 | blocksize 1 2 | parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 3 | 2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 4 | 3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 5 | 4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 6 | content bench/content 7 | content bench/1-content 8 | content bench/2-content 9 | content bench/3-content 10 | content bench/4-content 11 | disk disk1 bench/disk1/ 12 | disk disk2 bench/disk2/ 13 | disk disk3 bench/disk3/ 14 | disk disk4 bench/disk4/ 15 | disk disk5 bench/disk5/ 16 | disk disk6 bench/disk6/ 17 | include *.hidden 18 | exclude *.unrecoverable 19 | 20 | -------------------------------------------------------------------------------- /test/test-par5.conf: -------------------------------------------------------------------------------- 1 | blocksize 1 2 | parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 3 | 2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 4 | 3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 5 | 4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 6 | 5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 7 | content bench/content 8 | content bench/1-content 9 | content bench/2-content 10 | content bench/3-content 11 | content bench/4-content 12 | content bench/5-content 13 | disk disk1 bench/disk1/ 14 | disk disk2 bench/disk2/ 15 | disk disk3 bench/disk3/ 16 | disk disk4 bench/disk4/ 17 | disk disk5 bench/disk5/ 18 | disk disk6 bench/disk6/ 19 | include *.hidden 20 | exclude *.unrecoverable 21 | 22 | -------------------------------------------------------------------------------- /test/test-par6-hole.conf: -------------------------------------------------------------------------------- 1 | blocksize 1 2 | parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 3 | 2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 4 | 3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 5 | 4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 6 | 5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 7 | 6-parity bench/6-parity.0,bench/6-parity.1,bench/6-parity.2,bench/6-parity.3 8 | content bench/content 9 | content bench/1-content 10 | content bench/2-content 11 | content bench/3-content 12 | content bench/4-content 13 | content bench/5-content 14 | content bench/6-content 15 | disk disk1 bench/disk1/ 16 | disk disk3 bench/disk3/ 17 | disk disk4 bench/disk4/ 18 | disk disk5 bench/disk5/ 19 | disk disk6 bench/disk6/ 20 | include *.hidden 21 | exclude *.unrecoverable 22 | 23 | -------------------------------------------------------------------------------- /test/test-par6-noaccess.conf: -------------------------------------------------------------------------------- 1 | blocksize 1 2 | parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 3 | 2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 4 | 3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 5 | 4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 6 | 5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 7 | 6-parity bench/no-access/6-parity.0,bench/no-access/6-parity.1,bench/no-access/6-parity.2,bench/no-access/6-parity.3 8 | content bench/content 9 | content bench/1-content 10 | content bench/2-content 11 | content bench/3-content 12 | content bench/4-content 13 | content bench/5-content 14 | content bench/no-access/6-content 15 | disk disk1 bench/disk1/ 16 | disk disk2 bench/disk2/ 17 | disk disk3 bench/disk3/ 18 | disk disk4 bench/disk4/ 19 | disk disk5 bench/disk5/ 20 | disk disk6 bench/disk6/no-access/ 21 | include *.hidden 22 | exclude *.unrecoverable 23 | smartctl disk1 %s 24 | smartctl parity /dev/sda 25 | 26 | -------------------------------------------------------------------------------- /test/test-par6-rename.conf: -------------------------------------------------------------------------------- 1 | blocksize 1 2 | parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 3 | 2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 4 | 3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 5 | 4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 6 | 5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 7 | 6-parity bench/6-parity.0,bench/6-parity.1,bench/6-parity.2,bench/6-parity.3 8 | content bench/content 9 | content bench/1-content 10 | content bench/2-content 11 | content bench/3-content 12 | content bench/4-content 13 | content bench/5-content 14 | content bench/6-content 15 | disk disk1-rename bench/disk1/ 16 | disk disk2 bench/disk2/ 17 | disk disk3 bench/disk3/ 18 | disk disk4 bench/disk4/ 19 | disk disk5 bench/disk5/ 20 | disk disk6 bench/disk6/ 21 | include *.hidden 22 | exclude *.unrecoverable 23 | 24 | 25 | -------------------------------------------------------------------------------- /test/test-par6.conf: -------------------------------------------------------------------------------- 1 | blocksize 1 2 | parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 3 | 2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 4 | 3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 5 | 4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 6 | 5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 7 | 6-parity bench/6-parity.0,bench/6-parity.1,bench/6-parity.2,bench/6-parity.3 8 | content bench/content 9 | content bench/1-content 10 | content bench/2-content 11 | content bench/3-content 12 | content bench/4-content 13 | content bench/5-content 14 | content bench/6-content 15 | disk disk1 bench/disk1/ 16 | disk disk2 bench/disk2/ 17 | disk disk3 bench/disk3/ 18 | disk disk4 bench/disk4/ 19 | disk disk5 bench/disk5/ 20 | disk disk6 bench/disk6/ 21 | include *.hidden 22 | exclude *.unrecoverable 23 | smartctl disk1 %s 24 | smartctl parity /dev/sda 25 | 26 | -------------------------------------------------------------------------------- /tommyds/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions 5 | are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 18 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 21 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 22 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 23 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /tommyds/tommy.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /* redefine the malloc for tommy use */ 29 | #define tommy_malloc malloc_nofail 30 | #define tommy_calloc calloc_nofail 31 | #define tommy_free free 32 | 33 | #include "cmdline/portable.h" 34 | #include "cmdline/support.h" /* for malloc/calloc_nofail() */ 35 | 36 | #include "tommyhash.c" 37 | #include "tommyarray.c" 38 | #include "tommyarrayblkof.c" 39 | #include "tommylist.c" 40 | #include "tommytree.c" 41 | #include "tommyhashdyn.c" 42 | 43 | -------------------------------------------------------------------------------- /tommyds/tommyarray.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyarray.h" 29 | 30 | /******************************************************************************/ 31 | /* array */ 32 | 33 | void tommy_array_init(tommy_array* array) 34 | { 35 | tommy_uint_t i; 36 | 37 | /* fixed initial size */ 38 | array->bucket_bit = TOMMY_ARRAY_BIT; 39 | array->bucket_max = (tommy_size_t)1 << array->bucket_bit; 40 | array->bucket[0] = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); 41 | for (i = 1; i < TOMMY_ARRAY_BIT; ++i) 42 | array->bucket[i] = array->bucket[0]; 43 | 44 | array->count = 0; 45 | } 46 | 47 | void tommy_array_done(tommy_array* array) 48 | { 49 | tommy_uint_t i; 50 | 51 | tommy_free(array->bucket[0]); 52 | for (i = TOMMY_ARRAY_BIT; i < array->bucket_bit; ++i) { 53 | void** segment = array->bucket[i]; 54 | tommy_free(&segment[(tommy_ptrdiff_t)1 << i]); 55 | } 56 | } 57 | 58 | void tommy_array_grow(tommy_array* array, tommy_size_t count) 59 | { 60 | if (array->count >= count) 61 | return; 62 | array->count = count; 63 | 64 | while (count > array->bucket_max) { 65 | void** segment; 66 | 67 | /* allocate one more segment */ 68 | segment = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); 69 | 70 | /* store it adjusting the offset */ 71 | /* cast to ptrdiff_t to ensure to get a negative value */ 72 | array->bucket[array->bucket_bit] = &segment[-(tommy_ptrdiff_t)array->bucket_max]; 73 | 74 | ++array->bucket_bit; 75 | array->bucket_max = (tommy_size_t)1 << array->bucket_bit; 76 | } 77 | } 78 | 79 | tommy_size_t tommy_array_memory_usage(tommy_array* array) 80 | { 81 | return array->bucket_max * (tommy_size_t)sizeof(void*); 82 | } 83 | 84 | -------------------------------------------------------------------------------- /tommyds/tommyarray.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Dynamic array based on segments of exponential growing size. 30 | * 31 | * This array is able to grow dynamically upon request, without any reallocation. 32 | * 33 | * The grow operation involves an allocation of a new array segment, without reallocating 34 | * the already used memory, and then not increasing the heap fragmentation. 35 | * This also implies that the address of the stored elements never change. 36 | * 37 | * Allocated segments grow in size exponentially. 38 | */ 39 | 40 | #ifndef __TOMMYARRAY_H 41 | #define __TOMMYARRAY_H 42 | 43 | #include "tommytypes.h" 44 | 45 | #include /* for assert */ 46 | 47 | /******************************************************************************/ 48 | /* array */ 49 | 50 | /** 51 | * Initial and minimal size of the array expressed as a power of 2. 52 | * The initial size is 2^TOMMY_ARRAY_BIT. 53 | */ 54 | #define TOMMY_ARRAY_BIT 6 55 | 56 | /** 57 | * Array container type. 58 | * \note Don't use internal fields directly, but access the container only using functions. 59 | */ 60 | typedef struct tommy_array_struct { 61 | void** bucket[TOMMY_SIZE_BIT]; /**< Dynamic array of buckets. */ 62 | tommy_size_t bucket_max; /**< Number of buckets. */ 63 | tommy_size_t count; /**< Number of initialized elements in the array. */ 64 | tommy_uint_t bucket_bit; /**< Bits used in the bit mask. */ 65 | } tommy_array; 66 | 67 | /** 68 | * Initializes the array. 69 | */ 70 | void tommy_array_init(tommy_array* array); 71 | 72 | /** 73 | * Deinitializes the array. 74 | */ 75 | void tommy_array_done(tommy_array* array); 76 | 77 | /** 78 | * Grows the size up to the specified value. 79 | * All the new elements in the array are initialized with the 0 value. 80 | */ 81 | void tommy_array_grow(tommy_array* array, tommy_size_t size); 82 | 83 | /** 84 | * Gets a reference of the element at the specified position. 85 | * You must be sure that space for this position is already 86 | * allocated calling tommy_array_grow(). 87 | */ 88 | tommy_inline void** tommy_array_ref(tommy_array* array, tommy_size_t pos) 89 | { 90 | tommy_uint_t bsr; 91 | 92 | assert(pos < array->count); 93 | 94 | /* get the highest bit set, in case of all 0, return 0 */ 95 | bsr = tommy_ilog2(pos | 1); 96 | 97 | return &array->bucket[bsr][pos]; 98 | } 99 | 100 | /** 101 | * Sets the element at the specified position. 102 | * You must be sure that space for this position is already 103 | * allocated calling tommy_array_grow(). 104 | */ 105 | tommy_inline void tommy_array_set(tommy_array* array, tommy_size_t pos, void* element) 106 | { 107 | *tommy_array_ref(array, pos) = element; 108 | } 109 | 110 | /** 111 | * Gets the element at the specified position. 112 | * You must be sure that space for this position is already 113 | * allocated calling tommy_array_grow(). 114 | */ 115 | tommy_inline void* tommy_array_get(tommy_array* array, tommy_size_t pos) 116 | { 117 | return *tommy_array_ref(array, pos); 118 | } 119 | 120 | /** 121 | * Grows and inserts a new element at the end of the array. 122 | */ 123 | tommy_inline void tommy_array_insert(tommy_array* array, void* element) 124 | { 125 | tommy_size_t pos = array->count; 126 | 127 | tommy_array_grow(array, pos + 1); 128 | 129 | tommy_array_set(array, pos, element); 130 | } 131 | 132 | /** 133 | * Gets the initialized size of the array. 134 | */ 135 | tommy_inline tommy_size_t tommy_array_size(tommy_array* array) 136 | { 137 | return array->count; 138 | } 139 | 140 | /** 141 | * Gets the size of allocated memory. 142 | */ 143 | tommy_size_t tommy_array_memory_usage(tommy_array* array); 144 | 145 | #endif 146 | 147 | -------------------------------------------------------------------------------- /tommyds/tommyarrayblkof.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyarrayblkof.h" 29 | 30 | /******************************************************************************/ 31 | /* array */ 32 | 33 | void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size) 34 | { 35 | tommy_array_init(&array->block); 36 | 37 | array->element_size = element_size; 38 | array->count = 0; 39 | } 40 | 41 | void tommy_arrayblkof_done(tommy_arrayblkof* array) 42 | { 43 | tommy_size_t i; 44 | 45 | for (i = 0; i < tommy_array_size(&array->block); ++i) 46 | tommy_free(tommy_array_get(&array->block, i)); 47 | 48 | tommy_array_done(&array->block); 49 | } 50 | 51 | void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t count) 52 | { 53 | tommy_size_t block_max; 54 | tommy_size_t block_mac; 55 | 56 | if (array->count >= count) 57 | return; 58 | array->count = count; 59 | 60 | block_max = (count + TOMMY_ARRAYBLKOF_SIZE - 1) / TOMMY_ARRAYBLKOF_SIZE; 61 | block_mac = tommy_array_size(&array->block); 62 | 63 | if (block_mac < block_max) { 64 | /* grow the block array */ 65 | tommy_array_grow(&array->block, block_max); 66 | 67 | /* allocate new blocks */ 68 | while (block_mac < block_max) { 69 | void** ptr = tommy_cast(void**, tommy_calloc(TOMMY_ARRAYBLKOF_SIZE, array->element_size)); 70 | 71 | /* set the new block */ 72 | tommy_array_set(&array->block, block_mac, ptr); 73 | 74 | ++block_mac; 75 | } 76 | } 77 | } 78 | 79 | tommy_size_t tommy_arrayblkof_memory_usage(tommy_arrayblkof* array) 80 | { 81 | return tommy_array_memory_usage(&array->block) + tommy_array_size(&array->block) * TOMMY_ARRAYBLKOF_SIZE * array->element_size; 82 | } 83 | 84 | -------------------------------------------------------------------------------- /tommyds/tommyarrayblkof.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Dynamic array based on blocks of fixed size. 30 | * 31 | * This array is able to grow dynamically upon request, without any reallocation. 32 | * 33 | * This is very similar at ::tommy_arrayblk, but it allows to store elements of any 34 | * size and not just pointers. 35 | * 36 | * Note that in this case tommy_arrayblkof_ref() returns a pointer to the element, 37 | * that should be used for getting and setting elements in the array, 38 | * as generic getter and setter are not available. 39 | */ 40 | 41 | #ifndef __TOMMYARRAYBLKOF_H 42 | #define __TOMMYARRAYBLKOF_H 43 | 44 | #include "tommytypes.h" 45 | #include "tommyarray.h" 46 | 47 | #include /* for assert */ 48 | 49 | /******************************************************************************/ 50 | /* array */ 51 | 52 | /** 53 | * Elements for each block. 54 | */ 55 | #define TOMMY_ARRAYBLKOF_SIZE (4 * 1024) 56 | 57 | /** 58 | * Array container type. 59 | * \note Don't use internal fields directly, but access the container only using functions. 60 | */ 61 | typedef struct tommy_arrayblkof_struct { 62 | tommy_array block; /**< Array of blocks. */ 63 | tommy_size_t element_size; /**< Size of the stored element in bytes. */ 64 | tommy_size_t count; /**< Number of initialized elements in the array. */ 65 | } tommy_arrayblkof; 66 | 67 | /** 68 | * Initializes the array. 69 | * \param element_size Size in byte of the element to store in the array. 70 | */ 71 | void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size); 72 | 73 | /** 74 | * Deinitializes the array. 75 | */ 76 | void tommy_arrayblkof_done(tommy_arrayblkof* array); 77 | 78 | /** 79 | * Grows the size up to the specified value. 80 | * All the new elements in the array are initialized with the 0 value. 81 | */ 82 | void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t size); 83 | 84 | /** 85 | * Gets a reference of the element at the specified position. 86 | * You must be sure that space for this position is already 87 | * allocated calling tommy_arrayblkof_grow(). 88 | */ 89 | tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_size_t pos) 90 | { 91 | unsigned char* base; 92 | 93 | assert(pos < array->count); 94 | 95 | base = tommy_cast(unsigned char*, tommy_array_get(&array->block, pos / TOMMY_ARRAYBLKOF_SIZE)); 96 | 97 | return base + (pos % TOMMY_ARRAYBLKOF_SIZE) * array->element_size; 98 | } 99 | 100 | /** 101 | * Gets the initialized size of the array. 102 | */ 103 | tommy_inline tommy_size_t tommy_arrayblkof_size(tommy_arrayblkof* array) 104 | { 105 | return array->count; 106 | } 107 | 108 | /** 109 | * Gets the size of allocated memory. 110 | */ 111 | tommy_size_t tommy_arrayblkof_memory_usage(tommy_arrayblkof* array); 112 | 113 | #endif 114 | 115 | -------------------------------------------------------------------------------- /tommyds/tommychain.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Chain of nodes. 30 | * A chain of nodes is an abstraction used to implements complex list operations 31 | * like sorting. 32 | * 33 | * Do not use this directly. Use lists instead. 34 | */ 35 | 36 | #ifndef __TOMMYCHAIN_H 37 | #define __TOMMYCHAIN_H 38 | 39 | #include "tommytypes.h" 40 | 41 | /******************************************************************************/ 42 | /* chain */ 43 | 44 | /** 45 | * Chain of nodes. 46 | * A chain of nodes is a sequence of nodes with the following properties: 47 | * - It contains at least one node. A chains of zero nodes cannot exist. 48 | * - The next field of the tail is of *undefined* value. 49 | * - The prev field of the head is of *undefined* value. 50 | * - All the other inner prev and next fields are correctly set. 51 | */ 52 | typedef struct tommy_chain_struct { 53 | tommy_node* head; /**< Pointer to the head of the chain. */ 54 | tommy_node* tail; /**< Pointer to the tail of the chain. */ 55 | } tommy_chain; 56 | 57 | /** 58 | * Splices a chain in the middle of another chain. 59 | */ 60 | tommy_inline void tommy_chain_splice(tommy_node* first_before, tommy_node* first_after, tommy_node* second_head, tommy_node* second_tail) 61 | { 62 | /* set the prev list */ 63 | first_after->prev = second_tail; 64 | second_head->prev = first_before; 65 | 66 | /* set the next list */ 67 | first_before->next = second_head; 68 | second_tail->next = first_after; 69 | } 70 | 71 | /** 72 | * Concats two chains. 73 | */ 74 | tommy_inline void tommy_chain_concat(tommy_node* first_tail, tommy_node* second_head) 75 | { 76 | /* set the prev list */ 77 | second_head->prev = first_tail; 78 | 79 | /* set the next list */ 80 | first_tail->next = second_head; 81 | } 82 | 83 | /** 84 | * Merges two chains. 85 | */ 86 | tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) 87 | { 88 | tommy_node* first_i = first->head; 89 | tommy_node* second_i = second->head; 90 | 91 | /* merge */ 92 | while (1) { 93 | if (cmp(first_i->data, second_i->data) > 0) { 94 | tommy_node* next = second_i->next; 95 | if (first_i == first->head) { 96 | tommy_chain_concat(second_i, first_i); 97 | first->head = second_i; 98 | } else { 99 | tommy_chain_splice(first_i->prev, first_i, second_i, second_i); 100 | } 101 | if (second_i == second->tail) 102 | break; 103 | second_i = next; 104 | } else { 105 | if (first_i == first->tail) { 106 | tommy_chain_concat(first_i, second_i); 107 | first->tail = second->tail; 108 | break; 109 | } 110 | first_i = first_i->next; 111 | } 112 | } 113 | } 114 | 115 | /** 116 | * Merges two chains managing special degenerated cases. 117 | * It's functionally equivalent at tommy_chain_merge() but faster with already ordered chains. 118 | */ 119 | tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) 120 | { 121 | /* identify the condition first <= second */ 122 | if (cmp(first->tail->data, second->head->data) <= 0) { 123 | tommy_chain_concat(first->tail, second->head); 124 | first->tail = second->tail; 125 | return; 126 | } 127 | 128 | /* identify the condition second < first */ 129 | /* here we must be strict on comparison to keep the sort stable */ 130 | if (cmp(second->tail->data, first->head->data) < 0) { 131 | tommy_chain_concat(second->tail, first->head); 132 | first->head = second->head; 133 | return; 134 | } 135 | 136 | tommy_chain_merge(first, second, cmp); 137 | } 138 | 139 | /** 140 | * Sorts a chain. 141 | * It's a stable merge sort using power of 2 buckets, with O(N*log(N)) complexity, 142 | * similar at the one used in the SGI STL libraries and in the Linux Kernel, 143 | * but faster on degenerated cases like already ordered lists. 144 | * 145 | * SGI STL stl_list.h 146 | * http://www.sgi.com/tech/stl/stl_list.h 147 | * 148 | * Linux Kernel lib/list_sort.c 149 | * http://lxr.linux.no/#linux+v2.6.36/lib/list_sort.c 150 | */ 151 | tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* cmp) 152 | { 153 | /* 154 | * Bit buckets of chains. 155 | * Each bucket contains 2^i nodes or it's empty. 156 | * The chain at address TOMMY_BIT_MAX is an independent variable operating as "carry". 157 | * We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck. 158 | */ 159 | tommy_chain bit[TOMMY_SIZE_BIT + 1]; 160 | 161 | /** 162 | * Value stored inside the bit bucket. 163 | * It's used to know which bucket is empty of full. 164 | */ 165 | tommy_size_t counter; 166 | tommy_node* node = chain->head; 167 | tommy_node* tail = chain->tail; 168 | tommy_size_t mask; 169 | tommy_size_t i; 170 | 171 | counter = 0; 172 | while (1) { 173 | tommy_node* next; 174 | tommy_chain* last; 175 | 176 | /* carry bit to add */ 177 | last = &bit[TOMMY_SIZE_BIT]; 178 | bit[TOMMY_SIZE_BIT].head = node; 179 | bit[TOMMY_SIZE_BIT].tail = node; 180 | next = node->next; 181 | 182 | /* add the bit, propagating the carry */ 183 | i = 0; 184 | mask = counter; 185 | while ((mask & 1) != 0) { 186 | tommy_chain_merge_degenerated(&bit[i], last, cmp); 187 | mask >>= 1; 188 | last = &bit[i]; 189 | ++i; 190 | } 191 | 192 | /* copy the carry in the first empty bit */ 193 | bit[i] = *last; 194 | 195 | /* add the carry in the counter */ 196 | ++counter; 197 | 198 | if (node == tail) 199 | break; 200 | node = next; 201 | } 202 | 203 | /* merge the buckets */ 204 | i = tommy_ctz(counter); 205 | mask = counter >> i; 206 | while (mask != 1) { 207 | mask >>= 1; 208 | if (mask & 1) 209 | tommy_chain_merge_degenerated(&bit[i + 1], &bit[i], cmp); 210 | else 211 | bit[i + 1] = bit[i]; 212 | ++i; 213 | } 214 | 215 | *chain = bit[i]; 216 | } 217 | 218 | #endif 219 | 220 | -------------------------------------------------------------------------------- /tommyds/tommyhash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyhash.h" 29 | 30 | /******************************************************************************/ 31 | /* hash */ 32 | 33 | tommy_inline tommy_uint32_t tommy_le_uint32_read(const void* ptr) 34 | { 35 | /* allow unaligned read on Intel x86 and x86_64 platforms */ 36 | #if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || defined(__x86_64__) || defined(_M_X64) 37 | /* defines from http://predef.sourceforge.net/ */ 38 | return *(const tommy_uint32_t*)ptr; 39 | #else 40 | const unsigned char* ptr8 = tommy_cast(const unsigned char*, ptr); 41 | return ptr8[0] + ((tommy_uint32_t)ptr8[1] << 8) + ((tommy_uint32_t)ptr8[2] << 16) + ((tommy_uint32_t)ptr8[3] << 24); 42 | #endif 43 | } 44 | 45 | #define tommy_rot(x, k) \ 46 | (((x) << (k)) | ((x) >> (32 - (k)))) 47 | 48 | #define tommy_mix(a, b, c) \ 49 | do { \ 50 | a -= c; a ^= tommy_rot(c, 4); c += b; \ 51 | b -= a; b ^= tommy_rot(a, 6); a += c; \ 52 | c -= b; c ^= tommy_rot(b, 8); b += a; \ 53 | a -= c; a ^= tommy_rot(c, 16); c += b; \ 54 | b -= a; b ^= tommy_rot(a, 19); a += c; \ 55 | c -= b; c ^= tommy_rot(b, 4); b += a; \ 56 | } while (0) 57 | 58 | #define tommy_final(a, b, c) \ 59 | do { \ 60 | c ^= b; c -= tommy_rot(b, 14); \ 61 | a ^= c; a -= tommy_rot(c, 11); \ 62 | b ^= a; b -= tommy_rot(a, 25); \ 63 | c ^= b; c -= tommy_rot(b, 16); \ 64 | a ^= c; a -= tommy_rot(c, 4); \ 65 | b ^= a; b -= tommy_rot(a, 14); \ 66 | c ^= b; c -= tommy_rot(b, 24); \ 67 | } while (0) 68 | 69 | tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len) 70 | { 71 | const unsigned char* key = tommy_cast(const unsigned char*, void_key); 72 | tommy_uint32_t a, b, c; 73 | 74 | a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + init_val; 75 | 76 | while (key_len > 12) { 77 | a += tommy_le_uint32_read(key + 0); 78 | b += tommy_le_uint32_read(key + 4); 79 | c += tommy_le_uint32_read(key + 8); 80 | 81 | tommy_mix(a, b, c); 82 | 83 | key_len -= 12; 84 | key += 12; 85 | } 86 | 87 | switch (key_len) { 88 | case 0 : 89 | return c; /* used only when called with a zero length */ 90 | case 12 : 91 | c += tommy_le_uint32_read(key + 8); 92 | b += tommy_le_uint32_read(key + 4); 93 | a += tommy_le_uint32_read(key + 0); 94 | break; 95 | case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ 96 | case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ 97 | case 9 : c += key[8]; /* fallthrough */ 98 | case 8 : 99 | b += tommy_le_uint32_read(key + 4); 100 | a += tommy_le_uint32_read(key + 0); 101 | break; 102 | case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ 103 | case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ 104 | case 5 : b += key[4]; /* fallthrough */ 105 | case 4 : 106 | a += tommy_le_uint32_read(key + 0); 107 | break; 108 | case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ 109 | case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ 110 | case 1 : a += key[0]; /* fallthrough */ 111 | } 112 | 113 | tommy_final(a, b, c); 114 | 115 | return c; 116 | } 117 | 118 | tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len) 119 | { 120 | const unsigned char* key = tommy_cast(const unsigned char*, void_key); 121 | tommy_uint32_t a, b, c; 122 | 123 | a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + (init_val & 0xffffffff); 124 | c += init_val >> 32; 125 | 126 | while (key_len > 12) { 127 | a += tommy_le_uint32_read(key + 0); 128 | b += tommy_le_uint32_read(key + 4); 129 | c += tommy_le_uint32_read(key + 8); 130 | 131 | tommy_mix(a, b, c); 132 | 133 | key_len -= 12; 134 | key += 12; 135 | } 136 | 137 | switch (key_len) { 138 | case 0 : 139 | return c + ((tommy_uint64_t)b << 32); /* used only when called with a zero length */ 140 | case 12 : 141 | c += tommy_le_uint32_read(key + 8); 142 | b += tommy_le_uint32_read(key + 4); 143 | a += tommy_le_uint32_read(key + 0); 144 | break; 145 | case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ 146 | case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ 147 | case 9 : c += key[8]; /* fallthrough */ 148 | case 8 : 149 | b += tommy_le_uint32_read(key + 4); 150 | a += tommy_le_uint32_read(key + 0); 151 | break; 152 | case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ 153 | case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ 154 | case 5 : b += key[4]; /* fallthrough */ 155 | case 4 : 156 | a += tommy_le_uint32_read(key + 0); 157 | break; 158 | case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ 159 | case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ 160 | case 1 : a += key[0]; /* fallthrough */ 161 | } 162 | 163 | tommy_final(a, b, c); 164 | 165 | return c + ((tommy_uint64_t)b << 32); 166 | } 167 | 168 | tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key) 169 | { 170 | const unsigned char* key = tommy_cast(const unsigned char*, void_key); 171 | tommy_uint32_t a, b, c; 172 | tommy_uint32_t m[3] = { 0xff, 0xff00, 0xff0000 }; 173 | 174 | a = b = c = 0xdeadbeef + init_val; 175 | /* this is different than original lookup3 and the result won't match */ 176 | 177 | while (1) { 178 | tommy_uint32_t v = tommy_le_uint32_read(key); 179 | 180 | if (tommy_haszero_u32(v)) { 181 | if (v & m[0]) { 182 | a += v & m[0]; 183 | if (v & m[1]) { 184 | a += v & m[1]; 185 | if (v & m[2]) 186 | a += v & m[2]; 187 | } 188 | } 189 | 190 | break; 191 | } 192 | 193 | a += v; 194 | 195 | v = tommy_le_uint32_read(key + 4); 196 | 197 | if (tommy_haszero_u32(v)) { 198 | if (v & m[0]) { 199 | b += v & m[0]; 200 | if (v & m[1]) { 201 | b += v & m[1]; 202 | if (v & m[2]) 203 | b += v & m[2]; 204 | } 205 | } 206 | 207 | break; 208 | } 209 | 210 | b += v; 211 | 212 | v = tommy_le_uint32_read(key + 8); 213 | 214 | if (tommy_haszero_u32(v)) { 215 | if (v & m[0]) { 216 | c += v & m[0]; 217 | if (v & m[1]) { 218 | c += v & m[1]; 219 | if (v & m[2]) 220 | c += v & m[2]; 221 | } 222 | } 223 | 224 | break; 225 | } 226 | 227 | c += v; 228 | 229 | tommy_mix(a, b, c); 230 | 231 | key += 12; 232 | } 233 | 234 | /* for lengths that are multipliers of 12 we already have called mix */ 235 | /* this is different than the original lookup3 and the result won't match */ 236 | 237 | tommy_final(a, b, c); 238 | 239 | return c; 240 | } 241 | 242 | -------------------------------------------------------------------------------- /tommyds/tommyhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Hash functions for the use with ::tommy_hashtable, ::tommy_hashdyn and ::tommy_hashlin. 30 | */ 31 | 32 | #ifndef __TOMMYHASH_H 33 | #define __TOMMYHASH_H 34 | 35 | #include "tommytypes.h" 36 | 37 | /******************************************************************************/ 38 | /* hash */ 39 | 40 | /** 41 | * Hash function with a 32 bits result. 42 | * Implementation of the Robert Jenkins "lookup3" hash 32 bits version, 43 | * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). 44 | * 45 | * This hash is designed to provide a good overall performance in all platforms, 46 | * including 32 bits. If you target only 64 bits, you can use faster hashes, 47 | * like SpookyHash or FarmHash. 48 | * 49 | * \param init_val Initialization value. 50 | * Using a different initialization value, you can generate a completely different set of hash values. 51 | * Use 0 if not relevant. 52 | * \param void_key Pointer to the data to hash. 53 | * \param key_len Size of the data to hash. 54 | * \note 55 | * This function is endianness independent. 56 | * \return The hash value of 32 bits. 57 | */ 58 | tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len); 59 | 60 | /** 61 | * Hash function with a 64 bits result. 62 | * Implementation of the Robert Jenkins "lookup3" hash 64 bits versions, 63 | * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle2(). 64 | * 65 | * This hash is designed to provide a good overall performance in all platforms, 66 | * including 32 bits. If you target only 64 bits, you can use faster hashes, 67 | * like SpookyHash or FarmHash. 68 | * 69 | * \param init_val Initialization value. 70 | * Using a different initialization value, you can generate a completely different set of hash values. 71 | * Use 0 if not relevant. 72 | * \param void_key Pointer to the data to hash. 73 | * \param key_len Size of the data to hash. 74 | * \note 75 | * This function is endianness independent. 76 | * \return The hash value of 64 bits. 77 | */ 78 | tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len); 79 | 80 | /** 81 | * String hash function with a 32 bits result. 82 | * Implementation is based on Robert Jenkins "lookup3" hash 32 bits version, 83 | * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). 84 | * 85 | * This hash is designed to handle strings with an unknown length. If you 86 | * know the string length, the other hash functions are surely faster. 87 | * 88 | * \param init_val Initialization value. 89 | * Using a different initialization value, you can generate a completely different set of hash values. 90 | * Use 0 if not relevant. 91 | * \param void_key Pointer to the string to hash. It has to be 0 terminated. 92 | * \note 93 | * This function is endianness independent. 94 | * \return The hash value of 32 bits. 95 | */ 96 | tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key); 97 | 98 | /** 99 | * Integer reversible hash function for 32 bits. 100 | * Implementation of the Robert Jenkins "4-byte Integer Hashing", 101 | * from http://burtleburtle.net/bob/hash/integer.html 102 | */ 103 | tommy_inline tommy_uint32_t tommy_inthash_u32(tommy_uint32_t key) 104 | { 105 | key -= key << 6; 106 | key ^= key >> 17; 107 | key -= key << 9; 108 | key ^= key << 4; 109 | key -= key << 3; 110 | key ^= key << 10; 111 | key ^= key >> 15; 112 | 113 | return key; 114 | } 115 | 116 | /** 117 | * Integer reversible hash function for 64 bits. 118 | * Implementation of the Thomas Wang "Integer Hash Function", 119 | * from http://web.archive.org/web/20071223173210/http://www.concentric.net/~Ttwang/tech/inthash.htm 120 | */ 121 | tommy_inline tommy_uint64_t tommy_inthash_u64(tommy_uint64_t key) 122 | { 123 | key = ~key + (key << 21); 124 | key = key ^ (key >> 24); 125 | key = key + (key << 3) + (key << 8); 126 | key = key ^ (key >> 14); 127 | key = key + (key << 2) + (key << 4); 128 | key = key ^ (key >> 28); 129 | key = key + (key << 31); 130 | 131 | return key; 132 | } 133 | 134 | #endif 135 | 136 | -------------------------------------------------------------------------------- /tommyds/tommyhashdyn.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyhashdyn.h" 29 | #include "tommylist.h" 30 | 31 | /******************************************************************************/ 32 | /* hashdyn */ 33 | 34 | void tommy_hashdyn_init(tommy_hashdyn* hashdyn) 35 | { 36 | /* fixed initial size */ 37 | hashdyn->bucket_bit = TOMMY_HASHDYN_BIT; 38 | hashdyn->bucket_max = (tommy_size_t)1 << hashdyn->bucket_bit; 39 | hashdyn->bucket_mask = hashdyn->bucket_max - 1; 40 | hashdyn->bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(hashdyn->bucket_max, sizeof(tommy_hashdyn_node*))); 41 | 42 | hashdyn->count = 0; 43 | } 44 | 45 | void tommy_hashdyn_done(tommy_hashdyn* hashdyn) 46 | { 47 | tommy_free(hashdyn->bucket); 48 | } 49 | 50 | /** 51 | * Resize the bucket vector. 52 | */ 53 | static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_size_t new_bucket_bit) 54 | { 55 | tommy_size_t bucket_bit; 56 | tommy_size_t bucket_max; 57 | tommy_size_t new_bucket_max; 58 | tommy_size_t new_bucket_mask; 59 | tommy_hashdyn_node** new_bucket; 60 | 61 | bucket_bit = hashdyn->bucket_bit; 62 | bucket_max = hashdyn->bucket_max; 63 | 64 | new_bucket_max = (tommy_size_t)1 << new_bucket_bit; 65 | new_bucket_mask = new_bucket_max - 1; 66 | 67 | /* allocate the new vector using malloc() and not calloc() */ 68 | /* because data is fully initialized in the update process */ 69 | new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); 70 | 71 | /* reinsert all the elements */ 72 | if (new_bucket_bit > bucket_bit) { 73 | tommy_size_t i; 74 | 75 | /* grow */ 76 | for (i = 0; i < bucket_max; ++i) { 77 | tommy_hashdyn_node* j; 78 | 79 | /* setup the new two buckets */ 80 | new_bucket[i] = 0; 81 | new_bucket[i + bucket_max] = 0; 82 | 83 | /* reinsert the bucket */ 84 | j = hashdyn->bucket[i]; 85 | while (j) { 86 | tommy_hashdyn_node* j_next = j->next; 87 | tommy_size_t pos = j->index & new_bucket_mask; 88 | if (new_bucket[pos]) 89 | tommy_list_insert_tail_not_empty(new_bucket[pos], j); 90 | else 91 | tommy_list_insert_first(&new_bucket[pos], j); 92 | j = j_next; 93 | } 94 | } 95 | } else { 96 | tommy_size_t i; 97 | 98 | /* shrink */ 99 | for (i = 0; i < new_bucket_max; ++i) { 100 | /* setup the new bucket with the lower bucket*/ 101 | new_bucket[i] = hashdyn->bucket[i]; 102 | 103 | /* concat the upper bucket */ 104 | tommy_list_concat(&new_bucket[i], &hashdyn->bucket[i + new_bucket_max]); 105 | } 106 | } 107 | 108 | tommy_free(hashdyn->bucket); 109 | 110 | /* setup */ 111 | hashdyn->bucket_bit = new_bucket_bit; 112 | hashdyn->bucket_max = new_bucket_max; 113 | hashdyn->bucket_mask = new_bucket_mask; 114 | hashdyn->bucket = new_bucket; 115 | } 116 | 117 | /** 118 | * Grow. 119 | */ 120 | tommy_inline void hashdyn_grow_step(tommy_hashdyn* hashdyn) 121 | { 122 | /* grow if more than 50% full */ 123 | if (hashdyn->count >= hashdyn->bucket_max / 2) 124 | tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit + 1); 125 | } 126 | 127 | /** 128 | * Shrink. 129 | */ 130 | tommy_inline void hashdyn_shrink_step(tommy_hashdyn* hashdyn) 131 | { 132 | /* shrink if less than 12.5% full */ 133 | if (hashdyn->count <= hashdyn->bucket_max / 8 && hashdyn->bucket_bit > TOMMY_HASHDYN_BIT) 134 | tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit - 1); 135 | } 136 | 137 | void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash) 138 | { 139 | tommy_size_t pos = hash & hashdyn->bucket_mask; 140 | 141 | tommy_list_insert_tail(&hashdyn->bucket[pos], node, data); 142 | 143 | node->index = hash; 144 | 145 | ++hashdyn->count; 146 | 147 | hashdyn_grow_step(hashdyn); 148 | } 149 | 150 | void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) 151 | { 152 | tommy_size_t pos = node->index & hashdyn->bucket_mask; 153 | 154 | tommy_list_remove_existing(&hashdyn->bucket[pos], node); 155 | 156 | --hashdyn->count; 157 | 158 | hashdyn_shrink_step(hashdyn); 159 | 160 | return node->data; 161 | } 162 | 163 | void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) 164 | { 165 | tommy_size_t pos = hash & hashdyn->bucket_mask; 166 | tommy_hashdyn_node* node = hashdyn->bucket[pos]; 167 | 168 | while (node) { 169 | /* we first check if the hash matches, as in the same bucket we may have multiples hash values */ 170 | if (node->index == hash && cmp(cmp_arg, node->data) == 0) { 171 | tommy_list_remove_existing(&hashdyn->bucket[pos], node); 172 | 173 | --hashdyn->count; 174 | 175 | hashdyn_shrink_step(hashdyn); 176 | 177 | return node->data; 178 | } 179 | node = node->next; 180 | } 181 | 182 | return 0; 183 | } 184 | 185 | void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func) 186 | { 187 | tommy_size_t bucket_max = hashdyn->bucket_max; 188 | tommy_hashdyn_node** bucket = hashdyn->bucket; 189 | tommy_size_t pos; 190 | 191 | for (pos = 0; pos < bucket_max; ++pos) { 192 | tommy_hashdyn_node* node = bucket[pos]; 193 | 194 | while (node) { 195 | void* data = node->data; 196 | node = node->next; 197 | func(data); 198 | } 199 | } 200 | } 201 | 202 | void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* func, void* arg) 203 | { 204 | tommy_size_t bucket_max = hashdyn->bucket_max; 205 | tommy_hashdyn_node** bucket = hashdyn->bucket; 206 | tommy_size_t pos; 207 | 208 | for (pos = 0; pos < bucket_max; ++pos) { 209 | tommy_hashdyn_node* node = bucket[pos]; 210 | 211 | while (node) { 212 | void* data = node->data; 213 | node = node->next; 214 | func(arg, data); 215 | } 216 | } 217 | } 218 | 219 | tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn) 220 | { 221 | return hashdyn->bucket_max * (tommy_size_t)sizeof(hashdyn->bucket[0]) 222 | + tommy_hashdyn_count(hashdyn) * (tommy_size_t)sizeof(tommy_hashdyn_node); 223 | } 224 | 225 | -------------------------------------------------------------------------------- /tommyds/tommylist.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommylist.h" 29 | #include "tommychain.h" 30 | 31 | /** \internal 32 | * Setup a list. 33 | */ 34 | tommy_inline void tommy_list_set(tommy_list* list, tommy_node* head, tommy_node* tail) 35 | { 36 | head->prev = tail; 37 | tail->next = 0; 38 | *list = head; 39 | } 40 | 41 | void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp) 42 | { 43 | tommy_chain chain; 44 | tommy_node* head; 45 | 46 | if (tommy_list_empty(list)) 47 | return; 48 | 49 | head = tommy_list_head(list); 50 | 51 | /* create a chain from the list */ 52 | chain.head = head; 53 | chain.tail = head->prev; 54 | 55 | tommy_chain_mergesort(&chain, cmp); 56 | 57 | /* restore the list */ 58 | tommy_list_set(list, chain.head, chain.tail); 59 | } 60 | 61 | -------------------------------------------------------------------------------- /tommyds/tommytree.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommytree.h" 29 | 30 | #include /* for assert */ 31 | 32 | /******************************************************************************/ 33 | /* tree */ 34 | 35 | void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp) 36 | { 37 | tree->root = 0; 38 | tree->count = 0; 39 | tree->cmp = cmp; 40 | } 41 | 42 | static tommy_ssize_t tommy_tree_delta(tommy_tree_node* root) 43 | { 44 | tommy_ssize_t left_height = root->prev ? root->prev->index : 0; 45 | tommy_ssize_t right_height = root->next ? root->next->index : 0; 46 | 47 | return left_height - right_height; 48 | } 49 | 50 | /* AVL tree operations */ 51 | static tommy_tree_node* tommy_tree_balance(tommy_tree_node*); 52 | 53 | static tommy_tree_node* tommy_tree_rotate_left(tommy_tree_node* root) 54 | { 55 | tommy_tree_node* next = root->next; 56 | 57 | root->next = next->prev; 58 | 59 | next->prev = tommy_tree_balance(root); 60 | 61 | return tommy_tree_balance(next); 62 | } 63 | 64 | static tommy_tree_node* tommy_tree_rotate_right(tommy_tree_node* root) 65 | { 66 | tommy_tree_node* prev = root->prev; 67 | 68 | root->prev = prev->next; 69 | 70 | prev->next = tommy_tree_balance(root); 71 | 72 | return tommy_tree_balance(prev); 73 | } 74 | 75 | static tommy_tree_node* tommy_tree_move_right(tommy_tree_node* root, tommy_tree_node* node) 76 | { 77 | if (!root) 78 | return node; 79 | 80 | root->next = tommy_tree_move_right(root->next, node); 81 | 82 | return tommy_tree_balance(root); 83 | } 84 | 85 | static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root) 86 | { 87 | tommy_ssize_t delta = tommy_tree_delta(root); 88 | 89 | if (delta < -1) { 90 | if (tommy_tree_delta(root->next) > 0) 91 | root->next = tommy_tree_rotate_right(root->next); 92 | return tommy_tree_rotate_left(root); 93 | } 94 | 95 | if (delta > 1) { 96 | if (tommy_tree_delta(root->prev) < 0) 97 | root->prev = tommy_tree_rotate_left(root->prev); 98 | return tommy_tree_rotate_right(root); 99 | } 100 | 101 | /* recompute key */ 102 | root->index = 0; 103 | 104 | if (root->prev && root->prev->index > root->index) 105 | root->index = root->prev->index; 106 | 107 | if (root->next && root->next->index > root->index) 108 | root->index = root->next->index; 109 | 110 | /* count itself */ 111 | root->index += 1; 112 | 113 | return root; 114 | } 115 | 116 | static tommy_tree_node* tommy_tree_insert_node(tommy_compare_func* cmp, tommy_tree_node* root, tommy_tree_node** let) 117 | { 118 | int c; 119 | 120 | if (!root) 121 | return *let; 122 | 123 | c = cmp((*let)->data, root->data); 124 | 125 | if (c < 0) { 126 | root->prev = tommy_tree_insert_node(cmp, root->prev, let); 127 | return tommy_tree_balance(root); 128 | } 129 | 130 | if (c > 0) { 131 | root->next = tommy_tree_insert_node(cmp, root->next, let); 132 | return tommy_tree_balance(root); 133 | } 134 | 135 | /* already present, set the return pointer */ 136 | *let = root; 137 | 138 | return root; 139 | } 140 | 141 | void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data) 142 | { 143 | tommy_tree_node* insert = node; 144 | 145 | insert->data = data; 146 | insert->prev = 0; 147 | insert->next = 0; 148 | insert->index = 0; 149 | 150 | tree->root = tommy_tree_insert_node(tree->cmp, tree->root, &insert); 151 | 152 | if (insert == node) 153 | ++tree->count; 154 | 155 | return insert->data; 156 | } 157 | 158 | static tommy_tree_node* tommy_tree_remove_node(tommy_compare_func* cmp, tommy_tree_node* root, void* data, tommy_tree_node** let) 159 | { 160 | int c; 161 | 162 | if (!root) 163 | return 0; 164 | 165 | c = cmp(data, root->data); 166 | 167 | if (c < 0) { 168 | root->prev = tommy_tree_remove_node(cmp, root->prev, data, let); 169 | return tommy_tree_balance(root); 170 | } 171 | 172 | if (c > 0) { 173 | root->next = tommy_tree_remove_node(cmp, root->next, data, let); 174 | return tommy_tree_balance(root); 175 | } 176 | 177 | /* found */ 178 | *let = root; 179 | 180 | return tommy_tree_move_right(root->prev, root->next); 181 | } 182 | 183 | void* tommy_tree_remove(tommy_tree* tree, void* data) 184 | { 185 | tommy_tree_node* node = 0; 186 | 187 | tree->root = tommy_tree_remove_node(tree->cmp, tree->root, data, &node); 188 | 189 | if (!node) 190 | return 0; 191 | 192 | --tree->count; 193 | 194 | return node->data; 195 | } 196 | 197 | static tommy_tree_node* tommy_tree_search_node(tommy_compare_func* cmp, tommy_tree_node* root, void* data) 198 | { 199 | int c; 200 | 201 | if (!root) 202 | return 0; 203 | 204 | c = cmp(data, root->data); 205 | 206 | if (c < 0) 207 | return tommy_tree_search_node(cmp, root->prev, data); 208 | 209 | if (c > 0) 210 | return tommy_tree_search_node(cmp, root->next, data); 211 | 212 | return root; 213 | } 214 | 215 | void* tommy_tree_search(tommy_tree* tree, void* data) 216 | { 217 | tommy_tree_node* node = tommy_tree_search_node(tree->cmp, tree->root, data); 218 | 219 | if (!node) 220 | return 0; 221 | 222 | return node->data; 223 | } 224 | 225 | void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg) 226 | { 227 | tommy_tree_node* node = tommy_tree_search_node(cmp, tree->root, cmp_arg); 228 | 229 | if (!node) 230 | return 0; 231 | 232 | return node->data; 233 | } 234 | 235 | void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node) 236 | { 237 | void* data = tommy_tree_remove(tree, node->data); 238 | 239 | assert(data != 0); 240 | 241 | return data; 242 | } 243 | 244 | static void tommy_tree_foreach_node(tommy_tree_node* root, tommy_foreach_func* func) 245 | { 246 | tommy_tree_node* next; 247 | 248 | if (!root) 249 | return; 250 | 251 | tommy_tree_foreach_node(root->prev, func); 252 | 253 | /* make a copy in case func is free() */ 254 | next = root->next; 255 | 256 | func(root->data); 257 | 258 | tommy_tree_foreach_node(next, func); 259 | } 260 | 261 | void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func) 262 | { 263 | tommy_tree_foreach_node(tree->root, func); 264 | } 265 | 266 | static void tommy_tree_foreach_arg_node(tommy_tree_node* root, tommy_foreach_arg_func* func, void* arg) 267 | { 268 | tommy_tree_node* next; 269 | 270 | if (!root) 271 | return; 272 | 273 | tommy_tree_foreach_arg_node(root->prev, func, arg); 274 | 275 | /* make a copy in case func is free() */ 276 | next = root->next; 277 | 278 | func(arg, root->data); 279 | 280 | tommy_tree_foreach_arg_node(next, func, arg); 281 | } 282 | 283 | void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg) 284 | { 285 | tommy_tree_foreach_arg_node(tree->root, func, arg); 286 | } 287 | 288 | tommy_size_t tommy_tree_memory_usage(tommy_tree* tree) 289 | { 290 | return tommy_tree_count(tree) * sizeof(tommy_tree_node); 291 | } 292 | 293 | -------------------------------------------------------------------------------- /tommyds/tommytree.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * AVL tree. 30 | * 31 | * This tree is a standard AVL tree implementation that stores elements in the 32 | * order defined by the comparison function. 33 | * 34 | * As difference than other tommy containers, duplicate elements cannot be inserted. 35 | * 36 | * To initialize a tree you have to call tommy_tree_init() specifying a comparison 37 | * function that will define the order in the tree. 38 | * 39 | * \code 40 | * tommy_tree tree; 41 | * 42 | * tommy_tree_init(&tree, cmp); 43 | * \endcode 44 | * 45 | * To insert elements in the tree you have to call tommy_tree_insert() for 46 | * each element. 47 | * In the insertion call you have to specify the address of the node and the 48 | * address of the object. 49 | * The address of the object is used to initialize the tommy_node::data field 50 | * of the node. 51 | * 52 | * \code 53 | * struct object { 54 | * int value; 55 | * // other fields 56 | * tommy_tree_node node; 57 | * }; 58 | * 59 | * struct object* obj = malloc(sizeof(struct object)); // creates the object 60 | * 61 | * obj->value = ...; // initializes the object 62 | * 63 | * tommy_tree_insert(&tree, &obj->node, obj); // inserts the object 64 | * \endcode 65 | * 66 | * To find and element in the tree you have to call tommy_tree_search() providing 67 | * the key to search. 68 | * 69 | * \code 70 | * struct object value_to_find = { 1 }; 71 | * struct object* obj = tommy_tree_search(&tree, &value_to_find); 72 | * if (!obj) { 73 | * // not found 74 | * } else { 75 | * // found 76 | * } 77 | * \endcode 78 | * 79 | * To remove an element from the tree you have to call tommy_tree_remove() 80 | * providing the key to search and remove. 81 | * 82 | * \code 83 | * struct object value_to_remove = { 1 }; 84 | * struct object* obj = tommy_tree_remove(&tree, &value_to_remove); 85 | * if (obj) { 86 | * free(obj); // frees the object allocated memory 87 | * } 88 | * \endcode 89 | * 90 | * To destroy the tree you have to remove or destroy all the contained elements. 91 | * The tree itself doesn't have or need a deallocation function. 92 | * 93 | * If you need to iterate over all the elements in the tree, you can use 94 | * tommy_tree_foreach() or tommy_tree_foreach_arg(). 95 | * If you need a more precise control with a real iteration, you have to insert 96 | * all the elements also in a ::tommy_list, and use the list to iterate. 97 | * See the \ref multiindex example for more detail. 98 | */ 99 | 100 | #ifndef __TOMMYTREE_H 101 | #define __TOMMYTREE_H 102 | 103 | #include "tommytypes.h" 104 | 105 | /******************************************************************************/ 106 | /* tree */ 107 | 108 | /** 109 | * Tree node. 110 | * This is the node that you have to include inside your objects. 111 | */ 112 | typedef tommy_node tommy_tree_node; 113 | 114 | /** 115 | * Tree container type. 116 | * \note Don't use internal fields directly, but access the container only using functions. 117 | */ 118 | typedef struct tommy_tree_struct { 119 | tommy_tree_node* root; /**< Root node. */ 120 | tommy_compare_func* cmp; /**< Comparison function. */ 121 | tommy_size_t count; /**< Number of elements. */ 122 | } tommy_tree; 123 | 124 | /** 125 | * Initializes the tree. 126 | * \param cmp The comparison function that defines the order in the tree. 127 | */ 128 | void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp); 129 | 130 | /** 131 | * Inserts an element in the tree. 132 | * If the element is already present, it's not inserted again. 133 | * Check the return value to identify if the element was already present or not. 134 | * You have to provide the pointer of the node embedded into the object and 135 | * the pointer to the object. 136 | * \param node Pointer to the node embedded into the object to insert. 137 | * \param data Pointer to the object to insert. 138 | * \return The element in the tree. Either the already existing one, or the one just inserted. 139 | */ 140 | void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data); 141 | 142 | /** 143 | * Searches and removes an element. 144 | * If the element is not found, 0 is returned. 145 | * \param data Element used for comparison. 146 | * \return The removed element, or 0 if not found. 147 | */ 148 | void* tommy_tree_remove(tommy_tree* tree, void* data); 149 | 150 | /** 151 | * Searches an element in the tree. 152 | * If the element is not found, 0 is returned. 153 | * \param data Element used for comparison. 154 | * \return The first element found, or 0 if none. 155 | */ 156 | void* tommy_tree_search(tommy_tree* tree, void* data); 157 | 158 | /** 159 | * Searches an element in the tree with a specific comparison function. 160 | * 161 | * Like tommy_tree_search() but you can specify a different comparison function. 162 | * Note that this function must define a suborder of the original one. 163 | * 164 | * The ::data argument will be the first argument of the comparison function, 165 | * and it can be of a different type of the objects in the tree. 166 | */ 167 | void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg); 168 | 169 | /** 170 | * Removes an element from the tree. 171 | * You must already have the address of the element to remove. 172 | * \return The tommy_node::data field of the node removed. 173 | */ 174 | void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node); 175 | 176 | /** 177 | * Calls the specified function for each element in the tree. 178 | * 179 | * The elements are processed in order. 180 | * 181 | * You cannot add or remove elements from the inside of the callback, 182 | * but can use it to deallocate them. 183 | * 184 | * \code 185 | * tommy_tree tree; 186 | * 187 | * // initializes the tree 188 | * tommy_tree_init(&tree, cmp); 189 | * 190 | * ... 191 | * 192 | * // creates an object 193 | * struct object* obj = malloc(sizeof(struct object)); 194 | * 195 | * ... 196 | * 197 | * // insert it in the tree 198 | * tommy_tree_insert(&tree, &obj->node, obj); 199 | * 200 | * ... 201 | * 202 | * // deallocates all the objects iterating the tree 203 | * tommy_tree_foreach(&tree, free); 204 | * \endcode 205 | */ 206 | void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func); 207 | 208 | /** 209 | * Calls the specified function with an argument for each element in the tree. 210 | */ 211 | void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg); 212 | 213 | /** 214 | * Gets the number of elements. 215 | */ 216 | tommy_inline tommy_size_t tommy_tree_count(tommy_tree* tree) 217 | { 218 | return tree->count; 219 | } 220 | 221 | /** 222 | * Gets the size of allocated memory. 223 | * It includes the size of the ::tommy_tree_node of the stored elements. 224 | */ 225 | tommy_size_t tommy_tree_memory_usage(tommy_tree* tree); 226 | 227 | #endif 228 | 229 | -------------------------------------------------------------------------------- /valgrind.supp: -------------------------------------------------------------------------------- 1 | { 2 | printf reports false positives when printing more than one variables at time 3 | exp-sgcheck:SorG 4 | ... 5 | fun:printf 6 | } 7 | 8 | --------------------------------------------------------------------------------