├── .gitignore ├── .gitreview ├── .mailmap ├── .zuul.yaml ├── AUTHORS ├── COPYING ├── ChangeLog ├── Makefile.am ├── NEWS ├── README ├── README.md ├── Xorcode.pc.in ├── autogen.sh ├── bindep.txt ├── check-symbols.sh ├── configure.ac ├── doc ├── Makefile.am ├── api_definition.md ├── code_organization.md └── doxygen.cfg.in ├── erasurecode.pc.in ├── erasurecode_rs_vand.pc.in ├── get_flags_from_cpuid.c ├── include ├── config.h.in ├── erasurecode │ ├── alg_sig.h │ ├── erasurecode.h │ ├── erasurecode_backend.h │ ├── erasurecode_helpers.h │ ├── erasurecode_helpers_ext.h │ ├── erasurecode_log.h │ ├── erasurecode_postprocessing.h │ ├── erasurecode_preprocessing.h │ ├── erasurecode_stdinc.h │ ├── erasurecode_version.h │ ├── list.h │ └── md5.h ├── isa_l │ └── isa_l_common.h ├── rs_vand │ ├── liberasurecode_rs_vand.h │ └── rs_galois.h └── xor_codes │ ├── xor_code.h │ └── xor_hd_code_defs.h ├── libXorcode.sym ├── liberasurecode.sym ├── liberasurecode_rs_vand.sym ├── libnullcode.sym ├── playbooks ├── enable-fips.yaml └── unittests │ └── run.yaml ├── roles ├── install_isal │ └── tasks │ │ └── main.yaml ├── install_jerasure │ └── tasks │ │ └── main.yaml └── test_liberasurecode │ └── tasks │ └── main.yaml ├── src ├── Makefile.am ├── backends │ ├── isa-l │ │ ├── isa_l_common.c │ │ ├── isa_l_rs_cauchy.c │ │ └── isa_l_rs_vand.c │ ├── jerasure │ │ ├── jerasure_rs_cauchy.c │ │ └── jerasure_rs_vand.c │ ├── null │ │ └── null.c │ ├── phazrio │ │ └── libphazr.c │ ├── rs_vand │ │ └── liberasurecode_rs_vand.c │ ├── shss │ │ └── shss.c │ └── xor │ │ └── flat_xor_hd.c ├── builtin │ ├── null_code │ │ ├── Makefile.am │ │ └── null_code.c │ ├── rs_vand │ │ ├── Makefile.am │ │ ├── liberasurecode_rs_vand.c │ │ └── rs_galois.c │ └── xor_codes │ │ ├── Makefile.am │ │ ├── xor_code.c │ │ └── xor_hd_code.c ├── erasurecode.c ├── erasurecode_helpers.c ├── erasurecode_postprocessing.c ├── erasurecode_preprocessing.c └── utils │ └── chksum │ ├── alg_sig.c │ ├── crc32.c │ └── md5.c └── test ├── Makefile.am ├── builtin ├── rs_vand │ ├── liberasurecode_rs_vand_test.c │ └── rs_galois_test.c └── xor_codes │ ├── test_xor_hd_code.c │ └── test_xor_hd_code.h ├── libec_slap.c ├── libec_slap.h ├── liberasurecode_test.c └── utils └── chksum └── test_alg_sig.c /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | *.obj 5 | *.elf 6 | 7 | # Libraries 8 | *.lib 9 | *.a 10 | *.la 11 | *.lo 12 | 13 | # Shared objects (inc. Windows DLLs) 14 | *.dll 15 | *.so 16 | *.so.* 17 | *.dylib 18 | 19 | # Executables 20 | *.exe 21 | *.out 22 | *.app 23 | *.i*86 24 | *.x86_64 25 | *.hex 26 | 27 | # pkgconfig files 28 | *.pc 29 | 30 | # cscope files 31 | cscope.*out 32 | 33 | # autoconf/automake files 34 | .dirstamp 35 | .libs 36 | aclocal.m4 37 | autom4te.cache 38 | config.guess 39 | config.log 40 | config.status 41 | config.sub 42 | Makefile 43 | config.h.in 44 | config_liberasurecode.h.in 45 | .deps 46 | config_liberasurecode.h 47 | stamp-h1 48 | *Makefile.in 49 | configure 50 | compile 51 | cscope.files 52 | install-sh 53 | libtool 54 | ltmain.sh 55 | 56 | # doxygen documentation 57 | ^doc/.*\.doxytag 58 | ^doc/html 59 | ^doc/latex 60 | *doxygen.cfg 61 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.opendev.org 3 | port=29418 4 | project=openstack/liberasurecode.git 5 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Mark Storer Mark W. Storer 2 | Kevin Greenan 3 | Eric Lambert 4 | Eric Lambert 5 | Tushar Gohad 6 | Kota Tsuyuzaki 7 | Kota Tsuyuzaki 8 | Alistair Coles 9 | Ondřej Nový 10 | Thiago da Silva 11 | -------------------------------------------------------------------------------- /.zuul.yaml: -------------------------------------------------------------------------------- 1 | - job: 2 | name: liberasurecode-unittests 3 | parent: unittests 4 | description: | 5 | Build and run test and valgrind-test for liberasurecode. 6 | run: playbooks/unittests/run.yaml 7 | 8 | - job: 9 | name: liberasurecode-unittests-arm64 10 | parent: liberasurecode-unittests 11 | nodeset: ubuntu-focal-arm64 12 | description: | 13 | Build and run test and valgrind-test for liberasurecode 14 | on arm64. 15 | 16 | - job: 17 | name: liberasurecode-unittests-centos-9-stream 18 | parent: liberasurecode-unittests 19 | nodeset: centos-9-stream 20 | description: | 21 | Build and run test and valgrind-test for liberasurecode. 22 | This job runs on CentOS 9 Stream. 23 | 24 | - job: 25 | name: liberasurecode-unittests-centos-9-stream-fips 26 | parent: liberasurecode-unittests-centos-9-stream 27 | pre-run: playbooks/enable-fips.yaml 28 | description: | 29 | Build and run test and valgrind-test for liberasurecode. 30 | This job runs on a FIPS-enabled CentOS 9 Stream system. 31 | 32 | - project: 33 | check: 34 | jobs: 35 | - liberasurecode-unittests 36 | - liberasurecode-unittests-centos-9-stream 37 | - liberasurecode-unittests-centos-9-stream-fips 38 | check-arm64: 39 | jobs: 40 | - liberasurecode-unittests-arm64 41 | gate: 42 | jobs: 43 | - liberasurecode-unittests 44 | - liberasurecode-unittests-centos-9-stream 45 | - liberasurecode-unittests-centos-9-stream-fips 46 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Original Authors 2 | ---------------- 3 | 4 | Tushar Gohad (tushar.gohad@intel.com) 5 | Kevin Greenan (kmgreen2@gmail.com) 6 | Eric Lambert (eric_lambert@xyratex.com) 7 | Mark Storer (Mark.Storer@evault.com) 8 | 9 | Contributors 10 | ---------------- 11 | 12 | Timur Alperovich (timuralp@swiftstack.com) 13 | Alistair Coles (alistairncoles@gmail.com) 14 | Thiago da Silva (thiagodasilva@gmail.com) 15 | Andreas Jaeger (aj@suse.com) 16 | Chris Lamb (lamby@debian.org) 17 | Dirk Mueller (dirk@dmllr.de) 18 | James Page (james.page@ubuntu.com) 19 | Kota Tsuyuzaki (kota.tsuyuzaki.pc@hco.ntt.co.jp) 20 | Pete Zaitcev (zaitcev@kotori.zaitcev.us) 21 | gengchc2 (geng.changcai2@zte.com.cn) 22 | Daniel Axtens (dja@axtens.net) 23 | Tim Burke (tim.burke@gmail.com) 24 | John Dickinson (me@not.mn) 25 | Jim Cheung (jim.cheung@phazr.io) 26 | Ondřej Nový (novy@ondrej.org) 27 | Corey Bryant (corey.bryant@canonical.com) 28 | donnydavis (donny@fortnebula.com) 29 | Takashi Kajinami (kajinamit@oss.nttdata.com) 30 | Eli Schwartz (eschwartz93@gmail.com) 31 | Martin Raiber (martin@urbackup.org) 32 | Dorian Burihabwa (dorian@burihabwa.com) 33 | utree (aproc@qq.com) 34 | Martin Kopec (mkopec@redhat.com) 35 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Tushar Gohad, Kevin M Greenan, Eric Lambert, Mark Storer 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 9 | Redistributions in binary form must reproduce the above copyright notice, this 10 | list of conditions and the following disclaimer in the documentation and/or 11 | other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 14 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 17 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 20 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 21 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 22 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | Release 1.6.5 2 | ------------- 3 | 4 | . Fixed a segmentation fault when multiple liberasurecode_rs_vand descriptors 5 | have been created then one of them is destroyed. 6 | . Fixed several compiler warnings regarding strict prototypes. 7 | . Other minor fixes and updates. 8 | 9 | Release 1.6.4 10 | ------------- 11 | 12 | . Fixed a bounds check in get_fragment_partition that could lead to stray 13 | writes or segmentation faults. 14 | . Fixed builds on non-Bash shells. 15 | . Other minor fixes and updates. 16 | 17 | Release 1.6.3 18 | ------------- 19 | 20 | . Fixed an underflow error when using flat_xor_hd. 21 | . Added a build option to allow a suffix to be added to library names passed 22 | to dlopen(). To use, define LIBERASURECODE_SO_SUFFIX via CFLAGS when 23 | building. This is useful when renaming libraries for PyEClib wheels, for 24 | example, so they don't conflict with system packages that may be installed. 25 | 26 | Release 1.6.2 27 | ------------- 28 | 29 | . Allow the writing of fragments with the legacy, non-standard CRC-32. 30 | This is intended to smooth upgrades from liberasurecode 1.5.0 and 31 | earlier in a system with multiple readers and writers. 32 | 33 | See https://bugs.launchpad.net/liberasurecode/+bug/1886088 for more 34 | information, including a script you can run on already-written 35 | fragments to determine whether you are affected. 36 | 37 | If you are affected: 38 | 39 | - Before upgrading, ensure every writer will have the environment variable 40 | LIBERASURECODE_WRITE_LEGACY_CRC=1 set upon restart. 41 | - Upgrade liberasurecode on all systems, restarting processes as needed. 42 | Upgraded writers will continue writing CRCs that not-yet-upgraded 43 | readers can use. 44 | - After liberasurecode is upgraded everywhere, remove the environment 45 | variable. zlib CRCs will be used for new writes, and data written with 46 | either CRC will still be readable. 47 | 48 | Release 1.6.1 49 | ------------- 50 | 51 | . Fixed CRC validation of little-endian fragments on big-endian 52 | . Fixed compile warning about unaligned pointers 53 | 54 | Release 1.6.0 55 | ------------- 56 | 57 | . Use zlib for CRC-32 58 | . Allow support for reading of little-endian framents on big-ending 59 | and vice-versa 60 | . Added check to return error for negative data or parity arguments 61 | . Added better argument validation for flat_xor_hd 62 | 63 | Release 1.5.0 64 | ------------- 65 | 66 | . Added support for Phazr.IO libphazr library 67 | . Fixed memory leaks and initialization errors in Jerasure 68 | . Fixed memory leak in ISA-L 69 | . Changed the Galois Field multiplication table in ISA-L to be calculated 70 | only on init. 71 | . Fixed valgrind-check to enable finding memory-leaks. 72 | . Fixed several memory leaks in tests and also in the built-in rs_vand 73 | implementation. 74 | . Fixed warning on automake tool 75 | . Added compiler flag to disable AVX optimizations. 76 | 77 | Release 1.4.0 78 | ------------- 79 | 80 | . Added support for ISA-L Cauchy 81 | . Added get_version functionality to library 82 | . Fixed reconstruct to return an error when memory allocation failed 83 | . Fixed posix_memalign error handling 84 | 85 | Release 1.3.1 86 | ------------- 87 | 88 | . Fixed the internal version number. It was inadvertently skipped in 1.3.0. 89 | 90 | Release 1.3.0 91 | ------------- 92 | 93 | . Fixed error handling on gf_ivnert_matrix in isa-l backend 94 | . Added get_by_desc return value handling to get_fragment_size 95 | . Other minor fixes and updates 96 | 97 | 98 | Release 1.2.0 99 | ------------- 100 | 101 | . Add functionality to enable consumers to check which backends are 102 | present on a system 103 | . Fix segfault when fragment index is out of range 104 | . Add fragment metadata checksumming support 105 | 106 | Release 1.1.0 107 | ------------- 108 | 109 | . Fix for a segfault related to Jerasure uninit() function that is being 110 | looked up by the jerasure EC backend (Issue#19) 111 | . Split helpers.h include for backward compatibility 112 | . Create header symlinks in std locations for backward compat (<= 1.0.8) 113 | . Eliminate erasurecode_stdinc.h dependency on log.h 114 | . Move fragment_header defn to main erasurecode header 115 | . doxygen documentation fixes 116 | 117 | Release 1.0.9 118 | ------------- 119 | 120 | . Minor bugfixes including removing offending MAJOR/MINOR version macros 121 | . Enforce upper limit 32 on the number of erasure coding fragments 122 | (#data + #chunks) < 32 123 | 124 | Release 1.0.8 125 | ------------- 126 | 127 | . Introduce 'liberasurecode_rs_vand', a native, software-based Reed-Soloman 128 | Vandermonde backend 129 | . Properly set W in the new internal RS backend. Without this change, the 130 | fragment length passed up is incorrect. 131 | . Remove all GPLv3 m4 references for CPUID checks 132 | . Properly dedupe fragments in fragments_to_string() function 133 | . Prevent backends from reconstructing an index when it is not missing, 134 | ie, is available 135 | . Make ./configure to obey CFLAGS 136 | . Add missing pkg-config templates 137 | . Remove autoconf installed files from git control 138 | . Fix get_supported_flags() arguments 139 | . Properly detect 64-bit architecture. 140 | . Add -f argument to autoreconf to regenerate aclocal macros 141 | . Silent autoconf warning for ac_cv_sizeof_long 142 | . Fix C++ build issues (add missing cplusplus macros definitions) 143 | . Make liberasurecode header installs to a specific include directory 144 | . Fix 'make test' to properly run null and installed backend tests. 145 | . Fix a uint < 0 warning reported by Clang 146 | . Fix memory leak in alg_sig init 147 | . Fix decode when m > k and all parities are chosen as input to decode 148 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | # Top-level liberasurecode automake configuration 2 | SUBDIRS = src test doc 3 | 4 | EXTRA_DIST = autogen.sh get_flags_from_cpuid.c 5 | 6 | INCLUDE = -I$(abs_top_builddir)/include \ 7 | -I$(abs_top_builddir)/include/erasurecode \ 8 | -I$(abs_top_builddir)/include/xor_codes 9 | 10 | AM_CPPFLAGS = $(CPPFLAGS) $(INCLUDE) 11 | AM_CPPFLAGS += -Werror -Wall 12 | 13 | AM_CFLAGS = -fPIC $(AM_CPPFLAGS) @GCOV_FLAGS@ -L/usr/local/lib 14 | 15 | thisincludedir = $(includedir)/liberasurecode 16 | thisinclude_HEADERS = \ 17 | include/erasurecode/alg_sig.h \ 18 | include/erasurecode/erasurecode.h \ 19 | include/erasurecode/erasurecode_backend.h \ 20 | include/erasurecode/erasurecode_helpers.h \ 21 | include/erasurecode/erasurecode_helpers_ext.h \ 22 | include/erasurecode/erasurecode_log.h \ 23 | include/erasurecode/erasurecode_preprocessing.h \ 24 | include/erasurecode/erasurecode_postprocessing.h \ 25 | include/erasurecode/erasurecode_stdinc.h \ 26 | include/erasurecode/erasurecode_version.h \ 27 | include/erasurecode/list.h \ 28 | include/xor_codes/xor_hd_code_defs.h \ 29 | include/xor_codes/xor_code.h \ 30 | include/config_liberasurecode.h \ 31 | include/rs_vand/rs_galois.h \ 32 | include/rs_vand/liberasurecode_rs_vand.h 33 | 34 | pkgconfig_DATA = erasurecode-$(LIBERASURECODE_API_VERSION).pc 35 | 36 | install-exec-hook: 37 | -(mkdir -p $(DESTDIR)$(includedir) && \ 38 | cd $(DESTDIR)$(includedir) && \ 39 | rm -f erasurecode.h erasurecode_version.h \ 40 | erasurecode_stdinc.h erasurecode_helpers.h \ 41 | config_liberasurecode.h && \ 42 | $(LN_S) liberasurecode/erasurecode.h && \ 43 | $(LN_S) liberasurecode/erasurecode_version.h && \ 44 | $(LN_S) liberasurecode/erasurecode_stdinc.h && \ 45 | $(LN_S) liberasurecode/erasurecode_helpers.h && \ 46 | $(LN_S) liberasurecode/config_liberasurecode.h) 47 | 48 | test: check 49 | @./test/liberasurecode_test 50 | @./test/alg_sig_test 51 | @./test/test_xor_hd_code 52 | @./test/libec_slap 53 | 54 | LIBTOOL_COMMAND = $(LIBTOOL) --mode execute 55 | VALGRIND_EXEC_COMMAND = $(LIBTOOL_COMMAND) valgrind --tool=memcheck \ 56 | --error-exitcode=1 --leak-check=yes --track-fds=yes \ 57 | --malloc-fill=A5 --free-fill=DE --fullpath-after=. --trace-children=yes 58 | 59 | valgrind-test: check 60 | @$(VALGRIND_EXEC_COMMAND) ./test/alg_sig_test 61 | @$(VALGRIND_EXEC_COMMAND) ./test/liberasurecode_test 62 | @$(VALGRIND_EXEC_COMMAND) ./test/test_xor_hd_code 63 | @$(VALGRIND_EXEC_COMMAND) ./test/libec_slap 64 | 65 | CLEANFILES = cscope.in.out cscope.out cscope.po.out 66 | 67 | .PHONY: cscope 68 | cscope: 69 | find src include -name "*.[ch]" > cscope.files 70 | cscope -q -b 71 | 72 | MOSTLYCLEANFILES = *.gcda *.gcno *.gcov 73 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- 1 | this is used by automake 2 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | README.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | liberasurecode 2 | ============== 3 | 4 | liberasurecode is an Erasure Code API library written in C with pluggable Erasure Code backends. 5 | 6 | ---- 7 | 8 | Highlights 9 | ========== 10 | 11 | * Unified Erasure Coding interface for common storage workloads. 12 | 13 | * Pluggable Erasure Code backends - liberasurecode supports the following backends: 14 | 15 | - 'liberasurecode_rs_vand' - Native, software-only Erasure Coding implementation that supports a Reed-Solomon backend 16 | - 'Jerasure' - Erasure Coding library that supports Reed-Solomon, Cauchy backends [1] 17 | - 'ISA-L' - Intel Storage Acceleration Library - SIMD accelerated Erasure Coding backends [2] 18 | - 'SHSS' - NTT Lab Japan's hybrid Erasure Coding backend [4] 19 | - 'Flat XOR HD' - built-in to liberasurecode, based on [3] 20 | - 'libphazr' - Phazr.IO's erasure code backend with built-in privacy [5] 21 | - 'NULL' template backend implemented to help future backend writers 22 | 23 | 24 | * True 'plugin' architecture - liberasurecode uses Dynamically Loaded (DL) 25 | libraries to realize a true 'plugin' architecture. This also allows one to 26 | build liberasurecode indepdendent of the Erasure Code backend libraries. 27 | 28 | * Cross-platform - liberasurecode is known to work on Linux (Fedora/Debian 29 | flavors), Solaris, BSD and Darwin/Mac OS X. 30 | 31 | * Community support - Developed alongside Erasure Code authority Kevin 32 | Greenan, liberasurecode is an actively maintained open-source project with 33 | growing community involvement (Openstack Swift, Ceph, PyECLib, NTT Labs). 34 | 35 | ---- 36 | 37 | Active Users 38 | ==================== 39 | 40 | * PyECLib - Python EC library: https://github.com/openstack/pyeclib 41 | * Openstack Swift Object Store - https://wiki.openstack.org/wiki/Swift 42 | 43 | 44 | ---- 45 | 46 | Build and Install 47 | ================= 48 | 49 | Install dependencies - 50 | 51 | Debian/Ubuntu hosts: 52 | 53 | ```sh 54 | $ sudo apt-get install build-essential autoconf automake libtool zlib1g-dev 55 | ``` 56 | 57 | Fedora/RedHat/CentOS hosts: 58 | 59 | ```sh 60 | $ sudo yum install -y gcc make autoconf automake libtool zlib-devel 61 | ``` 62 | 63 | To build the liberasurecode repository, perform the following from the 64 | top-level directory: 65 | 66 | ``` sh 67 | $ ./autogen.sh 68 | $ ./configure 69 | $ make 70 | $ make test 71 | $ sudo make install 72 | ``` 73 | 74 | ---- 75 | 76 | Getting Help 77 | ============ 78 | 79 | - Bugs: https://bugs.launchpad.net/liberasurecode/ 80 | - Mailing List: http://lists.openstack.org/pipermail/openstack-discuss/ (use tag `[swift][liberasurecode]`) 81 | - IRC: #openstack-swift on OFTC 82 | 83 | ---- 84 | 85 | References 86 | ========== 87 | 88 | [1] Jerasure, C library that supports erasure coding in storage applications, http://jerasure.org 89 | 90 | [2] Intel(R) Storage Acceleration Library (Open Source Version), https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version 91 | 92 | [3] Greenan, Kevin M et al, "Flat XOR-based erasure codes in storage systems", https://web.archive.org/web/20161001210233/https://www.kaymgee.com/Kevin_Greenan/Publications_files/greenan-msst10.pdf 93 | 94 | [4] Kota Tsuyuzaki , "NTT SHSS Erasure Coding backend" 95 | 96 | [5] Jim Cheung , "Phazr.IO libphazr erasure code backend with built-in privacy" 97 | -------------------------------------------------------------------------------- /Xorcode.pc.in: -------------------------------------------------------------------------------- 1 | # Xorcode pkg-config source file 2 | 3 | prefix=@prefix@ 4 | exec_prefix=@exec_prefix@ 5 | libdir=@libdir@ 6 | includedir=@includedir@ 7 | 8 | Name: Xorcode 9 | Description: Flat XOR-based erasure codes 10 | Version: @XORCODE_VERSION@ 11 | Requires: 12 | Conflicts: 13 | Libs: -L${libdir} @XORCODE_RLD_FLAGS@ -lXorcode 14 | Libs.private: @XORCODE_STATIC_LIBS@ 15 | Cflags: -I${includedir}/ -I${includedir}/xor_codes @XORCODE_CFLAGS@ 16 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | autoreconf -i -v -f 3 | -------------------------------------------------------------------------------- /bindep.txt: -------------------------------------------------------------------------------- 1 | # This is a cross-platform list tracking distribution packages needed by tests; 2 | # see http://docs.openstack.org/infra/bindep/ for additional information. 3 | 4 | zlib1g-dev [platform:dpkg] 5 | zlib-devel [platform:rpm] 6 | 7 | build-essential [platform:dpkg] 8 | gcc [platform:rpm] 9 | make [platform:rpm] 10 | autoconf 11 | automake 12 | libtool 13 | valgrind 14 | -------------------------------------------------------------------------------- /check-symbols.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #/ Usage: check-symbols.sh {check,build} 3 | #/ Commands: 4 | #/ 5 | #/ build - Update the *.sym files at the root of this repository 6 | #/ based on the *.so files that have been built. 7 | #/ check - Compare the *.sym files against the symbols exposed in 8 | #/ the *.so files that have been built. 9 | #/ 10 | #/ By checking in the *.sym files and running `check-symbols.sh check` 11 | #/ in the gate, we establish an auditable record of when symbols are 12 | #/ added or removed from the various *.so files we produce. 13 | set -e 14 | 15 | get() { 16 | if [ "$( uname -s )" == "Darwin" ]; then 17 | nm $1 | cut -c18- | grep -v '^[a-zU]' | sed -e 's/ _/ /' | LC_COLLATE=C sort 18 | else 19 | nm --dynamic --defined-only $1 | cut -c18- | LC_COLLATE=C sort 20 | fi 21 | } 22 | 23 | check() { 24 | if [ ! -e $2 ]; then 25 | touch $2 26 | fi 27 | diff -u $2 <( get $1 ) 28 | } 29 | 30 | build() { 31 | get $1 > $2 32 | } 33 | 34 | case $1 in 35 | check | build) 36 | func=$1 37 | ;; 38 | *) 39 | grep '^#/' "$0" |cut -c4- 40 | exit 1 41 | esac 42 | 43 | if [ "$( uname -s )" == "Darwin" ]; then 44 | ext=dylib 45 | else 46 | ext=so 47 | fi 48 | 49 | rc=0 50 | for lib in $(find . -name '*.so' -or -name '*.dylib' -not -name '*.1.dylib') ; do 51 | echo "${func}ing $( basename $lib )" 52 | if ! $func $lib $( basename ${lib/%.$ext/.sym} ) ; then 53 | rc=1 54 | echo 55 | fi 56 | done 57 | 58 | if [ $rc == 1 ]; then 59 | echo "Symbols don't match! Try running '$0 build' before committing." 60 | fi 61 | exit $rc 62 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Standard Stuff 3 | ################################################################################ 4 | 5 | AC_PREREQ([2.61]) 6 | AC_INIT(liberasurecode, [-], 7 | [tusharsg AT gmail DOT com, kmgreen2 AT gmail DOT com], 8 | [], [https://github.com/openstack/liberasurecode]) 9 | AM_MAINTAINER_MODE([disable]) 10 | m4_ifndef([AM_SILENT_RULES], [m4_define([AM_SILENT_RULES],[])]) 11 | AM_SILENT_RULES([yes]) 12 | 13 | # Get release version 14 | AC_PATH_PROG(SED, sed, "", $PATH:/bin:/usr/bin:/usr/local/bin) 15 | if test -z "$SED"; then 16 | AC_MSG_WARN([sed was not found]) 17 | fi 18 | VERINC="${srcdir}/include/erasurecode/erasurecode_version.h" 19 | AC_MSG_CHECKING([liberasurecode API version]) 20 | LIBERASURECODE_VERSION_MAJOR=`$SED -ne 's/^#define _MAJOR *\([0-9]*\)/\1/p' ${VERINC} 2>/dev/null` 21 | AC_MSG_RESULT($LIBERASURECODE_VERSION_MAJOR) 22 | if test -z "$LIBERASURECODE_VERSION_MAJOR"; then 23 | AC_MSG_ERROR([API version number can not be retrieved from $VERINC]) 24 | fi 25 | LIBERASURECODE_VERSION_MINOR=`$SED -ne 's/^#define _MINOR *\([0-9]*\)/\1/p' ${VERINC} 2>/dev/null` 26 | LIBERASURECODE_VERSION_MICRO=`$SED -ne 's/^#define _REV *\([0-9]*\)/\1/p' ${VERINC} 2>/dev/null` 27 | LIBERASURECODE_VERSION=${LIBERASURECODE_VERSION_MAJOR}.${LIBERASURECODE_VERSION_MINOR}.${LIBERASURECODE_VERSION_MICRO} 28 | LIBERASURECODE_API_VERSION=${LIBERASURECODE_VERSION_MAJOR} 29 | 30 | LIBERASURECODE_VERSION_INFO=$((${LIBERASURECODE_VERSION_MAJOR}+${LIBERASURECODE_VERSION_MINOR})):${LIBERASURECODE_VERSION_MICRO}:${LIBERASURECODE_VERSION_MINOR} 31 | 32 | AC_GNU_SOURCE 33 | 34 | AM_INIT_AUTOMAKE([subdir-objects]) 35 | LT_INIT # libtool 36 | 37 | AC_PROG_LN_S 38 | 39 | dnl Compiling with per-target flags requires AM_PROG_CC_C_O. 40 | AC_PROG_CC 41 | AM_PROG_CC_C_O 42 | AC_PROG_LIBTOOL 43 | AC_PROG_CXX 44 | AC_PROG_INSTALL 45 | AC_PROG_MAKE_SET 46 | 47 | PKG_PROG_PKG_CONFIG 48 | 49 | m4_ifndef([PKG_INSTALLDIR],[ 50 | # PKG_INSTALLDIR(DIRECTORY) 51 | # ------------------------- 52 | # Substitutes the variable pkgconfigdir as the location where a module 53 | # should install pkg-config .pc files. By default the directory is 54 | # $libdir/pkgconfig, but the default can be changed by passing 55 | # DIRECTORY. The user can override through the --with-pkgconfigdir 56 | # parameter. 57 | AC_DEFUN([PKG_INSTALLDIR], 58 | [m4_pushdef([pkg_default], [m4_default([$1], ['${libdir}/pkgconfig'])]) 59 | m4_pushdef([pkg_description], 60 | [pkg-config installation directory @<:@]pkg_default[@:>@]) 61 | AC_ARG_WITH([pkgconfigdir], 62 | [AS_HELP_STRING([--with-pkgconfigdir], pkg_description)],, 63 | [with_pkgconfigdir=]pkg_default) 64 | AC_SUBST([pkgconfigdir], [$with_pkgconfigdir]) 65 | m4_popdef([pkg_default]) 66 | m4_popdef([pkg_description]) 67 | ]) dnl PKG_INSTALLDIR 68 | ]) 69 | 70 | PKG_INSTALLDIR 71 | 72 | AC_SUBST(LIBERASURECODE_VERSION_MAJOR) 73 | AC_SUBST(LIBERASURECODE_API_VERSION) 74 | AC_SUBST(LIBERASURECODE_VERSION_MINOR) 75 | AC_SUBST(LIBERASURECODE_VERSION_MICRO) 76 | AC_SUBST(LIBERASURECODE_VERSION) 77 | AC_SUBST(LIBERASURECODE_VERSION_INFO) 78 | AC_SUBST([PACKAGE_VERSION], LIBERASURECODE_VERSION) 79 | 80 | AC_CONFIG_HEADER(include/config_liberasurecode.h) 81 | 82 | AX_EXT() 83 | 84 | ################################################################################ 85 | # System Headers 86 | ################################################################################ 87 | dnl Check for C library headers 88 | AC_HEADER_STDC 89 | AC_CHECK_HEADERS(sys/types.h stdio.h stdlib.h stddef.h stdarg.h \ 90 | malloc.h memory.h string.h strings.h inttypes.h \ 91 | stdint.h ctype.h iconv.h signal.h dlfcn.h \ 92 | pthread.h unistd.h limits.h errno.h syslog.h) 93 | AC_CHECK_FUNCS(malloc calloc realloc free openlog) 94 | 95 | ################################################################################# 96 | # Debug/coverage Options 97 | ################################################################################# 98 | AC_ARG_ENABLE([werror], 99 | [ --disable-werror Dont treat compilation warnings as failures], 100 | [case "${enableval}" in 101 | yes) werror=true ;; 102 | no) werror=false ;; 103 | *) AC_MSG_ERROR([bad value ${enableval} for --disable-werror]) ;; 104 | esac],[werror=true]) 105 | if test x$werror = xtrue ; then 106 | werror_flag="-Werror" 107 | else 108 | werror_flag="" 109 | fi 110 | 111 | AC_ARG_ENABLE([debug], 112 | [ --enable-debug Turn on debugging], 113 | [case "${enableval}" in 114 | yes) debug=true ;; 115 | no) debug=false ;; 116 | *) AC_MSG_ERROR([bad value ${enableval} for --enable-debug]) ;; 117 | esac],[debug=false]) 118 | if test x$debug = xtrue ; then 119 | DEBUG=1 120 | CXXFLAGS="" 121 | CFLAGS="-O0 -ggdb -g3 ${werror_flag} -D_GNU_SOURCE=1 -Wall -Wstrict-prototypes -pedantic -std=c99 ${CFLAGS}" 122 | else 123 | DEBUG=0 124 | CXXFLAGS="" 125 | CFLAGS="-O2 -g ${werror_flag} -D_GNU_SOURCE=1 -Wall -Wstrict-prototypes -pedantic -std=c99 ${CFLAGS}" 126 | fi 127 | 128 | AC_ARG_ENABLE([gcov], 129 | [ --enable-gcov Turn on code coverage], 130 | [case "${enableval}" in 131 | yes) gcov=true ;; 132 | no) gcov=false ;; 133 | *) AC_MSG_ERROR([bad value ${enableval} for --enable-gcov]) ;; 134 | esac],[gcov=false]) 135 | if test x$gcov = xtrue ; then 136 | AC_CHECK_LIB(gcov, main, 137 | [ 138 | GCOV_FLAGS="-fprofile-arcs -ftest-coverage" 139 | GCOV_LDFLAGS="-lgcov" 140 | CXXFLAGS="" 141 | CFLAGS="" 142 | ], [ 143 | AC_MSG_ERROR([failed to locate gcov library]) 144 | ]) 145 | fi 146 | AC_SUBST(GCOV_FLAGS) 147 | AC_SUBST(GCOV_LDFLAGS) 148 | 149 | dnl Expand the sources and objects needed to build the library 150 | AC_SUBST(ac_aux_dir) 151 | AC_SUBST(OBJECTS) 152 | 153 | dnl Do CPUID and associated compiler flag checks 154 | dnl but allow to disable all of this in order to build portable binaries 155 | 156 | AC_ARG_ENABLE([mmi], [ --disable-mmi do not use host-specific instructions], 157 | [case "${enableval}" in 158 | yes) mmi=true ;; 159 | no) mmi=false ;; 160 | *) AC_MSG_ERROR([bad value ${enableval} for --disable-mmi]) ;; 161 | esac],[mmi=true]) 162 | 163 | if test x$mmi = xtrue ; then 164 | 165 | SUPPORTED_FLAGS="" 166 | $CC - -E -mmmx /dev/null 2>&1 167 | if test $? = 0; then 168 | SUPPORTED_FLAGS="-mmmx" 169 | AC_MSG_RESULT([$CC supports -mmmx]) 170 | fi 171 | $CC - -E -msse /dev/null 2>&1 172 | if test $? = 0; then 173 | SUPPORTED_FLAGS="$SUPPORTED_FLAGS -msse" 174 | AC_MSG_RESULT([$CC supports -msse]) 175 | fi 176 | $CC - -E -msse2 /dev/null 2>&1 177 | if test $? = 0; then 178 | SUPPORTED_FLAGS="$SUPPORTED_FLAGS -msse2" 179 | AC_MSG_RESULT([$CC supports -msse2]) 180 | fi 181 | $CC - -E -msse3 /dev/null 2>&1 182 | if test $? = 0; then 183 | SUPPORTED_FLAGS="$SUPPORTED_FLAGS -msse3" 184 | AC_MSG_RESULT([$CC supports -msse3]) 185 | fi 186 | $CC - -E -mssse3 /dev/null 2>&1 187 | if test $? = 0; then 188 | SUPPORTED_FLAGS="$SUPPORTED_FLAGS -mssse3" 189 | AC_MSG_RESULT([$CC supports -mssse3]) 190 | fi 191 | $CC - -E -msse4.1 /dev/null 2>&1 192 | if test $? = 0; then 193 | SUPPORTED_FLAGS="$SUPPORTED_FLAGS -msse4.1" 194 | AC_MSG_RESULT([$CC supports -msse4.1]) 195 | fi 196 | $CC - -E -msse4.2 /dev/null 2>&1 197 | if test $? = 0; then 198 | SUPPORTED_FLAGS="$SUPPORTED_FLAGS -msse4.2" 199 | AC_MSG_RESULT([$CC supports -msse4.2]) 200 | fi 201 | $CC - -E -mavx /dev/null 2>&1 202 | if test $? = 0; then 203 | SUPPORTED_FLAGS="$SUPPORTED_FLAGS -mavx" 204 | AC_MSG_RESULT([$CC supports -mavx]) 205 | fi 206 | 207 | # Detect the SIMD features supported by both the compiler and the CPU 208 | SIMD_FLAGS="" 209 | cat "$srcdir/get_flags_from_cpuid.c" \ 210 | | sed "s/FLAGSFROMAUTOCONF/${SUPPORTED_FLAGS}/" \ 211 | | $CC -x c -g - -o get_flags_from_cpuid 212 | if [[ -e ./get_flags_from_cpuid ]]; then 213 | chmod 755 get_flags_from_cpuid 214 | ./get_flags_from_cpuid 215 | rm ./get_flags_from_cpuid 216 | if [[ -e compiler_flags ]]; then 217 | SIMD_FLAGS=`cat compiler_flags` 218 | rm -f compiler_flags 219 | else 220 | AC_MSG_WARN([Could not run the CPUID detection program]) 221 | fi 222 | else 223 | AC_MSG_WARN([Could not compile the CPUID detection program]) 224 | fi 225 | 226 | AC_MSG_RESULT([Generating with SIMD flags: $SIMD_FLAGS]) 227 | CFLAGS="$CFLAGS $SIMD_FLAGS" 228 | fi 229 | 230 | # Certain code may be dependent on 32 vs. 64-bit arch, so add a 231 | # flag for 64-bit 232 | AC_CHECK_SIZEOF([long]) 233 | if test "$ac_cv_sizeof_long" -eq 8; then 234 | CFLAGS="$CFLAGS -DARCH_64" 235 | AC_MSG_RESULT([Adding -DARCH_64 to CFLAGS]) 236 | fi 237 | 238 | 239 | ################################################################################# 240 | # Doxygen Documentation 241 | ################################################################################# 242 | AC_CHECK_PROG(DOXYGEN, doxygen, true, false) 243 | AM_CONDITIONAL(HAVE_DOXYGEN, $DOXYGEN) 244 | AC_SUBST(HAVE_DOXYGEN) 245 | 246 | dnl Let people disable the doxygen stuff. 247 | AC_ARG_ENABLE(doxygen, [ --enable-doxygen use doxygen to build documentation (default=auto)], 248 | enable_doxygen="$enableval", 249 | enable_doxygen=auto) 250 | 251 | if test x$enable_doxygen = xauto ; then 252 | if test x$DOXYGEN = xtrue ; then 253 | enable_doxygen=yes 254 | else 255 | enable_doxygen=no 256 | fi 257 | fi 258 | 259 | dnl NOTE: We need to use a separate automake conditional for this 260 | dnl to make this work with the tarballs. 261 | AM_CONDITIONAL(ENABLE_DOXYGEN, test x$enable_doxygen = xyes) 262 | 263 | ################################################################################ 264 | # Output Files 265 | ################################################################################ 266 | AC_CONFIG_FILES([\ 267 | src/builtin/null_code/Makefile \ 268 | src/builtin/xor_codes/Makefile \ 269 | src/builtin/rs_vand/Makefile \ 270 | src/Makefile \ 271 | test/Makefile \ 272 | doc/Makefile \ 273 | Makefile \ 274 | erasurecode.pc \ 275 | Xorcode.pc 276 | ]) 277 | 278 | AC_CONFIG_FILES([\ 279 | erasurecode-${LIBERASURECODE_API_VERSION}.pc:erasurecode.pc], [], 280 | [LIBERASURECODE_API_VERSION='$LIBERASURECODE_API_VERSION']) 281 | 282 | AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"]) 283 | AM_COND_IF([HAVE_DOXYGEN], 284 | [AC_CONFIG_FILES([doc/doxygen.cfg])]) 285 | 286 | AC_OUTPUT 287 | 288 | AC_MSG_NOTICE([ 289 | 290 | $PACKAGE configured: 291 | -------------------------- 292 | 293 | version: $LIBERASURECODE_VERSION 294 | interface revision: $LIBERASURECODE_VERSION_INFO 295 | generate documentation: $DOXYGEN 296 | installation prefix: $prefix 297 | CFLAGS: $CXXFLAGS $CFLAGS 298 | LDFLAGS: $LIBS $LDFLAGS 299 | 300 | type "make" followed by "make install" as root. 301 | $DOXMAKE 302 | ]) 303 | -------------------------------------------------------------------------------- /doc/Makefile.am: -------------------------------------------------------------------------------- 1 | DOC_MODULE=@PACKAGE@ 2 | HTML_DIR=$(datadir)/$(DOC_MODULE)/html 3 | TARGET_DIR=$(HTML_DIR)/$(DOC_MODULE) 4 | 5 | all-local: html/index.html 6 | 7 | if ENABLE_DOXYGEN 8 | html/index.html: doxygen.cfg 9 | doxygen doxygen.cfg 10 | else 11 | html/index.html: 12 | endif 13 | 14 | clean-local: 15 | rm -f *~ *.bak 16 | rm -rf html 17 | 18 | install-data-local: html 19 | $(top_srcdir)/install-sh -d $(DESTDIR)$(TARGET_DIR) 20 | (installfiles=`echo html/*`; \ 21 | if test "$$installfiles" = 'html/*'; \ 22 | then echo '-- Nothing to install' ; \ 23 | else \ 24 | for i in $$installfiles; do \ 25 | echo '-- Installing '$$i ; \ 26 | $(top_srcdir)/install-sh $$i $(DESTDIR)$(TARGET_DIR); \ 27 | done; \ 28 | fi) 29 | 30 | dist-hook: 31 | mkdir $(distdir)/html 32 | mkdir $(distdir)/man 33 | -cp html/* $(distdir)/html 34 | -cp man/* $(distdir)/man 35 | 36 | .PHONY : html latex man 37 | -------------------------------------------------------------------------------- /doc/api_definition.md: -------------------------------------------------------------------------------- 1 | liberasurecode API Definition 2 | ============================= 3 | 4 | ``` c 5 | 6 | /* liberasurecode frontend API functions */ 7 | 8 | /** 9 | * Create a liberasurecode instance and return a descriptor 10 | * for use with EC operations (encode, decode, reconstruct) 11 | * 12 | * @param id - one of the supported backends as 13 | * defined by ec_backend_id_t 14 | * @param ec_args - arguments to the EC backend 15 | * arguments common to all backends 16 | * k - number of data fragments 17 | * m - number of parity fragments 18 | * w - word size, in bits 19 | * hd - hamming distance (=m for Reed-Solomon) 20 | * ct - fragment checksum type (stored with the fragment metadata) 21 | * backend-specific arguments 22 | * null_args - arguments for the null backend 23 | * flat_xor_hd, jerasure do not require any special args 24 | * 25 | * @return liberasurecode instance descriptor (int > 0) 26 | */ 27 | int liberasurecode_instance_create(const ec_backend_id_t id, 28 | struct ec_args *args); 29 | 30 | /** 31 | * Close a liberasurecode instance 32 | * 33 | * @param desc - liberasurecode descriptor to close 34 | * 35 | * @return 0 on success, otherwise non-zero error code 36 | */ 37 | int liberasurecode_instance_destroy(int desc); 38 | 39 | 40 | /** 41 | * Erasure encode a data buffer 42 | * 43 | * @param desc - liberasurecode descriptor/handle 44 | * from liberasurecode_instance_create() 45 | * @param orig_data - data to encode 46 | * @param orig_data_size - length of data to encode 47 | * @param encoded_data - pointer to _output_ array (char **) of k data 48 | * fragments (char *), allocated by the callee 49 | * @param encoded_parity - pointer to _output_ array (char **) of m parity 50 | * fragments (char *), allocated by the callee 51 | * @param fragment_len - pointer to _output_ length of each fragment, assuming 52 | * all fragments are the same length 53 | * 54 | * @return 0 on success, -error code otherwise 55 | */ 56 | int liberasurecode_encode(int desc, 57 | const char *orig_data, uint64_t orig_data_size, /* input */ 58 | char ***encoded_data, char ***encoded_parity, /* output */ 59 | uint64_t *fragment_len); /* output */ 60 | 61 | /** 62 | * Cleanup structures allocated by librasurecode_encode 63 | * 64 | * The caller has no context, so cannot safely free memory 65 | * allocated by liberasurecode, so it must pass the 66 | * deallocation responsibility back to liberasurecode. 67 | * 68 | * @param desc - liberasurecode descriptor/handle 69 | * from liberasurecode_instance_create() 70 | * @param encoded_data - (char **) array of k data 71 | * fragments (char *), allocated by liberasurecode_encode 72 | * @param encoded_parity - (char **) array of m parity 73 | * fragments (char *), allocated by liberasurecode_encode 74 | * 75 | * @return 0 in success; -error otherwise 76 | */ 77 | int liberasurecode_encode_cleanup(int desc, char **encoded_data, 78 | char **encoded_parity); 79 | 80 | /** 81 | * Reconstruct original data from a set of k encoded fragments 82 | * 83 | * @param desc - liberasurecode descriptor/handle 84 | * from liberasurecode_instance_create() 85 | * @param fragments - erasure encoded fragments (> = k) 86 | * @param num_fragments - number of fragments being passed in 87 | * @param fragment_len - length of each fragment (assume they are the same) 88 | * @param force_metadata_checks - force fragment metadata checks (default: 0) 89 | * @param out_data - _output_ pointer to decoded data 90 | * @param out_data_len - _output_ length of decoded output 91 | * (both output data pointers are allocated by liberasurecode, 92 | * caller invokes liberasurecode_decode_clean() after it has 93 | * read decoded data in 'out_data') 94 | * 95 | * @return 0 on success, -error code otherwise 96 | */ 97 | int liberasurecode_decode(int desc, 98 | char **available_fragments, /* input */ 99 | int num_fragments, uint64_t fragment_len, /* input */ 100 | int force_metadata_checks, /* input */ 101 | char **out_data, uint64_t *out_data_len); /* output */ 102 | 103 | /** 104 | * Cleanup structures allocated by librasurecode_decode 105 | * 106 | * The caller has no context, so cannot safely free memory 107 | * allocated by liberasurecode, so it must pass the 108 | * deallocation responsibility back to liberasurecode. 109 | * 110 | * @param desc - liberasurecode descriptor/handle 111 | * from liberasurecode_instance_create() 112 | * @param data - (char *) buffer of data decoded by librasurecode_decode 113 | * 114 | * @return 0 on success; -error otherwise 115 | */ 116 | int liberasurecode_decode_cleanup(int desc, char *data); 117 | 118 | /** 119 | * Reconstruct a missing fragment from a subset of available fragments 120 | * 121 | * @param desc - liberasurecode descriptor/handle 122 | * from liberasurecode_instance_create() 123 | * @param available_fragments - erasure encoded fragments 124 | * @param num_fragments - number of fragments being passed in 125 | * @param fragment_len - size in bytes of the fragments 126 | * @param destination_idx - missing idx to reconstruct 127 | * @param out_fragment - output of reconstruct 128 | * 129 | * @return 0 on success, -error code otherwise 130 | */ 131 | int liberasurecode_reconstruct_fragment(int desc, 132 | char **available_fragments, /* input */ 133 | int num_fragments, uint64_t fragment_len, /* input */ 134 | int destination_idx, /* input */ 135 | char* out_fragment); /* output */ 136 | 137 | /** 138 | * Return a list of lists with valid rebuild indexes given 139 | * a list of missing indexes. 140 | * 141 | * @desc: liberasurecode instance descriptor (obtained with 142 | * liberasurecode_instance_create) 143 | * @fragments_to_reconstruct list of indexes to reconstruct 144 | * @fragments_to_exclude list of indexes to exclude from 145 | * reconstruction equation 146 | * @fragments_needed list of fragments needed to reconstruct 147 | * fragments in fragments_to_reconstruct 148 | * 149 | * @return 0 on success, non-zero on error 150 | */ 151 | int liberasurecode_fragments_needed(int desc, 152 | int *fragments_to_reconstruct, 153 | int *fragments_to_exclude, 154 | int *fragments_needed); 155 | 156 | ``` 157 | 158 | Erasure Code Fragment Checksum Types Supported 159 | ---------------------------------------------- 160 | 161 | ``` c 162 | 163 | /* Checksum types supported for fragment metadata stored in each fragment */ 164 | typedef enum { 165 | CHKSUM_NONE = 0, /* "none" (default) */ 166 | CHKSUM_CRC32 = 1, /* "crc32" */ 167 | CHKSUM_TYPES_MAX, 168 | } ec_checksum_type_t; 169 | 170 | ``` 171 | 172 | Erasure Code Fragment Checksum API 173 | ---------------------------------- 174 | 175 | ``` c 176 | 177 | struct __attribute__((__packed__)) 178 | fragment_metadata 179 | { 180 | uint32_t idx; /* 4 */ 181 | uint32_t size; /* 4 */ 182 | uint32_t frag_backend_metadata_size; /* 4 */ 183 | uint64_t orig_data_size; /* 8 */ 184 | uint8_t chksum_type; /* 1 */ 185 | uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */ 186 | uint8_t chksum_mismatch; /* 1 */ 187 | uint8_t backend_id; /* 1 */ 188 | uint32_t backend_version; /* 4 */ 189 | } fragment_metadata_t; 190 | 191 | #define FRAGSIZE_2_BLOCKSIZE(fragment_size) \ 192 | (fragment_size - sizeof(fragment_header_t)) 193 | 194 | /** 195 | * Get opaque metadata for a fragment. The metadata is opaque to the 196 | * client, but meaningful to the underlying library. It is used to verify 197 | * stripes in verify_stripe_metadata(). 198 | * 199 | * @param fragment - fragment data pointer 200 | * @param fragment_metadata - pointer to allocated buffer of size at least 201 | * sizeof(struct fragment_metadata) to hold fragment metadata struct 202 | * 203 | * @return 0 on success, non-zero on error 204 | */ 205 | //EDL: This needs to be implemented 206 | int liberasurecode_get_fragment_metadata(char *fragment, 207 | fragment_metadata_t *fragment_metadata); 208 | 209 | /** 210 | * Verify that the specified pointer points to a well formed fragment that can 211 | * be processed by both this instance of liberasurecode and the specified 212 | * backend. 213 | * 214 | * @param desc - liberasurecode descriptor/handle 215 | * from liberasurecode_instance_create() 216 | * @param fragment - fragment to verify 217 | * 218 | * @return 1 if fragment validation fails, 0 otherwise. 219 | */ 220 | int is_invalid_fragment(int desc, char *fragment); 221 | 222 | /** 223 | * Verify a subset of fragments generated by encode() 224 | * 225 | * @param desc - liberasurecode descriptor/handle 226 | * from liberasurecode_instance_create() 227 | * @param fragments - fragments part of the EC stripe to verify 228 | * @param num_fragments - number of fragments part of the EC stripe 229 | * 230 | * @return 1 if stripe checksum verification is successful, 0 otherwise 231 | */ 232 | int liberasurecode_verify_stripe_metadata(int desc, 233 | char **fragments, int num_fragments); 234 | 235 | /* ==~=*=~===~=*=~==~=*=~== liberasurecode Helpers ==~*==~=*=~==~=~=*=~==~= */ 236 | 237 | /** 238 | * This computes the aligned size of a buffer passed into 239 | * the encode function. The encode function must pad fragments 240 | * to be algined with the word size (w) and the last fragment also 241 | * needs to be aligned. This computes the sum of the algined fragment 242 | * sizes for a given buffer to encode. 243 | * 244 | * @param desc - liberasurecode descriptor/handle 245 | * from liberasurecode_instance_create() 246 | * @param data_len - original data length in bytes 247 | * 248 | * @return aligned length, or -error code on error 249 | */ 250 | int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len); 251 | 252 | /** 253 | * This will return the minimum encode size, which is the minimum 254 | * buffer size that can be encoded. 255 | * 256 | * @param desc - liberasurecode descriptor/handle 257 | * from liberasurecode_instance_create() 258 | * 259 | * @return minimum data length length, or -error code on error 260 | */ 261 | int liberasurecode_get_minimum_encode_size(int desc); 262 | 263 | /** 264 | * This will return the fragment size, which is each fragment data 265 | * length the backend will allocate when encoding. 266 | * 267 | * @param desc - liberasurecode descriptor/handle 268 | * from liberasurecode_instance_create() 269 | * @param data_len - original data length in bytes 270 | * 271 | * @return fragment size - sizeof(fragment_header) + size 272 | * + frag_backend_metadata_size 273 | */ 274 | int liberasurecode_get_fragment_size(int desc, int data_len); 275 | ``` -------------------------------------------------------------------------------- /doc/code_organization.md: -------------------------------------------------------------------------------- 1 | Code organization 2 | ================= 3 | ``` 4 | |-- include 5 | | +-- erasurecode 6 | | | +-- erasurecode.h --> liberasurecode frontend API header 7 | | | +-- erasurecode_backend.h --> liberasurecode backend API header 8 | | +-- xor_codes --> headers for the built-in XOR codes 9 | | 10 | |-- src 11 | | |-- erasurecode.c --> liberasurecode API implementation 12 | | | (frontend + backend) 13 | | |-- backends 14 | | | +-- null 15 | | | +--- null.c --> 'null' erasure code backend (template backend) 16 | | | +-- xor 17 | | | +--- flat_xor_hd.c --> 'flat_xor_hd' erasure code backend (built-in) 18 | | | +-- jerasure 19 | | | +-- jerasure_rs_cauchy.c --> 'jerasure_rs_vand' erasure code backend (jerasure.org) 20 | | | +-- jerasure_rs_vand.c --> 'jerasure_rs_cauchy' erasure code backend (jerasure.org) 21 | | | +-- isa-l 22 | | | +-- isa_l_rs_vand.c --> 'isa_l_rs_vand' erasure code backend (Intel) 23 | | | +-- shss 24 | | | +-- shss.c --> 'shss' erasure code backend (NTT Labs) 25 | | | +-- phazrio 26 | | | +-- libphazr.c --> 'libphazr' erasure code backend (Phazr.IO) 27 | | | 28 | | |-- builtin 29 | | | +-- xor_codes --> XOR HD code backend, built-in erasure 30 | | | | code implementation (shared library) 31 | | | +-- xor_code.c 32 | | | +-- xor_hd_code.c 33 | | | +-- rs_vand --> liberasurecode native Reed Soloman codes 34 | | | 35 | | +-- utils 36 | | +-- chksum --> fragment checksum utils for erasure 37 | | +-- alg_sig.c coded fragments 38 | | +-- crc32.c 39 | | 40 | |-- doc --> API Documentation 41 | | +-- Doxyfile 42 | | +-- html 43 | | 44 | |--- test --> Test routines 45 | | +-- builtin 46 | | | +-- xor_codes 47 | | +-- liberasurecode_test.c 48 | | +-- utils 49 | | 50 | |-- autogen.sh 51 | |-- configure.ac 52 | |-- Makefile.am 53 | |-- README 54 | |-- NEWS 55 | |-- COPYING 56 | |-- AUTHORS 57 | |-- INSTALL 58 | +-- ChangeLog 59 | ``` 60 | --- -------------------------------------------------------------------------------- /erasurecode.pc.in: -------------------------------------------------------------------------------- 1 | # erasurecode pkg-config source file 2 | 3 | prefix=@prefix@ 4 | exec_prefix=@exec_prefix@ 5 | libdir=@libdir@ 6 | includedir=@includedir@ 7 | 8 | Name: erasurecode 9 | Description: C library with pluggable erasure code backends 10 | Version: @LIBERASURECODE_VERSION@ 11 | Requires: 12 | Conflicts: 13 | Libs: -L${libdir} -lerasurecode -ldl 14 | Libs.private: @ERASURECODE_STATIC_LIBS@ -lz 15 | Cflags: -I${includedir}/ -I${includedir}/liberasurecode 16 | -------------------------------------------------------------------------------- /erasurecode_rs_vand.pc.in: -------------------------------------------------------------------------------- 1 | # liberasurecode_rs_vand pkg-config source file 2 | 3 | prefix=@prefix@ 4 | exec_prefix=@exec_prefix@ 5 | libdir=@libdir@ 6 | includedir=@includedir@ 7 | 8 | Name: erasurecode_rs_vand 9 | Description: Naive Reed-Soloman Vandermonde Backend built-in to liberasurecode 10 | Version: @ERASURECODE_RS_VAND_VERSION@ 11 | Requires: 12 | Conflicts: 13 | Libs: -L${libdir} @ERASURECODE_RS_VAND_RLD_FLAGS@ -lerasurecode 14 | Libs.private: @ERASURECODE_RS_VAND_STATIC_LIBS@ 15 | Cflags: -I${includedir}/ -I${includedir}/erasurecode @ERASURECODE_RS_VAND_CFLAGS@ 16 | -------------------------------------------------------------------------------- /get_flags_from_cpuid.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #define EDX_MMX_BIT 23 7 | #define EDX_SSE_BIT 25 8 | #define EDX_SSE2_BIT 26 9 | #define ECX_SSE3_BIT 0 10 | #define ECX_SSSE3_BIT 9 11 | #define ECX_SSE41_BIT 19 12 | #define ECX_SSE42_BIT 20 13 | #define ECX_AVXOS_BIT 27 14 | #define ECX_AVX_BIT 28 15 | 16 | 17 | uint64_t get_supported_flags() 18 | { 19 | uint64_t supp_comp_flgs = 0; 20 | char *comp_flag = NULL; 21 | char SUPPORTED_COMP_FLAGS[] ="FLAGSFROMAUTOCONF\0"; 22 | 23 | for (comp_flag = strtok(SUPPORTED_COMP_FLAGS, " "); comp_flag != NULL; comp_flag = strtok(NULL, " ")) { 24 | if (strncmp(comp_flag, "-m", 2) != 0) { 25 | fprintf(stderr, "Invalid comp_flag: %s\n", comp_flag); 26 | exit(2); 27 | } 28 | if (strcmp(comp_flag, "-mmmx\0") == 0) { 29 | supp_comp_flgs |= (1 << EDX_MMX_BIT); 30 | } 31 | if (strcmp(comp_flag, "-msse\0") == 0) { 32 | supp_comp_flgs |= (1 << EDX_SSE_BIT); 33 | } 34 | if (strcmp(comp_flag, "-msse2\0") == 0) { 35 | supp_comp_flgs |= (1 << EDX_SSE2_BIT); 36 | } 37 | if (strcmp(comp_flag, "-msse3\0") == 0) { 38 | supp_comp_flgs |= (1 << ECX_SSE3_BIT); 39 | } 40 | if (strcmp(comp_flag, "-mssse3\0") == 0) { 41 | supp_comp_flgs |= (1 << ECX_SSSE3_BIT); 42 | } 43 | if (strcmp(comp_flag, "-msse4.1\0") == 0) { 44 | supp_comp_flgs |= (1 << ECX_SSE41_BIT); 45 | } 46 | if (strcmp(comp_flag, "-msse4.2\0") == 0) { 47 | supp_comp_flgs |= (1 << ECX_SSE42_BIT); 48 | } 49 | if (strcmp(comp_flag, "-mavx\0") == 0) { 50 | supp_comp_flgs |= (1 << ECX_AVX_BIT); 51 | } 52 | } 53 | 54 | return supp_comp_flgs; 55 | } 56 | 57 | int is_supported(int cpuid_reg, uint64_t comp_flags, int feature_bit) 58 | { 59 | return ((cpuid_reg >> feature_bit) & 0x1) && ((comp_flags >> feature_bit) & 0x1); 60 | } 61 | 62 | int main(int argc, char** argv) 63 | { 64 | int feature_eax, feature_ebx, feature_ecx, feature_edx; 65 | uint64_t supp_comp_flgs = get_supported_flags(); 66 | FILE *f = fopen("compiler_flags", "w"); 67 | 68 | __asm__("cpuid" 69 | : "=a" (feature_eax), "=b" (feature_ebx), "=c" (feature_ecx), "=d" (feature_edx) 70 | : "a" (0x00000001)); 71 | 72 | if (is_supported(feature_edx, supp_comp_flgs, EDX_MMX_BIT)) { 73 | fprintf(f, "-mmmx -DINTEL_MMX "); 74 | } 75 | if (is_supported(feature_edx, supp_comp_flgs, EDX_SSE_BIT)) { 76 | fprintf(f, "-msse -DINTEL_SSE "); 77 | } 78 | if (is_supported(feature_edx, supp_comp_flgs, EDX_SSE2_BIT)) { 79 | fprintf(f, "-msse2 -DINTEL_SSE2 "); 80 | } 81 | if (is_supported(feature_ecx, supp_comp_flgs, ECX_SSE3_BIT)) { 82 | fprintf(f, "-msse3 -DINTEL_SSE3 "); 83 | } 84 | if (is_supported(feature_ecx, supp_comp_flgs, ECX_SSSE3_BIT)) { 85 | fprintf(f, "-mssse3 -DINTEL_SSSE3 "); 86 | } 87 | if (is_supported(feature_ecx, supp_comp_flgs, ECX_SSE41_BIT)) { 88 | fprintf(f, "-msse4.1 -DINTEL_SSE41 "); 89 | } 90 | if (is_supported(feature_ecx, supp_comp_flgs, ECX_SSE42_BIT)) { 91 | fprintf(f, "-msse4.2 -DINTEL_SSE42 "); 92 | } 93 | if (is_supported(feature_ecx, supp_comp_flgs, ECX_AVX_BIT)) { 94 | if ((feature_ecx >> ECX_AVXOS_BIT) & 0x1) { 95 | fprintf(f, "-mavx -DINTEL_AVX "); 96 | } 97 | } 98 | fclose(f); 99 | return 0; 100 | } 101 | 102 | -------------------------------------------------------------------------------- /include/config.h.in: -------------------------------------------------------------------------------- 1 | /* include/config.h.in. Generated from configure.ac by autoheader. */ 2 | 3 | /* Support Altivec instructions */ 4 | #undef HAVE_ALTIVEC 5 | 6 | /* Support AVX (Advanced Vector Extensions) instructions */ 7 | #undef HAVE_AVX 8 | 9 | /* Define to 1 if you have the `calloc' function. */ 10 | #undef HAVE_CALLOC 11 | 12 | /* Define to 1 if you have the header file. */ 13 | #undef HAVE_CTYPE_H 14 | 15 | /* Define to 1 if you have the header file. */ 16 | #undef HAVE_DLFCN_H 17 | 18 | /* Define to 1 if you have the header file. */ 19 | #undef HAVE_ERRNO_H 20 | 21 | /* Define to 1 if you have the `free' function. */ 22 | #undef HAVE_FREE 23 | 24 | /* Define to 1 if you have the header file. */ 25 | #undef HAVE_ICONV_H 26 | 27 | /* Define to 1 if you have the header file. */ 28 | #undef HAVE_INTTYPES_H 29 | 30 | /* Define to 1 if you have the header file. */ 31 | #undef HAVE_LIMITS_H 32 | 33 | /* Define to 1 if you have the `malloc' function. */ 34 | #undef HAVE_MALLOC 35 | 36 | /* Define to 1 if you have the header file. */ 37 | #undef HAVE_MALLOC_H 38 | 39 | /* Define to 1 if you have the header file. */ 40 | #undef HAVE_MATH_H 41 | 42 | /* Define to 1 if you have the header file. */ 43 | #undef HAVE_MEMORY_H 44 | 45 | /* Support mmx instructions */ 46 | #undef HAVE_MMX 47 | 48 | /* Define to 1 if you have the `openlog' function. */ 49 | #undef HAVE_OPENLOG 50 | 51 | /* Define to 1 if you have the header file. */ 52 | #undef HAVE_PTHREAD_H 53 | 54 | /* Define to 1 if you have the `realloc' function. */ 55 | #undef HAVE_REALLOC 56 | 57 | /* Define to 1 if you have the header file. */ 58 | #undef HAVE_SIGNAL_H 59 | 60 | /* Support SSE (Streaming SIMD Extensions) instructions */ 61 | #undef HAVE_SSE 62 | 63 | /* Support SSE2 (Streaming SIMD Extensions 2) instructions */ 64 | #undef HAVE_SSE2 65 | 66 | /* Support SSE3 (Streaming SIMD Extensions 3) instructions */ 67 | #undef HAVE_SSE3 68 | 69 | /* Support SSSE4.1 (Streaming SIMD Extensions 4.1) instructions */ 70 | #undef HAVE_SSE4_1 71 | 72 | /* Support SSSE4.2 (Streaming SIMD Extensions 4.2) instructions */ 73 | #undef HAVE_SSE4_2 74 | 75 | /* Support SSSE3 (Supplemental Streaming SIMD Extensions 3) instructions */ 76 | #undef HAVE_SSSE3 77 | 78 | /* Define to 1 if you have the header file. */ 79 | #undef HAVE_STDARG_H 80 | 81 | /* Define to 1 if you have the header file. */ 82 | #undef HAVE_STDDEF_H 83 | 84 | /* Define to 1 if you have the header file. */ 85 | #undef HAVE_STDINT_H 86 | 87 | /* Define to 1 if you have the header file. */ 88 | #undef HAVE_STDIO_H 89 | 90 | /* Define to 1 if you have the header file. */ 91 | #undef HAVE_STDLIB_H 92 | 93 | /* Define to 1 if you have the header file. */ 94 | #undef HAVE_STRINGS_H 95 | 96 | /* Define to 1 if you have the header file. */ 97 | #undef HAVE_STRING_H 98 | 99 | /* Define to 1 if you have the header file. */ 100 | #undef HAVE_SYSLOG_H 101 | 102 | /* Define to 1 if you have the header file. */ 103 | #undef HAVE_SYS_STAT_H 104 | 105 | /* Define to 1 if you have the header file. */ 106 | #undef HAVE_SYS_TYPES_H 107 | 108 | /* Define to 1 if you have the header file. */ 109 | #undef HAVE_UNISTD_H 110 | 111 | /* Define to the sub-directory in which libtool stores uninstalled libraries. 112 | */ 113 | #undef LT_OBJDIR 114 | 115 | /* Define to 1 if your C compiler doesn't accept -c and -o together. */ 116 | #undef NO_MINUS_C_MINUS_O 117 | 118 | /* Name of package */ 119 | #undef PACKAGE 120 | 121 | /* Define to the address where bug reports for this package should be sent. */ 122 | #undef PACKAGE_BUGREPORT 123 | 124 | /* Define to the full name of this package. */ 125 | #undef PACKAGE_NAME 126 | 127 | /* Define to the full name and version of this package. */ 128 | #undef PACKAGE_STRING 129 | 130 | /* Define to the one symbol short name of this package. */ 131 | #undef PACKAGE_TARNAME 132 | 133 | /* Define to the home page for this package. */ 134 | #undef PACKAGE_URL 135 | 136 | /* Define to the version of this package. */ 137 | #undef PACKAGE_VERSION 138 | 139 | /* Define to 1 if you have the ANSI C header files. */ 140 | #undef STDC_HEADERS 141 | 142 | /* Enable extensions on AIX 3, Interix. */ 143 | #ifndef _ALL_SOURCE 144 | # undef _ALL_SOURCE 145 | #endif 146 | /* Enable GNU extensions on systems that have them. */ 147 | #ifndef _GNU_SOURCE 148 | # undef _GNU_SOURCE 149 | #endif 150 | /* Enable threading extensions on Solaris. */ 151 | #ifndef _POSIX_PTHREAD_SEMANTICS 152 | # undef _POSIX_PTHREAD_SEMANTICS 153 | #endif 154 | /* Enable extensions on HP NonStop. */ 155 | #ifndef _TANDEM_SOURCE 156 | # undef _TANDEM_SOURCE 157 | #endif 158 | /* Enable general extensions on Solaris. */ 159 | #ifndef __EXTENSIONS__ 160 | # undef __EXTENSIONS__ 161 | #endif 162 | 163 | 164 | /* Version number of package */ 165 | #undef VERSION 166 | 167 | /* Define to 1 if on MINIX. */ 168 | #undef _MINIX 169 | 170 | /* Define to 2 if the system does not provide POSIX.1 features except with 171 | this defined. */ 172 | #undef _POSIX_1_SOURCE 173 | 174 | /* Define to 1 if you need to in order for `stat' and other things to work. */ 175 | #undef _POSIX_SOURCE 176 | -------------------------------------------------------------------------------- /include/erasurecode/alg_sig.h: -------------------------------------------------------------------------------- 1 | /* * Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | */ 24 | 25 | #ifndef _ALG_SIG_H 26 | #define _ALG_SIG_H 27 | 28 | #include 29 | typedef int (*galois_single_multiply_func)(int, int, int); 30 | typedef void (*galois_uninit_field_func)(int); 31 | 32 | struct jerasure_mult_routines { 33 | galois_single_multiply_func galois_single_multiply; 34 | galois_uninit_field_func galois_uninit_field; 35 | }; 36 | 37 | #if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) 38 | #define JERASURE_SONAME "libJerasure.dylib" 39 | #else 40 | #define JERASURE_SONAME "libJerasure.so.2" 41 | #endif 42 | 43 | typedef struct alg_sig_s 44 | { 45 | int gf_w; 46 | int sig_len; 47 | struct jerasure_mult_routines mult_routines; 48 | void *jerasure_sohandle; 49 | int *tbl1_l; 50 | int *tbl1_r; 51 | int *tbl2_l; 52 | int *tbl2_r; 53 | int *tbl3_l; 54 | int *tbl3_r; 55 | } alg_sig_t; 56 | 57 | alg_sig_t *init_alg_sig(int sig_len, int gf_w); 58 | void destroy_alg_sig(alg_sig_t* alg_sig_handle); 59 | 60 | int compute_alg_sig(alg_sig_t* alg_sig_handle, char *buf, int len, char *sig); 61 | int liberasurecode_crc32_alt(int crc, const void *buf, size_t size); 62 | 63 | #endif 64 | 65 | -------------------------------------------------------------------------------- /include/erasurecode/erasurecode_backend.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode backend API definition 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #ifndef _ERASURECODE_BACKEND_H_ 30 | #define _ERASURECODE_BACKEND_H_ 31 | 32 | #include "list.h" 33 | #include "erasurecode.h" 34 | #include "erasurecode_stdinc.h" 35 | 36 | /* ~=*=~===~=*=~==~=*=~==~=*=~= backend infrastructure =~=*=~==~=*=~==~=*=~ */ 37 | 38 | #ifdef __cplusplus 39 | extern "C" { 40 | #endif 41 | 42 | #if defined (__GNUC__) && __GNUC__ > 3 43 | #define dl_restrict __restrict 44 | #else 45 | #define dl_restrict 46 | #endif 47 | 48 | /* ==~=*=~===~=*=~==~=*=~==~=*=~= EC backend args =~==~=*=~==~=*=~===~=*=~== */ 49 | 50 | /* Arguments passed to the backend */ 51 | #define MAX_PRIV_ARGS 4 52 | struct ec_backend_args { 53 | struct ec_args uargs; /* common args passed in by the user */ 54 | void *pargs[MAX_PRIV_ARGS]; /* used for private backend args */ 55 | }; 56 | 57 | /* =~===~=*=~==~=*=~==~=*=~= backend stub definitions =~=*=~==~=*=~==~=*=~= */ 58 | 59 | #define INIT init 60 | #define EXIT exit 61 | #define ENCODE encode 62 | #define DECODE decode 63 | #define FRAGSNEEDED fragments_needed 64 | #define RECONSTRUCT reconstruct 65 | #define ELEMENTSIZE element_size 66 | #define ISCOMPATIBLEWITH is_compatible_with 67 | #define GETMETADATASIZE get_backend_metadata_size 68 | #define GETENCODEOFFSET get_encode_offset 69 | 70 | #define FN_NAME(s) str(s) 71 | #define str(s) #s 72 | 73 | /* EC backend stubs - implemented per backend */ 74 | struct ec_backend_op_stubs { 75 | /** Backend register/init, unregister/cleanup routines **/ 76 | 77 | /* Private backend init routine */ 78 | void * (*INIT)(struct ec_backend_args *args, void *sohandle); 79 | 80 | /* Private backend cleanup routine */ 81 | int (*EXIT)(void *); 82 | 83 | /* Backend stub declarations */ 84 | int (*ENCODE)(void *desc, 85 | char **data, char **parity, int blocksize); 86 | int (*DECODE)(void *desc, 87 | char **data, char **parity, int *missing_idxs, int blocksize); 88 | int (*FRAGSNEEDED)(void *desc, 89 | int *missing_idxs, int * fragments_to_exclude, int *fragments_needed); 90 | int (*RECONSTRUCT)(void *desc, 91 | char **data, char **parity, int *missing_idxs, int destination_idx, 92 | int blocksize); 93 | int (*ELEMENTSIZE)(void *desc); 94 | 95 | bool (*ISCOMPATIBLEWITH)(uint32_t version); 96 | 97 | size_t (*GETMETADATASIZE)(void *desc, int blocksize); 98 | size_t (*GETENCODEOFFSET)(void *desc, int metadata_size); 99 | }; 100 | 101 | /* ==~=*=~==~=*=~==~=*=~= backend struct definitions =~=*=~==~=*=~==~=*==~== */ 102 | 103 | struct ec_backend_desc { 104 | void *backend_desc; /* EC backend private descriptor */ 105 | void *backend_sohandle; /* EC backend shared lib handle */ 106 | }; 107 | 108 | #define MAX_LEN 64 109 | /* EC backend common attributes */ 110 | struct ec_backend_common { 111 | ec_backend_id_t id; /* EC backend type */ 112 | char name[MAX_LEN]; /* EC backend common name */ 113 | const char *soname; /* EC backend shared library path */ 114 | char soversion[MAX_LEN]; /* EC backend shared library version */ 115 | 116 | struct ec_backend_op_stubs *ops; /* EC backend stubs */ 117 | uint32_t ec_backend_version; /* The revision number of this back 118 | * end. Is used to determine whether 119 | * a specific instance of this backend 120 | * accepts fragments generated by 121 | * another version */ 122 | }; 123 | 124 | /* EC backend definition */ 125 | typedef struct ec_backend { 126 | struct ec_backend_common common; /* EC backend common attributes */ 127 | struct ec_backend_args args; /* EC backend instance data (private) */ 128 | 129 | int idesc; /* liberasurecode instance handle */ 130 | struct ec_backend_desc desc; /* EC backend instance handle */ 131 | 132 | SLIST_ENTRY(ec_backend) link; 133 | } *ec_backend_t; 134 | 135 | /* ~=*=~==~=*=~==~=*=~==~=*= frontend <-> backend API =*=~==~=*=~==~=*=~==~= */ 136 | 137 | /** 138 | * Look up a backend instance by descriptor 139 | * 140 | * Returns pointer to a registered liberasurecode instance 141 | * The caller must hold active_instances_rwlock 142 | */ 143 | ec_backend_t liberasurecode_backend_instance_get_by_desc(int desc); 144 | 145 | /* Common function for backends */ 146 | /** 147 | * A function to return 0 for generic usage on backends for get_encode_offset 148 | * 149 | * Returns 0 always 150 | */ 151 | static inline size_t get_encode_offset_zero(void *desc, int metadata_size){ return 0; } 152 | 153 | /** 154 | * A function to return 0 for generic usage on backends for get_backend_metadata_size 155 | * 156 | * Returns 0 always 157 | */ 158 | static inline size_t get_backend_metadata_size_zero(void *desc, int blocksize){ return 0; } 159 | 160 | /* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */ 161 | 162 | #ifdef __cplusplus 163 | } 164 | #endif 165 | 166 | #endif // _ERASURECODE_BACKEND_H_ 167 | 168 | -------------------------------------------------------------------------------- /include/erasurecode/erasurecode_helpers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Kevin Greenan, Tushar Gohad, All rights reserved 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode proprocssing helpers header 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #ifndef _ERASURECODE_HELPERS_H_ 30 | #define _ERASURECODE_HELPERS_H_ 31 | 32 | #include "erasurecode_stdinc.h" 33 | 34 | /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ 35 | 36 | #define talloc(type, num) (type *) malloc(sizeof(type) * (num)) 37 | 38 | /* Determine if an address is aligned to a particular boundary */ 39 | static inline 40 | int is_addr_aligned(unsigned long addr, int align) 41 | { 42 | return (addr & (align - 1)) == 0; 43 | } 44 | 45 | /* 46 | * Convert an int list into a bitmap 47 | * Assume the list is '-1' terminated. 48 | */ 49 | static inline 50 | unsigned long long convert_list_to_bitmap(int *list) 51 | { 52 | int i = 0; 53 | unsigned long long bm = 0; 54 | 55 | while (list[i] > -1) { 56 | /* 57 | * TODO: Assert list[i] < 64 58 | */ 59 | bm |= (1 << list[i]); 60 | i++; 61 | } 62 | 63 | return bm; 64 | } 65 | 66 | /* 67 | * Convert an index list int list into a bitmap 68 | * is_idx_in_erasure[] needs to be allocated by the caller 69 | * @returns number of idxs in error 70 | */ 71 | static inline 72 | int convert_idx_list_to_bitvalues( 73 | int *list_idxs, // input idx_list 74 | int *is_idx_in_erasure, // output idx list as boolean values (1/0) 75 | int num_idxs) // total number of indexes 76 | { 77 | int i = 0, n = 0; 78 | 79 | for (i = 0; i < num_idxs; i++) 80 | is_idx_in_erasure[i] = 0; 81 | for (i = 0, n = 0; (list_idxs[i] > -1) && (n < num_idxs); i++, n++) 82 | is_idx_in_erasure[list_idxs[i]] = 1; 83 | 84 | return n; 85 | } 86 | 87 | /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ 88 | 89 | void *alloc_zeroed_buffer(int size); 90 | void *alloc_and_set_buffer(int size, int value); 91 | void *check_and_free_buffer(void *buf); 92 | void *get_aligned_buffer16(int size); 93 | 94 | /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ 95 | 96 | #ifndef bswap_32 97 | static __inline uint32_t __libec_bswap_32(uint32_t __x) 98 | { 99 | return __x>>24 | (__x>>8&0xff00) | (__x<<8&0xff0000) | __x<<24; 100 | } 101 | #define bswap_32(x) __libec_bswap_32(x) 102 | #endif 103 | 104 | #ifndef bswap_64 105 | static __inline uint64_t __libec_bswap_64(uint64_t __x) 106 | { 107 | return (__libec_bswap_32(__x)+0ULL)<<32 | __libec_bswap_32(__x>>32); 108 | } 109 | #define bswap_64(x) __libec_bswap_64(x) 110 | #endif 111 | 112 | #endif // _ERASURECODE_HELPERS_H_ 113 | 114 | -------------------------------------------------------------------------------- /include/erasurecode/erasurecode_helpers_ext.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Kevin Greenan, Tushar Gohad, All rights reserved 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode proprocssing helpers header 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #ifndef _ERASURECODE_HELPERS_EXT_H_ 30 | #define _ERASURECODE_HELPERS_EXT_H_ 31 | 32 | #include "erasurecode_backend.h" 33 | #include "erasurecode_helpers.h" 34 | 35 | /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ 36 | 37 | static inline 38 | void init_fragment_header(char *buf) 39 | { 40 | fragment_header_t *header = (fragment_header_t *) buf; 41 | 42 | header->magic = LIBERASURECODE_FRAG_HEADER_MAGIC; 43 | } 44 | 45 | /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ 46 | 47 | char *alloc_fragment_buffer(int size); 48 | int free_fragment_buffer(char *buf); 49 | int get_aligned_data_size(ec_backend_t instance, int data_len); 50 | char *get_data_ptr_from_fragment(char *buf); 51 | int get_data_ptr_array_from_fragments(char **data_array, char **fragments, 52 | int num_fragments); 53 | int get_fragment_ptr_array_from_data(char **frag_array, char **data, 54 | int num_data); 55 | char *get_fragment_ptr_from_data_novalidate(char *buf); 56 | char *get_fragment_ptr_from_data(char *buf); 57 | uint64_t get_fragment_size(char *buf); 58 | int set_fragment_idx(char *buf, int idx); 59 | int get_fragment_idx(char *buf); 60 | int set_fragment_payload_size(char *buf, int size); 61 | int get_fragment_payload_size(char *buf); 62 | int set_fragment_backend_metadata_size(char *buf, int size); 63 | int get_fragment_backend_metadata_size(char *buf); 64 | int get_fragment_buffer_size(char *buf); 65 | int set_orig_data_size(char *buf, int orig_data_size); 66 | int get_orig_data_size(char *buf); 67 | int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize); 68 | int get_checksum(char *buf); 69 | int set_libec_version(char *fragment); 70 | int get_libec_version(char *fragment, uint32_t *ver); 71 | int set_backend_id(char *buf, ec_backend_id_t id); 72 | int get_backend_id(char *buf, ec_backend_id_t *id); 73 | int set_backend_version(char *buf, uint32_t version); 74 | int get_backend_version(char *buf, uint32_t *version); 75 | int is_invalid_fragment_header(fragment_header_t *header); 76 | 77 | /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ 78 | 79 | #endif // _ERASURECODE_HELPERS_EXT_H_ 80 | 81 | -------------------------------------------------------------------------------- /include/erasurecode/erasurecode_log.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode logging routines 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #ifndef _ERASURECODE_LOG_H_ 30 | #define _ERASURECODE_LOG_H_ 31 | 32 | /* ==~=*=~==~=*=~==~=*=~==~=*=~==~= Logging =~==~=*=~==~=*=~==~=*=~==~=*=~== */ 33 | 34 | #if __STDC_VERSION__ < 199901L 35 | #if __GNUC__ >= 2 36 | #define __func__ __FUNCTION__ 37 | #else 38 | #define __func__ "" 39 | #endif 40 | #endif 41 | 42 | #define _LOG1(level, ...) \ 43 | syslog (level, __VA_ARGS__) 44 | 45 | #define _LOG2(level, ...) \ 46 | syslog (level, "%s:%d:%s\n", __FILE__, __LINE__, __VA_ARGS__) 47 | 48 | #define log_info(...) _LOG1(LOG_INFO, __VA_ARGS__) 49 | #define log_warn(...) _LOG1(LOG_WARNING, __VA_ARGS__) 50 | #define log_error(...) _LOG1(LOG_ERR, __VA_ARGS__) 51 | #define log_debug(...) _LOG2(LOG_DEBUG, __VA_ARGS__) 52 | 53 | /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ 54 | 55 | #endif // _ERASURECODE_LOG_H_ 56 | -------------------------------------------------------------------------------- /include/erasurecode/erasurecode_postprocessing.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode API helpers header 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #ifndef _ERASURECODE_POSTPROCESSING_H_ 30 | #define _ERASURECODE_POSTPROCESSING_H_ 31 | 32 | int finalize_fragments_after_encode(ec_backend_t instance, 33 | int k, int m, int blocksize, uint64_t orig_data_size, 34 | char **encoded_data, char **encoded_parity); 35 | 36 | void add_fragment_metadata(ec_backend_t instance, char *fragment, 37 | int idx, uint64_t orig_data_size, int blocksize, 38 | ec_checksum_type_t ct, int add_chksum); 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /include/erasurecode/erasurecode_preprocessing.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode API helpers header 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #ifndef _ERASURECODE_PREPROCESSING_H_ 30 | #define _ERASURECODE_PREPROCESSING_H_ 31 | 32 | int prepare_fragments_for_encode( 33 | ec_backend_t instance, 34 | int k, int m, 35 | const char *orig_data, uint64_t orig_data_size, /* input */ 36 | char **encoded_data, char **encoded_parity, /* output */ 37 | int *blocksize); 38 | 39 | int prepare_fragments_for_decode( 40 | int k, int m, 41 | char **data, char **parity, 42 | int *missing_idxs, 43 | int *orig_size, int *fragment_payload_size, int fragment_size, 44 | uint64_t *realloc_bm); 45 | 46 | int get_fragment_partition( 47 | int k, int m, 48 | char **fragments, int num_fragments, 49 | char **data, char **parity, 50 | int *missing); 51 | 52 | int fragments_to_string( 53 | int k, int m, 54 | char **fragments, int num_fragments, 55 | char **orig_payload, uint64_t *payload_len); 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /include/erasurecode/erasurecode_stdinc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | */ 24 | 25 | #ifndef _ERASURECODE_STDINC_H_ 26 | #define _ERASURECODE_STDINC_H_ 27 | 28 | #ifndef _EXCLUDE_LIBERASURE_CODE_H_ 29 | #include "config_liberasurecode.h" 30 | #endif 31 | 32 | #ifdef HAVE_SYSLOG_H 33 | #include 34 | #endif 35 | #ifdef HAVE_SYS_TYPES_H 36 | #include 37 | #endif 38 | #ifdef HAVE_STDIO_H 39 | #include 40 | #endif 41 | #if defined(STDC_HEADERS) 42 | # include 43 | # include 44 | # include 45 | # include 46 | # include 47 | #else 48 | # if defined(HAVE_STDLIB_H) 49 | # include 50 | # elif defined(HAVE_MALLOC_H) 51 | # include 52 | # endif 53 | # if defined(HAVE_STDDEF_H) 54 | # include 55 | # endif 56 | # if defined(HAVE_STDARG_H) 57 | # include 58 | # endif 59 | # if defined(HAVE_UNISTD_H) 60 | # include 61 | # endif 62 | #endif 63 | #ifdef HAVE_STRING_H 64 | # if !defined(STDC_HEADERS) && defined(HAVE_MEMORY_H) 65 | # include 66 | # endif 67 | # include 68 | #endif 69 | #ifdef HAVE_STRINGS_H 70 | # include 71 | #endif 72 | #if defined(HAVE_INTTYPES_H) 73 | # include 74 | #elif defined(HAVE_STDINT_H) 75 | # include 76 | #endif 77 | #ifdef HAVE_CTYPE_H 78 | # include 79 | #endif 80 | #if defined(HAVE_ICONV) && defined(HAVE_ICONV_H) 81 | # include 82 | #endif 83 | #ifdef HAVE_DLFCN_H 84 | # include 85 | #endif 86 | #ifdef HAVE_DLFCN_H 87 | # include 88 | #endif 89 | #ifdef HAVE_PTHREAD_H 90 | # include 91 | #define RWLOCK_INITIALIZER PTHREAD_RWLOCK_INITIALIZER 92 | #define rwlock_t pthread_rwlock_t 93 | #define rwlock_rdlock pthread_rwlock_rdlock 94 | #define rwlock_wrlock pthread_rwlock_wrlock 95 | #define rwlock_tryrdlock pthread_rwlock_tryrdlock 96 | #define rwlock_trywrlock pthread_rwlock_trywrlock 97 | #define rwlock_unlock pthread_rwlock_unlock 98 | #define rwlock_destroy pthread_rwlock_destroy 99 | #endif 100 | #ifdef HAVE_ERRNO_H 101 | # include 102 | #endif 103 | 104 | #if defined(__GNUC__) && __GNUC__ >= 4 105 | # define DECLSPEC __attribute__ ((visibility("default"))) 106 | #else 107 | # define DECLSPEC 108 | #endif 109 | 110 | // FIXME - need to move these to the main liberasurecode header 111 | #ifdef HAVE_MALLOC 112 | #define ERASURECODE_malloc malloc 113 | #else 114 | extern DECLSPEC void * ERASURECODE_malloc(size_t size); 115 | #endif 116 | 117 | #ifdef HAVE_CALLOC 118 | #define ERASURECODE_calloc calloc 119 | #else 120 | extern DECLSPEC void * ERASURECODE_calloc(size_t nmemb, size_t size); 121 | #endif 122 | 123 | #ifdef HAVE_REALLOC 124 | #define ERASURECODE_realloc realloc 125 | #else 126 | extern DECLSPEC void * ERASURECODE_realloc(void *mem, size_t size); 127 | #endif 128 | 129 | #ifdef HAVE_FREE 130 | #define ERASURECODE_free free 131 | #else 132 | extern DECLSPEC void ERASURECODE_free(void *mem); 133 | #endif 134 | 135 | /* Redefine main() on MacOS */ 136 | 137 | #if defined(__MACOS__) || defined(__MACOSX__) 138 | 139 | #ifdef __cplusplus 140 | #define C_LINKAGE "C" 141 | #else 142 | #define C_LINKAGE 143 | #endif /* __cplusplus */ 144 | 145 | /** The application's main() function must be called with C linkage, 146 | * and should be declared like this: 147 | * @code 148 | * #ifdef __cplusplus 149 | * extern "C" 150 | * #endif 151 | * int main(int argc, char *argv[]) 152 | * { 153 | * } 154 | * @endcode 155 | */ 156 | #define main EC_main 157 | 158 | /** The prototype for the application's main() function */ 159 | extern C_LINKAGE int EC_main(int argc, char *argv[]); 160 | 161 | #endif // MACOSX 162 | 163 | #endif // _ERASURECODE_STDINC_H_ 164 | -------------------------------------------------------------------------------- /include/erasurecode/erasurecode_version.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | */ 24 | 25 | #ifndef _ERASURECODE_VERSION_H_ 26 | #define _ERASURECODE_VERSION_H_ 27 | 28 | #define _MAJOR 1 29 | #define _MINOR 6 30 | #define _REV 5 31 | #define _VERSION(x, y, z) ((x << 16) | (y << 8) | (z)) 32 | 33 | #define LIBERASURECODE_VERSION _VERSION(_MAJOR, _MINOR, _REV) 34 | 35 | #ifndef LIBERASURECODE_SO_SUFFIX 36 | #define LIBERASURECODE_SO_SUFFIX "" 37 | #endif // LIBERASURECODE_SO_SUFFIX 38 | 39 | #endif // _ERASURECODE_VERSION_H_ 40 | 41 | 42 | -------------------------------------------------------------------------------- /include/erasurecode/list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 1991, 1993 3 | * The Regents of the University of California. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 3. All advertising materials mentioning features or use of this software 14 | * must display the following acknowledgement: 15 | * This product includes software developed by the University of 16 | * California, Berkeley and its contributors. 17 | * 4. Neither the name of the University nor the names of its contributors 18 | * may be used to endorse or promote products derived from this software 19 | * without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 | * SUCH DAMAGE. 32 | * 33 | * BSD linked list implementation - extracted for use with liberasurecode 34 | * 35 | */ 36 | 37 | #ifndef _LIST_H 38 | #define _LIST_H 39 | 40 | /* 41 | * A singly-linked list is headed by a single forward pointer. The elements 42 | * are singly linked for minimum space and pointer manipulation overhead at 43 | * the expense of O(n) removal for arbitrary elements. New elements can be 44 | * added to the list after an existing element or at the head of the list. 45 | * Elements being removed from the head of the list should use the explicit 46 | * macro for this purpose for optimum efficiency. A singly-linked list may 47 | * only be traversed in the forward direction. Singly-linked lists are ideal 48 | * for applications with large datasets and few or no removals or for 49 | * implementing a LIFO queue. 50 | */ 51 | /* 52 | * Singly-linked List declarations. 53 | */ 54 | #define SLIST_HEAD(name, type) \ 55 | struct name { \ 56 | struct type *slh_first; /* first element */ \ 57 | } 58 | 59 | #define SLIST_HEAD_INITIALIZER(head) \ 60 | { NULL } 61 | 62 | #define SLIST_ENTRY(type) \ 63 | struct { \ 64 | struct type *sle_next; /* next element */ \ 65 | } 66 | 67 | /* 68 | * Singly-linked List functions. 69 | */ 70 | #define SLIST_EMPTY(head) ((head)->slh_first == NULL) 71 | 72 | #define SLIST_FIRST(head) ((head)->slh_first) 73 | 74 | #define SLIST_FOREACH(var, head, field) \ 75 | for ((var) = SLIST_FIRST((head)); \ 76 | (var); \ 77 | (var) = SLIST_NEXT((var), field)) 78 | 79 | #define SLIST_INIT(head) do { \ 80 | SLIST_FIRST((head)) = NULL; \ 81 | } while (0) 82 | 83 | #define SLIST_INSERT_AFTER(slistelm, elm, field) do { \ 84 | SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field); \ 85 | SLIST_NEXT((slistelm), field) = (elm); \ 86 | } while (0) 87 | 88 | #define SLIST_INSERT_HEAD(head, elm, field) do { \ 89 | SLIST_NEXT((elm), field) = SLIST_FIRST((head)); \ 90 | SLIST_FIRST((head)) = (elm); \ 91 | } while (0) 92 | 93 | #define SLIST_NEXT(elm, field) ((elm)->field.sle_next) 94 | 95 | #define SLIST_REMOVE(head, elm, type, field) do { \ 96 | if (SLIST_FIRST((head)) == (elm)) { \ 97 | SLIST_REMOVE_HEAD((head), field); \ 98 | } \ 99 | else { \ 100 | struct type *curelm = SLIST_FIRST((head)); \ 101 | while (SLIST_NEXT(curelm, field) != (elm)) \ 102 | curelm = SLIST_NEXT(curelm, field); \ 103 | SLIST_NEXT(curelm, field) = \ 104 | SLIST_NEXT(SLIST_NEXT(curelm, field), field); \ 105 | } \ 106 | } while (0) 107 | 108 | #define SLIST_REMOVE_HEAD(head, field) do { \ 109 | SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field); \ 110 | } while (0) 111 | 112 | #endif // _LIST_H 113 | -------------------------------------------------------------------------------- /include/erasurecode/md5.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. 3 | * MD5 Message-Digest Algorithm (RFC 1321). 4 | * 5 | * Homepage: 6 | * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 7 | * 8 | * Author: 9 | * Alexander Peslyak, better known as Solar Designer 10 | * 11 | * This software was written by Alexander Peslyak in 2001. No copyright is 12 | * claimed, and the software is hereby placed in the public domain. 13 | * In case this attempt to disclaim copyright and place the software in the 14 | * public domain is deemed null and void, then the software is 15 | * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the 16 | * general public under the following terms: 17 | * 18 | * Redistribution and use in source and binary forms, with or without 19 | * modification, are permitted. 20 | * 21 | * There's ABSOLUTELY NO WARRANTY, express or implied. 22 | * 23 | * See md5.c for more information. 24 | */ 25 | 26 | #ifdef HAVE_OPENSSL 27 | #include 28 | #elif !defined(_MD5_H) 29 | #define _MD5_H 30 | 31 | /* Any 32-bit or wider unsigned integer data type will do */ 32 | typedef unsigned int MD5_u32plus; 33 | 34 | typedef struct { 35 | MD5_u32plus lo, hi; 36 | MD5_u32plus a, b, c, d; 37 | unsigned char buffer[64]; 38 | MD5_u32plus block[16]; 39 | } MD5_CTX; 40 | 41 | extern void MD5_Init(MD5_CTX *ctx); 42 | extern void MD5_Update(MD5_CTX *ctx, void *data, unsigned long size); 43 | extern void MD5_Final(unsigned char *result, MD5_CTX *ctx); 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /include/isa_l/isa_l_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Kevin M Greenan 3 | * Copyright 2014 Tushar Gohad 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * Redistributions of source code must retain the above copyright notice, this 9 | * list of conditions and the following disclaimer. 10 | * 11 | * Redistributions in binary form must reproduce the above copyright notice, this 12 | * list of conditions and the following disclaimer in the documentation and/or 13 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 14 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 16 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 17 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 18 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 19 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 21 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 22 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * 25 | * isa_l_rs_vand backend implementation 26 | * 27 | * vi: set noai tw=79 ts=4 sw=4: 28 | */ 29 | 30 | #define ISA_L_W 8 31 | 32 | /* Forward declarations */ 33 | typedef void (*ec_encode_data_func)(int, int, int, unsigned char*, unsigned char **, unsigned char **); 34 | typedef void (*ec_init_tables_func)(int, int, unsigned char*, unsigned char *); 35 | typedef void (*gf_gen_encoding_matrix_func)(unsigned char*, int, int); 36 | typedef int (*gf_invert_matrix_func)(unsigned char*, unsigned char*, const int); 37 | typedef unsigned char (*gf_mul_func)(unsigned char, unsigned char); 38 | 39 | typedef struct { 40 | /* calls required for init */ 41 | ec_init_tables_func ec_init_tables; 42 | gf_gen_encoding_matrix_func gf_gen_encoding_matrix; 43 | 44 | /* calls required for encode */ 45 | ec_encode_data_func ec_encode_data; 46 | 47 | /* calls required for decode and reconstruct */ 48 | gf_invert_matrix_func gf_invert_matrix; 49 | 50 | /* multiplication function used by ISA-L */ 51 | gf_mul_func gf_mul; 52 | 53 | /* fields needed to hold state */ 54 | unsigned char *matrix; 55 | unsigned char *encode_tables; 56 | int k; 57 | int m; 58 | int w; 59 | } isa_l_descriptor; 60 | 61 | int isa_l_encode(void *desc, char **data, char **parity, int blocksize); 62 | int isa_l_decode(void *desc, char **data, char **parity, int *missing_idxs, 63 | int blocksize); 64 | int isa_l_reconstruct(void *desc, char **data, char **parity, 65 | int *missing_idxs, int destination_idx, int blocksize); 66 | int isa_l_min_fragments(void *desc, int *missing_idxs, 67 | int *fragments_to_exclude, int *fragments_needed); 68 | int isa_l_element_size(void* desc); 69 | int isa_l_exit(void *desc); 70 | void * isa_l_common_init(struct ec_backend_args *args, void *backend_sohandle, 71 | const char* gen_matrix_func_name); 72 | -------------------------------------------------------------------------------- /include/rs_vand/liberasurecode_rs_vand.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Kevin M Greenan 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * vi: set noai tw=79 ts=4 sw=4: 25 | */ 26 | 27 | void free_systematic_matrix(int *matrix); 28 | int* make_systematic_matrix(int k, int m); 29 | int is_missing(int *missing_idxs, int index_to_check); 30 | int gaussj_inversion(int *matrix, int *inverse, int n); 31 | int rs_galois_inverse(int x); 32 | int rs_galois_mult(int x, int y); 33 | void init_liberasurecode_rs_vand(int k, int m); 34 | void deinit_liberasurecode_rs_vand(void); 35 | void print_matrix(int *matrix, int rows, int cols); 36 | void square_matrix_multiply(int *m1, int *m2, int *prod, int n); 37 | int create_decoding_matrix(int *gen_matrix, int *dec_matrix, int *missing_idxs, int k, int m); 38 | int is_identity_matrix(int *matrix, int n); 39 | int liberasurecode_rs_vand_encode(int *generator_matrix, char **data, char **parity, int k, int m, int blocksize); 40 | int liberasurecode_rs_vand_decode(int *generator_matrix, char **data, char **parity, int k, int m, int *missing, int blocksize, int rebuild_parity); 41 | int liberasurecode_rs_vand_reconstruct(int *generator_matrix, char **data, char **parity, int k, int m, int *missing, int destination_idx, int blocksize); 42 | -------------------------------------------------------------------------------- /include/rs_vand/rs_galois.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Kevin M Greenan 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * vi: set noai tw=79 ts=4 sw=4: 25 | */ 26 | 27 | // DISCLAIMER: This is a totally basic implementation of RS used if a user does not 28 | // want to install one of the supported backends, such as Jerasure and ISA-L. 29 | // This is not expected to perform as well as the other supported backends, 30 | // but does not make any assumptions about the host system. Using a library 31 | // like Jerasure with GF-Complete will give users the ability to tune to their 32 | // architecture (Intel or ARM), CPU and memory (lots of options). 33 | 34 | // We are only implementing w=16 here. If you want to use something 35 | // else, then use Jerasure with GF-Complete or ISA-L. 36 | #define PRIM_POLY 0x1100b 37 | #define FIELD_SIZE (1 << 16) 38 | #define GROUP_SIZE (FIELD_SIZE - 1) 39 | 40 | void rs_galois_init_tables(void); 41 | void rs_galois_deinit_tables(void); 42 | int rs_galois_mult(int x, int y); 43 | int rs_galois_inverse(int x); 44 | 45 | -------------------------------------------------------------------------------- /include/xor_codes/xor_code.h: -------------------------------------------------------------------------------- 1 | /* * Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | */ 24 | 25 | #ifndef _XOR_CODE_H 26 | #define _XOR_CODE_H 27 | 28 | #define MAX_DATA 32 29 | #define MAX_PARITY MAX_DATA 30 | 31 | #define MEM_ALIGN_SIZE 16 32 | 33 | #define DECODED_MISSING_IDX MAX_DATA 34 | 35 | typedef enum { FAIL_PATTERN_GE_HD, // Num failures greater than or equal to HD 36 | FAIL_PATTERN_0D_0P, 37 | FAIL_PATTERN_1D_0P, 38 | FAIL_PATTERN_2D_0P, 39 | FAIL_PATTERN_3D_0P, 40 | FAIL_PATTERN_1D_1P, 41 | FAIL_PATTERN_1D_2P, 42 | FAIL_PATTERN_2D_1P, 43 | FAIL_PATTERN_0D_1P, 44 | FAIL_PATTERN_0D_2P, 45 | FAIL_PATTERN_0D_3P } failure_pattern_t; 46 | 47 | #define is_aligned(x) (((unsigned long)x & (MEM_ALIGN_SIZE-1)) == 0) 48 | #define num_unaligned_end(size) (size % MEM_ALIGN_SIZE) 49 | 50 | struct xor_code_s; 51 | 52 | typedef struct xor_code_s 53 | { 54 | int k; 55 | int m; 56 | int hd; 57 | unsigned int *parity_bms; 58 | unsigned int *data_bms; 59 | int (*decode)(struct xor_code_s *code_desc, char **data, char **parity, int *missing_idxs, int blocksize, int decode_parity); 60 | void (*encode)(struct xor_code_s *code_desc, char **data, char **parity, int blocksize); 61 | int (*fragments_needed)(struct xor_code_s *code_desc, int *missing_idxs, int *fragments_to_exclude, int *fragments_needed); 62 | } xor_code_t; 63 | 64 | int is_data_in_parity(int data_idx, unsigned int parity_bm); 65 | 66 | int does_parity_have_data(int parity_idx, unsigned int data_bm); 67 | 68 | int parity_bit_lookup(xor_code_t *code_desc, int index); 69 | 70 | int data_bit_lookup(xor_code_t *code_desc, int index); 71 | 72 | int missing_elements_bm(xor_code_t *code_desc, int *missing_elements, int (*bit_lookup_func)(xor_code_t *code_desc, int index)); 73 | 74 | failure_pattern_t get_failure_pattern(xor_code_t *code_desc, int *missing_idxs); 75 | 76 | void fast_memcpy(char *dst, char *src, int size); 77 | 78 | void xor_bufs_and_store(char *buf1, char *buf2, int blocksize); 79 | 80 | void xor_code_encode(xor_code_t *code_desc, char **data, char **parity, int blocksize); 81 | 82 | void selective_encode(xor_code_t *code_desc, char **data, char **parity, int *missing_parity, int blocksize); 83 | 84 | int * get_missing_parity(xor_code_t *code_desc, int *missing_idxs); 85 | 86 | int * get_missing_data(xor_code_t *code_desc, int *missing_idxs); 87 | 88 | int num_missing_data_in_parity(xor_code_t *code_desc, int parity_idx, int *missing_data); 89 | 90 | int index_of_connected_parity(xor_code_t *code_desc, int data_index, int *missing_parity, int *missing_data); 91 | 92 | void remove_from_missing_list(int element, int *missing_list); 93 | 94 | int* get_symbols_needed(xor_code_t *code_desc, int *missing_list, int *fragments_to_exclude); 95 | 96 | void xor_reconstruct_one(xor_code_t *code_desc, char **data, char **parity, int *missing_idxs, int index_to_reconstruct, int blocksize); 97 | 98 | xor_code_t* init_xor_hd_code(int k, int m, int hd); 99 | 100 | #endif 101 | -------------------------------------------------------------------------------- /include/xor_codes/xor_hd_code_defs.h: -------------------------------------------------------------------------------- 1 | /* * Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | */ 24 | 25 | #ifndef _XOR_HD_CODE_DEFS_H 26 | #define _XOR_HD_CODE_DEFS_H 27 | 28 | 29 | 30 | // I made these by hand... 31 | static unsigned int g_12_6_4_hd_code_parity_bms[] = { 1649, 3235, 2375, 718, 1436, 2872 }; 32 | static unsigned int g_12_6_4_hd_code_data_bms[] = { 7, 14, 28, 56, 49, 35, 13, 26, 52, 41, 19, 38 }; 33 | 34 | static unsigned int g_10_5_3_hd_code_parity_bms[] = { 163, 300, 337, 582, 664 }; 35 | static unsigned int g_10_5_3_hd_code_data_bms[] = { 5, 9, 10, 18, 20, 3, 12, 17, 6, 24 }; 36 | 37 | static unsigned int g_3_3_3_hd_code_parity_bms[] = { 5, 6, 3 }; 38 | static unsigned int g_3_3_3_hd_code_data_bms[] = { 5, 6, 3}; 39 | 40 | 41 | // The rest were generated via the "goldilocks" code algorithm 42 | static unsigned int g_6_6_3_hd_code_parity_bms[] = { 3, 48, 36, 24, 9, 6 }; 43 | static unsigned int g_6_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6 }; 44 | static unsigned int g_7_6_3_hd_code_parity_bms[] = { 67, 112, 36, 24, 9, 6 }; 45 | static unsigned int g_7_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6, 3 }; 46 | static unsigned int g_8_6_3_hd_code_parity_bms[] = { 67, 112, 164, 152, 9, 6 }; 47 | static unsigned int g_8_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6, 3, 12 }; 48 | static unsigned int g_9_6_3_hd_code_parity_bms[] = { 67, 112, 164, 152, 265, 262 }; 49 | static unsigned int g_9_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6, 3, 12, 48 }; 50 | static unsigned int g_10_6_3_hd_code_parity_bms[] = { 579, 112, 676, 152, 265, 262 }; 51 | static unsigned int g_10_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6, 3, 12, 48, 5 }; 52 | static unsigned int g_11_6_3_hd_code_parity_bms[] = { 579, 1136, 676, 152, 1289, 262 }; 53 | static unsigned int g_11_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6, 3, 12, 48, 5, 18 }; 54 | static unsigned int g_12_6_3_hd_code_parity_bms[] = { 579, 1136, 676, 2200, 1289, 2310 }; 55 | static unsigned int g_12_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6, 3, 12, 48, 5, 18, 40 }; 56 | static unsigned int g_13_6_3_hd_code_parity_bms[] = { 4675, 1136, 676, 6296, 1289, 2310 }; 57 | static unsigned int g_13_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6, 3, 12, 48, 5, 18, 40, 9 }; 58 | static unsigned int g_14_6_3_hd_code_parity_bms[] = { 4675, 9328, 676, 6296, 1289, 10502 }; 59 | static unsigned int g_14_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6, 3, 12, 48, 5, 18, 40, 9, 34 }; 60 | static unsigned int g_15_6_3_hd_code_parity_bms[] = { 4675, 9328, 17060, 6296, 17673, 10502 }; 61 | static unsigned int g_15_6_3_hd_code_data_bms[] = { 17, 33, 36, 24, 10, 6, 3, 12, 48, 5, 18, 40, 9, 34, 20 }; 62 | 63 | static unsigned int g_6_6_4_hd_code_parity_bms[] = { 7, 56, 56, 11, 21, 38 }; 64 | static unsigned int g_6_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38 }; 65 | static unsigned int g_7_6_4_hd_code_parity_bms[] = { 71, 120, 120, 11, 21, 38 }; 66 | static unsigned int g_7_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7 }; 67 | static unsigned int g_8_6_4_hd_code_parity_bms[] = { 71, 120, 120, 139, 149, 166 }; 68 | static unsigned int g_8_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56 }; 69 | static unsigned int g_9_6_4_hd_code_parity_bms[] = { 327, 376, 120, 395, 149, 166 }; 70 | static unsigned int g_9_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11 }; 71 | static unsigned int g_10_6_4_hd_code_parity_bms[] = { 327, 376, 632, 395, 661, 678 }; 72 | static unsigned int g_10_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52 }; 73 | static unsigned int g_11_6_4_hd_code_parity_bms[] = { 1351, 1400, 632, 395, 1685, 678 }; 74 | static unsigned int g_11_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52, 19 }; 75 | static unsigned int g_13_6_4_hd_code_parity_bms[] = { 5447, 5496, 2680, 2443, 1685, 6822 }; 76 | static unsigned int g_13_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52, 19, 44, 35 }; 77 | static unsigned int g_14_6_4_hd_code_parity_bms[] = { 5447, 5496, 10872, 10635, 9877, 6822 }; 78 | static unsigned int g_14_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52, 19, 44, 35, 28 }; 79 | static unsigned int g_15_6_4_hd_code_parity_bms[] = { 21831, 5496, 27256, 27019, 9877, 6822 }; 80 | static unsigned int g_15_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52, 19, 44, 35, 28, 13 }; 81 | static unsigned int g_16_6_4_hd_code_parity_bms[] = { 21831, 38264, 27256, 27019, 42645, 39590 }; 82 | static unsigned int g_16_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52, 19, 44, 35, 28, 13, 50 }; 83 | static unsigned int g_17_6_4_hd_code_parity_bms[] = { 87367, 38264, 92792, 27019, 108181, 39590 }; 84 | static unsigned int g_17_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52, 19, 44, 35, 28, 13, 50, 21 }; 85 | static unsigned int g_18_6_4_hd_code_parity_bms[] = { 87367, 169336, 92792, 158091, 108181, 170662 }; 86 | static unsigned int g_18_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52, 19, 44, 35, 28, 13, 50, 21, 42 }; 87 | static unsigned int g_19_6_4_hd_code_parity_bms[] = { 349511, 169336, 354936, 158091, 108181, 432806 }; 88 | static unsigned int g_19_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52, 19, 44, 35, 28, 13, 50, 21, 42, 37 }; 89 | static unsigned int g_20_6_4_hd_code_parity_bms[] = { 349511, 693624, 354936, 682379, 632469, 432806 }; 90 | static unsigned int g_20_6_4_hd_code_data_bms[] = { 25, 41, 49, 14, 22, 38, 7, 56, 11, 52, 19, 44, 35, 28, 13, 50, 21, 42, 37, 26 }; 91 | 92 | 93 | static unsigned int g_5_5_3_hd_code_parity_bms[] = { 3, 12, 17, 6, 24 }; 94 | static unsigned int g_5_5_3_hd_code_data_bms[] = { 5, 9, 10, 18, 20 }; 95 | static unsigned int g_6_5_3_hd_code_parity_bms[] = { 35, 44, 17, 6, 24 }; 96 | static unsigned int g_6_5_3_hd_code_data_bms[] = { 5, 9, 10, 18, 20, 3 }; 97 | static unsigned int g_7_5_3_hd_code_parity_bms[] = { 35, 44, 81, 70, 24 }; 98 | static unsigned int g_7_5_3_hd_code_data_bms[] = { 5, 9, 10, 18, 20, 3, 12 }; 99 | static unsigned int g_8_5_3_hd_code_parity_bms[] = { 163, 44, 81, 70, 152 }; 100 | static unsigned int g_8_5_3_hd_code_data_bms[] = { 5, 9, 10, 18, 20, 3, 12, 17 }; 101 | static unsigned int g_9_5_3_hd_code_parity_bms[] = { 163, 300, 337, 70, 152 }; 102 | static unsigned int g_9_5_3_hd_code_data_bms[] = { 5, 9, 10, 18, 20, 3, 12, 17, 6 }; 103 | 104 | static unsigned int g_5_5_4_hd_code_parity_bms[] = { 7, 25, 14, 19, 28 }; 105 | static unsigned int g_5_5_4_hd_code_data_bms[] = { 11, 13, 21, 22, 26 }; 106 | static unsigned int g_6_5_4_hd_code_parity_bms[] = { 39, 57, 46, 19, 28 }; 107 | static unsigned int g_6_5_4_hd_code_data_bms[] = { 11, 13, 21, 22, 26, 7 }; 108 | static unsigned int g_7_5_4_hd_code_parity_bms[] = { 103, 57, 46, 83, 92 }; 109 | static unsigned int g_7_5_4_hd_code_data_bms[] = { 11, 13, 21, 22, 26, 7, 25 }; 110 | static unsigned int g_8_5_4_hd_code_parity_bms[] = { 103, 185, 174, 211, 92 }; 111 | static unsigned int g_8_5_4_hd_code_data_bms[] = { 11, 13, 21, 22, 26, 7, 25, 14 }; 112 | static unsigned int g_9_5_4_hd_code_parity_bms[] = { 359, 441, 174, 211, 348 }; 113 | static unsigned int g_9_5_4_hd_code_data_bms[] = { 11, 13, 21, 22, 26, 7, 25, 14, 19 }; 114 | static unsigned int g_10_5_4_hd_code_parity_bms[] = { 359, 441, 686, 723, 860 }; 115 | static unsigned int g_10_5_4_hd_code_data_bms[] = { 11, 13, 21, 22, 26, 7, 25, 14, 19, 28 }; 116 | 117 | // Indexed by k 118 | static unsigned int * hd4_m5_parity[11] = { 0, 0, 0, 0, 0, g_5_5_4_hd_code_parity_bms, g_6_5_4_hd_code_parity_bms, g_7_5_4_hd_code_parity_bms, g_8_5_4_hd_code_parity_bms, g_9_5_4_hd_code_parity_bms, g_10_5_4_hd_code_parity_bms }; 119 | static unsigned int * hd4_m5_data[11] = { 0, 0, 0, 0, 0, g_5_5_4_hd_code_data_bms, g_6_5_4_hd_code_data_bms, g_7_5_4_hd_code_data_bms, g_8_5_4_hd_code_data_bms, g_9_5_4_hd_code_data_bms, g_10_5_4_hd_code_data_bms }; 120 | static unsigned int * hd4_m6_parity[21] = { 0, 0, 0, 0, 0, 0, g_6_6_4_hd_code_parity_bms, g_7_6_4_hd_code_parity_bms, g_8_6_4_hd_code_parity_bms, g_9_6_4_hd_code_parity_bms, g_10_6_4_hd_code_parity_bms, g_11_6_4_hd_code_parity_bms, g_12_6_4_hd_code_parity_bms, g_13_6_4_hd_code_parity_bms, g_14_6_4_hd_code_parity_bms, g_15_6_4_hd_code_parity_bms, g_16_6_4_hd_code_parity_bms, g_17_6_4_hd_code_parity_bms, g_18_6_4_hd_code_parity_bms, g_19_6_4_hd_code_parity_bms, g_20_6_4_hd_code_parity_bms }; 121 | 122 | static unsigned int * hd4_m6_data[21] = { 0, 0, 0, 0, 0, 0, g_6_6_4_hd_code_data_bms, g_7_6_4_hd_code_data_bms, g_8_6_4_hd_code_data_bms, g_9_6_4_hd_code_data_bms, g_10_6_4_hd_code_data_bms, g_11_6_4_hd_code_data_bms, g_12_6_4_hd_code_data_bms, g_13_6_4_hd_code_data_bms, g_14_6_4_hd_code_data_bms, g_15_6_4_hd_code_data_bms, g_16_6_4_hd_code_data_bms, g_17_6_4_hd_code_data_bms, g_18_6_4_hd_code_data_bms, g_19_6_4_hd_code_data_bms, g_20_6_4_hd_code_data_bms }; 123 | 124 | static unsigned int * hd3_m5_parity[11] = { 0, 0, 0, 0, 0, g_5_5_3_hd_code_parity_bms, g_6_5_3_hd_code_parity_bms, g_7_5_3_hd_code_parity_bms, g_8_5_3_hd_code_parity_bms, g_9_5_3_hd_code_parity_bms, g_10_5_3_hd_code_parity_bms }; 125 | static unsigned int * hd3_m5_data[11] = { 0, 0, 0, 0, 0, g_5_5_3_hd_code_data_bms, g_6_5_3_hd_code_data_bms, g_7_5_3_hd_code_data_bms, g_8_5_3_hd_code_data_bms, g_9_5_3_hd_code_data_bms, g_10_5_3_hd_code_data_bms }; 126 | static unsigned int * hd3_m6_parity[16] = { 0, 0, 0, 0, 0, 0, g_6_6_3_hd_code_parity_bms, g_7_6_3_hd_code_parity_bms, g_8_6_3_hd_code_parity_bms, g_9_6_3_hd_code_parity_bms, g_10_6_3_hd_code_parity_bms, g_11_6_3_hd_code_parity_bms, g_12_6_3_hd_code_parity_bms, g_13_6_3_hd_code_parity_bms, g_14_6_3_hd_code_parity_bms, g_15_6_3_hd_code_parity_bms }; 127 | static unsigned int * hd3_m6_data[16] = { 0, 0, 0, 0, 0, 0, g_6_6_3_hd_code_data_bms, g_7_6_3_hd_code_data_bms, g_8_6_3_hd_code_data_bms, g_9_6_3_hd_code_data_bms, g_10_6_3_hd_code_data_bms, g_11_6_3_hd_code_data_bms, g_12_6_3_hd_code_data_bms, g_13_6_3_hd_code_data_bms, g_14_6_3_hd_code_data_bms, g_15_6_3_hd_code_data_bms }; 128 | 129 | static unsigned int * hd3_m3_parity[4] = { 0, 0, 0, g_3_3_3_hd_code_parity_bms }; 130 | static unsigned int * hd3_m3_data[4] = { 0, 0, 0, g_3_3_3_hd_code_data_bms }; 131 | 132 | static unsigned int ** parity_bm_hd4 [7] = { 0, 0, 0, 0, 0, hd4_m5_parity, hd4_m6_parity }; 133 | static unsigned int ** data_bm_hd4 [7] = { 0, 0, 0, 0, 0, hd4_m5_data, hd4_m6_data }; 134 | static unsigned int ** parity_bm_hd3 [7] = { 0, 0, 0, hd3_m3_parity, 0, hd3_m5_parity, hd3_m6_parity }; 135 | static unsigned int ** data_bm_hd3 [7] = { 0, 0, 0, hd3_m3_data, 0, hd3_m5_data, hd3_m6_data }; 136 | 137 | #define PARITY_BM_ARY(k, m, hd) (hd == 3) ? parity_bm_hd3[m][k] : parity_bm_hd4[m][k] 138 | #define DATA_BM_ARY(k, m, hd) (hd == 3) ? data_bm_hd3[m][k] : data_bm_hd4[m][k] 139 | 140 | #endif 141 | -------------------------------------------------------------------------------- /libXorcode.sym: -------------------------------------------------------------------------------- 1 | T init_xor_hd_code 2 | T xor_code_encode 3 | T xor_hd_decode 4 | T xor_hd_fragments_needed 5 | T xor_reconstruct_one 6 | -------------------------------------------------------------------------------- /liberasurecode.sym: -------------------------------------------------------------------------------- 1 | T alloc_and_set_buffer 2 | T get_backend_id 3 | T get_backend_version 4 | T get_data_ptr_from_fragment 5 | T get_fragment_partition 6 | T get_libec_version 7 | T is_invalid_fragment 8 | T is_invalid_fragment_header 9 | T liberasurecode_backend_available 10 | T liberasurecode_backend_instance_get_by_desc 11 | T liberasurecode_crc32_alt 12 | T liberasurecode_decode 13 | T liberasurecode_decode_cleanup 14 | T liberasurecode_encode 15 | T liberasurecode_encode_cleanup 16 | T liberasurecode_exit 17 | T liberasurecode_fragments_needed 18 | T liberasurecode_get_aligned_data_size 19 | T liberasurecode_get_fragment_metadata 20 | T liberasurecode_get_fragment_size 21 | T liberasurecode_get_minimum_encode_size 22 | T liberasurecode_get_version 23 | T liberasurecode_init 24 | T liberasurecode_instance_create 25 | T liberasurecode_instance_destroy 26 | T liberasurecode_reconstruct_fragment 27 | T liberasurecode_verify_fragment_metadata 28 | T liberasurecode_verify_stripe_metadata 29 | -------------------------------------------------------------------------------- /liberasurecode_rs_vand.sym: -------------------------------------------------------------------------------- 1 | T create_decoding_matrix 2 | T deinit_liberasurecode_rs_vand 3 | T free_systematic_matrix 4 | T gaussj_inversion 5 | T init_liberasurecode_rs_vand 6 | T is_identity_matrix 7 | T is_missing 8 | T liberasurecode_rs_vand_decode 9 | T liberasurecode_rs_vand_encode 10 | T liberasurecode_rs_vand_reconstruct 11 | T make_systematic_matrix 12 | T print_matrix 13 | T square_matrix_multiply 14 | -------------------------------------------------------------------------------- /libnullcode.sym: -------------------------------------------------------------------------------- 1 | T null_code_decode 2 | T null_code_encode 3 | T null_code_fragments_needed 4 | T null_code_init 5 | T null_reconstruct 6 | -------------------------------------------------------------------------------- /playbooks/enable-fips.yaml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | roles: 3 | - role: enable-fips 4 | nslookup_target: 'opendev.org' 5 | -------------------------------------------------------------------------------- /playbooks/unittests/run.yaml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | roles: 3 | - install_isal 4 | - install_jerasure 5 | - test_liberasurecode 6 | 7 | -------------------------------------------------------------------------------- /roles/install_isal/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | - name: Ensure legacy workspace directory 2 | file: 3 | path: '{{ ansible_user_dir }}/workspace' 4 | state: directory 5 | 6 | - name: Build and Install nasm 7 | shell: 8 | cmd: | 9 | set -e 10 | set -x 11 | cd $WORKSPACE 12 | curl https://www.nasm.us/pub/nasm/releasebuilds/2.16.01/nasm-2.16.01.tar.gz | tar -xz 13 | cd nasm* 14 | ./autogen.sh 15 | ./configure 16 | make nasm 17 | sudo install -c nasm /usr/bin/nasm 18 | executable: /bin/bash 19 | chdir: '{{ ansible_user_dir }}/workspace' 20 | environment: '{{ zuul | zuul_legacy_vars }}' 21 | 22 | - name: Build and Install ISA-L 23 | shell: 24 | cmd: | 25 | set -e 26 | set -x 27 | cd $WORKSPACE 28 | git clone https://github.com/intel/isa-l.git 29 | cd isa-l 30 | ./autogen.sh 31 | ./configure 32 | make 33 | sudo make install 34 | executable: /bin/bash 35 | chdir: '{{ ansible_user_dir }}/workspace' 36 | environment: '{{ zuul | zuul_legacy_vars }}' 37 | -------------------------------------------------------------------------------- /roles/install_jerasure/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | - name: Ensure legacy workspace directory 2 | file: 3 | path: '{{ ansible_user_dir }}/workspace' 4 | state: directory 5 | 6 | - name: Build and Install GF-Complete 7 | shell: 8 | cmd: | 9 | set -e 10 | set -x 11 | cd $WORKSPACE 12 | git clone https://github.com/ceph/gf-complete.git 13 | cd gf-complete 14 | ./autogen.sh 15 | ./configure 16 | make 17 | sudo make install 18 | executable: /bin/bash 19 | chdir: '{{ ansible_user_dir }}/workspace' 20 | environment: '{{ zuul | zuul_legacy_vars }}' 21 | 22 | - name: Build and Install Jerasure 23 | shell: 24 | cmd: | 25 | set -e 26 | set -x 27 | cd $WORKSPACE 28 | git clone https://github.com/ceph/jerasure.git 29 | cd jerasure 30 | autoreconf --force --install 31 | ./configure 32 | make 33 | sudo make install 34 | executable: /bin/bash 35 | chdir: '{{ ansible_user_dir }}/workspace' 36 | environment: '{{ zuul | zuul_legacy_vars }}' 37 | -------------------------------------------------------------------------------- /roles/test_liberasurecode/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | - name: Build and test liberasurecode 2 | shell: 3 | cmd: | 4 | set -e 5 | set -x 6 | sudo bash -c "echo /usr/local/lib >> /etc/ld.so.conf" 7 | sudo ldconfig 8 | ./autogen.sh 9 | CONFIG_SHELL=/bin/sh ./configure 10 | make 11 | make test 12 | make valgrind-test 13 | ./check-symbols.sh check 14 | executable: /bin/bash 15 | chdir: '{{ zuul.project.src_dir }}' 16 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = builtin/xor_codes builtin/null_code builtin/rs_vand 2 | 3 | lib_LTLIBRARIES = liberasurecode.la 4 | 5 | INCLUDE = \ 6 | -I$(abs_top_srcdir)/include/erasurecode \ 7 | -I$(abs_top_srcdir)/include/xor_codes \ 8 | -I$(abs_top_srcdir)/include/rs_vand \ 9 | -I$(abs_top_srcdir)/include/isa_l \ 10 | -I$(abs_top_srcdir)/include/shss 11 | 12 | AM_CPPFLAGS = $(CPPFLAGS) $(INCLUDE) 13 | AM_CFLAGS = $(AM_CPPFLAGS) 14 | 15 | # liberasurecode params 16 | liberasurecode_la_SOURCES = \ 17 | erasurecode.c \ 18 | erasurecode_helpers.c \ 19 | erasurecode_preprocessing.c \ 20 | erasurecode_postprocessing.c \ 21 | utils/chksum/crc32.c \ 22 | utils/chksum/alg_sig.c \ 23 | backends/null/null.c \ 24 | backends/xor/flat_xor_hd.c \ 25 | backends/jerasure/jerasure_rs_vand.c \ 26 | backends/jerasure/jerasure_rs_cauchy.c \ 27 | backends/isa-l/isa_l_common.c \ 28 | backends/isa-l/isa_l_rs_vand.c \ 29 | backends/isa-l/isa_l_rs_cauchy.c \ 30 | backends/rs_vand/liberasurecode_rs_vand.c \ 31 | backends/shss/shss.c \ 32 | backends/phazrio/libphazr.c 33 | 34 | liberasurecode_la_CPPFLAGS = -Werror @GCOV_FLAGS@ 35 | liberasurecode_la_LIBADD = \ 36 | builtin/null_code/libnullcode.la \ 37 | builtin/xor_codes/libXorcode.la \ 38 | builtin/rs_vand/liberasurecode_rs_vand.la \ 39 | -lpthread -lm -lz @GCOV_LDFLAGS@ 40 | 41 | # Version format (C - A).(A).(R) for C:R:A input 42 | liberasurecode_la_LDFLAGS = -rpath '$(libdir)' -version-info @LIBERASURECODE_VERSION_INFO@ 43 | 44 | MOSTLYCLEANFILES = *.gcda *.gcno *.gcov utils/chksum/*.gcda utils/chksum/*.gcno utils/chksum/*.gcov \ 45 | backends/null/*.gcda backends/null/*.gcno backends/null/*.gcov \ 46 | backends/xor/*.gcda backends/xor/*.gcno backends/xor/*.gcov \ 47 | backends/jerasure/*.gcda backends/jerasure/*.gcno backends/jerasure/*.gcov \ 48 | backends/shss/*.gcda backends/shss/*.gcno backends/shss/*.gcov \ 49 | backends/rs_vand/*.gcda backends/rs_vand/*.gcno backends/rs_vand/*.gcov \ 50 | backends/phazrio/*.gcda backends/phazrio/*.gcno backends/phazrio/*.gcov 51 | -------------------------------------------------------------------------------- /src/backends/isa-l/isa_l_rs_cauchy.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Kevin M Greenan 3 | * Copyright 2014 Tushar Gohad 4 | * Copyright 2016 Kota Tsuyuzaki 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * Redistributions of source code must retain the above copyright notice, this 10 | * list of conditions and the following disclaimer. 11 | * 12 | * Redistributions in binary form must reproduce the above copyright notice, this 13 | * list of conditions and the following disclaimer in the documentation and/or 14 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 15 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 16 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 17 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 18 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 19 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 20 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 22 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 23 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 24 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | * 26 | * isa_l_rs_cauchy backend implementation 27 | * 28 | * vi: set noai tw=79 ts=4 sw=4: 29 | */ 30 | 31 | #include 32 | #include "erasurecode_backend.h" 33 | #include "isa_l_common.h" 34 | 35 | #define ISA_L_RS_CAUCHY_LIB_MAJOR 2 36 | #define ISA_L_RS_CAUCHY_LIB_MINOR 14 37 | #define ISA_L_RS_CAUCHY_LIB_REV 1 38 | #define ISA_L_RS_CAUCHY_LIB_VER_STR "2.14" 39 | #define ISA_L_RS_CAUCHY_LIB_NAME "isa_l_rs_cauchy" 40 | #if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) 41 | #define ISA_L_RS_CAUCHY_SO_NAME "libisal" LIBERASURECODE_SO_SUFFIX ".dylib" 42 | #else 43 | #define ISA_L_RS_CAUCHY_SO_NAME "libisal" LIBERASURECODE_SO_SUFFIX ".so.2" 44 | #endif 45 | 46 | /* Forward declarations */ 47 | struct ec_backend_common backend_isa_l_rs_cauchy; 48 | 49 | static void * isa_l_rs_cauchy_init(struct ec_backend_args *args, 50 | void *backend_sohandle) 51 | { 52 | return isa_l_common_init(args, backend_sohandle, "gf_gen_cauchy1_matrix"); 53 | } 54 | 55 | /* 56 | * For the time being, we only claim compatibility with versions that 57 | * match exactly 58 | */ 59 | static bool isa_l_rs_cauchy_is_compatible_with(uint32_t version) { 60 | return version == backend_isa_l_rs_cauchy.ec_backend_version; 61 | } 62 | 63 | static struct ec_backend_op_stubs isa_l_rs_cauchy_op_stubs = { 64 | .INIT = isa_l_rs_cauchy_init, 65 | .EXIT = isa_l_exit, 66 | .ENCODE = isa_l_encode, 67 | .DECODE = isa_l_decode, 68 | .FRAGSNEEDED = isa_l_min_fragments, 69 | .RECONSTRUCT = isa_l_reconstruct, 70 | .ELEMENTSIZE = isa_l_element_size, 71 | .ISCOMPATIBLEWITH = isa_l_rs_cauchy_is_compatible_with, 72 | .GETMETADATASIZE = get_backend_metadata_size_zero, 73 | .GETENCODEOFFSET = get_encode_offset_zero, 74 | }; 75 | 76 | __attribute__ ((visibility ("internal"))) 77 | struct ec_backend_common backend_isa_l_rs_cauchy = { 78 | .id = EC_BACKEND_ISA_L_RS_CAUCHY, 79 | .name = ISA_L_RS_CAUCHY_LIB_NAME, 80 | .soname = ISA_L_RS_CAUCHY_SO_NAME, 81 | .soversion = ISA_L_RS_CAUCHY_LIB_VER_STR, 82 | .ops = &isa_l_rs_cauchy_op_stubs, 83 | .ec_backend_version = _VERSION(ISA_L_RS_CAUCHY_LIB_MAJOR, 84 | ISA_L_RS_CAUCHY_LIB_MINOR, 85 | ISA_L_RS_CAUCHY_LIB_REV), 86 | }; 87 | -------------------------------------------------------------------------------- /src/backends/isa-l/isa_l_rs_vand.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Kevin M Greenan 3 | * Copyright 2014 Tushar Gohad 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * Redistributions of source code must retain the above copyright notice, this 9 | * list of conditions and the following disclaimer. 10 | * 11 | * Redistributions in binary form must reproduce the above copyright notice, this 12 | * list of conditions and the following disclaimer in the documentation and/or 13 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 14 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 16 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 17 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 18 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 19 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 21 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 22 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * 25 | * isa_l_rs_vand backend implementation 26 | * 27 | * vi: set noai tw=79 ts=4 sw=4: 28 | */ 29 | 30 | #include 31 | #include "erasurecode_backend.h" 32 | #include "isa_l_common.h" 33 | 34 | #define ISA_L_RS_VAND_LIB_MAJOR 2 35 | #define ISA_L_RS_VAND_LIB_MINOR 13 36 | #define ISA_L_RS_VAND_LIB_REV 0 37 | #define ISA_L_RS_VAND_LIB_VER_STR "2.13" 38 | #define ISA_L_RS_VAND_LIB_NAME "isa_l_rs_vand" 39 | #if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) 40 | #define ISA_L_RS_VAND_SO_NAME "libisal" LIBERASURECODE_SO_SUFFIX ".dylib" 41 | #else 42 | #define ISA_L_RS_VAND_SO_NAME "libisal" LIBERASURECODE_SO_SUFFIX ".so.2" 43 | #endif 44 | 45 | /* Forward declarations */ 46 | struct ec_backend_common backend_isa_l_rs_vand; 47 | 48 | static void * isa_l_rs_vand_init(struct ec_backend_args *args, 49 | void *backend_sohandle) 50 | { 51 | return isa_l_common_init(args, backend_sohandle, "gf_gen_rs_matrix"); 52 | } 53 | 54 | /* 55 | * For the time being, we only claim compatibility with versions that 56 | * match exactly 57 | */ 58 | static bool isa_l_rs_vand_is_compatible_with(uint32_t version) { 59 | return version == backend_isa_l_rs_vand.ec_backend_version; 60 | } 61 | 62 | static struct ec_backend_op_stubs isa_l_rs_vand_op_stubs = { 63 | .INIT = isa_l_rs_vand_init, 64 | .EXIT = isa_l_exit, 65 | .ENCODE = isa_l_encode, 66 | .DECODE = isa_l_decode, 67 | .FRAGSNEEDED = isa_l_min_fragments, 68 | .RECONSTRUCT = isa_l_reconstruct, 69 | .ELEMENTSIZE = isa_l_element_size, 70 | .ISCOMPATIBLEWITH = isa_l_rs_vand_is_compatible_with, 71 | .GETMETADATASIZE = get_backend_metadata_size_zero, 72 | .GETENCODEOFFSET = get_encode_offset_zero, 73 | }; 74 | 75 | __attribute__ ((visibility ("internal"))) 76 | struct ec_backend_common backend_isa_l_rs_vand = { 77 | .id = EC_BACKEND_ISA_L_RS_VAND, 78 | .name = ISA_L_RS_VAND_LIB_NAME, 79 | .soname = ISA_L_RS_VAND_SO_NAME, 80 | .soversion = ISA_L_RS_VAND_LIB_VER_STR, 81 | .ops = &isa_l_rs_vand_op_stubs, 82 | .ec_backend_version = _VERSION(ISA_L_RS_VAND_LIB_MAJOR, 83 | ISA_L_RS_VAND_LIB_MINOR, 84 | ISA_L_RS_VAND_LIB_REV), 85 | }; 86 | -------------------------------------------------------------------------------- /src/backends/null/null.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Tushar Gohad 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode null backend 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #include 30 | #include 31 | 32 | #include "erasurecode.h" 33 | #include "erasurecode_backend.h" 34 | #define NULL_LIB_MAJOR 1 35 | #define NULL_LIB_MINOR 0 36 | #define NULL_LIB_REV 0 37 | #define NULL_LIB_VER_STR "1.0" 38 | #define NULL_LIB_NAME "null" 39 | #if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) 40 | #define NULL_SO_NAME "libnullcode" LIBERASURECODE_SO_SUFFIX ".dylib" 41 | #else 42 | #define NULL_SO_NAME "libnullcode" LIBERASURECODE_SO_SUFFIX ".so.1" 43 | #endif 44 | /* Forward declarations */ 45 | 46 | typedef void* (*init_null_code_func)(int, int, int); 47 | typedef int (*null_code_encode_func)(void *, char **, char **, int); 48 | typedef int (*null_code_decode_func)(void *, char **, char **, int *, int, int); 49 | typedef int (*null_reconstruct_func)(char **, int, uint64_t, int, char *); 50 | typedef int (*null_code_fragments_needed_func)(void *, int *, int *, int *); 51 | struct null_descriptor { 52 | /* calls required for init */ 53 | init_null_code_func init_null_code; 54 | 55 | /* calls required for encode */ 56 | null_code_encode_func null_code_encode; 57 | 58 | /* calls required for decode */ 59 | null_code_decode_func null_code_decode; 60 | 61 | /* calls required for reconstruct */ 62 | null_reconstruct_func null_reconstruct; 63 | 64 | /* set of fragments needed to reconstruct at a minimum */ 65 | null_code_fragments_needed_func null_code_fragments_needed; 66 | 67 | /* fields needed to hold state */ 68 | int *matrix; 69 | int k; 70 | int m; 71 | int w; 72 | int arg1; 73 | }; 74 | 75 | #define DEFAULT_W 32 76 | 77 | static int null_encode(void *desc, char **data, char **parity, int blocksize) 78 | { 79 | return 0; 80 | } 81 | 82 | static int null_decode(void *desc, char **data, char **parity, 83 | int *missing_idxs, int blocksize) 84 | { 85 | return 0; 86 | } 87 | 88 | static int null_reconstruct(void *desc, char **data, char **parity, 89 | int *missing_idxs, int destination_idx, int blocksize) 90 | { 91 | return 0; 92 | } 93 | 94 | static int null_min_fragments(void *desc, int *missing_idxs, 95 | int *fragments_to_exclude, int *fragments_needed) 96 | { 97 | return 0; 98 | } 99 | 100 | /** 101 | * Return the element-size, which is the number of bits stored 102 | * on a given device, per codeword. This is usually just 'w'. 103 | */ 104 | static int 105 | null_element_size(void* desc) 106 | { 107 | return DEFAULT_W; 108 | } 109 | 110 | static void * null_init(struct ec_backend_args *args, void *backend_sohandle) 111 | { 112 | struct null_descriptor *xdesc = NULL; 113 | 114 | /* allocate and fill in null_descriptor */ 115 | xdesc = (struct null_descriptor *) malloc(sizeof(struct null_descriptor)); 116 | if (NULL == xdesc) { 117 | return NULL; 118 | } 119 | memset(xdesc, 0, sizeof(struct null_descriptor)); 120 | 121 | xdesc->k = args->uargs.k; 122 | xdesc->m = args->uargs.m; 123 | xdesc->w = args->uargs.w; 124 | 125 | if (xdesc->w <= 0) 126 | xdesc->w = DEFAULT_W; 127 | 128 | /* Sample on how to pass extra args to the backend */ 129 | xdesc->arg1 = args->uargs.priv_args1.null_args.arg1; 130 | 131 | /* store w back in args so upper layer can get to it */ 132 | args->uargs.w = DEFAULT_W; 133 | 134 | /* validate EC arguments */ 135 | { 136 | long long max_symbols; 137 | if (xdesc->w != 8 && xdesc->w != 16 && xdesc->w != 32) { 138 | goto error; 139 | } 140 | max_symbols = 1LL << xdesc->w; 141 | if ((xdesc->k + xdesc->m) > max_symbols) { 142 | goto error; 143 | } 144 | } 145 | 146 | /* 147 | * ISO C forbids casting a void* to a function pointer. 148 | * Since dlsym return returns a void*, we use this union to 149 | * "transform" the void* to a function pointer. 150 | */ 151 | union { 152 | init_null_code_func initp; 153 | null_code_encode_func encodep; 154 | null_code_decode_func decodep; 155 | null_reconstruct_func reconp; 156 | null_code_fragments_needed_func fragsneededp; 157 | void *vptr; 158 | } func_handle = {.vptr = NULL}; 159 | 160 | /* fill in function addresses */ 161 | func_handle.vptr = NULL; 162 | func_handle.vptr = dlsym(backend_sohandle, "null_code_init"); 163 | xdesc->init_null_code = func_handle.initp; 164 | if (NULL == xdesc->init_null_code) { 165 | goto error; 166 | } 167 | 168 | func_handle.vptr = NULL; 169 | func_handle.vptr = dlsym(backend_sohandle, "null_code_encode"); 170 | xdesc->null_code_encode = func_handle.encodep; 171 | if (NULL == xdesc->null_code_encode) { 172 | goto error; 173 | } 174 | 175 | func_handle.vptr = NULL; 176 | func_handle.vptr = dlsym(backend_sohandle, "null_code_decode"); 177 | xdesc->null_code_decode = func_handle.decodep; 178 | if (NULL == xdesc->null_code_decode) { 179 | goto error; 180 | } 181 | 182 | func_handle.vptr = NULL; 183 | func_handle.vptr = dlsym(backend_sohandle, "null_reconstruct"); 184 | xdesc->null_reconstruct = func_handle.reconp; 185 | if (NULL == xdesc->null_reconstruct) { 186 | goto error; 187 | } 188 | 189 | func_handle.vptr = NULL; 190 | func_handle.vptr = dlsym(backend_sohandle, "null_code_fragments_needed"); 191 | xdesc->null_code_fragments_needed = func_handle.fragsneededp; 192 | if (NULL == xdesc->null_code_fragments_needed) { 193 | goto error; 194 | } 195 | 196 | return (void *) xdesc; 197 | 198 | error: 199 | free (xdesc); 200 | 201 | return NULL; 202 | } 203 | 204 | static int null_exit(void *desc) 205 | { 206 | struct null_descriptor *xdesc = (struct null_descriptor *) desc; 207 | 208 | free (xdesc); 209 | return 0; 210 | } 211 | 212 | static bool null_is_compatible_with(uint32_t version) { 213 | return true; 214 | } 215 | 216 | static struct ec_backend_op_stubs null_op_stubs = { 217 | .INIT = null_init, 218 | .EXIT = null_exit, 219 | .ENCODE = null_encode, 220 | .DECODE = null_decode, 221 | .FRAGSNEEDED = null_min_fragments, 222 | .RECONSTRUCT = null_reconstruct, 223 | .ELEMENTSIZE = null_element_size, 224 | .ISCOMPATIBLEWITH = null_is_compatible_with, 225 | .GETMETADATASIZE = get_backend_metadata_size_zero, 226 | .GETENCODEOFFSET = get_encode_offset_zero, 227 | }; 228 | 229 | __attribute__ ((visibility ("internal"))) 230 | struct ec_backend_common backend_null = { 231 | .id = EC_BACKEND_NULL, 232 | .name = NULL_LIB_NAME, 233 | .soname = NULL_SO_NAME, 234 | .soversion = NULL_LIB_VER_STR, 235 | .ops = &null_op_stubs, 236 | .ec_backend_version = _VERSION(NULL_LIB_MAJOR, NULL_LIB_MINOR, 237 | NULL_LIB_REV), 238 | }; 239 | 240 | -------------------------------------------------------------------------------- /src/backends/rs_vand/liberasurecode_rs_vand.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Kevin M Greenan 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * vi: set noai tw=79 ts=4 sw=4: 25 | */ 26 | 27 | #include 28 | #include 29 | 30 | #include "erasurecode.h" 31 | #include "erasurecode_backend.h" 32 | #include "erasurecode_helpers.h" 33 | #include "erasurecode_helpers_ext.h" 34 | 35 | #define LIBERASURECODE_RS_VAND_LIB_MAJOR 1 36 | #define LIBERASURECODE_RS_VAND_LIB_MINOR 0 37 | #define LIBERASURECODE_RS_VAND_LIB_REV 0 38 | #define LIBERASURECODE_RS_VAND_LIB_VER_STR "1.0" 39 | #define LIBERASURECODE_RS_VAND_LIB_NAME "liberasurecode_rs_vand" 40 | #if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) 41 | #define LIBERASURECODE_RS_VAND_SO_NAME "liberasurecode_rs_vand" LIBERASURECODE_SO_SUFFIX ".dylib" 42 | #else 43 | #define LIBERASURECODE_RS_VAND_SO_NAME "liberasurecode_rs_vand" LIBERASURECODE_SO_SUFFIX ".so.1" 44 | #endif 45 | 46 | /* Forward declarations */ 47 | struct ec_backend_common backend_liberasurecode_rs_vand; 48 | 49 | typedef int (*liberasurecode_rs_vand_encode_func)(int *, char **, char **, int, int, int); 50 | typedef int (*liberasurecode_rs_vand_decode_func)(int *, char **, char **, int, int, int *, int, int); 51 | typedef int (*liberasurecode_rs_vand_reconstruct_func)(int *, char **, char **, int, int, int *, int, int); 52 | typedef void (*init_liberasurecode_rs_vand_func)(int, int); 53 | typedef void (*deinit_liberasurecode_rs_vand_func)(void); 54 | typedef void (*free_systematic_matrix_func)(int *); 55 | typedef int* (*make_systematic_matrix_func)(int, int); 56 | 57 | 58 | struct liberasurecode_rs_vand_descriptor { 59 | /* calls required for init */ 60 | init_liberasurecode_rs_vand_func init_liberasurecode_rs_vand; 61 | deinit_liberasurecode_rs_vand_func deinit_liberasurecode_rs_vand; 62 | free_systematic_matrix_func free_systematic_matrix; 63 | make_systematic_matrix_func make_systematic_matrix; 64 | 65 | /* calls required for encode */ 66 | liberasurecode_rs_vand_encode_func liberasurecode_rs_vand_encode; 67 | 68 | /* calls required for decode */ 69 | liberasurecode_rs_vand_decode_func liberasurecode_rs_vand_decode; 70 | 71 | /* calls required for reconstruct */ 72 | liberasurecode_rs_vand_reconstruct_func liberasurecode_rs_vand_reconstruct; 73 | 74 | /* fields needed to hold state */ 75 | int *matrix; 76 | int k; 77 | int m; 78 | int w; 79 | }; 80 | 81 | static int liberasurecode_rs_vand_encode(void *desc, char **data, char **parity, 82 | int blocksize) 83 | { 84 | struct liberasurecode_rs_vand_descriptor *rs_vand_desc = 85 | (struct liberasurecode_rs_vand_descriptor*) desc; 86 | 87 | /* FIXME: Should this return something? */ 88 | rs_vand_desc->liberasurecode_rs_vand_encode(rs_vand_desc->matrix, data, parity, 89 | rs_vand_desc->k, rs_vand_desc->m, blocksize); 90 | return 0; 91 | } 92 | 93 | static int liberasurecode_rs_vand_decode(void *desc, char **data, char **parity, 94 | int *missing_idxs, int blocksize) 95 | { 96 | struct liberasurecode_rs_vand_descriptor *rs_vand_desc = 97 | (struct liberasurecode_rs_vand_descriptor*) desc; 98 | 99 | /* FIXME: Should this return something? */ 100 | rs_vand_desc->liberasurecode_rs_vand_decode(rs_vand_desc->matrix, data, parity, 101 | rs_vand_desc->k, rs_vand_desc->m, missing_idxs, blocksize, 1); 102 | 103 | return 0; 104 | } 105 | 106 | static int liberasurecode_rs_vand_reconstruct(void *desc, char **data, char **parity, 107 | int *missing_idxs, int destination_idx, int blocksize) 108 | { 109 | struct liberasurecode_rs_vand_descriptor *rs_vand_desc = 110 | (struct liberasurecode_rs_vand_descriptor*) desc; 111 | 112 | /* FIXME: Should this return something? */ 113 | rs_vand_desc->liberasurecode_rs_vand_reconstruct(rs_vand_desc->matrix, data, parity, 114 | rs_vand_desc->k, rs_vand_desc->m, missing_idxs, destination_idx, blocksize); 115 | 116 | return 0; 117 | } 118 | 119 | static int liberasurecode_rs_vand_min_fragments(void *desc, int *missing_idxs, 120 | int *fragments_to_exclude, int *fragments_needed) 121 | { 122 | struct liberasurecode_rs_vand_descriptor *rs_vand_desc = 123 | (struct liberasurecode_rs_vand_descriptor*)desc; 124 | 125 | uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude); 126 | uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm; 127 | int i; 128 | int j = 0; 129 | int ret = -1; 130 | 131 | for (i = 0; i < (rs_vand_desc->k + rs_vand_desc->m); i++) { 132 | if (!(missing_bm & (1 << i))) { 133 | fragments_needed[j] = i; 134 | j++; 135 | } 136 | if (j == rs_vand_desc->k) { 137 | ret = 0; 138 | fragments_needed[j] = -1; 139 | break; 140 | } 141 | } 142 | 143 | return ret; 144 | } 145 | 146 | static void * liberasurecode_rs_vand_init(struct ec_backend_args *args, 147 | void *backend_sohandle) 148 | { 149 | struct liberasurecode_rs_vand_descriptor *desc = NULL; 150 | 151 | desc = (struct liberasurecode_rs_vand_descriptor *) 152 | malloc(sizeof(struct liberasurecode_rs_vand_descriptor)); 153 | if (NULL == desc) { 154 | return NULL; 155 | } 156 | 157 | desc->k = args->uargs.k; 158 | desc->m = args->uargs.m; 159 | 160 | /* store w back in args so upper layer can get to it */ 161 | args->uargs.w = desc->w = 16; // w is currently hard-coded at 16 162 | 163 | // This check should not matter, since 64K is way higher 164 | // than anyone should ever use 165 | if ((desc->k + desc->m) > 65536) { 166 | goto error; 167 | } 168 | 169 | /* 170 | * ISO C forbids casting a void* to a function pointer. 171 | * Since dlsym return returns a void*, we use this union to 172 | * "transform" the void* to a function pointer. 173 | */ 174 | union { 175 | init_liberasurecode_rs_vand_func initp; 176 | deinit_liberasurecode_rs_vand_func deinitp; 177 | free_systematic_matrix_func freematrixp; 178 | make_systematic_matrix_func makematrixp; 179 | liberasurecode_rs_vand_encode_func encodep; 180 | liberasurecode_rs_vand_decode_func decodep; 181 | liberasurecode_rs_vand_reconstruct_func reconstructp; 182 | void *vptr; 183 | } func_handle = {.vptr = NULL}; 184 | 185 | 186 | /* fill in function addresses */ 187 | func_handle.vptr = NULL; 188 | func_handle.vptr = dlsym(backend_sohandle, "init_liberasurecode_rs_vand"); 189 | desc->init_liberasurecode_rs_vand = func_handle.initp; 190 | if (NULL == desc->init_liberasurecode_rs_vand) { 191 | goto error; 192 | } 193 | 194 | func_handle.vptr = NULL; 195 | func_handle.vptr = dlsym(backend_sohandle, "deinit_liberasurecode_rs_vand"); 196 | desc->deinit_liberasurecode_rs_vand = func_handle.deinitp; 197 | if (NULL == desc->deinit_liberasurecode_rs_vand) { 198 | goto error; 199 | } 200 | 201 | func_handle.vptr = NULL; 202 | func_handle.vptr = dlsym(backend_sohandle, "make_systematic_matrix"); 203 | desc->make_systematic_matrix = func_handle.makematrixp; 204 | if (NULL == desc->make_systematic_matrix) { 205 | goto error; 206 | } 207 | 208 | func_handle.vptr = NULL; 209 | func_handle.vptr = dlsym(backend_sohandle, "free_systematic_matrix"); 210 | desc->free_systematic_matrix = func_handle.freematrixp; 211 | if (NULL == desc->free_systematic_matrix) { 212 | goto error; 213 | } 214 | 215 | func_handle.vptr = NULL; 216 | func_handle.vptr = dlsym(backend_sohandle, "liberasurecode_rs_vand_encode"); 217 | desc->liberasurecode_rs_vand_encode = func_handle.encodep; 218 | if (NULL == desc->liberasurecode_rs_vand_encode) { 219 | goto error; 220 | } 221 | 222 | func_handle.vptr = NULL; 223 | func_handle.vptr = dlsym(backend_sohandle, "liberasurecode_rs_vand_decode"); 224 | desc->liberasurecode_rs_vand_decode = func_handle.decodep; 225 | if (NULL == desc->liberasurecode_rs_vand_decode) { 226 | goto error; 227 | } 228 | 229 | func_handle.vptr = NULL; 230 | func_handle.vptr = dlsym(backend_sohandle, "liberasurecode_rs_vand_reconstruct"); 231 | desc->liberasurecode_rs_vand_reconstruct = func_handle.reconstructp; 232 | if (NULL == desc->liberasurecode_rs_vand_reconstruct) { 233 | goto error; 234 | } 235 | 236 | desc->init_liberasurecode_rs_vand(desc->k, desc->m); 237 | 238 | desc->matrix = desc->make_systematic_matrix(desc->k, desc->m); 239 | 240 | if (NULL == desc->matrix) { 241 | goto error; 242 | } 243 | 244 | return desc; 245 | 246 | error: 247 | free(desc); 248 | 249 | return NULL; 250 | } 251 | 252 | /** 253 | * Return the element-size, which is the number of bits stored 254 | * on a given device, per codeword. For Vandermonde, this is 255 | * 'w'. For somthing like cauchy, this is packetsize * w. 256 | * 257 | * Returns the size in bits! 258 | */ 259 | static int 260 | liberasurecode_rs_vand_element_size(void* desc) 261 | { 262 | struct liberasurecode_rs_vand_descriptor *rs_vand_desc = NULL; 263 | 264 | rs_vand_desc = (struct liberasurecode_rs_vand_descriptor*) desc; 265 | 266 | return rs_vand_desc->w; 267 | } 268 | 269 | static int liberasurecode_rs_vand_exit(void *desc) 270 | { 271 | struct liberasurecode_rs_vand_descriptor *rs_vand_desc = NULL; 272 | 273 | rs_vand_desc = (struct liberasurecode_rs_vand_descriptor*) desc; 274 | 275 | rs_vand_desc->free_systematic_matrix(rs_vand_desc->matrix); 276 | rs_vand_desc->deinit_liberasurecode_rs_vand(); 277 | free(rs_vand_desc); 278 | 279 | return 0; 280 | } 281 | 282 | /* 283 | * For the time being, we only claim compatibility with versions that 284 | * match exactly 285 | */ 286 | static bool liberasurecode_rs_vand_is_compatible_with(uint32_t version) { 287 | return version == backend_liberasurecode_rs_vand.ec_backend_version; 288 | } 289 | 290 | static struct ec_backend_op_stubs liberasurecode_rs_vand_op_stubs = { 291 | .INIT = liberasurecode_rs_vand_init, 292 | .EXIT = liberasurecode_rs_vand_exit, 293 | .ENCODE = liberasurecode_rs_vand_encode, 294 | .DECODE = liberasurecode_rs_vand_decode, 295 | .FRAGSNEEDED = liberasurecode_rs_vand_min_fragments, 296 | .RECONSTRUCT = liberasurecode_rs_vand_reconstruct, 297 | .ELEMENTSIZE = liberasurecode_rs_vand_element_size, 298 | .ISCOMPATIBLEWITH = liberasurecode_rs_vand_is_compatible_with, 299 | .GETMETADATASIZE = get_backend_metadata_size_zero, 300 | .GETENCODEOFFSET = get_encode_offset_zero, 301 | }; 302 | 303 | __attribute__ ((visibility ("internal"))) 304 | struct ec_backend_common backend_liberasurecode_rs_vand = { 305 | .id = EC_BACKEND_LIBERASURECODE_RS_VAND, 306 | .name = LIBERASURECODE_RS_VAND_LIB_NAME, 307 | .soname = LIBERASURECODE_RS_VAND_SO_NAME, 308 | .soversion = LIBERASURECODE_RS_VAND_LIB_VER_STR, 309 | .ops = &liberasurecode_rs_vand_op_stubs, 310 | .ec_backend_version = _VERSION(LIBERASURECODE_RS_VAND_LIB_MAJOR, 311 | LIBERASURECODE_RS_VAND_LIB_MINOR, 312 | LIBERASURECODE_RS_VAND_LIB_REV), 313 | }; 314 | -------------------------------------------------------------------------------- /src/backends/shss/shss.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2015 NTT corp. All Rights Reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;LOSS OF USE, 19 | * DATA, OR PROFITS;OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode shss backend 25 | * 26 | * Please contact us if you are insterested in the NTT backend (welcome!): 27 | * Kota Tsuyuzaki 28 | * 29 | * vi: set noai tw=79 ts=4 sw=4: 30 | */ 31 | 32 | #include 33 | #include 34 | 35 | #include "erasurecode.h" 36 | #include "erasurecode_helpers.h" 37 | #include "erasurecode_helpers_ext.h" 38 | #include "erasurecode_backend.h" 39 | 40 | /* Forward declarations */ 41 | struct ec_backend_common backend_shss; 42 | 43 | typedef int (*shss_encode_func)(char **, size_t, int, int, int, int, long long *); 44 | typedef int (*shss_decode_func)(char **, size_t, int *, int, int, int, int, int, long long *); 45 | typedef int (*shss_reconstruct_func)(char **, size_t, int *, int, int *, int, int, int, int, int, long long *); 46 | 47 | struct shss_descriptor { 48 | /* calls required for init */ 49 | shss_encode_func ssencode; 50 | shss_decode_func ssdecode; 51 | shss_reconstruct_func ssreconst; 52 | 53 | /* fields needed to hold state */ 54 | int k; 55 | int m; 56 | int n; 57 | int w; 58 | int aes_bit_length; 59 | }; 60 | 61 | #define SHSS_LIB_VER_STR "1.0" 62 | #define SHSS_LIB_NAME "shss" 63 | #if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) 64 | #define SHSS_SO_NAME "libshss" LIBERASURECODE_SO_SUFFIX ".dylib" 65 | #else 66 | #define SHSS_SO_NAME "libshss" LIBERASURECODE_SO_SUFFIX ".so.1" 67 | #endif 68 | #define DEFAULT_W 128 69 | #define METADATA 32 70 | 71 | static int shss_encode(void *desc, char **data, char **parity, 72 | int blocksize) 73 | { 74 | int i; 75 | int ret = 0; 76 | int priv_bitnum = 128; // privacy bit number 0 or 128(default) or 256 77 | int chksum = 0; // chksum 0 or 64 78 | char **encoded; 79 | long long einfo; 80 | struct shss_descriptor *xdesc = 81 | (struct shss_descriptor *) desc; 82 | 83 | if (xdesc->aes_bit_length != -1) { 84 | priv_bitnum = xdesc->aes_bit_length; 85 | } 86 | 87 | encoded = alloca(sizeof(char*)*xdesc->n); 88 | 89 | for (i = 0; ik; i++) encoded[i] = (char*)data[i]; 90 | for (i = 0; im; i++) encoded[i+xdesc->k] = (char*)parity[i]; 91 | 92 | ret = xdesc->ssencode((char**)encoded, (size_t)blocksize, 93 | xdesc->k, xdesc->m, priv_bitnum, chksum, &einfo); 94 | 95 | if (ret > 0) { 96 | return -ret; 97 | } 98 | 99 | return 0; 100 | } 101 | 102 | static int shss_decode(void *desc, char **data, char **parity, 103 | int *missing_idxs, int blocksize) 104 | { 105 | int i; 106 | int missing_size = 0; 107 | int ret = 0; 108 | int priv_bitnum = 128; // privacy bit number 0 or 128(default) or 256 109 | int chksum = 0; // chksum 0 or 64 110 | char **decoded; 111 | long long einfo; 112 | struct shss_descriptor *xdesc = 113 | (struct shss_descriptor *) desc; 114 | 115 | if (xdesc->aes_bit_length != -1) { 116 | priv_bitnum = xdesc->aes_bit_length; 117 | } 118 | 119 | decoded = alloca(sizeof(char*)*xdesc->n); 120 | 121 | for (i = 0; ik; i++) decoded[i] = (char*)data[i]; 122 | for (i = 0; im; i++) decoded[i+xdesc->k] = (char*)parity[i]; 123 | for (i = 0; in; i++) { 124 | if (i == missing_idxs[missing_size]) { 125 | missing_size++; 126 | } 127 | } 128 | 129 | ret = xdesc->ssdecode((char**)decoded, (size_t)blocksize, missing_idxs, missing_size, 130 | xdesc->k, xdesc->m, priv_bitnum, chksum, &einfo); 131 | 132 | if (ret > 0) { 133 | return -ret; 134 | } 135 | 136 | return 0; 137 | } 138 | 139 | static int shss_reconstruct(void *desc, char **data, char **parity, 140 | int *missing_idxs, int destination_idx, int blocksize) 141 | { 142 | int i; 143 | int missing_size = 0; 144 | int ret = 0; 145 | int priv_bitnum = 128; // privacy bit number 0 or 128(default) or 256 146 | int chksum = 0; // chksum 0 or 64 147 | int dst_size = 1; 148 | char **reconstructed; 149 | long long einfo; 150 | struct shss_descriptor *xdesc = 151 | (struct shss_descriptor *) desc; 152 | 153 | if (xdesc->aes_bit_length != -1) { 154 | priv_bitnum = xdesc->aes_bit_length; 155 | } 156 | 157 | reconstructed = alloca(sizeof(char*)*xdesc->n); 158 | 159 | for (i = 0; ik; i++) reconstructed[i] = (char*)data[i]; 160 | for (i = 0; im; i++) reconstructed[i+xdesc->k] = (char*)parity[i]; 161 | for (i = 0; in; i++) { 162 | if (i == missing_idxs[missing_size]) { 163 | missing_size++; 164 | } 165 | } 166 | 167 | ret = xdesc->ssreconst((char**)reconstructed, (size_t)blocksize, 168 | &destination_idx, dst_size, missing_idxs, missing_size, xdesc->k, 169 | xdesc->m, priv_bitnum, chksum, &einfo); 170 | 171 | if (ret > 0) { 172 | return -ret; 173 | } 174 | 175 | return 0; 176 | } 177 | 178 | static int shss_fragments_needed(void *desc, int *missing_idxs, 179 | int *fragments_to_exclude, int *fragments_needed) 180 | { 181 | struct shss_descriptor *xdesc = 182 | (struct shss_descriptor *) desc; 183 | uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude); 184 | uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm; 185 | int i; 186 | int j = 0; 187 | int ret = -101; 188 | 189 | for (i = 0; i < xdesc->n; i++) { 190 | if (!(missing_bm & (1 << i))) { 191 | fragments_needed[j] = i; 192 | j++; 193 | } 194 | if (j == xdesc->k) { 195 | ret = 0; 196 | fragments_needed[j] = -1; 197 | break; 198 | } 199 | } 200 | 201 | return ret; 202 | } 203 | 204 | /** 205 | * Return the element-size, which is the number of bits stored 206 | * on a given device, per codeword. This is usually just 'w'. 207 | */ 208 | static int shss_element_size(void* desc) 209 | { 210 | return DEFAULT_W; 211 | } 212 | 213 | static void * shss_init(struct ec_backend_args *args, void *backend_sohandle) 214 | { 215 | struct shss_descriptor *desc = NULL; 216 | 217 | desc = (struct shss_descriptor *) 218 | malloc(sizeof(struct shss_descriptor)); 219 | if (NULL == desc) { 220 | return NULL; 221 | } 222 | 223 | desc->k = args->uargs.k; 224 | desc->m = args->uargs.m; 225 | desc->n = args->uargs.k + args->uargs.m; 226 | desc->w = DEFAULT_W; 227 | args->uargs.w = DEFAULT_W; 228 | 229 | /* Sample on how to pass extra args to the backend */ 230 | // TODO: Need discussion how to pass extra args. 231 | // tentatively we could pass with priv_args2 as the bit_length 232 | int *priv = (int *)args->uargs.priv_args2; 233 | if(priv != NULL){ 234 | desc->aes_bit_length = priv[0]; // AES bit number 235 | }else{ 236 | desc->aes_bit_length = 128; 237 | } 238 | 239 | union { 240 | shss_encode_func encodep; 241 | shss_decode_func decodep; 242 | shss_reconstruct_func reconp; 243 | void *vptr; 244 | } func_handle; 245 | 246 | func_handle.vptr = NULL; 247 | func_handle.vptr = dlsym(backend_sohandle, "ssencode"); 248 | desc->ssencode = func_handle.encodep; 249 | if (NULL == desc->ssencode) { 250 | goto error; 251 | } 252 | 253 | func_handle.vptr = NULL; 254 | func_handle.vptr = dlsym(backend_sohandle, "ssdecode"); 255 | desc->ssdecode = func_handle.decodep; 256 | if (NULL == desc->ssdecode) { 257 | goto error; 258 | } 259 | 260 | func_handle.vptr = NULL; 261 | func_handle.vptr = dlsym(backend_sohandle, "ssreconst"); 262 | desc->ssreconst = func_handle.reconp; 263 | if (NULL == desc->ssreconst) { 264 | goto error; 265 | } 266 | 267 | return desc; 268 | 269 | error: 270 | free(desc); 271 | 272 | return NULL; 273 | } 274 | 275 | static int shss_exit(void *desc) 276 | { 277 | if (desc != NULL) { 278 | free(desc); 279 | } 280 | return 0; 281 | } 282 | 283 | static bool shss_is_compatible_with(uint32_t version) { 284 | return version == backend_shss.ec_backend_version; 285 | } 286 | 287 | static size_t shss_get_backend_metadata_size(void *desc, int blocksize) { 288 | return METADATA; 289 | } 290 | 291 | static struct ec_backend_op_stubs shss_op_stubs = { 292 | .INIT = shss_init, 293 | .EXIT = shss_exit, 294 | .ENCODE = shss_encode, 295 | .DECODE = shss_decode, 296 | .FRAGSNEEDED = shss_fragments_needed, 297 | .RECONSTRUCT = shss_reconstruct, 298 | .ELEMENTSIZE = shss_element_size, 299 | .ISCOMPATIBLEWITH = shss_is_compatible_with, 300 | .GETMETADATASIZE = shss_get_backend_metadata_size, 301 | .GETENCODEOFFSET = get_encode_offset_zero, 302 | }; 303 | 304 | __attribute__ ((visibility ("internal"))) 305 | struct ec_backend_common backend_shss = { 306 | .id = EC_BACKEND_SHSS, 307 | .name = SHSS_LIB_NAME, 308 | .soname = SHSS_SO_NAME, 309 | .soversion = SHSS_LIB_VER_STR, 310 | .ops = &shss_op_stubs, 311 | }; 312 | -------------------------------------------------------------------------------- /src/backends/xor/flat_xor_hd.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Tushar Gohad 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode flat_xor_hd backend 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | #include "erasurecode.h" 34 | #include "erasurecode_backend.h" 35 | 36 | #define FLAT_XOR_LIB_MAJOR 1 37 | #define FLAT_XOR_LIB_MINOR 0 38 | #define FLAT_XOR_LIB_REV 0 39 | #define FLAT_XOR_LIB_VER_STR "1.0" 40 | #define FLAT_XOR_LIB_NAME "flat_xor_hd" 41 | #if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) 42 | #define FLAT_XOR_SO_NAME "libXorcode" LIBERASURECODE_SO_SUFFIX ".dylib" 43 | #else 44 | #define FLAT_XOR_SO_NAME "libXorcode" LIBERASURECODE_SO_SUFFIX ".so.1" 45 | #endif 46 | #define DEFAULT_W 32 47 | 48 | /* Forward declarations */ 49 | struct ec_backend_common backend_flat_xor_hd; 50 | 51 | typedef xor_code_t* (*init_xor_hd_code_func)(int, int, int); 52 | typedef void (*xor_code_encode_func)(xor_code_t *, char **, char **, int); 53 | typedef int (*xor_code_decode_func)(xor_code_t *, char **, char **, int *, int, int); 54 | typedef int (*xor_hd_fragments_needed_func)(xor_code_t *, int *, int *, int *); 55 | 56 | struct flat_xor_hd_descriptor { 57 | xor_code_t *xor_desc; 58 | init_xor_hd_code_func init_xor_hd_code; 59 | xor_code_encode_func xor_code_encode; 60 | xor_code_decode_func xor_code_decode; 61 | xor_hd_fragments_needed_func xor_hd_fragments_needed; 62 | }; 63 | 64 | static int flat_xor_hd_encode(void *desc, 65 | char **data, char **parity, int blocksize) 66 | { 67 | struct flat_xor_hd_descriptor *xdesc = 68 | (struct flat_xor_hd_descriptor *) desc; 69 | 70 | xor_code_t *xor_desc = (xor_code_t *) xdesc->xor_desc; 71 | xor_desc->encode(xor_desc, data, parity, blocksize); 72 | return 0; 73 | } 74 | 75 | static int flat_xor_hd_decode(void *desc, 76 | char **data, char **parity, int *missing_idxs, 77 | int blocksize) 78 | { 79 | struct flat_xor_hd_descriptor *xdesc = 80 | (struct flat_xor_hd_descriptor *) desc; 81 | 82 | xor_code_t *xor_desc = (xor_code_t *) xdesc->xor_desc; 83 | return xor_desc->decode(xor_desc, data, parity, missing_idxs, blocksize, 1); 84 | } 85 | 86 | static int flat_xor_hd_reconstruct(void *desc, 87 | char **data, char **parity, int *missing_idxs, 88 | int destination_idx, int blocksize) 89 | { 90 | struct flat_xor_hd_descriptor *xdesc = 91 | (struct flat_xor_hd_descriptor *) desc; 92 | 93 | xor_code_t *xor_desc = (xor_code_t *) xdesc->xor_desc; 94 | xor_reconstruct_one(xor_desc, data, parity, 95 | missing_idxs, destination_idx, blocksize); 96 | return 0; 97 | } 98 | 99 | static int flat_xor_hd_min_fragments(void *desc, 100 | int *missing_idxs, int *fragments_to_exclude, 101 | int *fragments_needed) 102 | { 103 | struct flat_xor_hd_descriptor *xdesc = 104 | (struct flat_xor_hd_descriptor *) desc; 105 | 106 | xor_code_t *xor_desc = (xor_code_t *) xdesc->xor_desc; 107 | xor_desc->fragments_needed(xor_desc, missing_idxs, fragments_to_exclude, fragments_needed); 108 | return 0; 109 | } 110 | 111 | /** 112 | * Return the element-size, which is the number of bits stored 113 | * on a given device, per codeword. This is usually just 'w'. 114 | */ 115 | static int 116 | flar_xor_hd_element_size(void* desc) 117 | { 118 | return DEFAULT_W; 119 | } 120 | 121 | static void * flat_xor_hd_init(struct ec_backend_args *args, void *sohandle) 122 | { 123 | int k = args->uargs.k; 124 | int m = args->uargs.m; 125 | int hd = args->uargs.hd; 126 | 127 | xor_code_t *xor_desc = NULL; 128 | struct flat_xor_hd_descriptor *bdesc = NULL; 129 | 130 | /* store w back in args so upper layer can get to it */ 131 | args->uargs.w = DEFAULT_W; 132 | 133 | /* init xor_code_t descriptor */ 134 | xor_desc = init_xor_hd_code(k, m, hd); 135 | if (NULL == xor_desc) { 136 | return NULL; 137 | } 138 | 139 | /* fill in flat_xor_hd_descriptor */ 140 | bdesc = (struct flat_xor_hd_descriptor *) 141 | malloc(sizeof(struct flat_xor_hd_descriptor)); 142 | if (NULL == bdesc) { 143 | free (xor_desc); 144 | return NULL; 145 | } 146 | 147 | bdesc->xor_desc = xor_desc; 148 | 149 | return (void *) bdesc; 150 | } 151 | 152 | static int flat_xor_hd_exit(void *desc) 153 | { 154 | struct flat_xor_hd_descriptor *bdesc = 155 | (struct flat_xor_hd_descriptor *) desc; 156 | 157 | free (bdesc->xor_desc); 158 | free (bdesc); 159 | return 0; 160 | } 161 | 162 | /* 163 | * For the time being, we only claim compatibility with versions that 164 | * match exactly 165 | */ 166 | static bool flat_xor_is_compatible_with(uint32_t version) { 167 | return version == backend_flat_xor_hd.ec_backend_version; 168 | } 169 | 170 | static struct ec_backend_op_stubs flat_xor_hd_op_stubs = { 171 | .INIT = flat_xor_hd_init, 172 | .EXIT = flat_xor_hd_exit, 173 | .ENCODE = flat_xor_hd_encode, 174 | .DECODE = flat_xor_hd_decode, 175 | .FRAGSNEEDED = flat_xor_hd_min_fragments, 176 | .RECONSTRUCT = flat_xor_hd_reconstruct, 177 | .ELEMENTSIZE = flar_xor_hd_element_size, 178 | .ISCOMPATIBLEWITH = flat_xor_is_compatible_with, 179 | .GETMETADATASIZE = get_backend_metadata_size_zero, 180 | .GETENCODEOFFSET = get_encode_offset_zero, 181 | }; 182 | 183 | __attribute__ ((visibility ("internal"))) 184 | struct ec_backend_common backend_flat_xor_hd = { 185 | .id = EC_BACKEND_FLAT_XOR_HD, 186 | .name = FLAT_XOR_LIB_NAME, 187 | .soname = FLAT_XOR_SO_NAME, 188 | .soversion = FLAT_XOR_LIB_VER_STR, 189 | .ops = &flat_xor_hd_op_stubs, 190 | .ec_backend_version = _VERSION(FLAT_XOR_LIB_MAJOR, 191 | FLAT_XOR_LIB_MINOR, 192 | FLAT_XOR_LIB_REV), 193 | }; 194 | 195 | -------------------------------------------------------------------------------- /src/builtin/null_code/Makefile.am: -------------------------------------------------------------------------------- 1 | lib_LTLIBRARIES = libnullcode.la 2 | 3 | # libnullcode params 4 | libnullcode_la_SOURCES = null_code.c 5 | libnullcode_la_CPPFLAGS = -I$(top_srcdir)/include/null_code @GCOV_FLAGS@ 6 | 7 | # Version format (C - A).(A).(R) for C:R:A input 8 | libnullcode_la_LDFLAGS = @GCOV_LDFLAGS@ -rpath '$(libdir)' -version-info 1:1:0 9 | 10 | MOSTLYCLEANFILES = *.gcda *.gcno *.gcov 11 | 12 | -------------------------------------------------------------------------------- /src/builtin/null_code/null_code.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * null EC backend 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #include 30 | #include 31 | 32 | /* calls required for init */ 33 | void* null_code_init(int k, int m, int hd) 34 | { 35 | /* add your code here */ 36 | return NULL; 37 | } 38 | 39 | /* calls required for encode */ 40 | int null_code_encode(void *code_desc, char **data, char **parity, 41 | int blocksize) 42 | { 43 | /* add your code here */ 44 | return 0; 45 | } 46 | 47 | /* calls required for decode */ 48 | int null_code_decode(void *code_desc, char **data, char **parity, 49 | int *missing_idxs, int blocksize, int decode_parity) 50 | { 51 | /* add your code here */ 52 | return 0; 53 | } 54 | 55 | /* calls required for reconstruct */ 56 | int null_reconstruct(char **available_fragments, int num_fragments, 57 | uint64_t fragment_len, int destination_idx, char* out_fragment) 58 | { 59 | /* add your code here */ 60 | return 0; 61 | } 62 | 63 | /* set of fragments needed to reconstruct at a minimum */ 64 | int null_code_fragments_needed(void *code_desc, int *missing_idxs, 65 | int *fragments_needed) 66 | { 67 | /* add your code here */ 68 | return 0; 69 | } 70 | 71 | -------------------------------------------------------------------------------- /src/builtin/rs_vand/Makefile.am: -------------------------------------------------------------------------------- 1 | lib_LTLIBRARIES = liberasurecode_rs_vand.la 2 | 3 | # liberasurecode_rs_vand params 4 | liberasurecode_rs_vand_la_SOURCES = rs_galois.c liberasurecode_rs_vand.c 5 | liberasurecode_rs_vand_la_CPPFLAGS = -I$(top_srcdir)/include/rs_vand @GCOV_FLAGS@ 6 | 7 | # Version format (C - A).(A).(R) for C:R:A input 8 | liberasurecode_rs_vand_la_LDFLAGS = @GCOV_LDFLAGS@ -rpath '$(libdir)' -version-info 1:1:0 9 | 10 | MOSTLYCLEANFILES = *.gcda *.gcno *.gcov 11 | -------------------------------------------------------------------------------- /src/builtin/rs_vand/rs_galois.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Kevin M Greenan 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * vi: set noai tw=79 ts=4 sw=4: 25 | */ 26 | 27 | // DISCLAIMER: This is a totally basic implementation of RS used if a user does not 28 | // want to install one of the supported backends, such as Jerasure and ISA-L. 29 | // This is not expected to perform as well as the other supported backends, 30 | // but does not make any assumptions about the host system. Using a library 31 | // like Jerasure with GF-Complete will give users the ability to tune to their 32 | // architecture (Intel or ARM), CPU and memory (lots of options). 33 | 34 | #include 35 | #include 36 | #include 37 | 38 | // We are only implementing w=16 here. If you want to use something 39 | // else, then use Jerasure with GF-Complete or ISA-L. 40 | #define PRIM_POLY 0x1100b 41 | #define FIELD_SIZE (1 << 16) 42 | #define GROUP_SIZE (FIELD_SIZE - 1) 43 | 44 | static int *log_table = NULL; 45 | static int *ilog_table = NULL; 46 | static int *ilog_table_begin = NULL; 47 | static int init_counter = 0; 48 | 49 | __attribute__ ((visibility ("internal"))) 50 | void rs_galois_init_tables(void) 51 | { 52 | if (init_counter++ > 0) { 53 | /* already initialized */ 54 | return; 55 | } 56 | log_table = (int*)malloc(sizeof(int)*FIELD_SIZE); 57 | ilog_table_begin = (int*)malloc(sizeof(int)*FIELD_SIZE*3); 58 | int i = 0; 59 | int x = 1; 60 | 61 | for (i = 0; i < GROUP_SIZE; i++) { 62 | log_table[x] = i; 63 | ilog_table_begin[i] = x; 64 | ilog_table_begin[i + GROUP_SIZE] = x; 65 | ilog_table_begin[i + (GROUP_SIZE*2)] = x; 66 | x = x << 1; 67 | if (x & FIELD_SIZE) { 68 | x ^= PRIM_POLY; 69 | } 70 | } 71 | ilog_table = &ilog_table_begin[GROUP_SIZE]; 72 | } 73 | 74 | __attribute__ ((visibility ("internal"))) 75 | void rs_galois_deinit_tables(void) 76 | { 77 | init_counter--; 78 | if (init_counter < 0) { 79 | /* deinit when not initialized?? */ 80 | init_counter = 0; 81 | } else if (init_counter > 0) { 82 | /* still at least one desc using it */ 83 | return; 84 | } else { 85 | free(log_table); 86 | log_table = NULL; 87 | free(ilog_table_begin); 88 | ilog_table_begin = NULL; 89 | } 90 | } 91 | 92 | __attribute__ ((visibility ("internal"))) 93 | int rs_galois_mult(int x, int y) 94 | { 95 | int sum; 96 | if (x == 0 || y == 0) return 0; 97 | // This can 'overflow' beyond 255. This is 98 | // handled by positive overflow of ilog_table 99 | sum = log_table[x] + log_table[y]; 100 | 101 | return ilog_table[sum]; 102 | } 103 | 104 | static int rs_galois_div(int x, int y) 105 | { 106 | int diff; 107 | if (x == 0) return 0; 108 | if (y == 0) return -1; 109 | 110 | // This can 'underflow'. This is handled 111 | // by negative overflow of ilog_table 112 | diff = log_table[x] - log_table[y]; 113 | 114 | return ilog_table[diff]; 115 | } 116 | 117 | __attribute__ ((visibility ("internal"))) 118 | int rs_galois_inverse(int x) 119 | { 120 | return rs_galois_div(1, x); 121 | } 122 | -------------------------------------------------------------------------------- /src/builtin/xor_codes/Makefile.am: -------------------------------------------------------------------------------- 1 | lib_LTLIBRARIES = libXorcode.la 2 | 3 | # libXorcode params 4 | libXorcode_la_SOURCES = xor_code.c xor_hd_code.c 5 | libXorcode_la_CPPFLAGS = -I$(top_srcdir)/include/xor_codes $(SIMD_FLAGS) @GCOV_FLAGS@ 6 | 7 | # Version format (C - A).(A).(R) for C:R:A input 8 | libXorcode_la_LDFLAGS = @GCOV_LDFLAGS@ -rpath '$(libdir)' -version-info 1:1:0 9 | 10 | MOSTLYCLEANFILES = *.gcda *.gcno *.gcov 11 | 12 | -------------------------------------------------------------------------------- /src/erasurecode_postprocessing.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Tushar Gohad, Kevin M Greenan, Eric Lambert 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * liberasurecode postprocessing helpers implementation 25 | * 26 | * vi: set noai tw=79 ts=4 sw=4: 27 | */ 28 | 29 | #include 30 | #include "erasurecode_backend.h" 31 | #include "erasurecode_helpers.h" 32 | #include "erasurecode_helpers_ext.h" 33 | #include "erasurecode_log.h" 34 | #include "erasurecode_stdinc.h" 35 | #include "alg_sig.h" 36 | 37 | __attribute__ ((visibility ("internal"))) 38 | void add_fragment_metadata(ec_backend_t be, char *fragment, 39 | int idx, uint64_t orig_data_size, int blocksize, 40 | ec_checksum_type_t ct, int add_chksum) 41 | { 42 | //TODO EDL we are ignoring the return codes here, fix that 43 | set_libec_version(fragment); 44 | set_fragment_idx(fragment, idx); 45 | set_orig_data_size(fragment, orig_data_size); 46 | set_fragment_payload_size(fragment, blocksize); 47 | set_backend_id(fragment, be->common.id); 48 | set_backend_version(fragment, be->common.ec_backend_version); 49 | set_fragment_backend_metadata_size(fragment, be->common.ops->get_backend_metadata_size( 50 | be->desc.backend_desc, 51 | blocksize)); 52 | 53 | if (add_chksum) { 54 | set_checksum(ct, fragment, blocksize); 55 | } 56 | 57 | fragment_header_t* header = (fragment_header_t*) fragment; 58 | 59 | if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { 60 | log_error("Invalid fragment header (add fragment metadata)!\n"); 61 | return; 62 | } 63 | 64 | char *flag = getenv("LIBERASURECODE_WRITE_LEGACY_CRC"); 65 | if (flag && !(flag[0] == '\0' || (flag[0] == '0' && flag[1] == '\0'))) { 66 | header->metadata_chksum = liberasurecode_crc32_alt( 67 | 0, &header->meta, sizeof(fragment_metadata_t)); 68 | } else { 69 | header->metadata_chksum = crc32(0, (unsigned char *) &header->meta, 70 | sizeof(fragment_metadata_t)); 71 | } 72 | } 73 | 74 | __attribute__ ((visibility ("internal"))) 75 | int finalize_fragments_after_encode(ec_backend_t instance, 76 | int k, int m, int blocksize, uint64_t orig_data_size, 77 | char **encoded_data, char **encoded_parity) 78 | { 79 | int i, set_chksum = 1; 80 | ec_checksum_type_t ct = instance->args.uargs.ct; 81 | 82 | /* finalize data fragments */ 83 | for (i = 0; i < k; i++) { 84 | char *fragment = get_fragment_ptr_from_data(encoded_data[i]); 85 | add_fragment_metadata(instance, fragment, i, orig_data_size, 86 | blocksize, ct, set_chksum); 87 | encoded_data[i] = fragment; 88 | } 89 | 90 | /* finalize parity fragments */ 91 | for (i = 0; i < m; i++) { 92 | char *fragment = get_fragment_ptr_from_data(encoded_parity[i]); 93 | add_fragment_metadata(instance, fragment, i + k, orig_data_size, 94 | blocksize, ct, set_chksum); 95 | encoded_parity[i] = fragment; 96 | } 97 | 98 | return 0; 99 | } 100 | -------------------------------------------------------------------------------- /src/utils/chksum/crc32.c: -------------------------------------------------------------------------------- 1 | /*- 2 | * COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or 3 | * code or tables extracted from it, as desired without restriction. 4 | * 5 | * First, the polynomial itself and its table of feedback terms. The 6 | * polynomial is 7 | * X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0 8 | * 9 | * Note that we take it "backwards" and put the highest-order term in 10 | * the lowest-order bit. The X^32 term is "implied"; the LSB is the 11 | * X^31 term, etc. The X^0 term (usually shown as "+1") results in 12 | * the MSB being 1 13 | * 14 | * Note that the usual hardware shift register implementation, which 15 | * is what we're using (we're merely optimizing it by doing eight-bit 16 | * chunks at a time) shifts bits into the lowest-order term. In our 17 | * implementation, that means shifting towards the right. Why do we 18 | * do it this way? Because the calculated CRC must be transmitted in 19 | * order from highest-order term to lowest-order term. UARTs transmit 20 | * characters in order from LSB to MSB. By storing the CRC this way 21 | * we hand it to the UART in the order low-byte to high-byte; the UART 22 | * sends each low-bit to hight-bit; and the result is transmission bit 23 | * by bit from highest- to lowest-order term without requiring any bit 24 | * shuffling on our part. Reception works similarly 25 | * 26 | * The feedback terms table consists of 256, 32-bit entries. Notes 27 | * 28 | * The table can be generated at runtime if desired; code to do so 29 | * is shown later. It might not be obvious, but the feedback 30 | * terms simply represent the results of eight shift/xor opera 31 | * tions for all combinations of data and CRC register values 32 | * 33 | * The values must be right-shifted by eight bits by the "updcrc 34 | * logic; the shift must be unsigned (bring in zeroes). On some 35 | * hardware you could probably optimize the shift in assembler by 36 | * using byte-swap instructions 37 | * polynomial $edb88320 38 | * 39 | * 40 | * CRC32 code derived from work by Gary S. Brown. 41 | */ 42 | 43 | #include 44 | 45 | static int crc32_tab[] = { 46 | 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 47 | 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 48 | 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, 49 | 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 50 | 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 51 | 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 52 | 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, 53 | 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 54 | 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 55 | 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 56 | 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 57 | 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 58 | 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 59 | 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 60 | 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 61 | 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 62 | 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 63 | 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 64 | 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 65 | 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 66 | 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 67 | 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 68 | 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 69 | 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 70 | 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 71 | 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 72 | 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 73 | 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 74 | 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 75 | 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 76 | 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 77 | 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 78 | 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 79 | 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 80 | 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 81 | 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 82 | 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 83 | 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 84 | 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 85 | 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 86 | 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 87 | 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 88 | 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d 89 | }; 90 | 91 | int 92 | liberasurecode_crc32_alt(int crc, const void *buf, size_t size) 93 | { 94 | const char *p; 95 | 96 | p = buf; 97 | crc = crc ^ ~0U; 98 | 99 | while (size--) 100 | crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ 101 | ((((crc >> 8) & 0x00FFFFFF) ^ 0x00800000) - 0x00800000); 102 | 103 | return crc ^ ~0U; 104 | } 105 | -------------------------------------------------------------------------------- /src/utils/chksum/md5.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. 3 | * MD5 Message-Digest Algorithm (RFC 1321). 4 | * 5 | * Homepage: 6 | * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 7 | * 8 | * Author: 9 | * Alexander Peslyak, better known as Solar Designer 10 | * 11 | * This software was written by Alexander Peslyak in 2001. No copyright is 12 | * claimed, and the software is hereby placed in the public domain. 13 | * In case this attempt to disclaim copyright and place the software in the 14 | * public domain is deemed null and void, then the software is 15 | * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the 16 | * general public under the following terms: 17 | * 18 | * Redistribution and use in source and binary forms, with or without 19 | * modification, are permitted. 20 | * 21 | * There's ABSOLUTELY NO WARRANTY, express or implied. 22 | * 23 | * (This is a heavily cut-down "BSD license".) 24 | * 25 | * This differs from Colin Plumb's older public domain implementation in that 26 | * no exactly 32-bit integer data type is required (any 32-bit or wider 27 | * unsigned integer data type will do), there's no compile-time endianness 28 | * configuration, and the function prototypes match OpenSSL's. No code from 29 | * Colin Plumb's implementation has been reused; this comment merely compares 30 | * the properties of the two independent implementations. 31 | * 32 | * The primary goals of this implementation are portability and ease of use. 33 | * It is meant to be fast, but not as fast as possible. Some known 34 | * optimizations are not included to reduce source code size and avoid 35 | * compile-time configuration. 36 | */ 37 | 38 | #ifndef HAVE_OPENSSL 39 | 40 | #include 41 | 42 | #include "md5.h" 43 | 44 | /* 45 | * The basic MD5 functions. 46 | * 47 | * F and G are optimized compared to their RFC 1321 definitions for 48 | * architectures that lack an AND-NOT instruction, just like in Colin Plumb's 49 | * implementation. 50 | */ 51 | #define F(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) 52 | #define G(x, y, z) ((y) ^ ((z) & ((x) ^ (y)))) 53 | #define H(x, y, z) ((x) ^ (y) ^ (z)) 54 | #define I(x, y, z) ((y) ^ ((x) | ~(z))) 55 | 56 | /* 57 | * The MD5 transformation for all four rounds. 58 | */ 59 | #define STEP(f, a, b, c, d, x, t, s) \ 60 | (a) += f((b), (c), (d)) + (x) + (t); \ 61 | (a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \ 62 | (a) += (b); 63 | 64 | /* 65 | * SET reads 4 input bytes in little-endian byte order and stores them 66 | * in a properly aligned word in host byte order. 67 | * 68 | * The check for little-endian architectures that tolerate unaligned 69 | * memory accesses is just an optimization. Nothing will break if it 70 | * doesn't work. 71 | */ 72 | #if defined(__i386__) || defined(__x86_64__) || defined(__vax__) 73 | #define SET(n) (*(MD5_u32plus *)&ptr[(n) * 4]) 74 | #define GET(n) SET(n) 75 | #else 76 | #define SET(n) \ 77 | (ctx->block[(n)] = \ 78 | (MD5_u32plus)ptr[(n) * 4] | \ 79 | ((MD5_u32plus)ptr[(n) * 4 + 1] << 8) | \ 80 | ((MD5_u32plus)ptr[(n) * 4 + 2] << 16) | \ 81 | ((MD5_u32plus)ptr[(n) * 4 + 3] << 24)) 82 | #define GET(n) (ctx->block[(n)]) 83 | #endif 84 | 85 | /* 86 | * This processes one or more 64-byte data blocks, but does NOT update 87 | * the bit counters. There are no alignment requirements. 88 | */ 89 | static void *body(MD5_CTX *ctx, void *data, unsigned long size) 90 | { 91 | unsigned char *ptr; 92 | MD5_u32plus a, b, c, d; 93 | MD5_u32plus saved_a, saved_b, saved_c, saved_d; 94 | 95 | ptr = data; 96 | 97 | a = ctx->a; 98 | b = ctx->b; 99 | c = ctx->c; 100 | d = ctx->d; 101 | 102 | do { 103 | saved_a = a; 104 | saved_b = b; 105 | saved_c = c; 106 | saved_d = d; 107 | 108 | /* Round 1 */ 109 | STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7) 110 | STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12) 111 | STEP(F, c, d, a, b, SET(2), 0x242070db, 17) 112 | STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22) 113 | STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7) 114 | STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12) 115 | STEP(F, c, d, a, b, SET(6), 0xa8304613, 17) 116 | STEP(F, b, c, d, a, SET(7), 0xfd469501, 22) 117 | STEP(F, a, b, c, d, SET(8), 0x698098d8, 7) 118 | STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12) 119 | STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17) 120 | STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22) 121 | STEP(F, a, b, c, d, SET(12), 0x6b901122, 7) 122 | STEP(F, d, a, b, c, SET(13), 0xfd987193, 12) 123 | STEP(F, c, d, a, b, SET(14), 0xa679438e, 17) 124 | STEP(F, b, c, d, a, SET(15), 0x49b40821, 22) 125 | 126 | /* Round 2 */ 127 | STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5) 128 | STEP(G, d, a, b, c, GET(6), 0xc040b340, 9) 129 | STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14) 130 | STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20) 131 | STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5) 132 | STEP(G, d, a, b, c, GET(10), 0x02441453, 9) 133 | STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14) 134 | STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20) 135 | STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5) 136 | STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9) 137 | STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14) 138 | STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20) 139 | STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5) 140 | STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9) 141 | STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14) 142 | STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20) 143 | 144 | /* Round 3 */ 145 | STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4) 146 | STEP(H, d, a, b, c, GET(8), 0x8771f681, 11) 147 | STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16) 148 | STEP(H, b, c, d, a, GET(14), 0xfde5380c, 23) 149 | STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4) 150 | STEP(H, d, a, b, c, GET(4), 0x4bdecfa9, 11) 151 | STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16) 152 | STEP(H, b, c, d, a, GET(10), 0xbebfbc70, 23) 153 | STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4) 154 | STEP(H, d, a, b, c, GET(0), 0xeaa127fa, 11) 155 | STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16) 156 | STEP(H, b, c, d, a, GET(6), 0x04881d05, 23) 157 | STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4) 158 | STEP(H, d, a, b, c, GET(12), 0xe6db99e5, 11) 159 | STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16) 160 | STEP(H, b, c, d, a, GET(2), 0xc4ac5665, 23) 161 | 162 | /* Round 4 */ 163 | STEP(I, a, b, c, d, GET(0), 0xf4292244, 6) 164 | STEP(I, d, a, b, c, GET(7), 0x432aff97, 10) 165 | STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15) 166 | STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21) 167 | STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6) 168 | STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10) 169 | STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15) 170 | STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21) 171 | STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6) 172 | STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10) 173 | STEP(I, c, d, a, b, GET(6), 0xa3014314, 15) 174 | STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21) 175 | STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6) 176 | STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10) 177 | STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15) 178 | STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21) 179 | 180 | a += saved_a; 181 | b += saved_b; 182 | c += saved_c; 183 | d += saved_d; 184 | 185 | ptr += 64; 186 | } while (size -= 64); 187 | 188 | ctx->a = a; 189 | ctx->b = b; 190 | ctx->c = c; 191 | ctx->d = d; 192 | 193 | return ptr; 194 | } 195 | 196 | void MD5_Init(MD5_CTX *ctx) 197 | { 198 | ctx->a = 0x67452301; 199 | ctx->b = 0xefcdab89; 200 | ctx->c = 0x98badcfe; 201 | ctx->d = 0x10325476; 202 | 203 | ctx->lo = 0; 204 | ctx->hi = 0; 205 | } 206 | 207 | void MD5_Update(MD5_CTX *ctx, void *data, unsigned long size) 208 | { 209 | MD5_u32plus saved_lo; 210 | unsigned long used, free; 211 | 212 | saved_lo = ctx->lo; 213 | if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo) 214 | ctx->hi++; 215 | ctx->hi += size >> 29; 216 | 217 | used = saved_lo & 0x3f; 218 | 219 | if (used) { 220 | free = 64 - used; 221 | 222 | if (size < free) { 223 | memcpy(&ctx->buffer[used], data, size); 224 | return; 225 | } 226 | 227 | memcpy(&ctx->buffer[used], data, free); 228 | data = (unsigned char *)data + free; 229 | size -= free; 230 | body(ctx, ctx->buffer, 64); 231 | } 232 | 233 | if (size >= 64) { 234 | data = body(ctx, data, size & ~(unsigned long)0x3f); 235 | size &= 0x3f; 236 | } 237 | 238 | memcpy(ctx->buffer, data, size); 239 | } 240 | 241 | void MD5_Final(unsigned char *result, MD5_CTX *ctx) 242 | { 243 | unsigned long used, free; 244 | 245 | used = ctx->lo & 0x3f; 246 | 247 | ctx->buffer[used++] = 0x80; 248 | 249 | free = 64 - used; 250 | 251 | if (free < 8) { 252 | memset(&ctx->buffer[used], 0, free); 253 | body(ctx, ctx->buffer, 64); 254 | used = 0; 255 | free = 64; 256 | } 257 | 258 | memset(&ctx->buffer[used], 0, free - 8); 259 | 260 | ctx->lo <<= 3; 261 | ctx->buffer[56] = ctx->lo; 262 | ctx->buffer[57] = ctx->lo >> 8; 263 | ctx->buffer[58] = ctx->lo >> 16; 264 | ctx->buffer[59] = ctx->lo >> 24; 265 | ctx->buffer[60] = ctx->hi; 266 | ctx->buffer[61] = ctx->hi >> 8; 267 | ctx->buffer[62] = ctx->hi >> 16; 268 | ctx->buffer[63] = ctx->hi >> 24; 269 | 270 | body(ctx, ctx->buffer, 64); 271 | 272 | result[0] = ctx->a; 273 | result[1] = ctx->a >> 8; 274 | result[2] = ctx->a >> 16; 275 | result[3] = ctx->a >> 24; 276 | result[4] = ctx->b; 277 | result[5] = ctx->b >> 8; 278 | result[6] = ctx->b >> 16; 279 | result[7] = ctx->b >> 24; 280 | result[8] = ctx->c; 281 | result[9] = ctx->c >> 8; 282 | result[10] = ctx->c >> 16; 283 | result[11] = ctx->c >> 24; 284 | result[12] = ctx->d; 285 | result[13] = ctx->d >> 8; 286 | result[14] = ctx->d >> 16; 287 | result[15] = ctx->d >> 24; 288 | 289 | memset(ctx, 0, sizeof(*ctx)); 290 | } 291 | 292 | #endif 293 | -------------------------------------------------------------------------------- /test/Makefile.am: -------------------------------------------------------------------------------- 1 | noinst_HEADERS = builtin/xor_codes/test_xor_hd_code.h 2 | noinst_PROGRAMS = test_xor_hd_code alg_sig_test liberasurecode_test libec_slap rs_galois_test liberasurecode_rs_vand_test 3 | 4 | test_xor_hd_code_SOURCES = \ 5 | builtin/xor_codes/test_xor_hd_code.c \ 6 | builtin/xor_codes/test_xor_hd_code.h 7 | test_xor_hd_code_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/include/erasurecode -I$(top_srcdir)/include/xor_codes @GCOV_FLAGS@ 8 | test_xor_hd_code_LDFLAGS = @GCOV_LDFLAGS@ -static-libtool-libs $(top_builddir)/src/liberasurecode.la $(top_builddir)/src/builtin/xor_codes/libXorcode.la -ldl 9 | check_PROGRAMS = test_xor_hd_code 10 | 11 | alg_sig_test_SOURCES = utils/chksum/test_alg_sig.c 12 | alg_sig_test_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/include/erasurecode -I$(top_srcdir)/include/xor_codes @GCOV_FLAGS@ 13 | alg_sig_test_LDFLAGS = @GCOV_LDFLAGS@ -static-libtool-libs $(top_builddir)/src/liberasurecode.la -ldl 14 | check_PROGRAMS += alg_sig_test 15 | 16 | liberasurecode_test_SOURCES = liberasurecode_test.c 17 | liberasurecode_test_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/include/erasurecode @GCOV_FLAGS@ 18 | liberasurecode_test_LDFLAGS = @GCOV_LDFLAGS@ $(top_builddir)/src/liberasurecode.la -ldl -lpthread -lz 19 | check_PROGRAMS += liberasurecode_test 20 | 21 | libec_slap_SOURCES = libec_slap.c 22 | libec_slap_CPPFLAGS = -I. -I$(top_srcdir)/include -I$(top_srcdir)/include/erasurecode @GCOV_FLAGS@ 23 | libec_slap_LDFLAGS = @GCOV_LDFLAGS@ $(top_builddir)/src/liberasurecode.la -ldl -lpthread 24 | check_PROGRAMS += libec_slap 25 | 26 | rs_galois_test_SOURCES = builtin/rs_vand/rs_galois_test.c 27 | rs_galois_test_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/include/rs_vand @GCOV_FLAGS@ 28 | rs_galois_test_LDFLAGS = @GCOV_LDFLAGS@ -static-libtool-libs $(top_builddir)/src/builtin/rs_vand/liberasurecode_rs_vand.la 29 | check_PROGRAMS += rs_galois_test 30 | 31 | liberasurecode_rs_vand_test_SOURCES = builtin/rs_vand/liberasurecode_rs_vand_test.c 32 | liberasurecode_rs_vand_test_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/include/rs_vand @GCOV_FLAGS@ 33 | liberasurecode_rs_vand_test_LDFLAGS = @GCOV_LDFLAGS@ -static-libtool-libs $(top_builddir)/src/builtin/rs_vand/liberasurecode_rs_vand.la 34 | check_PROGRAMS += liberasurecode_rs_vand_test 35 | 36 | MOSTLYCLEANFILES = *.gcda *.gcno *.gcov \ 37 | ./builtin/xor_codes/*.gcda ./builtin/xor_codes/*.gcno ./builtin/xor_codes/*.gcov \ 38 | ./utils/chksum/*.gcda ./utils/chksum/*.gcno ./utils/chksum/*.gcov 39 | -------------------------------------------------------------------------------- /test/builtin/rs_vand/liberasurecode_rs_vand_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Kevin M Greenan 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * vi: set noai tw=79 ts=4 sw=4: 25 | */ 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | 36 | int test_make_systematic_matrix(int k, int m) 37 | { 38 | int *matrix = make_systematic_matrix(k, m); 39 | int is_identity = is_identity_matrix(matrix, k); 40 | 41 | if (!is_identity) { 42 | printf("Generating systematic matrix did not work!\n"); 43 | printf("Generator matrix: \n\n"); 44 | print_matrix(matrix, m+k, k); 45 | } 46 | 47 | free_systematic_matrix(matrix); 48 | 49 | return is_identity; 50 | } 51 | 52 | void dump_buffer(char *buf, int size, const char* filename) 53 | { 54 | int fd = open(filename, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 55 | ssize_t nbytes = write(fd, buf, size); 56 | if (nbytes < 0) { 57 | printf("dump_buffer: write error!\n"); 58 | } 59 | close(fd); 60 | } 61 | 62 | int test_invert_systematic_matrix(int k, int m, int num_missing) 63 | { 64 | int *inverse = (int*)malloc(sizeof(int)*k*k); 65 | int *prod = (int*)malloc(sizeof(int)*k*k); 66 | int *missing = (int*)malloc(sizeof(int)*(num_missing+1)); 67 | int *matrix = make_systematic_matrix(k, m); 68 | int *decoding_matrix = (int*)malloc(sizeof(int)*k*k); 69 | int *decoding_matrix_cpy = (int*)malloc(sizeof(int)*k*k); 70 | int n = k+m; 71 | int i, res; 72 | 73 | srand((unsigned int)time(0)); 74 | 75 | for (i = 0;i < num_missing+1; i++) { 76 | missing[i] = -1; 77 | } 78 | 79 | for (i = 0;i < num_missing; i++) { 80 | int idx = rand() % n; 81 | while (is_missing(missing, idx)) { 82 | idx = rand() % n; 83 | } 84 | missing[i] = idx; 85 | } 86 | 87 | create_decoding_matrix(matrix, decoding_matrix, missing, k, m); 88 | create_decoding_matrix(matrix, decoding_matrix_cpy, missing, k, m); 89 | 90 | gaussj_inversion(decoding_matrix, inverse, k); 91 | 92 | square_matrix_multiply(decoding_matrix_cpy, inverse, prod, k); 93 | 94 | res = is_identity_matrix(prod, k); 95 | 96 | if (!res) { 97 | printf("Inverting decoding matrix did not work!\n"); 98 | printf("Generator matrix: \n\n"); 99 | print_matrix(matrix, n, k); 100 | printf("Decoding matrix: \n\n"); 101 | print_matrix(decoding_matrix_cpy, k, k); 102 | printf("Inverse Decoding matrix: \n\n"); 103 | print_matrix(inverse, k, k); 104 | printf("Missing: \n\n"); 105 | print_matrix(missing, 1, num_missing); 106 | } 107 | 108 | free(inverse); 109 | free(prod); 110 | free(missing); 111 | free(decoding_matrix); 112 | free(decoding_matrix_cpy); 113 | free_systematic_matrix(matrix); 114 | 115 | return res; 116 | } 117 | 118 | char* gen_random_buffer(int blocksize) 119 | { 120 | int i; 121 | char *buf = (char*)malloc(blocksize); 122 | 123 | for (i = 0; i < blocksize; i++) { 124 | buf[i] = (char)(rand() % 255); 125 | } 126 | 127 | return buf; 128 | } 129 | 130 | int test_encode_decode(int k, int m, int num_missing, int blocksize) 131 | { 132 | char **data = (char**)malloc(sizeof(char*)*k); 133 | char **parity = (char**)malloc(sizeof(char*)*m); 134 | char **missing_bufs = (char**)malloc(sizeof(char*)*num_missing); 135 | int *missing = (int*)malloc(sizeof(int)*(num_missing+1)); 136 | int *matrix = make_systematic_matrix(k, m); 137 | int n = k + m; 138 | int i; 139 | int ret = 1; 140 | 141 | srand((unsigned int)time(0)); 142 | 143 | for (i = 0; i < k; i++) { 144 | data[i] = gen_random_buffer(blocksize); 145 | } 146 | 147 | for (i = 0; i < m; i++) { 148 | parity[i] = (char*)malloc(blocksize); 149 | } 150 | 151 | for (i = 0;i < num_missing+1; i++) { 152 | missing[i] = -1; 153 | } 154 | 155 | // Encode 156 | liberasurecode_rs_vand_encode(matrix, data, parity, k, m, blocksize); 157 | 158 | // Copy data and parity 159 | for (i = 0;i < num_missing; i++) { 160 | int idx = rand() % n; 161 | while (is_missing(missing, idx)) { 162 | idx = rand() % n; 163 | } 164 | missing_bufs[i] = (char*)malloc(blocksize); 165 | memcpy(missing_bufs[i], idx < k ? data[idx] : parity[idx - k], blocksize); 166 | missing[i] = idx; 167 | } 168 | 169 | // Zero missing bufs 170 | for (i = 0;i < num_missing; i++) { 171 | if (missing[i] < k) { 172 | memset(data[missing[i]], 0, blocksize); 173 | } else { 174 | memset(parity[missing[i] - k], 0, blocksize); 175 | } 176 | } 177 | 178 | // Decode and check 179 | liberasurecode_rs_vand_decode(matrix, data, parity, k, m, missing, blocksize, 1); 180 | 181 | for (i = 0; i < num_missing; i++) { 182 | int idx = missing[i]; 183 | if (idx < k) { 184 | if (memcmp(data[idx], missing_bufs[i], blocksize)) { 185 | dump_buffer(data[idx], blocksize, "decoded_buffer"); 186 | dump_buffer(missing_bufs[i], blocksize, "orig_buffer"); 187 | ret = 0; 188 | } 189 | } else if (memcmp(parity[idx - k], missing_bufs[i], blocksize)) { 190 | ret = 0; 191 | } 192 | } 193 | 194 | for (i = 0; i < k; i++) { 195 | free(data[i]); 196 | } 197 | free(data); 198 | for (i = 0; i < m; i++) { 199 | free(parity[i]); 200 | } 201 | free(parity); 202 | for (i = 0; i < num_missing; i++) { 203 | free(missing_bufs[i]); 204 | } 205 | free(missing_bufs); 206 | free(missing); 207 | free(matrix); 208 | 209 | return ret; 210 | } 211 | 212 | int test_reconstruct(int k, int m, int num_missing, int blocksize) 213 | { 214 | char **data = (char**)malloc(sizeof(char*)*k); 215 | char **parity = (char**)malloc(sizeof(char*)*m); 216 | char **missing_bufs = (char**)malloc(sizeof(char*)*num_missing); 217 | int *missing = (int*)malloc(sizeof(int)*(num_missing+1)); 218 | int *matrix = make_systematic_matrix(k, m); 219 | int destination_idx = 0; 220 | int n = k + m; 221 | int i; 222 | int ret = 1; 223 | 224 | srand((unsigned int)time(0)); 225 | 226 | for (i = 0; i < k; i++) { 227 | data[i] = gen_random_buffer(blocksize); 228 | } 229 | 230 | for (i = 0; i < m; i++) { 231 | parity[i] = (char*)malloc(blocksize); 232 | } 233 | 234 | for (i = 0;i < num_missing+1; i++) { 235 | missing[i] = -1; 236 | } 237 | 238 | // Encode 239 | liberasurecode_rs_vand_encode(matrix, data, parity, k, m, blocksize); 240 | 241 | // Copy data and parity 242 | for (i = 0; i < num_missing; i++) { 243 | int idx = rand() % n; 244 | while (is_missing(missing, idx)) { 245 | idx = rand() % n; 246 | } 247 | missing_bufs[i] = (char*)malloc(blocksize); 248 | memcpy(missing_bufs[i], idx < k ? data[idx] : parity[idx - k], blocksize); 249 | missing[i] = idx; 250 | if (i == 0) { 251 | destination_idx = missing[i]; 252 | } 253 | } 254 | 255 | // Zero missing bufs 256 | for (i = 0;i < num_missing; i++) { 257 | if (missing[i] < k) { 258 | memset(data[missing[i]], 0, blocksize); 259 | } else { 260 | memset(parity[missing[i] - k], 0, blocksize); 261 | } 262 | } 263 | 264 | // Reconstruct and check destination buffer 265 | liberasurecode_rs_vand_reconstruct(matrix, data, parity, k, m, missing, destination_idx, blocksize); 266 | 267 | // The original copy of the destination buffer is in the 0th buffer (see above) 268 | if (destination_idx < k) { 269 | if (memcmp(data[destination_idx], missing_bufs[0], blocksize)) { 270 | dump_buffer(data[destination_idx], blocksize, "decoded_buffer"); 271 | dump_buffer(missing_bufs[0], blocksize, "orig_buffer"); 272 | ret = 0; 273 | } 274 | } else if (memcmp(parity[destination_idx - k], missing_bufs[0], blocksize)) { 275 | ret = 0; 276 | } 277 | 278 | for (i = 0; i < k; i++) { 279 | free(data[i]); 280 | } 281 | free(data); 282 | for (i = 0; i < m; i++) { 283 | free(parity[i]); 284 | } 285 | free(parity); 286 | for (i = 0; i < num_missing; i++) { 287 | free(missing_bufs[i]); 288 | } 289 | free(missing_bufs); 290 | free(missing); 291 | free(matrix); 292 | 293 | return ret; 294 | } 295 | 296 | int matrix_dimensions[][2] = { {12, 6}, {12, 3}, {12, 2}, {12, 1}, {5, 3}, {5, 2}, {5, 1}, {1, 1}, {-1, -1} }; 297 | 298 | int main(void) 299 | { 300 | int i = 0; 301 | int blocksize = 4096; 302 | 303 | while (matrix_dimensions[i][0] >= 0) { 304 | int k = matrix_dimensions[i][0], m = matrix_dimensions[i][1]; 305 | 306 | init_liberasurecode_rs_vand(k, m); 307 | 308 | int make_systematic_res = test_make_systematic_matrix(k, m); 309 | if (!make_systematic_res) { 310 | fprintf(stderr, "Error running make systematic matrix for k=%d, m=%d\n", k, m); 311 | return 1; 312 | } 313 | 314 | int invert_res = test_invert_systematic_matrix(k, m, m); 315 | if (!invert_res) { 316 | fprintf(stderr, "Error running inversion test for k=%d, m=%d\n", k, m); 317 | return 1; 318 | } 319 | 320 | int enc_dec_res = test_encode_decode(k, m, m, blocksize); 321 | if (!enc_dec_res) { 322 | fprintf(stderr, "Error running encode/decode test for k=%d, m=%d, bs=%d\n", k, m, blocksize); 323 | return 1; 324 | } 325 | 326 | int reconstr_res = test_reconstruct(k, m, m, blocksize); 327 | if (!reconstr_res) { 328 | fprintf(stderr, "Error running reconstruction test for k=%d, m=%d, bs=%d\n", k, m, blocksize); 329 | return 1; 330 | } 331 | 332 | 333 | deinit_liberasurecode_rs_vand(); 334 | i++; 335 | } 336 | 337 | return 0; 338 | } 339 | -------------------------------------------------------------------------------- /test/builtin/rs_vand/rs_galois_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Kevin M Greenan 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | * 24 | * vi: set noai tw=79 ts=4 sw=4: 25 | */ 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | int test_inverse(void) 33 | { 34 | int *uniq = (int*)malloc(sizeof(int)*FIELD_SIZE); 35 | int i = 0; 36 | 37 | memset(uniq, 0, sizeof(int)*FIELD_SIZE); 38 | 39 | rs_galois_init_tables(); 40 | 41 | for (i = 1; i < FIELD_SIZE; i++) { 42 | if (uniq[i] != 0) { 43 | fprintf(stderr, "Duplicate %d: %d , %d \n", i, uniq[i], rs_galois_inverse(i)); 44 | return 1; 45 | } 46 | uniq[i] = rs_galois_inverse(i); 47 | int one = rs_galois_mult(rs_galois_inverse(i), i); 48 | if (one != 1) { 49 | fprintf(stderr, "%d is not the inverse of %d = %d\n", rs_galois_inverse(i), i, one); 50 | return 1; 51 | } 52 | } 53 | return 0; 54 | } 55 | 56 | int main(int argc, char **argv) 57 | { 58 | int ret = 0; 59 | if (test_inverse() != 0) { 60 | fprintf(stderr, "test_inverse() failed\n"); 61 | ret = 1; 62 | } 63 | return ret; 64 | } 65 | -------------------------------------------------------------------------------- /test/utils/chksum/test_alg_sig.c: -------------------------------------------------------------------------------- 1 | /* * Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * 7 | * Redistributions of source code must retain the above copyright notice, this 8 | * list of conditions and the following disclaimer. 9 | * 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | * list of conditions and the following disclaimer in the documentation and/or 12 | * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 | * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | */ 24 | 25 | #include "alg_sig.h" 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | // Max is 8 bytes right now (64-bits) 32 | #define MAX_SIG_LEN 8 33 | 34 | void fill_random_buffer(char *buf, int size) 35 | { 36 | int i; 37 | 38 | for (i=0; i < size; i++) { 39 | buf[i] = (char)(rand() % 256); 40 | } 41 | } 42 | 43 | void compute_parity(char **data, char* parity, int num_data, int size) 44 | { 45 | int i, j; 46 | 47 | bzero(parity, size); 48 | 49 | for (i=0; i < num_data; i++) { 50 | for (j=0; j < size; j++) { 51 | parity[j] ^= data[i][j]; 52 | } 53 | } 54 | } 55 | 56 | int check_parity_of_sigs(char **sigs, int num_data, int size) 57 | { 58 | int i, j; 59 | int ret = 0; 60 | char *parity_sig = (char*)malloc(MAX_SIG_LEN); 61 | 62 | bzero(parity_sig, MAX_SIG_LEN); 63 | 64 | for (i=0; i < num_data; i++) { 65 | for (j=0; j < MAX_SIG_LEN; j++) { 66 | parity_sig[j] ^= sigs[i][j]; 67 | } 68 | } 69 | 70 | if (memcmp(parity_sig, sigs[num_data], MAX_SIG_LEN) != 0) { 71 | fprintf(stderr, "Signatures do not match:\n"); 72 | for (i=0; i < MAX_SIG_LEN; i++) { 73 | fprintf(stderr, "parity_sig[%d] = 0x%x, sigs[%d][%d] = 0x%x\n", i, parity_sig[i], num_data, i, sigs[num_data][i]); 74 | } 75 | ret = 1; 76 | } 77 | 78 | free(parity_sig); 79 | 80 | return ret; 81 | } 82 | 83 | static int basic_xor_test_8_32(void) 84 | { 85 | int blocksize = 65536; 86 | int num_data = 12; 87 | char **data; 88 | char *parity; 89 | char **sigs; 90 | int i; 91 | int ret = 0; 92 | 93 | alg_sig_t* sig_handle = init_alg_sig(32, 8); 94 | if (NULL == sig_handle) { 95 | goto out; 96 | } 97 | data = (char**)malloc(sizeof(char*) * num_data); 98 | sigs = (char**)malloc(sizeof(char*) * (num_data + 1)); 99 | for (i=0; i < num_data; i++) { 100 | data[i] = (char*)malloc(sizeof(char)*blocksize); 101 | fill_random_buffer(data[i], blocksize); 102 | sigs[i] = (char*)malloc(MAX_SIG_LEN); 103 | 104 | } 105 | parity = (char*)malloc(sizeof(char)*blocksize); 106 | sigs[i] = (char*)malloc(MAX_SIG_LEN); 107 | 108 | compute_parity(data, parity, num_data, blocksize); 109 | 110 | for (i=0; i < num_data; i++) { 111 | bzero(sigs[i], MAX_SIG_LEN); 112 | compute_alg_sig(sig_handle, data[i], blocksize, sigs[i]); 113 | } 114 | bzero(sigs[i], MAX_SIG_LEN); 115 | compute_alg_sig(sig_handle, parity, blocksize, sigs[i]); 116 | 117 | ret = check_parity_of_sigs(sigs, num_data, blocksize); 118 | 119 | for (i=0; i < num_data; i++) { 120 | free(data[i]); 121 | free(sigs[i]); 122 | } 123 | 124 | free(parity); 125 | free(sigs[num_data]); 126 | free(sigs); 127 | free(data); 128 | destroy_alg_sig(sig_handle); 129 | 130 | out: 131 | return ret; 132 | } 133 | 134 | static int basic_xor_test_16_64(void) 135 | { 136 | int blocksize = 65536; 137 | int num_data = 12; 138 | char **data; 139 | char *parity; 140 | char **sigs; 141 | int i; 142 | int ret = 0; 143 | 144 | alg_sig_t* sig_handle = init_alg_sig(64, 16); 145 | if (NULL == sig_handle) { 146 | goto out; 147 | } 148 | 149 | data = (char**)malloc(sizeof(char*) * num_data); 150 | sigs = (char**)malloc(sizeof(char*) * (num_data + 1)); 151 | for (i=0; i < num_data; i++) { 152 | data[i] = (char*)malloc(sizeof(char)*blocksize); 153 | fill_random_buffer(data[i], blocksize); 154 | sigs[i] = (char*)malloc(MAX_SIG_LEN); 155 | } 156 | parity = (char*)malloc(sizeof(char)*blocksize); 157 | sigs[i] = (char*)malloc(MAX_SIG_LEN); 158 | 159 | compute_parity(data, parity, num_data, blocksize); 160 | 161 | for (i=0; i < num_data; i++) { 162 | bzero(sigs[i], MAX_SIG_LEN); 163 | compute_alg_sig(sig_handle, data[i], blocksize, sigs[i]); 164 | } 165 | bzero(sigs[i], MAX_SIG_LEN); 166 | compute_alg_sig(sig_handle, parity, blocksize, sigs[i]); 167 | 168 | ret = check_parity_of_sigs(sigs, num_data, blocksize); 169 | 170 | for (i=0; i < num_data; i++) { 171 | free(data[i]); 172 | free(sigs[i]); 173 | } 174 | 175 | free(parity); 176 | free(sigs[num_data]); 177 | free(sigs); 178 | free(data); 179 | destroy_alg_sig(sig_handle); 180 | 181 | out: 182 | return ret; 183 | } 184 | 185 | static int basic_xor_test_16_32(void) 186 | { 187 | int blocksize = 65536; 188 | int num_data = 12; 189 | char **data; 190 | char *parity; 191 | char **sigs; 192 | int i; 193 | int ret = 0; 194 | 195 | alg_sig_t* sig_handle = init_alg_sig(32, 16); 196 | if (NULL == sig_handle) { 197 | goto out; 198 | } 199 | data = (char**)malloc(sizeof(char*) * num_data); 200 | sigs = (char**)malloc(sizeof(char*) * (num_data + 1)); 201 | for (i=0; i < num_data; i++) { 202 | data[i] = (char*)malloc(sizeof(char)*blocksize); 203 | fill_random_buffer(data[i], blocksize); 204 | sigs[i] = (char*)malloc(MAX_SIG_LEN); 205 | 206 | } 207 | parity = (char*)malloc(sizeof(char)*blocksize); 208 | sigs[i] = (char*)malloc(MAX_SIG_LEN); 209 | 210 | compute_parity(data, parity, num_data, blocksize); 211 | 212 | for (i=0; i < num_data; i++) { 213 | bzero(sigs[i], MAX_SIG_LEN); 214 | compute_alg_sig(sig_handle, data[i], blocksize, sigs[i]); 215 | } 216 | bzero(sigs[i], MAX_SIG_LEN); 217 | compute_alg_sig(sig_handle, parity, blocksize, sigs[i]); 218 | 219 | ret = check_parity_of_sigs(sigs, num_data, blocksize); 220 | 221 | for (i=0; i < num_data; i++) { 222 | free(data[i]); 223 | free(sigs[i]); 224 | } 225 | 226 | free(parity); 227 | free(sigs[num_data]); 228 | free(sigs); 229 | free(data); 230 | destroy_alg_sig(sig_handle); 231 | 232 | out: 233 | return ret; 234 | } 235 | 236 | int main(int argc, char**argv) 237 | { 238 | int ret; 239 | int num_failed = 0; 240 | 241 | srand(time(NULL)); 242 | 243 | ret = basic_xor_test_16_32(); 244 | if (ret) { 245 | fprintf(stderr, "basic_xor_test_16_32 has failed!\n"); 246 | num_failed++; 247 | } 248 | ret = basic_xor_test_16_64(); 249 | if (ret) { 250 | fprintf(stderr, "basic_xor_test_16_64 has failed!\n"); 251 | num_failed++; 252 | } 253 | ret = basic_xor_test_8_32(); 254 | if (ret) { 255 | fprintf(stderr, "basic_xor_test_8_32 has failed!\n"); 256 | num_failed++; 257 | } 258 | 259 | if (num_failed == 0) { 260 | fprintf(stderr, "Tests pass!!!\n"); 261 | } 262 | return num_failed; 263 | } 264 | 265 | --------------------------------------------------------------------------------