├── .gitignore ├── examples ├── CMakeLists.txt ├── test_rs.go ├── test_003_output.txt ├── simple-encoder.c ├── simple-encoder.go └── fec_test.c ├── LICENSE ├── rs.h ├── README.md ├── test_rs.c └── rs.c /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | *.obj 5 | *.elf 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Libraries 12 | *.lib 13 | *.a 14 | *.la 15 | *.lo 16 | 17 | # Shared objects (inc. Windows DLLs) 18 | *.dll 19 | *.so 20 | *.so.* 21 | *.dylib 22 | 23 | # Executables 24 | *.exe 25 | *.out 26 | *.app 27 | *.i*86 28 | *.x86_64 29 | *.hex 30 | 31 | # Debug files 32 | *.dSYM/ 33 | *.su 34 | 35 | build* 36 | tags 37 | cscope.* 38 | 39 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.6) 2 | project (examples) 3 | #set(CMAKE_VERBOSE_MAKEFILE ON) 4 | #add_definitions( -O0 -g -rdynamic -fno-inline -Wstrict-prototypes -std=gnu99 -Wall) 5 | add_definitions( -Os -Wstrict-prototypes -Wstrict-prototypes -std=gnu99 -Wall) 6 | 7 | set(CMAKE_SYSTEM_NAME Linux) 8 | 9 | SET(CMAKE_BUILD_TYPE debug) 10 | #SET(CMAKE_EXE_LINKER_FLAGS "-static") 11 | 12 | include_directories(${PROJECT_SOURCE_DIR}/src) 13 | 14 | add_executable(test_rs ../test_rs.c) 15 | target_link_libraries(test_rs) 16 | 17 | #add_executable(fec_test fec_test.c ../rs.c) 18 | #target_link_libraries(fec_test) 19 | 20 | add_executable(simple-encoder simple-encoder.c ../rs.c) 21 | target_link_libraries(simple-encoder) 22 | 23 | -------------------------------------------------------------------------------- /examples/test_rs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "fmt" 7 | 8 | "github.com/klauspost/reedsolomon" 9 | ) 10 | 11 | func main() { 12 | dataShards := 12 13 | parShards := 6 14 | enc, err := reedsolomon.New(dataShards, parShards) 15 | checkErr(err) 16 | 17 | text := "hello world hello world " 18 | data := make([]byte, len(text)) 19 | copy(data, []byte(text)) 20 | fmt.Println("data len:", len(data)) 21 | 22 | shards, err := enc.Split(data) 23 | checkErr(err) 24 | fmt.Println("split shards:", shards) 25 | 26 | err = enc.Encode(shards) 27 | checkErr(err) 28 | fmt.Println("encode shards:", shards) 29 | 30 | dec, err := reedsolomon.New(dataShards, parShards) 31 | checkErr(err) 32 | //fmt.Println(len(shards), len(shards[0])) 33 | 34 | shards[1] = nil 35 | shards[3] = nil 36 | shards[4] = nil 37 | //fmt.Println(shards) 38 | 39 | err = dec.Reconstruct(shards) 40 | checkErr(err) 41 | fmt.Println("decode:", shards) 42 | } 43 | 44 | func checkErr(err error) { 45 | if err != nil { 46 | fmt.Fprintf(os.Stderr, "Error: %s", err.Error()) 47 | os.Exit(2) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 janson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/test_003_output.txt: -------------------------------------------------------------------------------- 1 | $ go build -o go_test_rs ../test_rs.go 2 | $ ./go_test_rs 3 | data len: 24 4 | split shards: [[104 101] [108 108] [111 32] [119 111] [114 108] [100 32] [104 101] [108 108] [111 32] [119 111] [114 108] [100 32] [0 0] [0 0] [0 0] [0 0] [0 0] [0 0]] 5 | encode shards: [[104 101] [108 108] [111 32] [119 111] [114 108] [100 32] [104 101] [108 108] [111 32] [119 111] [114 108] [100 32] [157 178] [83 31] [48 240] [254 93] [31 89] [151 184]] 6 | decode: [[104 101] [108 108] [111 32] [119 111] [114 108] [100 32] [104 101] [108 108] [111 32] [119 111] [114 108] [100 32] [157 178] [83 31] [48 240] [254 93] [31 89] [151 184]] 7 | 8 | $ ./test_rs 9 | test_003: 10 | text size=24 11 | matrix (18,2): 12 | 104 101 13 | 108 108 14 | 111 32 15 | 119 111 16 | 114 108 17 | 100 32 18 | 104 101 19 | 108 108 20 | 111 32 21 | 119 111 22 | 114 108 23 | 100 32 24 | 0 0 25 | 0 0 26 | 0 0 27 | 0 0 28 | 0 0 29 | 0 0 30 | matrix (18,2): 31 | 104 101 32 | 108 108 33 | 111 32 34 | 119 111 35 | 114 108 36 | 100 32 37 | 104 101 38 | 108 108 39 | 111 32 40 | 119 111 41 | 114 108 42 | 100 32 43 | 157 178 44 | 83 31 45 | 48 240 46 | 254 93 47 | 31 89 48 | 151 184 49 | -------------------------------------------------------------------------------- /rs.h: -------------------------------------------------------------------------------- 1 | #ifndef __RS_H_ 2 | #define __RS_H_ 3 | 4 | /* use small value to save memory */ 5 | #ifndef DATA_SHARDS_MAX 6 | #define DATA_SHARDS_MAX (255) 7 | #endif 8 | 9 | /* use other memory allocator */ 10 | #ifndef RS_MALLOC 11 | #define RS_MALLOC(x) malloc(x) 12 | #endif 13 | 14 | #ifndef RS_FREE 15 | #define RS_FREE(x) free(x) 16 | #endif 17 | 18 | #ifndef RS_CALLOC 19 | #define RS_CALLOC(n, x) calloc(n, x) 20 | #endif 21 | 22 | typedef struct _reed_solomon { 23 | int data_shards; 24 | int parity_shards; 25 | int shards; 26 | unsigned char* m; 27 | unsigned char* parity; 28 | } reed_solomon; 29 | 30 | /** 31 | * MUST initial one time 32 | * */ 33 | void fec_init(void); 34 | 35 | reed_solomon* reed_solomon_new(int data_shards, int parity_shards); 36 | void reed_solomon_release(reed_solomon* rs); 37 | 38 | /** 39 | * encode one shard 40 | * input: 41 | * rs 42 | * data_blocks[rs->data_shards][block_size] 43 | * fec_blocks[rs->data_shards][block_size] 44 | * */ 45 | int reed_solomon_encode(reed_solomon* rs, 46 | unsigned char** data_blocks, 47 | unsigned char** fec_blocks, 48 | int block_size); 49 | 50 | 51 | /** 52 | * decode one shard 53 | * input: 54 | * rs 55 | * original data_blocks[rs->data_shards][block_size] 56 | * dec_fec_blocks[nr_fec_blocks][block_size] 57 | * fec_block_nos: fec pos number in original fec_blocks 58 | * erased_blocks: erased blocks in original data_blocks 59 | * nr_fec_blocks: the number of erased blocks 60 | * */ 61 | int reed_solomon_decode(reed_solomon* rs, 62 | unsigned char **data_blocks, 63 | int block_size, 64 | unsigned char **dec_fec_blocks, 65 | unsigned int *fec_block_nos, 66 | unsigned int *erased_blocks, 67 | int nr_fec_blocks); 68 | 69 | /** 70 | * encode a big size of buffer 71 | * input: 72 | * rs 73 | * nr_shards: assert(0 == nr_shards % rs->data_shards) 74 | * shards[nr_shards][block_size] 75 | * */ 76 | int reed_solomon_encode2(reed_solomon* rs, unsigned char** shards, int nr_shards, int block_size); 77 | 78 | /** 79 | * reconstruct a big size of buffer 80 | * input: 81 | * rs 82 | * nr_shards: assert(0 == nr_shards % rs->data_shards) 83 | * shards[nr_shards][block_size] 84 | * marks[nr_shards] marks as errors 85 | * */ 86 | int reed_solomon_reconstruct(reed_solomon* rs, unsigned char** shards, unsigned char* marks, int nr_shards, int block_size); 87 | #endif 88 | 89 | -------------------------------------------------------------------------------- /examples/simple-encoder.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "../rs.h" 13 | 14 | int main(int argc, char *argv[]) { 15 | struct stat st; 16 | int parity_shards = 0; 17 | int data_shards = 0; 18 | int nr_shards; 19 | int block_size; 20 | char* out = NULL; 21 | char* filename = NULL, f[256]; 22 | int i, n; 23 | int fd, size; 24 | unsigned char* data; 25 | reed_solomon* rs = NULL; 26 | unsigned char **data_blocks = NULL; 27 | char output[256]; 28 | 29 | while(-1 != (i = getopt(argc, argv, "d:p:o:f:"))) { 30 | switch(i) { 31 | case 'd': 32 | data_shards = atoi(optarg); 33 | break; 34 | case 'p': 35 | parity_shards = atoi(optarg); 36 | break; 37 | case 'o': 38 | out = optarg; 39 | break; 40 | case 'f': 41 | strcpy(f, optarg); 42 | filename = f; 43 | break; 44 | default: 45 | fprintf(stderr, "simple-encoder -d 10 -p 3 -o output -f filename.ext\n"); 46 | exit(1); 47 | } 48 | } 49 | 50 | if(0 == parity_shards || 0 == data_shards || NULL == filename) { 51 | fprintf(stderr, "error input, example:\nsimple-encoder -d 10 -p 3 -o output -f filename.ext\n"); 52 | exit(1); 53 | } 54 | if(out == NULL) { 55 | out = "./"; 56 | } 57 | 58 | fec_init(); 59 | 60 | fd = open(filename, O_RDONLY); 61 | if(fd < 0) { 62 | fprintf(stderr, "input file: %s not found\n", filename); 63 | exit(1); 64 | } 65 | 66 | fstat(fd, &st); 67 | size = st.st_size; 68 | block_size = (size+data_shards-1) / data_shards; 69 | nr_shards = data_shards + parity_shards; 70 | printf("filename=%s size=%d block_size=%d nr_shards=%d\n", filename, size, block_size, nr_shards); 71 | 72 | data = malloc(nr_shards*block_size); 73 | n = read(fd, data, size); 74 | if(n < size) { 75 | fprintf(stderr, "file read error!\n"); 76 | close(fd); 77 | exit(1); 78 | } 79 | close(fd); 80 | 81 | filename = basename(filename); 82 | 83 | memset(data+size, 0, nr_shards*block_size - size); 84 | //printf("data=%s\n", data); 85 | data_blocks = (unsigned char**)malloc(nr_shards * sizeof(unsigned char*)); 86 | for(i = 0; i < nr_shards; i++) { 87 | data_blocks[i] = data + i*block_size; 88 | } 89 | 90 | rs = reed_solomon_new(data_shards, parity_shards); 91 | reed_solomon_encode2(rs, data_blocks, nr_shards, block_size); 92 | 93 | for(i = 0; i < nr_shards; i++) { 94 | sprintf(output, "%s/%s.%d", out, filename, i); 95 | fd = open(output, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); 96 | write(fd, data_blocks[i], block_size); 97 | close(fd); 98 | } 99 | 100 | free(data_blocks); 101 | free(data); 102 | reed_solomon_release(rs); 103 | return 0; 104 | } 105 | 106 | -------------------------------------------------------------------------------- /examples/simple-encoder.go: -------------------------------------------------------------------------------- 1 | //+build ignore 2 | 3 | // Copyright 2015, Klaus Post, see LICENSE for details. 4 | // 5 | // Simple encoder example 6 | // 7 | // The encoder encodes a simgle file into a number of shards 8 | // To reverse the process see "simpledecoder.go" 9 | // 10 | // To build an executable use: 11 | // 12 | // go build simple-decoder.go 13 | // 14 | // Simple Encoder/Decoder Shortcomings: 15 | // * If the file size of the input isn't diviable by the number of data shards 16 | // the output will contain extra zeroes 17 | // 18 | // * If the shard numbers isn't the same for the decoder as in the 19 | // encoder, invalid output will be generated. 20 | // 21 | // * If values have changed in a shard, it cannot be reconstructed. 22 | // 23 | // * If two shards have been swapped, reconstruction will always fail. 24 | // You need to supply the shards in the same order as they were given to you. 25 | // 26 | // The solution for this is to save a metadata file containing: 27 | // 28 | // * File size. 29 | // * The number of data/parity shards. 30 | // * HASH of each shard. 31 | // * Order of the shards. 32 | // 33 | // If you save these properties, you should abe able to detect file corruption 34 | // in a shard and be able to reconstruct your data if you have the needed number of shards left. 35 | 36 | package main 37 | 38 | import ( 39 | "flag" 40 | "fmt" 41 | "io/ioutil" 42 | "os" 43 | "path/filepath" 44 | 45 | "github.com/klauspost/reedsolomon" 46 | ) 47 | 48 | var dataShards = flag.Int("data", 4, "Number of shards to split the data into, must be below 257.") 49 | var parShards = flag.Int("par", 2, "Number of parity shards") 50 | var outDir = flag.String("out", "", "Alternative output directory") 51 | 52 | func init() { 53 | flag.Usage = func() { 54 | fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) 55 | fmt.Fprintf(os.Stderr, " simple-encoder [-flags] filename.ext\n\n") 56 | fmt.Fprintf(os.Stderr, "Valid flags:\n") 57 | flag.PrintDefaults() 58 | } 59 | } 60 | 61 | func main() { 62 | // Parse command line parameters. 63 | flag.Parse() 64 | args := flag.Args() 65 | if len(args) != 1 { 66 | fmt.Fprintf(os.Stderr, "Error: No input filename given\n") 67 | flag.Usage() 68 | os.Exit(1) 69 | } 70 | if *dataShards > 257 { 71 | fmt.Fprintf(os.Stderr, "Error: Too many data shards\n") 72 | os.Exit(1) 73 | } 74 | fname := args[0] 75 | 76 | // Create encoding matrix. 77 | enc, err := reedsolomon.New(*dataShards, *parShards) 78 | checkErr(err) 79 | 80 | fmt.Println("Opening", fname) 81 | b, err := ioutil.ReadFile(fname) 82 | checkErr(err) 83 | 84 | // Split the file into equally sized shards. 85 | shards, err := enc.Split(b) 86 | checkErr(err) 87 | fmt.Printf("File split into %d data+parity shards with %d bytes/shard.\n", len(shards), len(shards[0])) 88 | 89 | // Encode parity 90 | err = enc.Encode(shards) 91 | checkErr(err) 92 | 93 | // Write out the resulting files. 94 | dir, file := filepath.Split(fname) 95 | if *outDir != "" { 96 | dir = *outDir 97 | } 98 | for i, shard := range shards { 99 | outfn := fmt.Sprintf("%s.%d", file, i) 100 | 101 | fmt.Println("Writing to", outfn) 102 | err = ioutil.WriteFile(filepath.Join(dir, outfn), shard, os.ModePerm) 103 | checkErr(err) 104 | } 105 | } 106 | 107 | func checkErr(err error) { 108 | if err != nil { 109 | fmt.Fprintf(os.Stderr, "Error: %s", err.Error()) 110 | os.Exit(2) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /examples/fec_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "../rs.h" 10 | 11 | #ifndef PROFILE 12 | #ifdef __x86_64__ 13 | static long long rdtsc(void) 14 | { 15 | unsigned long low, hi; 16 | asm volatile ("rdtsc" : "=d" (hi), "=a" (low)); 17 | return ( (((long long)hi) << 32) | ((long long) low)); 18 | } 19 | #elif __arm__ 20 | static long long rdtsc(void) 21 | { 22 | u64 val; 23 | asm volatile("mrs %0, cntvct_el0" : "=r" (val)); 24 | return val; 25 | } 26 | #endif 27 | #endif 28 | 29 | /* 30 | * ./fec_test dataShards parityShards blockSize input.txt 31 | * example: ./fec_test 10 3 256 input.txt > output.txt 32 | * */ 33 | int main(int argc, char **argv) { 34 | struct stat st; 35 | int size; 36 | struct timeval tv; 37 | int seed; 38 | int fd; 39 | int dataShards, parityShards, nrShards, blockSize, nrBlocks, nrFecBlocks; 40 | int i, n; 41 | int corrupted; 42 | reed_solomon* rs; 43 | unsigned long long begin, end; 44 | unsigned char *data; 45 | unsigned char **data_blocks; 46 | unsigned char *zilch; 47 | 48 | if(argc != 5) { 49 | fprintf(stderr, "example: ./fec_test 10 3 256 input.txt > output.txt\n"); 50 | exit(1); 51 | } 52 | 53 | gettimeofday(&tv, 0); 54 | seed = tv.tv_sec ^ tv.tv_usec; 55 | srandom(seed); 56 | 57 | begin = rdtsc(); 58 | fec_init(); 59 | end = rdtsc(); 60 | fprintf(stderr, "%d cycles to create FEC buffer\n", (unsigned int) (end-begin)); 61 | 62 | fd = open(argv[4], O_RDONLY); 63 | if(fd < 0) { 64 | fprintf(stderr, "input file not found\n"); 65 | exit(1); 66 | } 67 | 68 | dataShards = atoi(argv[1]); 69 | parityShards = atoi(argv[2]); 70 | blockSize = atoi(argv[3]); 71 | rs = reed_solomon_new(dataShards, parityShards); 72 | nrShards = rs->shards; 73 | 74 | fstat(fd, &st); 75 | size = st.st_size; 76 | nrBlocks = (size+blockSize-1) / blockSize; 77 | nrBlocks = ((nrBlocks+dataShards-1)/dataShards) * dataShards; 78 | i = nrBlocks / dataShards; 79 | nrFecBlocks = i*parityShards; 80 | nrShards = nrBlocks + nrFecBlocks; 81 | fprintf(stderr, "size=%d nr=%d\n", size, nrBlocks); 82 | 83 | data = (unsigned char*)malloc(nrShards * blockSize); 84 | n = read(fd, data, size); 85 | if(n < size) { 86 | fprintf(stderr, "Short read\n"); 87 | close(fd); 88 | exit(1); 89 | } 90 | close(fd); 91 | memset(data+size, 0, nrShards*blockSize - size); 92 | 93 | data_blocks = (unsigned char**)malloc( nrShards * sizeof(unsigned char**) ); 94 | for(i = 0; i < nrShards; i++) { 95 | data_blocks[i] = data + i*blockSize; 96 | } 97 | 98 | begin = rdtsc(); 99 | reed_solomon_encode2(rs, data_blocks, nrBlocks+nrFecBlocks, blockSize); 100 | end = rdtsc(); 101 | fprintf(stderr, "times %ld\n", (unsigned long) (end-begin)); 102 | 103 | zilch = (unsigned char*)calloc(1, nrShards); 104 | memset(zilch, 0, sizeof(zilch)); 105 | corrupted = parityShards; 106 | for(i = 0; i < corrupted; i++) { 107 | int corr = random() % nrBlocks; 108 | memset(data + blockSize * corr, 137, blockSize); 109 | fprintf(stderr, "Corrupting %d\n", corr); 110 | zilch[corr] = 1; 111 | } 112 | 113 | begin = rdtsc(); 114 | reed_solomon_reconstruct(rs, data_blocks, zilch, nrShards, blockSize); 115 | end = rdtsc(); 116 | fprintf(stderr, "times %ld\n", (unsigned long) (end-begin)); 117 | 118 | write(1, data, size); 119 | 120 | free(data); 121 | free(data_blocks); 122 | free(zilch); 123 | reed_solomon_release(rs); 124 | 125 | return 0; 126 | } 127 | 128 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # reedsolomon-c 2 | C compatible version for [golang-klauspost-reedsolomon](https://github.com/klauspost/reedsolomon). 3 | 4 | It can run in platform of MIPS/ARM/i386/x64 of linux, especially for some embedded projects. 5 | 6 | It's really simple, with a little change, it can run in other platform. 7 | 8 | # Installation 9 | Copy rs.c and rs.h to your projects and make it. 10 | 11 | # Usage 12 | 13 | This section assumes you know the basics of Reed-Solomon encoding. A good start is this [Backblaze blog post](https://www.backblaze.com/blog/reed-solomon/). 14 | 15 | To create an encoder/decoder with 10 data shards (where your data goes) and 3 parity shards (calculated): 16 | ```C 17 | reed_solomon* rs = reed_solomon_new(10, 3); 18 | ``` 19 | 20 | Destroy: 21 | ```C 22 | reed_solomon_relelase(rs); 23 | ``` 24 | 25 | Encode: 26 | 27 | Look at [test_create_encoding](https://github.com/jannson/reedsolomon-c/blob/master/test_rs.c#L163) 28 | 29 | ```C 30 | int test_create_encoding( 31 | reed_solomon *rs, 32 | unsigned char *data, 33 | int data_size, 34 | int block_size 35 | ) { 36 | unsigned char **data_blocks; 37 | int data_shards, parity_shards; 38 | int i, n, nr_shards, nr_blocks, nr_fec_blocks; 39 | 40 | data_shards = rs->data_shards; 41 | parity_shards = rs->parity_shards; 42 | nr_blocks = (data_size+block_size-1)/block_size; 43 | nr_blocks = ((nr_blocks+data_shards-1)/data_shards) * data_shards; 44 | n = nr_blocks / data_shards; 45 | nr_fec_blocks = n * parity_shards; 46 | nr_shards = nr_blocks + nr_fec_blocks; 47 | 48 | data_blocks = (unsigned char**)malloc(nr_shards * sizeof(unsigned char*)); 49 | for(i = 0; i < nr_shards; i++) { 50 | data_blocks[i] = data + i*block_size; 51 | } 52 | 53 | n = reed_solomon_encode2(rs, data_blocks, nr_shards, block_size); 54 | free(data_blocks); 55 | 56 | return n; 57 | } 58 | ``` 59 | 60 | Decode: 61 | 62 | Look at [test_data_decode](https://github.com/jannson/reedsolomon-c/blob/master/test_rs.c#L192) 63 | 64 | ```C 65 | int test_data_decode( 66 | reed_solomon *rs, 67 | unsigned char *data, 68 | int data_size, 69 | int block_size, 70 | int *erases, 71 | int erase_count) { 72 | unsigned char **data_blocks; 73 | unsigned char *zilch; 74 | int data_shards, parity_shards; 75 | int i, j, n, nr_shards, nr_blocks, nr_fec_blocks; 76 | 77 | data_shards = rs->data_shards; 78 | parity_shards = rs->parity_shards; 79 | nr_blocks = (data_size+block_size-1)/block_size; 80 | nr_blocks = ((nr_blocks+data_shards-1)/data_shards) * data_shards; 81 | n = nr_blocks / data_shards; 82 | nr_fec_blocks = n * parity_shards; 83 | nr_shards = nr_blocks + nr_fec_blocks; 84 | 85 | data_blocks = (unsigned char**)malloc(nr_shards * sizeof(unsigned char*)); 86 | for(i = 0; i < nr_shards; i++) { 87 | data_blocks[i] = data + i*block_size; 88 | } 89 | 90 | zilch = (unsigned char*)calloc(1, nr_shards); 91 | for(i = 0; i < erase_count; i++) { 92 | j = erases[i]; 93 | memset(data + j*block_size, 137, block_size); 94 | zilch[j] = 1; //mark as erased 95 | } 96 | 97 | n = reed_solomon_reconstruct(rs, data_blocks, zilch, nr_shards, block_size); 98 | free(data_blocks); 99 | free(zilch); 100 | 101 | return n; 102 | } 103 | ``` 104 | 105 | # Performance 106 | 107 | I have implemented a benchmarkEncode in test_rs.c. but is this implement right ? it's so quick, I think there maybe some mistake in benchmark test. 108 | 109 | # Compatible Test 110 | 111 | Implement a tool to test C and golang version. 112 | 113 | [simple-encoder.c](https://github.com/jannson/reedsolomon-c/raw/master/examples/simple-encoder.c) VS [simple-encoder.go](https://github.com/klauspost/reedsolomon/raw/master/examples/simple-encoder.go) 114 | 115 | ``` 116 | go build -o go-simple-encoder ../simple-encoder.go 117 | ./go-simple-encoder --data 12 --par 6 --out o input.txt 118 | ./simple-encoder -d 12 -p 6 -o o2 -f input.txt 119 | meld o o2 120 | ``` 121 | 122 | # License 123 | 124 | published under an MIT license. See LICENSE file for more information. 125 | 126 | -------------------------------------------------------------------------------- /test_rs.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define PROFILE 11 | #include "rs.h" 12 | #include "rs.c" 13 | 14 | void print_matrix1(gf* matrix, int nrows, int ncols); 15 | void print_matrix2(gf** matrix, int nrows, int ncols); 16 | 17 | void print_buf(gf* buf, char *fmt, size_t len) { 18 | size_t i = 0; 19 | while(i < len) { 20 | printf(fmt, buf[i]); 21 | i++; 22 | if((i % 16) == 0) { 23 | printf("\n"); 24 | } 25 | } 26 | printf("\n"); 27 | } 28 | 29 | void test_galois(void) { 30 | printf("%s:\n", __FUNCTION__); 31 | 32 | //copy from golang rs version 33 | assert(galMultiply(3, 4) == 12); 34 | assert(galMultiply(7, 7) == 21); 35 | assert(galMultiply(23, 45) == 41); 36 | 37 | { 38 | gf in[] = {0, 1, 2, 3, 4, 5, 6, 10, 50, 100, 150, 174, 201, 255, 99, 32, 67, 85}; 39 | gf out[sizeof(in)/sizeof(gf)] = {0}; 40 | gf expect[] = {0x0, 0x19, 0x32, 0x2b, 0x64, 0x7d, 0x56, 0xfa, 0xb8, 0x6d, 0xc7, 0x85, 0xc3, 0x1f, 0x22, 0x7, 0x25, 0xfe}; 41 | gf expect2[] = {0x0, 0xb1, 0x7f, 0xce, 0xfe, 0x4f, 0x81, 0x9e, 0x3, 0x6, 0xe8, 0x75, 0xbd, 0x40, 0x36, 0xa3, 0x95, 0xcb}; 42 | int rlt; 43 | addmul(out, in, 25, sizeof(int)/sizeof(gf)); 44 | rlt = memcmp(out, expect, sizeof(int)/sizeof(gf)); 45 | assert(0 == rlt); 46 | 47 | memset(out, 0, sizeof(in)/sizeof(gf)); 48 | addmul(out, in, 177, sizeof(in)/sizeof(gf)); 49 | rlt = memcmp(out, expect2, sizeof(int)/sizeof(gf)); 50 | assert(0 == rlt); 51 | } 52 | 53 | assert(galExp(2,2) == 4); 54 | assert(galExp(5,20) == 235); 55 | assert(galExp(13,7) == 43); 56 | } 57 | 58 | void test_sub_matrix(void) { 59 | int r, c, ptr, nrows = 10, ncols = 20; 60 | gf* m1 = (gf*)RS_MALLOC(nrows * ncols); 61 | gf *test1; 62 | 63 | printf("%s:\n", __FUNCTION__); 64 | 65 | ptr = 0; 66 | for(r = 0; r < nrows; r++) { 67 | for(c = 0; c < ncols; c++) { 68 | m1[ptr] = ptr; 69 | ptr++; 70 | } 71 | } 72 | test1 = sub_matrix(m1, 0, 0, 3, 4, nrows, ncols); 73 | for(r = 0; r < 3; r++) { 74 | for(c = 0; c < 4; c++) { 75 | assert(test1[r*4 + c] == (r*ncols + c)); 76 | } 77 | } 78 | free(test1); 79 | 80 | test1 = sub_matrix(m1, 3, 2, 7, 9, nrows, ncols); 81 | for(r = 0; r < (7-3); r++) { 82 | for(c = 0; c < (9-2); c++) { 83 | assert(test1[r*(9-2) + c] == ((r+3)*ncols + (c+2))); 84 | } 85 | } 86 | 87 | free(m1); 88 | } 89 | 90 | void test_multiply(void) { 91 | gf a[] = {1,2,3,4}; 92 | gf b[] = {5,6,7,8}; 93 | gf exp[] = {11,22,19,42}; 94 | gf *out; 95 | int rlt; 96 | 97 | printf("%s:\n", __FUNCTION__); 98 | 99 | out = multiply1(a, 2, 2, b, 2, 2); 100 | rlt = memcmp(out, exp, 4); 101 | assert(0 == rlt); 102 | } 103 | 104 | void test_inverse(void) { 105 | printf("%s:\n", __FUNCTION__); 106 | { 107 | gf a[] = {56, 23, 98, 3, 100, 200, 45, 201, 123}; 108 | gf ae[] = {175, 133, 33, 130, 13, 245, 112, 35, 126}; 109 | int rlt = invert_mat(a, 3); 110 | assert(0 == rlt); 111 | rlt = memcmp(a, ae, 3*3); 112 | assert(0 == rlt); 113 | } 114 | 115 | { 116 | gf a[] = { 1, 0, 0, 0, 0, 117 | 0, 1, 0, 0, 0, 118 | 0, 0, 0, 1, 0, 119 | 0, 0, 0, 0, 1, 120 | 7, 7, 6, 6, 1}; 121 | gf ae[] = {1, 0, 0, 0, 0, 122 | 0, 1, 0, 0, 0, 123 | 123, 123, 1, 122, 122, 124 | 0, 0, 1, 0, 0, 125 | 0, 0, 0, 1, 0}; 126 | int rlt = invert_mat(a, 5); 127 | assert(0 == rlt); 128 | rlt = memcmp(a, ae, 5*5); 129 | assert(0 == rlt); 130 | } 131 | 132 | { 133 | /* error matrix */ 134 | gf a[] = {4,2,12,6}; 135 | int rlt = invert_mat(a, 2); 136 | assert(0 != rlt); 137 | } 138 | } 139 | 140 | unsigned char* test_create_random(reed_solomon *rs, int data_size, int block_size) { 141 | struct timeval tv; 142 | unsigned char* data; 143 | int i, n, seed, nr_blocks; 144 | 145 | gettimeofday(&tv, 0); 146 | seed = tv.tv_sec ^ tv.tv_usec; 147 | srandom(seed); 148 | 149 | nr_blocks = (data_size+block_size-1)/block_size; 150 | nr_blocks = ((nr_blocks + rs->data_shards - 1)/ rs->data_shards) * rs->data_shards; 151 | n = nr_blocks / rs->data_shards; 152 | nr_blocks += n * rs->parity_shards; 153 | 154 | data = (unsigned char *) malloc(nr_blocks * block_size); 155 | for(i = 0; i < data_size; i++) { 156 | data[i] = (unsigned char)(random() % 255); 157 | } 158 | memset(data + data_size, 0, nr_blocks*block_size - data_size); 159 | 160 | return data; 161 | } 162 | 163 | int test_create_encoding( 164 | reed_solomon *rs, 165 | unsigned char *data, 166 | int data_size, 167 | int block_size 168 | ) { 169 | unsigned char **data_blocks; 170 | int data_shards, parity_shards; 171 | int i, n, nr_shards, nr_blocks, nr_fec_blocks; 172 | 173 | data_shards = rs->data_shards; 174 | parity_shards = rs->parity_shards; 175 | nr_blocks = (data_size+block_size-1)/block_size; 176 | nr_blocks = ((nr_blocks+data_shards-1)/data_shards) * data_shards; 177 | n = nr_blocks / data_shards; 178 | nr_fec_blocks = n * parity_shards; 179 | nr_shards = nr_blocks + nr_fec_blocks; 180 | 181 | data_blocks = (unsigned char**)malloc(nr_shards * sizeof(unsigned char*)); 182 | for(i = 0; i < nr_shards; i++) { 183 | data_blocks[i] = data + i*block_size; 184 | } 185 | 186 | n = reed_solomon_encode2(rs, data_blocks, nr_shards, block_size); 187 | free(data_blocks); 188 | 189 | return n; 190 | } 191 | 192 | int test_data_decode( 193 | reed_solomon *rs, 194 | unsigned char *data, 195 | int data_size, 196 | int block_size, 197 | int *erases, 198 | int erase_count) { 199 | unsigned char **data_blocks; 200 | unsigned char *zilch; 201 | int data_shards, parity_shards; 202 | int i, j, n, nr_shards, nr_blocks, nr_fec_blocks; 203 | 204 | data_shards = rs->data_shards; 205 | parity_shards = rs->parity_shards; 206 | nr_blocks = (data_size+block_size-1)/block_size; 207 | nr_blocks = ((nr_blocks+data_shards-1)/data_shards) * data_shards; 208 | n = nr_blocks / data_shards; 209 | nr_fec_blocks = n * parity_shards; 210 | nr_shards = nr_blocks + nr_fec_blocks; 211 | 212 | data_blocks = (unsigned char**)malloc(nr_shards * sizeof(unsigned char*)); 213 | for(i = 0; i < nr_shards; i++) { 214 | data_blocks[i] = data + i*block_size; 215 | } 216 | 217 | zilch = (unsigned char*)calloc(1, nr_shards); 218 | for(i = 0; i < erase_count; i++) { 219 | j = erases[i]; 220 | memset(data + j*block_size, 137, block_size); 221 | zilch[j] = 1; //mark as erased 222 | } 223 | 224 | n = reed_solomon_reconstruct(rs, data_blocks, zilch, nr_shards, block_size); 225 | free(data_blocks); 226 | free(zilch); 227 | 228 | return n; 229 | } 230 | 231 | void test_one_encoding(void) { 232 | reed_solomon *rs; 233 | unsigned char* data; 234 | int block_size = 50000; 235 | int data_size = 10*block_size; 236 | int err; 237 | 238 | printf("%s:\n", __FUNCTION__); 239 | 240 | rs = reed_solomon_new(10, 3); 241 | data = test_create_random(rs, data_size, block_size); 242 | err = test_create_encoding(rs, data, data_size, block_size); 243 | 244 | free(data); 245 | reed_solomon_release(rs); 246 | 247 | assert(0 == err); 248 | } 249 | 250 | int test_one_decoding_13(int *erases, int erase_count) { 251 | reed_solomon *rs; 252 | unsigned char *data, *origin; 253 | int block_size = 50000; 254 | int data_size = 10*block_size; 255 | int err, err2; 256 | 257 | rs = reed_solomon_new(10, 3); 258 | data = test_create_random(rs, data_size, block_size); 259 | err = test_create_encoding(rs, data, data_size, block_size); 260 | assert(0 == err); 261 | 262 | origin = (unsigned char*)malloc(data_size); 263 | memcpy(origin, data, data_size); 264 | 265 | err = test_data_decode(rs, data, data_size, block_size, erases, erase_count); 266 | if(0 == err) { 267 | err2 = memcmp(origin, data, data_size); 268 | assert(0 == err2); 269 | } else { 270 | //failed here 271 | err2 = memcmp(origin, data, data_size); 272 | assert(0 != err2); 273 | } 274 | 275 | free(data); 276 | free(origin); 277 | reed_solomon_release(rs); 278 | 279 | return err; 280 | } 281 | 282 | void test_one_decoding(void) { 283 | printf("%s:\n", __FUNCTION__); 284 | 285 | { 286 | int erases[] = {0}; 287 | int err; 288 | 289 | // lost nothing 290 | err = test_one_decoding_13(erases, 0); 291 | assert(0 == err); 292 | } 293 | 294 | { 295 | int erases[] = {0}; 296 | int erases_count = sizeof(erases)/sizeof(int); 297 | int err; 298 | 299 | // lost only one 300 | err = test_one_decoding_13(erases, erases_count); 301 | assert(0 == err); 302 | 303 | erases[0] = 5; 304 | err = test_one_decoding_13(erases, erases_count); 305 | assert(0 == err); 306 | 307 | erases[0] = 9; 308 | err = test_one_decoding_13(erases, erases_count); 309 | assert(0 == err); 310 | 311 | erases[0] = 11; 312 | err = test_one_decoding_13(erases, erases_count); 313 | assert(0 == err); 314 | } 315 | 316 | { 317 | int erases[] = {0, 1}; 318 | int erases_count = sizeof(erases)/sizeof(int); 319 | int err; 320 | 321 | // lost two 322 | err = test_one_decoding_13(erases, erases_count); 323 | assert(0 == err); 324 | 325 | erases[0] = 3; 326 | erases[1] = 7; 327 | err = test_one_decoding_13(erases, erases_count); 328 | assert(0 == err); 329 | 330 | erases[0] = 11; 331 | erases[1] = 9; 332 | err = test_one_decoding_13(erases, erases_count); 333 | assert(0 == err); 334 | 335 | erases[0] = 11; 336 | erases[1] = 12; 337 | err = test_one_decoding_13(erases, erases_count); 338 | assert(0 == err); 339 | } 340 | 341 | { 342 | int erases[] = {0, 1, 4}; 343 | int erases_count = sizeof(erases)/sizeof(int); 344 | int err; 345 | 346 | // lost three 347 | err = test_one_decoding_13(erases, erases_count); 348 | assert(0 == err); 349 | 350 | erases[0] = 3; 351 | erases[1] = 8; 352 | erases[2] = 7; 353 | err = test_one_decoding_13(erases, erases_count); 354 | assert(0 == err); 355 | 356 | erases[0] = 11; 357 | erases[1] = 9; 358 | erases[2] = 1; 359 | err = test_one_decoding_13(erases, erases_count); 360 | assert(0 == err); 361 | 362 | erases[0] = 11; 363 | erases[1] = 12; 364 | erases[2] = 9; 365 | err = test_one_decoding_13(erases, erases_count); 366 | assert(0 == err); 367 | 368 | erases[0] = 11; 369 | erases[1] = 12; 370 | erases[2] = 9; 371 | err = test_one_decoding_13(erases, erases_count); 372 | assert(0 == err); 373 | 374 | erases[0] = 11; 375 | erases[1] = 12; 376 | erases[2] = 10; 377 | err = test_one_decoding_13(erases, erases_count); 378 | assert(0 == err); 379 | } 380 | 381 | { 382 | int erases[] = {0, 1, 4, 8}; 383 | int erases_count = sizeof(erases)/sizeof(int); 384 | int err; 385 | 386 | // lost 4, failed! 387 | err = test_one_decoding_13(erases, erases_count); 388 | assert(0 != err); 389 | 390 | erases[0] = 11; 391 | erases[1] = 12; 392 | erases[2] = 10; 393 | erases[3] = 9; 394 | err = test_one_decoding_13(erases, erases_count); 395 | assert(0 != err); 396 | } 397 | } 398 | 399 | int test_one_decoding_13_6(int *erases, int erase_count) { 400 | reed_solomon *rs; 401 | unsigned char *data, *origin; 402 | int block_size = 50000; 403 | int data_size = 10*block_size*6; 404 | int err, err2; 405 | 406 | rs = reed_solomon_new(10, 3); 407 | data = test_create_random(rs, data_size, block_size); 408 | err = test_create_encoding(rs, data, data_size, block_size); 409 | assert(0 == err); 410 | 411 | origin = (unsigned char*)malloc(data_size); 412 | memcpy(origin, data, data_size); 413 | 414 | err = test_data_decode(rs, data, data_size, block_size, erases, erase_count); 415 | if(0 == err) { 416 | err2 = memcmp(origin, data, data_size); 417 | assert(0 == err2); 418 | } else { 419 | //failed here 420 | err2 = memcmp(origin, data, data_size); 421 | assert(0 != err2); 422 | } 423 | 424 | free(data); 425 | free(origin); 426 | reed_solomon_release(rs); 427 | 428 | return err; 429 | } 430 | 431 | void test_encoding(void) { 432 | reed_solomon *rs; 433 | unsigned char *data; 434 | int block_size = 50000; 435 | //multi shards encoding 436 | int data_size = 13*block_size*6; 437 | int err; 438 | 439 | printf("%s:\n", __FUNCTION__); 440 | 441 | rs = reed_solomon_new(10, 3); 442 | data = test_create_random(rs, data_size, block_size); 443 | err = test_create_encoding(rs, data, data_size, block_size); 444 | 445 | free(data); 446 | reed_solomon_release(rs); 447 | 448 | assert(0 == err); 449 | } 450 | 451 | void test_reconstruct(void) { 452 | #define FEC_START (10*6) 453 | printf("%s:\n", __FUNCTION__); 454 | 455 | { 456 | int erases[] = {0}; 457 | int err; 458 | 459 | // lost nothing 460 | err = test_one_decoding_13_6(erases, 0); 461 | assert(0 == err); 462 | } 463 | 464 | { 465 | int erases[] = {0, 1, 9, 10+2, 10+4, 10+9}; 466 | int erases_count = sizeof(erases)/sizeof(int); 467 | int err; 468 | 469 | // shard1 shard2 both lost three 470 | err = test_one_decoding_13_6(erases, erases_count); 471 | assert(0 == err); 472 | } 473 | 474 | { 475 | int erases[] = {0, 9, FEC_START + 1, 10+2, 10+9, FEC_START + 3 + 2}; 476 | int erases_count = sizeof(erases)/sizeof(int); 477 | int err; 478 | 479 | // shard1 shard2 both lost three, and both lost one in fec 480 | err = test_one_decoding_13_6(erases, erases_count); 481 | assert(0 == err); 482 | } 483 | 484 | { 485 | int erases[] = {11, 12, 10, 9}; 486 | int erases_count = sizeof(erases)/sizeof(int); 487 | int err; 488 | 489 | /* this is ok. not lost 4 but shard1 lost 1, shard2 lost 3, we can reconstruct it */ 490 | err = test_one_decoding_13_6(erases, erases_count); 491 | assert(0 == err); 492 | } 493 | 494 | { 495 | int erases[] = {0, 1, 4, 8}; 496 | int erases_count = sizeof(erases)/sizeof(int); 497 | int err; 498 | 499 | // shard1 lost 4, failed! 500 | err = test_one_decoding_13_6(erases, erases_count); 501 | assert(0 != err); 502 | } 503 | 504 | { 505 | int erases[] = {10, 11, 14, 18}; 506 | int erases_count = sizeof(erases)/sizeof(int); 507 | int err; 508 | 509 | // shard2 lost 4, failed! 510 | err = test_one_decoding_13_6(erases, erases_count); 511 | assert(0 != err); 512 | } 513 | 514 | { 515 | int erases[] = {0, 1, 4, 8, 10, 11, 14, 18}; 516 | int erases_count = sizeof(erases)/sizeof(int); 517 | int err; 518 | 519 | // shard1 and shard2 both lost 4, failed! 520 | err = test_one_decoding_13_6(erases, erases_count); 521 | assert(0 != err); 522 | } 523 | 524 | { 525 | int erases[] = {11, 12, 10, 9, FEC_START+3+0, FEC_START+3+1, FEC_START+3+2}; 526 | int erases_count = sizeof(erases)/sizeof(int); 527 | int err; 528 | 529 | err = test_one_decoding_13_6(erases, erases_count); 530 | assert(0 != err); 531 | } 532 | } 533 | 534 | double benchmarkEncodeTest(int n, int dataShards, int parityShards, int shardSize) { 535 | clock_t start, end; 536 | double millis; 537 | unsigned char* data; 538 | int i; 539 | int dataSize = shardSize*dataShards; 540 | reed_solomon* rs = reed_solomon_new(dataShards, parityShards); 541 | 542 | data = test_create_random(rs, dataSize, shardSize); 543 | 544 | start = clock(); 545 | for(i = 0; i < n; i++) { 546 | test_create_encoding(rs, data, dataSize, shardSize); 547 | } 548 | end = clock(); 549 | millis = (double)(end - start) * 1000.0 / CLOCKS_PER_SEC; 550 | return (millis); 551 | } 552 | 553 | /* TODO please check, is this benchmark ok? */ 554 | void benchmarkEncode(void) { 555 | double millis; 556 | double per_sec_in_bytes; 557 | double MB = 1024.0 * 1024.0; 558 | double millis_to_sec = 1000*1000; 559 | int n; 560 | int size; 561 | 562 | printf("%s:\n", __FUNCTION__); 563 | 564 | n = 20000; 565 | size = 10000; 566 | millis = benchmarkEncodeTest(n, 10, 2, size); 567 | 568 | per_sec_in_bytes = (10*2*size/MB) * millis_to_sec * n / millis; 569 | printf("10x2x10000, test_count=%d millis=%lf per_sec_in_bytes=%lfMB/s\n", n, millis, per_sec_in_bytes); 570 | 571 | n = 200; 572 | millis = benchmarkEncodeTest(n, 100, 20, size); 573 | per_sec_in_bytes = (100*20*size/MB) * millis_to_sec * n / millis; 574 | printf("100x20x10000, test_count=%d millis=%lf per_sec_in_bytes=%lfMB/s\n", n, millis, per_sec_in_bytes); 575 | 576 | n = 200; 577 | size = 1024*1024; 578 | millis = benchmarkEncodeTest(n, 17, 3, size); 579 | per_sec_in_bytes = (17*3*size/MB) * millis_to_sec * n / millis; 580 | printf("17x3x(1024*1024), test_count=%d millis=%lf per_sec_in_bytes=%lfMB/s\n", n, millis, per_sec_in_bytes); 581 | } 582 | 583 | void test_001(void) { 584 | reed_solomon* rs = reed_solomon_new(11, 6); 585 | print_matrix1(rs->m, rs->data_shards, rs->data_shards); 586 | print_matrix1(rs->parity, rs->parity_shards, rs->data_shards); 587 | reed_solomon_release(rs); 588 | } 589 | 590 | void test_002(void) { 591 | char text[] = "hello world", output[256]; 592 | int block_size = 1; 593 | int nrDataBlocks = sizeof(text)/sizeof(char) - 1; 594 | unsigned char* data_blocks[128]; 595 | unsigned char* fec_blocks[128]; 596 | int nrFecBlocks = 6; 597 | 598 | //decode 599 | unsigned int fec_block_nos[128], erased_blocks[128]; 600 | unsigned char* dec_fec_blocks[128]; 601 | int nr_fec_blocks; 602 | 603 | int i; 604 | reed_solomon* rs = reed_solomon_new(nrDataBlocks, nrFecBlocks); 605 | 606 | printf("%s:\n", __FUNCTION__); 607 | 608 | for(i = 0; i < nrDataBlocks; i++) { 609 | data_blocks[i] = (unsigned char*)&text[i]; 610 | } 611 | 612 | memset(output, 0, sizeof(output)); 613 | memcpy(output, text, nrDataBlocks); 614 | for(i = 0; i < nrFecBlocks; i++) { 615 | fec_blocks[i] = (unsigned char*)&output[i + nrDataBlocks]; 616 | } 617 | 618 | reed_solomon_encode(rs, data_blocks, fec_blocks, block_size); 619 | print_buf((gf*)output, "%d ", nrFecBlocks+nrDataBlocks); 620 | 621 | text[1] = 'x'; 622 | text[3] = 'y'; 623 | text[4] = 'z'; 624 | erased_blocks[0] = 4; 625 | erased_blocks[1] = 1; 626 | erased_blocks[2] = 3; 627 | 628 | fec_block_nos[0] = 1; 629 | fec_block_nos[1] = 3; 630 | fec_block_nos[2] = 5; 631 | dec_fec_blocks[0] = fec_blocks[1]; 632 | dec_fec_blocks[1] = fec_blocks[3]; 633 | dec_fec_blocks[2] = fec_blocks[5]; 634 | nr_fec_blocks = 3; 635 | 636 | printf("erased:%s\n", text); 637 | 638 | reed_solomon_decode(rs, data_blocks, block_size, dec_fec_blocks, 639 | fec_block_nos, erased_blocks, nr_fec_blocks); 640 | 641 | printf("fixed:%s\n", text); 642 | 643 | reed_solomon_release(rs); 644 | } 645 | 646 | void test_003(void) { 647 | char text[] = "hello world hello world ", output[256]; 648 | int block_size = 2; 649 | int nrDataBlocks = (sizeof(text)/sizeof(char) - 1) / block_size; 650 | unsigned char* data_blocks[128]; 651 | unsigned char* fec_blocks[128]; 652 | int nrFecBlocks = 6; 653 | 654 | //decode 655 | unsigned int fec_block_nos[128], erased_blocks[128]; 656 | unsigned char* dec_fec_blocks[128]; 657 | int nr_fec_blocks; 658 | 659 | int i; 660 | reed_solomon* rs = reed_solomon_new(nrDataBlocks, nrFecBlocks); 661 | 662 | printf("%s:\n", __FUNCTION__); 663 | //printf("text size=%d\n", (int)(sizeof(text)/sizeof(char) - 1) ); 664 | 665 | for(i = 0; i < nrDataBlocks; i++) { 666 | data_blocks[i] = (unsigned char*)&text[i*block_size]; 667 | } 668 | 669 | memset(output, 0, sizeof(output)); 670 | memcpy(output, text, nrDataBlocks*block_size); 671 | //print_matrix1((gf*)output, nrDataBlocks + nrFecBlocks, block_size); 672 | for(i = 0; i < nrFecBlocks; i++) { 673 | fec_blocks[i] = (unsigned char*)&output[i*block_size + nrDataBlocks*block_size]; 674 | } 675 | reed_solomon_encode(rs, data_blocks, fec_blocks, block_size); 676 | printf("golang output(example/test_rs.go):\n [[104 101] [108 108] [111 32] [119 111] [114 108] [100 32] [104 101] [108 108] [111 32] [119 111] [114 108] [100 32] \n[157 178] [83 31] [48 240] [254 93] [31 89] [151 184]]\n"); 677 | printf("c verion output:\n"); 678 | print_buf((gf*)output, "%d ", nrFecBlocks*block_size + nrDataBlocks*block_size); 679 | //print_matrix1((gf*)output, nrDataBlocks + nrFecBlocks, block_size); 680 | 681 | //decode 682 | text[1*block_size] = 'x'; 683 | text[10*block_size+1] = 'y'; 684 | text[4*block_size] = 'z'; 685 | erased_blocks[0] = 4; 686 | erased_blocks[1] = 1; 687 | erased_blocks[2] = 10; 688 | 689 | fec_block_nos[0] = 1; 690 | fec_block_nos[1] = 3; 691 | fec_block_nos[2] = 5; 692 | dec_fec_blocks[0] = fec_blocks[1]; 693 | dec_fec_blocks[1] = fec_blocks[3]; 694 | dec_fec_blocks[2] = fec_blocks[5]; 695 | nr_fec_blocks = 3; 696 | 697 | printf("erased:%s\n", text); 698 | 699 | reed_solomon_decode(rs, data_blocks, block_size, dec_fec_blocks, 700 | fec_block_nos, erased_blocks, nr_fec_blocks); 701 | 702 | printf("fixed:%s\n", text); 703 | 704 | reed_solomon_release(rs); 705 | } 706 | 707 | void test_004(void) { 708 | //char text[] = "hello world hello world "; 709 | int dataShards = 30; 710 | int parityShards = 21; 711 | int blockSize = 280; 712 | struct timeval tv; 713 | int i, j, n, seed, size, nrShards, nrBlocks, nrFecBlocks; 714 | unsigned char *origin, *data; 715 | unsigned char **data_blocks; 716 | unsigned char *zilch; 717 | reed_solomon *rs; 718 | 719 | gettimeofday(&tv, 0); 720 | seed = tv.tv_sec ^ tv.tv_usec; 721 | srandom(seed); 722 | 723 | fec_init(); 724 | 725 | //size = sizeof(text)/sizeof(char)-1; 726 | size = 1024*1024; 727 | origin = (unsigned char *) malloc(size); 728 | //memcpy(origin, text, size); 729 | for(i = 0; i < size; i++) { 730 | origin[i] = (unsigned char)(random() % 255); 731 | } 732 | 733 | nrBlocks = (size+blockSize-1) / blockSize; 734 | nrBlocks = ((nrBlocks+dataShards-1)/dataShards) * dataShards; 735 | n = nrBlocks / dataShards; 736 | nrFecBlocks = n*parityShards; 737 | nrShards = nrBlocks + nrFecBlocks; 738 | data = (unsigned char *) malloc(nrShards * blockSize); 739 | memcpy(data, origin, size); 740 | memset(data + size, 0, nrShards*blockSize - size); 741 | printf("nrBlocks=%d nrFecBlocks=%d nrShards=%d n=%d left=%d\n", nrBlocks, nrFecBlocks, nrShards, n, nrShards*blockSize - size); 742 | //print_buf(origin, "%d ", size); 743 | //print_buf(data, "%d ", nrShards*blockSize); 744 | 745 | data_blocks = (unsigned char**)malloc( nrShards * sizeof(unsigned char**) ); 746 | for(i = 0; i < nrShards; i++) { 747 | data_blocks[i] = data + i*blockSize; 748 | } 749 | 750 | rs = reed_solomon_new(dataShards, parityShards); 751 | reed_solomon_encode2(rs, data_blocks, nrShards, blockSize); 752 | i = memcmp(origin, data, size); 753 | assert(0 == i); 754 | //print_matrix2(data_blocks, nrShards, blockSize); 755 | 756 | zilch = (unsigned char*)calloc(1, nrShards); 757 | n = parityShards; 758 | 759 | /* int es[100]; 760 | es[0] = 3; 761 | es[1] = 3; 762 | es[2] = 2; 763 | es[3] = 8; */ 764 | 765 | for(i = 0; i < n-2; i++) { 766 | j = random() % (nrBlocks-1); 767 | //j = es[i]; 768 | memset(data + j*blockSize, 137, blockSize); 769 | zilch[j] = 1; //erased! 770 | printf("erased %d\n", j); 771 | } 772 | if(nrFecBlocks > 2) { 773 | for(i = 0; i < 2; i++) { 774 | j = nrBlocks + (random() % nrFecBlocks); 775 | memset(data + j*blockSize, 139, blockSize); 776 | zilch[j] = 1; 777 | printf("erased %d\n", j); 778 | } 779 | } 780 | 781 | reed_solomon_reconstruct(rs, data_blocks, zilch, nrShards, blockSize); 782 | i = memcmp(origin, data, size); 783 | //print_buf(origin, "%d ", nrBlocks); 784 | //print_buf(data, "%d ", nrBlocks); 785 | printf("rlt=%d\n", i); 786 | assert(0 == i); 787 | 788 | free(origin); 789 | free(data); 790 | free(data_blocks); 791 | free(zilch); 792 | reed_solomon_release(rs); 793 | } 794 | 795 | int main(void) { 796 | fec_init(); 797 | 798 | test_galois(); 799 | test_sub_matrix(); 800 | test_multiply(); 801 | test_inverse(); 802 | test_one_encoding(); 803 | test_one_decoding(); 804 | test_encoding(); 805 | test_reconstruct(); 806 | printf("reach here means test all ok\n"); 807 | 808 | benchmarkEncode(); 809 | 810 | //test_001(); 811 | //test_002(); 812 | test_003(); 813 | //test_004(); 814 | 815 | return 0; 816 | } 817 | -------------------------------------------------------------------------------- /rs.c: -------------------------------------------------------------------------------- 1 | /*#define PROFILE*/ 2 | /* 3 | * fec.c -- forward error correction based on Vandermonde matrices 4 | * 980624 5 | * (C) 1997-98 Luigi Rizzo (luigi@iet.unipi.it) 6 | * (C) 2001 Alain Knaff (alain@knaff.lu) 7 | * 8 | * Portions derived from code by Phil Karn (karn@ka9q.ampr.org), 9 | * Robert Morelos-Zaragoza (robert@spectra.eng.hawaii.edu) and Hari 10 | * Thirumoorthy (harit@spectra.eng.hawaii.edu), Aug 1995 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions 14 | * are met: 15 | * 16 | * 1. Redistributions of source code must retain the above copyright 17 | * notice, this list of conditions and the following disclaimer. 18 | * 2. Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 25 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 26 | * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS 27 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, 28 | * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 29 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, 30 | * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 31 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 32 | * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 33 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 34 | * OF SUCH DAMAGE. 35 | * 36 | * Reimplement by Jannson (20161018): compatible for golang version of https://github.com/klauspost/reedsolomon 37 | */ 38 | 39 | /* 40 | * The following parameter defines how many bits are used for 41 | * field elements. The code supports any value from 2 to 16 42 | * but fastest operation is achieved with 8 bit elements 43 | * This is the only parameter you may want to change. 44 | */ 45 | #define GF_BITS 8 /* code over GF(2**GF_BITS) - change to suit */ 46 | 47 | #include 48 | #include 49 | #include 50 | 51 | #include 52 | #include "rs.h" 53 | 54 | /* 55 | * stuff used for testing purposes only 56 | */ 57 | 58 | #ifdef TEST 59 | #define DEB(x) 60 | #define DDB(x) x 61 | #define DEBUG 0 /* minimal debugging */ 62 | 63 | #include 64 | #define DIFF_T(a,b) \ 65 | (1+ 1000000*(a.tv_sec - b.tv_sec) + (a.tv_usec - b.tv_usec) ) 66 | 67 | #define TICK(t) \ 68 | {struct timeval x ; \ 69 | gettimeofday(&x, NULL) ; \ 70 | t = x.tv_usec + 1000000* (x.tv_sec & 0xff ) ; \ 71 | } 72 | #define TOCK(t) \ 73 | { u_long t1 ; TICK(t1) ; \ 74 | if (t1 < t) t = 256000000 + t1 - t ; \ 75 | else t = t1 - t ; \ 76 | if (t == 0) t = 1 ;} 77 | 78 | u_long ticks[10]; /* vars for timekeeping */ 79 | #else 80 | #define DEB(x) 81 | #define DDB(x) 82 | #define TICK(x) 83 | #define TOCK(x) 84 | #endif /* TEST */ 85 | 86 | /* 87 | * You should not need to change anything beyond this point. 88 | * The first part of the file implements linear algebra in GF. 89 | * 90 | * gf is the type used to store an element of the Galois Field. 91 | * Must constain at least GF_BITS bits. 92 | * 93 | * Note: unsigned char will work up to GF(256) but int seems to run 94 | * faster on the Pentium. We use int whenever have to deal with an 95 | * index, since they are generally faster. 96 | */ 97 | /* 98 | * AK: Udpcast only uses GF_BITS=8. Remove other possibilities 99 | */ 100 | #if (GF_BITS != 8) 101 | #error "GF_BITS must be 8" 102 | #endif 103 | typedef unsigned char gf; 104 | 105 | #define GF_SIZE ((1 << GF_BITS) - 1) /* powers of \alpha */ 106 | 107 | /* 108 | * Primitive polynomials - see Lin & Costello, Appendix A, 109 | * and Lee & Messerschmitt, p. 453. 110 | */ 111 | static char *allPp[] = { /* GF_BITS polynomial */ 112 | NULL, /* 0 no code */ 113 | NULL, /* 1 no code */ 114 | "111", /* 2 1+x+x^2 */ 115 | "1101", /* 3 1+x+x^3 */ 116 | "11001", /* 4 1+x+x^4 */ 117 | "101001", /* 5 1+x^2+x^5 */ 118 | "1100001", /* 6 1+x+x^6 */ 119 | "10010001", /* 7 1 + x^3 + x^7 */ 120 | "101110001", /* 8 1+x^2+x^3+x^4+x^8 */ 121 | "1000100001", /* 9 1+x^4+x^9 */ 122 | "10010000001", /* 10 1+x^3+x^10 */ 123 | "101000000001", /* 11 1+x^2+x^11 */ 124 | "1100101000001", /* 12 1+x+x^4+x^6+x^12 */ 125 | "11011000000001", /* 13 1+x+x^3+x^4+x^13 */ 126 | "110000100010001", /* 14 1+x+x^6+x^10+x^14 */ 127 | "1100000000000001", /* 15 1+x+x^15 */ 128 | "11010000000010001" /* 16 1+x+x^3+x^12+x^16 */ 129 | }; 130 | 131 | 132 | /* 133 | * To speed up computations, we have tables for logarithm, exponent 134 | * and inverse of a number. If GF_BITS <= 8, we use a table for 135 | * multiplication as well (it takes 64K, no big deal even on a PDA, 136 | * especially because it can be pre-initialized an put into a ROM!), 137 | * otherwhise we use a table of logarithms. 138 | * In any case the macro gf_mul(x,y) takes care of multiplications. 139 | */ 140 | 141 | static gf gf_exp[2*GF_SIZE]; /* index->poly form conversion table */ 142 | static int gf_log[GF_SIZE + 1]; /* Poly->index form conversion table */ 143 | static gf inverse[GF_SIZE+1]; /* inverse of field elem. */ 144 | /* inv[\alpha**i]=\alpha**(GF_SIZE-i-1) */ 145 | 146 | /* 147 | * modnn(x) computes x % GF_SIZE, where GF_SIZE is 2**GF_BITS - 1, 148 | * without a slow divide. 149 | */ 150 | static inline gf 151 | modnn(int x) 152 | { 153 | while (x >= GF_SIZE) { 154 | x -= GF_SIZE; 155 | x = (x >> GF_BITS) + (x & GF_SIZE); 156 | } 157 | return x; 158 | } 159 | 160 | #define SWAP(a,b,t) {t tmp; tmp=a; a=b; b=tmp;} 161 | 162 | /* 163 | * gf_mul(x,y) multiplies two numbers. If GF_BITS<=8, it is much 164 | * faster to use a multiplication table. 165 | * 166 | * USE_GF_MULC, GF_MULC0(c) and GF_ADDMULC(x) can be used when multiplying 167 | * many numbers by the same constant. In this case the first 168 | * call sets the constant, and others perform the multiplications. 169 | * A value related to the multiplication is held in a local variable 170 | * declared with USE_GF_MULC . See usage in addmul1(). 171 | */ 172 | static gf gf_mul_table[(GF_SIZE + 1)*(GF_SIZE + 1)] 173 | #ifdef WINDOWS 174 | __attribute__((aligned (16))) 175 | #else 176 | __attribute__((aligned (256))) 177 | #endif 178 | ; 179 | 180 | #define gf_mul(x,y) gf_mul_table[(x<<8)+y] 181 | 182 | #define USE_GF_MULC register gf * __gf_mulc_ 183 | #define GF_MULC0(c) __gf_mulc_ = &gf_mul_table[(c)<<8] 184 | #define GF_ADDMULC(dst, x) dst ^= __gf_mulc_[x] 185 | #define GF_MULC(dst, x) dst = __gf_mulc_[x] 186 | 187 | static void 188 | init_mul_table(void) 189 | { 190 | int i, j; 191 | for (i=0; i< GF_SIZE+1; i++) 192 | for (j=0; j< GF_SIZE+1; j++) 193 | gf_mul_table[(i<<8)+j] = gf_exp[modnn(gf_log[i] + gf_log[j]) ] ; 194 | 195 | for (j=0; j< GF_SIZE+1; j++) 196 | gf_mul_table[j] = gf_mul_table[j<<8] = 0; 197 | } 198 | 199 | /* 200 | * Generate GF(2**m) from the irreducible polynomial p(X) in p[0]..p[m] 201 | * Lookup tables: 202 | * index->polynomial form gf_exp[] contains j= \alpha^i; 203 | * polynomial form -> index form gf_log[ j = \alpha^i ] = i 204 | * \alpha=x is the primitive element of GF(2^m) 205 | * 206 | * For efficiency, gf_exp[] has size 2*GF_SIZE, so that a simple 207 | * multiplication of two numbers can be resolved without calling modnn 208 | */ 209 | 210 | 211 | 212 | /* 213 | * initialize the data structures used for computations in GF. 214 | */ 215 | static void 216 | generate_gf(void) 217 | { 218 | int i; 219 | gf mask; 220 | char *Pp = allPp[GF_BITS] ; 221 | 222 | mask = 1; /* x ** 0 = 1 */ 223 | gf_exp[GF_BITS] = 0; /* will be updated at the end of the 1st loop */ 224 | /* 225 | * first, generate the (polynomial representation of) powers of \alpha, 226 | * which are stored in gf_exp[i] = \alpha ** i . 227 | * At the same time build gf_log[gf_exp[i]] = i . 228 | * The first GF_BITS powers are simply bits shifted to the left. 229 | */ 230 | for (i = 0; i < GF_BITS; i++, mask <<= 1 ) { 231 | gf_exp[i] = mask; 232 | gf_log[gf_exp[i]] = i; 233 | /* 234 | * If Pp[i] == 1 then \alpha ** i occurs in poly-repr 235 | * gf_exp[GF_BITS] = \alpha ** GF_BITS 236 | */ 237 | if ( Pp[i] == '1' ) 238 | gf_exp[GF_BITS] ^= mask; 239 | } 240 | /* 241 | * now gf_exp[GF_BITS] = \alpha ** GF_BITS is complete, so can als 242 | * compute its inverse. 243 | */ 244 | gf_log[gf_exp[GF_BITS]] = GF_BITS; 245 | /* 246 | * Poly-repr of \alpha ** (i+1) is given by poly-repr of 247 | * \alpha ** i shifted left one-bit and accounting for any 248 | * \alpha ** GF_BITS term that may occur when poly-repr of 249 | * \alpha ** i is shifted. 250 | */ 251 | mask = 1 << (GF_BITS - 1 ) ; 252 | for (i = GF_BITS + 1; i < GF_SIZE; i++) { 253 | if (gf_exp[i - 1] >= mask) 254 | gf_exp[i] = gf_exp[GF_BITS] ^ ((gf_exp[i - 1] ^ mask) << 1); 255 | else 256 | gf_exp[i] = gf_exp[i - 1] << 1; 257 | gf_log[gf_exp[i]] = i; 258 | } 259 | /* 260 | * log(0) is not defined, so use a special value 261 | */ 262 | gf_log[0] = GF_SIZE ; 263 | /* set the extended gf_exp values for fast multiply */ 264 | for (i = 0 ; i < GF_SIZE ; i++) 265 | gf_exp[i + GF_SIZE] = gf_exp[i] ; 266 | 267 | /* 268 | * again special cases. 0 has no inverse. This used to 269 | * be initialized to GF_SIZE, but it should make no difference 270 | * since noone is supposed to read from here. 271 | */ 272 | inverse[0] = 0 ; 273 | inverse[1] = 1; 274 | for (i=2; i<=GF_SIZE; i++) 275 | inverse[i] = gf_exp[GF_SIZE-gf_log[i]]; 276 | } 277 | 278 | /* 279 | * Various linear algebra operations that i use often. 280 | */ 281 | 282 | /* 283 | * addmul() computes dst[] = dst[] + c * src[] 284 | * This is used often, so better optimize it! Currently the loop is 285 | * unrolled 16 times, a good value for 486 and pentium-class machines. 286 | * The case c=0 is also optimized, whereas c=1 is not. These 287 | * calls are unfrequent in my typical apps so I did not bother. 288 | * 289 | * Note that gcc on 290 | */ 291 | #if 0 292 | #define addmul(dst, src, c, sz) \ 293 | if (c != 0) addmul1(dst, src, c, sz) 294 | #endif 295 | 296 | 297 | 298 | #define UNROLL 16 /* 1, 4, 8, 16 */ 299 | static void 300 | slow_addmul1(gf *dst1, gf *src1, gf c, int sz) 301 | { 302 | USE_GF_MULC ; 303 | register gf *dst = dst1, *src = src1 ; 304 | gf *lim = &dst[sz - UNROLL + 1] ; 305 | 306 | GF_MULC0(c) ; 307 | 308 | #if (UNROLL > 1) /* unrolling by 8/16 is quite effective on the pentium */ 309 | for (; dst < lim ; dst += UNROLL, src += UNROLL ) { 310 | GF_ADDMULC( dst[0] , src[0] ); 311 | GF_ADDMULC( dst[1] , src[1] ); 312 | GF_ADDMULC( dst[2] , src[2] ); 313 | GF_ADDMULC( dst[3] , src[3] ); 314 | #if (UNROLL > 4) 315 | GF_ADDMULC( dst[4] , src[4] ); 316 | GF_ADDMULC( dst[5] , src[5] ); 317 | GF_ADDMULC( dst[6] , src[6] ); 318 | GF_ADDMULC( dst[7] , src[7] ); 319 | #endif 320 | #if (UNROLL > 8) 321 | GF_ADDMULC( dst[8] , src[8] ); 322 | GF_ADDMULC( dst[9] , src[9] ); 323 | GF_ADDMULC( dst[10] , src[10] ); 324 | GF_ADDMULC( dst[11] , src[11] ); 325 | GF_ADDMULC( dst[12] , src[12] ); 326 | GF_ADDMULC( dst[13] , src[13] ); 327 | GF_ADDMULC( dst[14] , src[14] ); 328 | GF_ADDMULC( dst[15] , src[15] ); 329 | #endif 330 | } 331 | #endif 332 | lim += UNROLL - 1 ; 333 | for (; dst < lim; dst++, src++ ) /* final components */ 334 | GF_ADDMULC( *dst , *src ); 335 | } 336 | 337 | # define addmul1 slow_addmul1 338 | 339 | static void addmul(gf *dst, gf *src, gf c, int sz) { 340 | // fprintf(stderr, "Dst=%p Src=%p, gf=%02x sz=%d\n", dst, src, c, sz); 341 | if (c != 0) addmul1(dst, src, c, sz); 342 | } 343 | 344 | /* 345 | * mul() computes dst[] = c * src[] 346 | * This is used often, so better optimize it! Currently the loop is 347 | * unrolled 16 times, a good value for 486 and pentium-class machines. 348 | * The case c=0 is also optimized, whereas c=1 is not. These 349 | * calls are unfrequent in my typical apps so I did not bother. 350 | * 351 | * Note that gcc on 352 | */ 353 | #if 0 354 | #define mul(dst, src, c, sz) \ 355 | do { if (c != 0) mul1(dst, src, c, sz); else memset(dst, 0, c); } while(0) 356 | #endif 357 | 358 | #define UNROLL 16 /* 1, 4, 8, 16 */ 359 | static void 360 | slow_mul1(gf *dst1, gf *src1, gf c, int sz) 361 | { 362 | USE_GF_MULC ; 363 | register gf *dst = dst1, *src = src1 ; 364 | gf *lim = &dst[sz - UNROLL + 1] ; 365 | 366 | GF_MULC0(c) ; 367 | 368 | #if (UNROLL > 1) /* unrolling by 8/16 is quite effective on the pentium */ 369 | for (; dst < lim ; dst += UNROLL, src += UNROLL ) { 370 | GF_MULC( dst[0] , src[0] ); 371 | GF_MULC( dst[1] , src[1] ); 372 | GF_MULC( dst[2] , src[2] ); 373 | GF_MULC( dst[3] , src[3] ); 374 | #if (UNROLL > 4) 375 | GF_MULC( dst[4] , src[4] ); 376 | GF_MULC( dst[5] , src[5] ); 377 | GF_MULC( dst[6] , src[6] ); 378 | GF_MULC( dst[7] , src[7] ); 379 | #endif 380 | #if (UNROLL > 8) 381 | GF_MULC( dst[8] , src[8] ); 382 | GF_MULC( dst[9] , src[9] ); 383 | GF_MULC( dst[10] , src[10] ); 384 | GF_MULC( dst[11] , src[11] ); 385 | GF_MULC( dst[12] , src[12] ); 386 | GF_MULC( dst[13] , src[13] ); 387 | GF_MULC( dst[14] , src[14] ); 388 | GF_MULC( dst[15] , src[15] ); 389 | #endif 390 | } 391 | #endif 392 | lim += UNROLL - 1 ; 393 | for (; dst < lim; dst++, src++ ) /* final components */ 394 | GF_MULC( *dst , *src ); 395 | } 396 | 397 | # define mul1 slow_mul1 398 | 399 | static inline void mul(gf *dst, gf *src, gf c, int sz) { 400 | /*fprintf(stderr, "%p = %02x * %p\n", dst, c, src);*/ 401 | if (c != 0) mul1(dst, src, c, sz); else memset(dst, 0, c); 402 | } 403 | 404 | /* 405 | * invert_mat() takes a matrix and produces its inverse 406 | * k is the size of the matrix. 407 | * (Gauss-Jordan, adapted from Numerical Recipes in C) 408 | * Return non-zero if singular. 409 | */ 410 | DEB( int pivloops=0; int pivswaps=0 ; /* diagnostic */) 411 | static int 412 | invert_mat(gf *src, int k) 413 | { 414 | gf c, *p ; 415 | int irow, icol, row, col, i, ix ; 416 | 417 | int error = 1 ; 418 | int indxc[k]; 419 | int indxr[k]; 420 | int ipiv[k]; 421 | gf id_row[k]; 422 | 423 | memset(id_row, 0, k*sizeof(gf)); 424 | DEB( pivloops=0; pivswaps=0 ; /* diagnostic */ ) 425 | /* 426 | * ipiv marks elements already used as pivots. 427 | */ 428 | for (i = 0; i < k ; i++) 429 | ipiv[i] = 0 ; 430 | 431 | for (col = 0; col < k ; col++) { 432 | gf *pivot_row ; 433 | /* 434 | * Zeroing column 'col', look for a non-zero element. 435 | * First try on the diagonal, if it fails, look elsewhere. 436 | */ 437 | irow = icol = -1 ; 438 | if (ipiv[col] != 1 && src[col*k + col] != 0) { 439 | irow = col ; 440 | icol = col ; 441 | goto found_piv ; 442 | } 443 | for (row = 0 ; row < k ; row++) { 444 | if (ipiv[row] != 1) { 445 | for (ix = 0 ; ix < k ; ix++) { 446 | DEB( pivloops++ ; ) 447 | if (ipiv[ix] == 0) { 448 | if (src[row*k + ix] != 0) { 449 | irow = row ; 450 | icol = ix ; 451 | goto found_piv ; 452 | } 453 | } else if (ipiv[ix] > 1) { 454 | fprintf(stderr, "singular matrix\n"); 455 | goto fail ; 456 | } 457 | } 458 | } 459 | } 460 | if (icol == -1) { 461 | fprintf(stderr, "XXX pivot not found!\n"); 462 | goto fail ; 463 | } 464 | found_piv: 465 | ++(ipiv[icol]) ; 466 | /* 467 | * swap rows irow and icol, so afterwards the diagonal 468 | * element will be correct. Rarely done, not worth 469 | * optimizing. 470 | */ 471 | if (irow != icol) { 472 | for (ix = 0 ; ix < k ; ix++ ) { 473 | SWAP( src[irow*k + ix], src[icol*k + ix], gf) ; 474 | } 475 | } 476 | indxr[col] = irow ; 477 | indxc[col] = icol ; 478 | pivot_row = &src[icol*k] ; 479 | c = pivot_row[icol] ; 480 | if (c == 0) { 481 | fprintf(stderr, "singular matrix 2\n"); 482 | goto fail ; 483 | } 484 | if (c != 1 ) { /* otherwhise this is a NOP */ 485 | /* 486 | * this is done often , but optimizing is not so 487 | * fruitful, at least in the obvious ways (unrolling) 488 | */ 489 | DEB( pivswaps++ ; ) 490 | c = inverse[ c ] ; 491 | pivot_row[icol] = 1 ; 492 | for (ix = 0 ; ix < k ; ix++ ) 493 | pivot_row[ix] = gf_mul(c, pivot_row[ix] ); 494 | } 495 | /* 496 | * from all rows, remove multiples of the selected row 497 | * to zero the relevant entry (in fact, the entry is not zero 498 | * because we know it must be zero). 499 | * (Here, if we know that the pivot_row is the identity, 500 | * we can optimize the addmul). 501 | */ 502 | id_row[icol] = 1; 503 | if (memcmp(pivot_row, id_row, k*sizeof(gf)) != 0) { 504 | for (p = src, ix = 0 ; ix < k ; ix++, p += k ) { 505 | if (ix != icol) { 506 | c = p[icol] ; 507 | p[icol] = 0 ; 508 | addmul(p, pivot_row, c, k ); 509 | } 510 | } 511 | } 512 | id_row[icol] = 0; 513 | } /* done all columns */ 514 | for (col = k-1 ; col >= 0 ; col-- ) { 515 | if (indxr[col] <0 || indxr[col] >= k) 516 | fprintf(stderr, "AARGH, indxr[col] %d\n", indxr[col]); 517 | else if (indxc[col] <0 || indxc[col] >= k) 518 | fprintf(stderr, "AARGH, indxc[col] %d\n", indxc[col]); 519 | else 520 | if (indxr[col] != indxc[col] ) { 521 | for (row = 0 ; row < k ; row++ ) { 522 | SWAP( src[row*k + indxr[col]], src[row*k + indxc[col]], gf) ; 523 | } 524 | } 525 | } 526 | error = 0 ; 527 | fail: 528 | return error ; 529 | } 530 | 531 | static int fec_initialized = 0 ; 532 | 533 | void fec_init(void) 534 | { 535 | TICK(ticks[0]); 536 | generate_gf(); 537 | TOCK(ticks[0]); 538 | DDB(fprintf(stderr, "generate_gf took %ldus\n", ticks[0]);) 539 | TICK(ticks[0]); 540 | init_mul_table(); 541 | TOCK(ticks[0]); 542 | DDB(fprintf(stderr, "init_mul_table took %ldus\n", ticks[0]);) 543 | fec_initialized = 1 ; 544 | } 545 | 546 | 547 | #ifdef PROFILE 548 | #ifdef __x86_64__ 549 | static long long rdtsc(void) 550 | { 551 | unsigned long low, hi; 552 | asm volatile ("rdtsc" : "=d" (hi), "=a" (low)); 553 | return ( (((long long)hi) << 32) | ((long long) low)); 554 | } 555 | #elif __arm__ 556 | static long long rdtsc(void) 557 | { 558 | u64 val; 559 | asm volatile("mrs %0, cntvct_el0" : "=r" (val)); 560 | return val; 561 | } 562 | #endif 563 | 564 | void print_matrix1(gf* matrix, int nrows, int ncols) { 565 | int i, j; 566 | printf("matrix (%d,%d):\n", nrows, ncols); 567 | for(i = 0; i < nrows; i++) { 568 | for(j = 0; j < ncols; j++) { 569 | printf("%6d ", matrix[i*ncols + j]); 570 | } 571 | printf("\n"); 572 | } 573 | } 574 | 575 | void print_matrix2(gf** matrix, int nrows, int ncols) { 576 | int i, j; 577 | printf("matrix (%d,%d):\n", nrows, ncols); 578 | for(i = 0; i < nrows; i++) { 579 | for(j = 0; j < ncols; j++) { 580 | printf("%6d ", matrix[i][j]); 581 | } 582 | printf("\n"); 583 | } 584 | } 585 | 586 | #endif 587 | 588 | /* y = a**n */ 589 | static gf galExp(gf a, gf n) { 590 | int logA; 591 | int logResult; 592 | if(0 == n) { 593 | return 1; 594 | } 595 | if(0 == a) { 596 | return 0; 597 | } 598 | logA = gf_log[a]; 599 | logResult = logA * n; 600 | while(logResult >= 255) { 601 | logResult -= 255; 602 | } 603 | 604 | return gf_exp[logResult]; 605 | } 606 | 607 | static inline gf galMultiply(gf a, gf b) { 608 | return gf_mul_table[ ((int)a << 8) + (int)b ]; 609 | } 610 | 611 | static gf* vandermonde(int nrows, int ncols) { 612 | int row, col, ptr; 613 | gf* matrix = (gf*)RS_MALLOC(nrows * ncols); 614 | if(NULL != matrix) { 615 | ptr = 0; 616 | for(row = 0; row < nrows; row++) { 617 | for(col = 0; col < ncols; col++) { 618 | matrix[ptr++] = galExp((gf)row, (gf)col); 619 | } 620 | } 621 | } 622 | 623 | return matrix; 624 | } 625 | 626 | /* 627 | * Not check for input params 628 | * */ 629 | static gf* sub_matrix(gf* matrix, int rmin, int cmin, int rmax, int cmax, int nrows, int ncols) { 630 | int i, j, ptr = 0; 631 | gf* new_m = (gf*)RS_MALLOC( (rmax-rmin) * (cmax-cmin) ); 632 | if(NULL != new_m) { 633 | for(i = rmin; i < rmax; i++) { 634 | for(j = cmin; j < cmax; j++) { 635 | new_m[ptr++] = matrix[i*ncols + j]; 636 | } 637 | } 638 | } 639 | 640 | return new_m; 641 | } 642 | 643 | /* y = a.dot(b) */ 644 | static gf* multiply1(gf *a, int ar, int ac, gf *b, int br, int bc) { 645 | gf *new_m, tg; 646 | int r, c, i, ptr = 0; 647 | 648 | assert(ac == br); 649 | new_m = (gf*)RS_CALLOC(1, ar*bc); 650 | if(NULL != new_m) { 651 | 652 | /* this multiply is slow */ 653 | for(r = 0; r < ar; r++) { 654 | for(c = 0; c < bc; c++) { 655 | tg = 0; 656 | for(i = 0; i < ac; i++) { 657 | /* tg ^= gf_mul_table[ ((int)a[r*ac+i] << 8) + (int)b[i*bc+c] ]; */ 658 | tg ^= galMultiply(a[r*ac+i], b[i*bc+c]); 659 | } 660 | 661 | new_m[ptr++] = tg; 662 | } 663 | } 664 | 665 | } 666 | 667 | return new_m; 668 | } 669 | 670 | /* copy from golang rs version */ 671 | static inline int code_some_shards(gf* matrixRows, gf** inputs, gf** outputs, 672 | int dataShards, int outputCount, int byteCount) { 673 | gf* in; 674 | int iRow, c; 675 | for(c = 0; c < dataShards; c++) { 676 | in = inputs[c]; 677 | for(iRow = 0; iRow < outputCount; iRow++) { 678 | if(0 == c) { 679 | mul(outputs[iRow], in, matrixRows[iRow*dataShards+c], byteCount); 680 | } else { 681 | addmul(outputs[iRow], in, matrixRows[iRow*dataShards+c], byteCount); 682 | } 683 | } 684 | } 685 | 686 | return 0; 687 | } 688 | 689 | reed_solomon* reed_solomon_new(int data_shards, int parity_shards) { 690 | gf* vm = NULL; 691 | gf* top = NULL; 692 | int err = 0; 693 | reed_solomon* rs = NULL; 694 | 695 | /* MUST use fec_init once time first */ 696 | assert(fec_initialized); 697 | 698 | do { 699 | rs = (reed_solomon*) RS_MALLOC(sizeof(reed_solomon)); 700 | if(NULL == rs) { 701 | return NULL; 702 | } 703 | rs->data_shards = data_shards; 704 | rs->parity_shards = parity_shards; 705 | rs->shards = (data_shards + parity_shards); 706 | rs->m = NULL; 707 | rs->parity = NULL; 708 | 709 | if(rs->shards > DATA_SHARDS_MAX || data_shards <= 0 || parity_shards <= 0) { 710 | err = 1; 711 | break; 712 | } 713 | 714 | vm = vandermonde(rs->shards, rs->data_shards); 715 | if(NULL == vm) { 716 | err = 2; 717 | break; 718 | } 719 | 720 | top = sub_matrix(vm, 0, 0, data_shards, data_shards, rs->shards, data_shards); 721 | if(NULL == top) { 722 | err = 3; 723 | break; 724 | } 725 | 726 | err = invert_mat(top, data_shards); 727 | assert(0 == err); 728 | 729 | rs->m = multiply1(vm, rs->shards, data_shards, top, data_shards, data_shards); 730 | if(NULL == rs->m) { 731 | err = 4; 732 | break; 733 | } 734 | 735 | rs->parity = sub_matrix(rs->m, data_shards, 0, rs->shards, data_shards, rs->shards, data_shards); 736 | if(NULL == rs->parity) { 737 | err = 5; 738 | break; 739 | } 740 | 741 | RS_FREE(vm); 742 | RS_FREE(top); 743 | vm = NULL; 744 | top = NULL; 745 | return rs; 746 | 747 | } while(0); 748 | 749 | fprintf(stderr, "err=%d\n", err); 750 | if(NULL != vm) { 751 | RS_FREE(vm); 752 | } 753 | if(NULL != top) { 754 | RS_FREE(top); 755 | } 756 | if(NULL != rs) { 757 | if(NULL != rs->m) { 758 | RS_FREE(rs->m); 759 | } 760 | if(NULL != rs->parity) { 761 | RS_FREE(rs->parity); 762 | } 763 | RS_FREE(rs); 764 | } 765 | 766 | return NULL; 767 | } 768 | 769 | void reed_solomon_release(reed_solomon* rs) { 770 | if(NULL != rs) { 771 | if(NULL != rs->m) { 772 | RS_FREE(rs->m); 773 | } 774 | if(NULL != rs->parity) { 775 | RS_FREE(rs->parity); 776 | } 777 | RS_FREE(rs); 778 | } 779 | } 780 | 781 | /** 782 | * encode one shard 783 | * input: 784 | * rs 785 | * data_blocks[rs->data_shards][block_size] 786 | * fec_blocks[rs->data_shards][block_size] 787 | * */ 788 | int reed_solomon_encode(reed_solomon* rs, 789 | unsigned char** data_blocks, 790 | unsigned char** fec_blocks, 791 | int block_size) { 792 | assert(NULL != rs && NULL != rs->parity); 793 | 794 | return code_some_shards(rs->parity, data_blocks, fec_blocks 795 | , rs->data_shards, rs->parity_shards, block_size); 796 | } 797 | 798 | /** 799 | * decode one shard 800 | * input: 801 | * rs 802 | * original data_blocks[rs->data_shards][block_size] 803 | * dec_fec_blocks[nr_fec_blocks][block_size] 804 | * fec_block_nos: fec pos number in original fec_blocks 805 | * erased_blocks: erased blocks in original data_blocks 806 | * nr_fec_blocks: the number of erased blocks 807 | * */ 808 | int reed_solomon_decode(reed_solomon* rs, 809 | unsigned char **data_blocks, 810 | int block_size, 811 | unsigned char **dec_fec_blocks, 812 | unsigned int *fec_block_nos, 813 | unsigned int *erased_blocks, 814 | int nr_fec_blocks) { 815 | /* use stack instead of malloc, define a small number of DATA_SHARDS_MAX to save memory */ 816 | gf dataDecodeMatrix[DATA_SHARDS_MAX*DATA_SHARDS_MAX]; 817 | unsigned char* subShards[DATA_SHARDS_MAX]; 818 | unsigned char* outputs[DATA_SHARDS_MAX]; 819 | gf* m = rs->m; 820 | int i, j, c, swap, subMatrixRow, dataShards, nos, nshards; 821 | 822 | /* the erased_blocks should always sorted 823 | * if sorted, nr_fec_blocks times to check it 824 | * if not, sort it here 825 | * */ 826 | for(i = 0; i < nr_fec_blocks; i++) { 827 | swap = 0; 828 | for(j = i+1; j < nr_fec_blocks; j++) { 829 | if(erased_blocks[i] > erased_blocks[j]) { 830 | /* the prefix is bigger than the following, swap */ 831 | c = erased_blocks[i]; 832 | erased_blocks[i] = erased_blocks[j]; 833 | erased_blocks[j] = c; 834 | 835 | swap = 1; 836 | } 837 | } 838 | //printf("swap:%d\n", swap); 839 | if(!swap) { 840 | //already sorted or sorted ok 841 | break; 842 | } 843 | } 844 | 845 | j = 0; 846 | subMatrixRow = 0; 847 | nos = 0; 848 | nshards = 0; 849 | dataShards = rs->data_shards; 850 | for(i = 0; i < dataShards; i++) { 851 | if(j < nr_fec_blocks && i == erased_blocks[j]) { 852 | //ignore the invalid block 853 | j++; 854 | } else { 855 | /* this row is ok */ 856 | for(c = 0; c < dataShards; c++) { 857 | dataDecodeMatrix[subMatrixRow*dataShards + c] = m[i*dataShards + c]; 858 | } 859 | subShards[subMatrixRow] = data_blocks[i]; 860 | subMatrixRow++; 861 | } 862 | } 863 | 864 | for(i = 0; i < nr_fec_blocks && subMatrixRow < dataShards; i++) { 865 | subShards[subMatrixRow] = dec_fec_blocks[i]; 866 | j = dataShards + fec_block_nos[i]; 867 | for(c = 0; c < dataShards; c++) { 868 | dataDecodeMatrix[subMatrixRow*dataShards + c] = m[j*dataShards + c]; //use spefic pos of original fec_blocks 869 | } 870 | subMatrixRow++; 871 | } 872 | 873 | if(subMatrixRow < dataShards) { 874 | //cannot correct 875 | return -1; 876 | } 877 | 878 | invert_mat(dataDecodeMatrix, dataShards); 879 | //printf("invert:\n"); 880 | //print_matrix1(dataDecodeMatrix, dataShards, dataShards); 881 | //printf("nShards:\n"); 882 | //print_matrix2(subShards, dataShards, block_size); 883 | 884 | for(i = 0; i < nr_fec_blocks; i++) { 885 | j = erased_blocks[i]; 886 | outputs[i] = data_blocks[j]; 887 | //data_blocks[j][0] = 0; 888 | memmove(dataDecodeMatrix+i*dataShards, dataDecodeMatrix+j*dataShards, dataShards); 889 | } 890 | //printf("subMatrixRow:\n"); 891 | //print_matrix1(dataDecodeMatrix, nr_fec_blocks, dataShards); 892 | 893 | //printf("outputs:\n"); 894 | //print_matrix2(outputs, nr_fec_blocks, block_size); 895 | 896 | return code_some_shards(dataDecodeMatrix, subShards, outputs, 897 | dataShards, nr_fec_blocks, block_size); 898 | } 899 | 900 | /** 901 | * encode a big size of buffer 902 | * input: 903 | * rs 904 | * nr_shards: assert(0 == nr_shards % rs->shards) 905 | * shards[nr_shards][block_size] 906 | * */ 907 | int reed_solomon_encode2(reed_solomon* rs, unsigned char** shards, int nr_shards, int block_size) { 908 | unsigned char** data_blocks; 909 | unsigned char** fec_blocks; 910 | int i, ds = rs->data_shards, ps = rs->parity_shards, ss = rs->shards; 911 | i = nr_shards / ss; 912 | data_blocks = shards; 913 | fec_blocks = &shards[(i*ds)]; 914 | 915 | for(i = 0; i < nr_shards; i += ss) { 916 | reed_solomon_encode(rs, data_blocks, fec_blocks, block_size); 917 | data_blocks += ds; 918 | fec_blocks += ps; 919 | } 920 | return 0; 921 | } 922 | 923 | /** 924 | * reconstruct a big size of buffer 925 | * input: 926 | * rs 927 | * nr_shards: assert(0 == nr_shards % rs->data_shards) 928 | * shards[nr_shards][block_size] 929 | * marks[nr_shards] marks as errors 930 | * */ 931 | int reed_solomon_reconstruct(reed_solomon* rs, 932 | unsigned char** shards, 933 | unsigned char* marks, 934 | int nr_shards, 935 | int block_size) { 936 | unsigned char *dec_fec_blocks[DATA_SHARDS_MAX]; 937 | unsigned int fec_block_nos[DATA_SHARDS_MAX]; 938 | unsigned int erased_blocks[DATA_SHARDS_MAX]; 939 | unsigned char* fec_marks; 940 | unsigned char **data_blocks, **fec_blocks; 941 | int i, j, dn, pn, n; 942 | int ds = rs->data_shards; 943 | int ps = rs->parity_shards; 944 | int err = 0; 945 | 946 | data_blocks = shards; 947 | n = nr_shards / rs->shards; 948 | fec_marks = marks + n*ds; //after all data, is't fec marks 949 | fec_blocks = shards + n*ds; 950 | 951 | for(j = 0; j < n; j++) { 952 | dn = 0; 953 | for(i = 0; i < ds; i++) { 954 | if(marks[i]) { 955 | //errors 956 | erased_blocks[dn++] = i; 957 | } 958 | } 959 | if(dn > 0) { 960 | pn = 0; 961 | for(i = 0; i < ps && pn < dn; i++) { 962 | if(!fec_marks[i]) { 963 | //got valid fec row 964 | fec_block_nos[pn] = i; 965 | dec_fec_blocks[pn] = fec_blocks[i]; 966 | pn++; 967 | } 968 | } 969 | 970 | if(dn == pn) { 971 | reed_solomon_decode(rs 972 | , data_blocks 973 | , block_size 974 | , dec_fec_blocks 975 | , fec_block_nos 976 | , erased_blocks 977 | , dn); 978 | } else { 979 | //error but we continue 980 | err = -1; 981 | } 982 | } 983 | data_blocks += ds; 984 | marks += ds; 985 | fec_blocks += ps; 986 | fec_marks += ps; 987 | } 988 | 989 | return err; 990 | } 991 | 992 | --------------------------------------------------------------------------------