├── faketypes.h ├── util ├── mkfile ├── edit.h ├── uuid.h ├── calc.y ├── disklabel64.h ├── label.c └── edit.c ├── mkfile ├── thread.c ├── 9phammer.h ├── LICENSE ├── hammer2fs.c ├── README.txt ├── uuid.h ├── cons.c ├── hammer2.c ├── xxhash.h ├── hammer2.h ├── 9p.c ├── xxhash.c ├── lz4.h └── hammer2_disk.h /faketypes.h: -------------------------------------------------------------------------------- 1 | /* Fake size_t; copied from /sys/include/ape/stddef.h */ 2 | typedef unsigned long size_t; 3 | typedef unsigned int uintptr_t; 4 | 5 | typedef u32int uint32_t; 6 | typedef uchar uint8_t; 7 | -------------------------------------------------------------------------------- /util/mkfile: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include <9p.h> 6 | 7 | /* FIXME: This hack shouldn't be required. We allocate some big things on the 8 | stack in Srv, so we need a big stack. Srv should just queue and not 9 | need the stack, a different thread with a more appropriate stack size should be 10 | the one reading from the queue doing the heavy lifting instead of hacking up 11 | threadpostmountsrv. */ 12 | static void 13 | tforker(void (*fn)(void*), void *arg, int rflag) 14 | { 15 | procrfork(fn, arg, 512*1024, rflag); 16 | } 17 | 18 | void 19 | mythreadpostmountsrv(Srv *s, char *name, char *mtpt, int flag) 20 | { 21 | _forker = tforker; 22 | _postmountsrv(s, name, mtpt, flag); 23 | } 24 | 25 | -------------------------------------------------------------------------------- /9phammer.h: -------------------------------------------------------------------------------- 1 | void loadinode(hammer2_blockref_t *block, hammer2_inode_data_t *inode); 2 | void dumpinode(hammer2_inode_data_t *i); 3 | void dumpblock(hammer2_blockref_t*); 4 | void readvolume(int fd, hammer2_dev_t *hd); 5 | char* loadblock(hammer2_blockref_t *block, void *dst, int dstsize, int *rsize); 6 | hammer2_crc32_t icrc32(void *buf, int n); 7 | 8 | void initcons(char *service); 9 | void fsstart(Srv *); 10 | void fsattach(Req *r); 11 | void fsopen(Req *r); 12 | char* fswalk(Fid *fid, char *name, Qid *qid); 13 | char* fswalkclone(Fid *old, Fid *new); 14 | void fsstat(Req *r); 15 | void fsread(Req *r); 16 | 17 | // constantly writing out the whole name is annoying, so we make a type 18 | typedef hammer2_inode_data_t inode; 19 | typedef struct{ 20 | // A pointer to the start of a DIRENT array. 21 | hammer2_blockref_t *entry; 22 | 23 | // the number of DIRENT blocks populated 24 | vlong count; 25 | // The maximum number of DIRENT blocks entry[] can contain. 26 | vlong cap; 27 | } DirEnts; 28 | 29 | typedef struct{ 30 | Qid; 31 | inode; 32 | DirEnts; 33 | char *pfsname; 34 | } root_t; 35 | 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Dave MacFarlane 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /util/edit.h: -------------------------------------------------------------------------------- 1 | typedef struct Part Part; 2 | struct Part { 3 | char *name; 4 | char *ctlname; 5 | vlong start; 6 | vlong end; 7 | vlong ctlstart; 8 | vlong ctlend; 9 | int changed; 10 | }; 11 | 12 | enum { 13 | Maxpart = 32 14 | }; 15 | 16 | typedef struct Edit Edit; 17 | struct Edit { 18 | Disk *disk; 19 | 20 | Part *ctlpart[Maxpart]; 21 | int nctlpart; 22 | 23 | Part *part[Maxpart]; 24 | int npart; 25 | 26 | char *(*add)(Edit*, char*, vlong, vlong); 27 | char *(*del)(Edit*, Part*); 28 | char *(*ext)(Edit*, int, char**); 29 | char *(*help)(Edit*); 30 | char *(*okname)(Edit*, char*); 31 | void (*sum)(Edit*, Part*, vlong, vlong); 32 | char *(*write)(Edit*); 33 | void (*printctl)(Edit*, int); 34 | 35 | char *unit; 36 | void *aux; 37 | vlong dot; 38 | vlong end; 39 | vlong unitsz; 40 | 41 | /* do not use fields below this line */ 42 | int changed; 43 | int warned; 44 | int lastcmd; 45 | }; 46 | 47 | char *getline(Edit*); 48 | void runcmd(Edit*, char*); 49 | Part *findpart(Edit*, char*); 50 | char *addpart(Edit*, Part*); 51 | char *delpart(Edit*, Part*); 52 | char *parseexpr(char *s, vlong xdot, vlong xdollar, vlong xsize, vlong xunit, vlong *result); 53 | int ctldiff(Edit *edit, int ctlfd); 54 | void *emalloc(ulong); 55 | char *estrdup(char*); 56 | -------------------------------------------------------------------------------- /hammer2fs.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include <9p.h> 8 | 9 | #include "uuid.h" 10 | #include "hammer2_disk.h" 11 | #include "hammer2.h" 12 | #include "9phammer.h" 13 | #include "lz4.h" 14 | 15 | extern char *filename; 16 | extern root_t root; 17 | extern int devfd; 18 | 19 | void mythreadpostmountsrv(Srv *s, char *name, char *mtpt, int flag); 20 | 21 | 22 | static Srv fs = { 23 | .open = fsopen, 24 | .attach = fsattach, 25 | .start = fsstart, 26 | .read = fsread, 27 | .walk1 = fswalk, 28 | .clone = fswalkclone, 29 | .stat = fsstat, 30 | }; 31 | 32 | void usage(void) { 33 | fprint(2, "usage: %s [-r root] [-S srvname] [-f devicename]\n", argv0); 34 | } 35 | 36 | void threadmain(int argc, char *argv[]) 37 | { 38 | char *srvname = "hammer2"; 39 | 40 | ARGBEGIN{ 41 | case 'D': 42 | chatty9p++; 43 | break; 44 | case 'S': 45 | srvname = EARGF(usage()); 46 | break; 47 | case 'f': 48 | filename = strdup(EARGF(usage())); 49 | break; 50 | case 'r': 51 | root.pfsname = EARGF(usage()); 52 | break; 53 | default: 54 | usage(); 55 | }ARGEND; 56 | 57 | if (argc > 0) 58 | usage(); 59 | if (filename == nil) { 60 | filename = "/dev/sdE0/hammer2"; 61 | } 62 | if (root.pfsname == nil) { 63 | root.pfsname = "ROOT"; 64 | } 65 | initcons(srvname); 66 | mythreadpostmountsrv(&fs, srvname, nil, 0); 67 | threadexits(nil); 68 | } 69 | 70 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | # hammer2 9p 2 | 3 | This contains a 9p implementation of the hammer2 filesystem, to make 4 | it possible to read the DragonFly filesystem on Plan 9. (I've only 5 | tried it on 9front, but as far as I know I didn't do anything 9front 6 | specific.) 7 | 8 | The util directory contains a hacked up version of disk/prep which 9 | installs as disk/label. disk/label reads BSD disklabel64 labels and 10 | outputs in a similar fashion to disk/prep -p. You can use it as 11 | "disk/label /dev/sdE0/bsd386 > /dev/sdE0/ctl" to see your BSD 12 | partitions as regular files under /dev/sdE0. (Assuming you're using a 13 | 64-bit disklabel and not the 32-bit disklabel format and your disk 14 | partition is named sdE0.) 15 | 16 | hammer2fs reads a hammer2 partition (by default /dev/sdE0/hammer2) and 17 | serves it over 9p. It posts to /srv/hammer2. It's currently 18 | read-only and and will likely remain read-only for the foreseeable 19 | future. 20 | 21 | lz4.^(c h) are a port of the basic lz4 library. I mostly just removed 22 | #ifdefs for other operating systems/compilers and changed the types to 23 | be compatible with the Plan 9 compiler. You should be able to just 24 | copy them elsewhere and use them if you need lz4 (de)compression 25 | support under Plan 9. 26 | 27 | hammer2.h and hammer2_disk.h are similarly ported from DragonFly (and 28 | mostly just have the types changed and packed pragma syntax changed.) 29 | They contain the disk structures and documentation about the 30 | filesystem. 31 | 32 | -------------------------------------------------------------------------------- /uuid.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * Copyright (c) 2002 Marcel Moolenaar 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 9 | * 1. Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | * 26 | * $FreeBSD: src/sys/sys/uuid.h,v 1.6 2005/10/07 13:37:10 marcel Exp $ 27 | * $DragonFly: src/sys/sys/uuid.h,v 1.5 2007/10/09 17:19:09 dillon Exp $ 28 | */ 29 | 30 | #define _UUID_NODE_LEN 6 31 | 32 | /* 33 | * See also: 34 | * http://www.opengroup.org/dce/info/draft-leach-uuids-guids-01.txt 35 | * http://www.opengroup.org/onlinepubs/009629399/apdxa.htm 36 | * 37 | * A DCE 1.1 compatible source representation of UUIDs. 38 | */ 39 | 40 | typedef struct { 41 | u32int time_low; 42 | u16int time_mid; 43 | u16int time_hi_and_version; 44 | u8int clock_seq_hi_and_reserved; 45 | u8int clock_seq_low; 46 | u8int node[_UUID_NODE_LEN]; 47 | } uuid_t ; 48 | 49 | typedef uuid_t uuid; 50 | 51 | 52 | -------------------------------------------------------------------------------- /util/uuid.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * Copyright (c) 2002 Marcel Moolenaar 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 9 | * 1. Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | * 26 | * $FreeBSD: src/sys/sys/uuid.h,v 1.6 2005/10/07 13:37:10 marcel Exp $ 27 | * $DragonFly: src/sys/sys/uuid.h,v 1.5 2007/10/09 17:19:09 dillon Exp $ 28 | */ 29 | 30 | #ifndef _SYS_UUID_H_ 31 | #define _SYS_UUID_H_ 32 | 33 | /* Length of a node address (an IEEE 802 address). */ 34 | #define _UUID_NODE_LEN 6 35 | 36 | /* 37 | * See also: 38 | * http://www.opengroup.org/dce/info/draft-leach-uuids-guids-01.txt 39 | * http://www.opengroup.org/onlinepubs/009629399/apdxa.htm 40 | * 41 | * A DCE 1.1 compatible source representation of UUIDs. 42 | */ 43 | struct uuid { 44 | uint32_t time_low; 45 | uint16_t time_mid; 46 | uint16_t time_hi_and_version; 47 | uint8_t clock_seq_hi_and_reserved; 48 | uint8_t clock_seq_low; 49 | uint8_t node[_UUID_NODE_LEN]; 50 | }; 51 | 52 | typedef struct uuid uuid_t; 53 | 54 | struct sbuf; 55 | 56 | #endif /* _SYS_UUID_H_ */ 57 | -------------------------------------------------------------------------------- /cons.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "uuid.h" 7 | #include "hammer2_disk.h" 8 | 9 | extern hammer2_volume_data_t volumehdr; 10 | 11 | // This is mostly adapted from hjfs. 12 | enum {MAXARGS = 16}; 13 | 14 | typedef struct Cmd Cmd; 15 | struct Cmd { 16 | char *name; 17 | int nargs; 18 | void (*f)(int, char**); 19 | }; 20 | 21 | void printfriendly(uvlong x) { 22 | const uvlong tb = 1024ULL*1024ULL*1024ULL*1024ULL; 23 | const uvlong gb = 1024ULL*1024ULL*1024ULL; 24 | const uvlong mb = 1024ULL*1024ULL; 25 | const uvlong kb = 1024ULL; 26 | if (x >= 10000*gb) 27 | print("%ulldTB", x / tb); 28 | else if(x >= 10000*mb) 29 | print("%ulldGB", x / gb); 30 | else if(x >= 10000*kb) 31 | print("%ulldMB", x / mb); 32 | else if(x >= 1024*10000) 33 | print("%ulldKB", x / 1024); 34 | else 35 | print("%ulld bytes", x); 36 | } 37 | void cmddf(int, char**) { 38 | print("Size\tUsed\tAvail\tCapacity\n"); 39 | printfriendly(volumehdr.volu_size); 40 | print("\t"); 41 | printfriendly(volumehdr.allocator_size-volumehdr.allocator_free); 42 | print("\t"); 43 | printfriendly(volumehdr.allocator_free); 44 | print("\t"); 45 | print("%ulld%%\n", 100-(volumehdr.allocator_free*100/volumehdr.allocator_size)); 46 | } 47 | void cmdhelp(int, char**) { 48 | print("Command\tDescription\n"); 49 | print("df\tShow free disk space\n"); 50 | print("help\tThis message\n"); 51 | } 52 | 53 | Cmd cmds[] = { 54 | { "df", 0, cmddf}, 55 | { "help", 0, cmdhelp}, 56 | }; 57 | 58 | static Biobuf bio; 59 | static void consproc(void *v){ 60 | Biobuf *in; 61 | char *args[MAXARGS]; 62 | char *s; 63 | Cmd *c; 64 | int rc; 65 | 66 | in = (Biobuf*) v; 67 | for(;;){ 68 | s = Brdstr(in, '\n', 1); 69 | if(s == nil) 70 | continue; 71 | rc = tokenize(s, args, MAXARGS); 72 | if(rc == 0) { 73 | // if we got an empty line, don't print a bad command error 74 | // but free s. 75 | print("\n"); 76 | goto good; 77 | } 78 | for(c = cmds; c < cmds + nelem(cmds); c++) { 79 | if(strcmp(c->name, args[0]) == 0) { 80 | if (c->nargs != rc-1) 81 | goto bad; 82 | c->f(rc, args); 83 | goto good; 84 | } 85 | } 86 | 87 | bad: 88 | print("bad command: %s\n", s); 89 | good: 90 | free(s); 91 | } 92 | 93 | } 94 | 95 | void initcons(char *service){ 96 | int fd, pfd[2]; 97 | char buf[512]; 98 | snprint(buf, sizeof(buf), "/srv/%s.cmd", service); 99 | fd = create(buf, OWRITE|ORCLOSE, 0600); 100 | if(fd < 0) 101 | return; 102 | pipe(pfd); 103 | fprint(fd, "%d", pfd[1]); 104 | Binit(&bio, pfd[0], OREAD); 105 | // send print to the command pipe 106 | dup(pfd[0], 1); 107 | procrfork(consproc, &bio, mainstacksize,0); 108 | } 109 | -------------------------------------------------------------------------------- /util/calc.y: -------------------------------------------------------------------------------- 1 | %{ 2 | typedef struct Exp Exp; 3 | enum { 4 | NUM, 5 | DOT, 6 | DOLLAR, 7 | ADD, 8 | SUB, 9 | MUL, 10 | DIV, 11 | FRAC, 12 | NEG, 13 | }; 14 | 15 | struct Exp { 16 | int ty; 17 | long long n; 18 | Exp *e1; 19 | Exp *e2; 20 | }; 21 | 22 | typedef Exp* Expptr; 23 | #define YYSTYPE Expptr 24 | Exp *yyexp; 25 | %} 26 | 27 | %token NUMBER 28 | 29 | %left '+' '-' 30 | %left '*' '/' 31 | %left UNARYMINUS '%' 32 | %% 33 | top: expr { yyexp = $1; return 0; } 34 | 35 | expr: NUMBER 36 | | '.' { $$ = mkOP(DOT, nil, nil); } 37 | | '$' { $$ = mkOP(DOLLAR, nil, nil); } 38 | | '(' expr ')' { $$ = $2; } 39 | | expr '+' expr { $$ = mkOP(ADD, $1, $3); } 40 | | expr '-' expr { $$ = mkOP(SUB, $1, $3); } 41 | | expr '*' expr { $$ = mkOP(MUL, $1, $3); } 42 | | expr '/' expr { $$ = mkOP(DIV, $1, $3); } 43 | | expr '%' { $$ = mkOP(FRAC, $1, nil); } 44 | | '-' expr %prec UNARYMINUS { $$ = mkOP(NEG, $2, nil); } 45 | ; 46 | 47 | %% 48 | 49 | #include 50 | #include 51 | #include 52 | #include "disk.h" 53 | #include "edit.h" 54 | 55 | static Exp* 56 | mkNUM(vlong x) 57 | { 58 | Exp *n; 59 | 60 | n = emalloc(sizeof *n); 61 | 62 | n->ty = NUM; 63 | n->n = x; 64 | return n; 65 | } 66 | 67 | static Exp* 68 | mkOP(int ty, Exp *e1, Exp *e2) 69 | { 70 | Exp *n; 71 | 72 | n = emalloc(sizeof *n); 73 | n->ty = ty; 74 | n->e1 = e1; 75 | n->e2 = e2; 76 | 77 | return n; 78 | } 79 | 80 | static char *inp; 81 | static jmp_buf jmp; 82 | static vlong dot, size, dollar, unit; 83 | static char** errp; 84 | 85 | static int 86 | yylex(void) 87 | { 88 | int c; 89 | uvlong n; 90 | 91 | while(isspace(*inp)) 92 | inp++; 93 | 94 | if(*inp == 0) 95 | return 0; 96 | 97 | if(isdigit(*inp)) { 98 | n = strtoull(inp, &inp, 0); /* default unit is sectors */ 99 | c = *inp++; 100 | if(isascii(c) && isupper(c)) 101 | c = tolower(c); 102 | switch(c) { 103 | case 't': 104 | n *= 1024; 105 | /* fall through */ 106 | case 'g': 107 | n *= 1024; 108 | /* fall through */ 109 | case 'm': 110 | n *= 1024; 111 | /* fall through */ 112 | case 'k': 113 | n *= 1024; 114 | n /= unit; /* convert to sectors */ 115 | break; 116 | default: 117 | --inp; 118 | break; 119 | } 120 | yylval = mkNUM(n); 121 | return NUMBER; 122 | } 123 | return *inp++; 124 | } 125 | 126 | static void 127 | yyerror(char *s) 128 | { 129 | *errp = s; 130 | longjmp(jmp, 1); 131 | } 132 | 133 | static vlong 134 | eval(Exp *e) 135 | { 136 | vlong i; 137 | 138 | switch(e->ty) { 139 | case NUM: 140 | return e->n; 141 | case DOT: 142 | return dot; 143 | case DOLLAR: 144 | return dollar; 145 | case ADD: 146 | return eval(e->e1)+eval(e->e2); 147 | case SUB: 148 | return eval(e->e1)-eval(e->e2); 149 | case MUL: 150 | return eval(e->e1)*eval(e->e2); 151 | case DIV: 152 | i = eval(e->e2); 153 | if(i == 0) 154 | yyerror("division by zero"); 155 | return eval(e->e1)/i; 156 | case FRAC: 157 | return (size*eval(e->e1))/100; 158 | case NEG: 159 | return -eval(e->e1); 160 | } 161 | assert(0); 162 | return 0; 163 | } 164 | 165 | int yyparse(void); 166 | 167 | char* 168 | parseexpr(char *s, vlong xdot, vlong xdollar, vlong xsize, vlong xunit, vlong *result) 169 | { 170 | char *err; 171 | 172 | errp = &err; 173 | if(setjmp(jmp)) 174 | return err; 175 | 176 | inp = s; 177 | dot = xdot; 178 | size = xsize; 179 | dollar = xdollar; 180 | unit = xunit; 181 | yyparse(); 182 | if(yyexp == nil) 183 | return "nil yylval?"; 184 | *result = eval(yyexp); 185 | return nil; 186 | } 187 | 188 | #ifdef TEST 189 | void 190 | main(int argc, char **argv) 191 | { 192 | int i; 193 | vlong r; 194 | char *e; 195 | 196 | for(i=1; i 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include <9p.h> 7 | 8 | // for crc32 9 | #include 10 | 11 | #include "uuid.h" 12 | #include "hammer2_disk.h" 13 | #include "hammer2.h" 14 | #include "9phammer.h" 15 | 16 | hammer2_volume_data_t volumehdr; 17 | 18 | // Loads an inode from dataoff in fd into the inode argument. 19 | void loadinode(hammer2_blockref_t *block, hammer2_inode_data_t *inode) { 20 | char *err = loadblock(block, inode, sizeof(hammer2_inode_data_t), nil); 21 | if(err != nil) { 22 | fprint(2, "error loading inode: %s\n", err); 23 | } 24 | }; 25 | 26 | void dumpinode(hammer2_inode_data_t *i) { 27 | if (i == nil) { 28 | fprintf(stderr, "No inode\n"); 29 | return; 30 | } 31 | fprintf(stderr, "Inode: %d\nSize: %d\n", i->meta.inum, i->meta.size); 32 | fprintf(stderr, "Type: %d\n", i->meta.type); 33 | fprintf(stderr, "Name_key: %d\n", i->meta.name_key); 34 | fprintf(stderr, "OpFlags: %d\n", i->meta.op_flags); 35 | fprintf(stderr, "CapFlags: %d\n", i->meta.cap_flags); 36 | fprintf(stderr, "NLinks: %d\n", i->meta.nlinks); 37 | fprintf(stderr, "pfs_inum: %d: %d\n", i->meta.pfs_inum); 38 | fprintf(stderr, "Comp_Algo: %d\n", i->meta.comp_algo); 39 | } 40 | void dumpblock(hammer2_blockref_t *b) { 41 | if (b == nil) { 42 | fprintf(stderr, "No blockref\n"); 43 | return; 44 | } 45 | fprintf(stderr, "Block type: %d\n", b->type); 46 | fprintf(stderr, "Methods: %d (Check: %d Comp: %d)\n", b->methods, HAMMER2_DEC_CHECK(b->methods), HAMMER2_DEC_COMP(b->methods)); 47 | fprintf(stderr, "Keybits: %d\n", b->keybits); 48 | fprintf(stderr, "vradix: %d\n", b->vradix); 49 | fprintf(stderr, "flags: %d\n", b->flags); 50 | fprintf(stderr, "leaf_count: %d\n", b->leaf_count); 51 | fprintf(stderr, "key: %d\n", b->key); 52 | fprintf(stderr, "key: %d\n", b->key); 53 | fprintf(stderr, "(if applicable) data_count %d inode_count: %d\n", b->embed.stats.data_count, b->embed.stats.inode_count); 54 | } 55 | 56 | void readvolume(int fd, hammer2_dev_t *hd) { 57 | hammer2_volume_data_t vol; 58 | int valid; 59 | int i; 60 | valid = 0; 61 | for(i = 0; i < 4; i++){ 62 | seek(fd, i*HAMMER2_ZONE_BYTES64, SEEK_SET); 63 | read(fd, &vol, sizeof(hammer2_volume_data_t)); 64 | if(vol.magic != HAMMER2_VOLUME_ID_HBO) continue; 65 | 66 | // printf("good magic"); 67 | // printf(" Total size: %u Free: %u\n", vol.allocator_size, vol.allocator_free); 68 | // printf("Mirror_tid: %16d\n", vol.mirror_tid); 69 | if(icrc32(&vol, 512-4) != vol.icrc_sects[7]) { 70 | fprint(2,"Invalid CRC for volume header sector 1\n"); 71 | continue; 72 | } 73 | if(icrc32(&vol.sroot_blockset, 512) != vol.icrc_sects[6]) { 74 | fprint(2,"Invalid CRC for superoot block\n"); 75 | continue; 76 | } 77 | /* 78 | These seem to always be 0 on real DragonFly systems, despite 79 | the documentation in hammer2_disk.h claiming otherwise. 80 | if(icrc32(vol.sector2, 512) != vol.icrc_sects[5]) { 81 | fprint(2,"Invalid CRC for volume sector 2 got %ux want %x\n", icrc32(vol.sector2, 512), vol.icrc_sects[5]); 82 | continue; 83 | } 84 | if(icrc32(vol.sector3, 512) != vol.icrc_sects[4]) { 85 | fprint(2,"Invalid CRC for volume sector 3\n"); 86 | continue; 87 | } 88 | if(icrc32(&vol.freemap_blockset, 512) != vol.icrc_sects[3]) { 89 | fprint(2,"Invalid CRC for volume freemap got %ux want %ux\n", icrc32(&vol.freemap_blockset, 512), vol.icrc_sects[3]); 90 | continue; 91 | } 92 | // sector 5? 6? 7? 93 | */ 94 | 95 | if(icrc32(&vol, sizeof(hammer2_volume_data_t)-4) != vol.icrc_volheader) { 96 | fprint(2, "Invalid full volume header CRC\n"); 97 | continue; 98 | } 99 | if(valid == 0 || hd->voldata.mirror_tid < vol.mirror_tid){ 100 | valid = 1; 101 | hd->voldata = vol; 102 | hd->volhdrno = i; 103 | } 104 | } 105 | volumehdr = vol; 106 | /* 107 | print("Using volume header #%d\n", hd->volhdrno); 108 | print("Volume size: %ulld\n", hd->voldata.allocator_size); 109 | print("Freemap version: %d\n", hd->voldata.freemap_version); 110 | print("Allocator size: %ulld\n", hd->voldata.volu_size); 111 | print("Allocator free: %ulld\n", hd->voldata.allocator_free); 112 | print("Allocator beg: %ulld\n", hd->voldata.allocator_beg); 113 | */ 114 | 115 | /* 116 | For looking into the freemap on start/debugging 117 | char buf[HAMMER2_BLOCKREF_LEAF_MAX]; 118 | int j = 0; 119 | for(i = 0; i < 4; i++) { 120 | hammer2_blockref_t *block =&(hd->voldata.freemap_blockset.blockref[i]); 121 | print("Freemap 0.%d Type %d\n", i, block->type); 122 | print("Key: %ulld keysize: %d\n", block->key, 1<keybits); 123 | assert(HAMMER2_DEC_COMP(block->methods) == 0); 124 | pread(fd, buf, HAMMER2_BLOCKREF_LEAF_MAX, block->data_off & HAMMER2_OFF_MASK); 125 | for(j = 0; j < 1<keybits;j++) { 126 | block = ((hammer2_blockref_t*) buf) + j; 127 | print("Freemap 1 Level: %d.%d Type %d\n", i, j, block->type); 128 | print("Key: %ulld keysize: %uld\n", block->key, 1<keybits); 129 | print("Available: %ulld Radixes: %ulb\n", block->check.freemap.avail, block->check.freemap.bigmask); 130 | } 131 | } 132 | //print("mirror_tid: %x freemap tid: %8.x bulkfree tid: %x\n", hd->voldata.mirror_tid, hd->voldata.freemap_tid, hd->voldata.bulkfree_tid); 133 | */ 134 | } 135 | 136 | -------------------------------------------------------------------------------- /util/disklabel64.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007 The DragonFly Project. All rights reserved. 3 | * 4 | * This code is derived from software contributed to The DragonFly Project 5 | * by Matthew Dillon 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * 1. Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 2. Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * 3. Neither the name of The DragonFly Project nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific, prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 | * SUCH DAMAGE. 33 | * 34 | * $DragonFly: src/sys/sys/disklabel64.h,v 1.4 2007/06/19 06:39:10 dillon Exp $ 35 | */ 36 | 37 | #ifndef _SYS_DISKLABEL64_H_ 38 | #define _SYS_DISKLABEL64_H_ 39 | 40 | /* 41 | * disklabel64's start at offset 0 on the disk or slice they reside. All 42 | * values are byte offsets, not block numbers, in order to allow portability. 43 | * Unlike the original 32 bit disklabels, the on-disk format for a 64 bit 44 | * disklabel is slice-relative and does not have to be translated. 45 | * 46 | * Currently the number of partitions is limited to 16, but the virgin 47 | * disklabel code will leave enough space for 32. 48 | */ 49 | #define DISKMAGIC64 ((u_int32_t)0xc4464c59) /* The disk magic number */ 50 | #ifndef MAXPARTITIONS64 51 | #define MAXPARTITIONS64 16 52 | #endif 53 | #ifndef RESPARTITIONS64 54 | #define RESPARTITIONS64 32 55 | #endif 56 | 57 | #define u_int32_t ulong 58 | #define uint32_t ulong 59 | #define int32_t long 60 | #define uint16_t ushort 61 | #define u_int64_t uvlong 62 | #define u_char unsigned char 63 | #define u_int8_t unsigned char 64 | #define uint8_t unsigned char 65 | #define ufs_daddr_t uint32_t 66 | #define ufs_time_t uint32_t 67 | #define int8_t char 68 | #define int16_t short 69 | #define uint64_t uvlong 70 | #define int64_t vlong 71 | 72 | #include "uuid.h" 73 | 74 | #ifndef LOCORE 75 | 76 | /* 77 | * A disklabel64 starts at slice relative offset 0, NOT SECTOR 1. In 78 | * otherwords, d_magic is at byte offset 512 within the slice, regardless 79 | * of the sector size. 80 | * 81 | * The d_reserved0 area is not included in the crc and any kernel writeback 82 | * of the label will not change the d_reserved area on-disk. It is purely 83 | * a shim to allow us to avoid sector calculations when reading or 84 | * writing the label. Since byte offsets are used in our 64 bit disklabel, 85 | * the entire disklabel and the I/O required to access it becomes 86 | * sector-agnostic. 87 | */ 88 | struct disklabel64 { 89 | char d_reserved0[512]; /* reserved or unused */ 90 | u_int32_t d_magic; /* the magic number */ 91 | u_int32_t d_crc; /* crc32() d_magic thru last part */ 92 | u_int32_t d_align; /* partition alignment requirement */ 93 | u_int32_t d_npartitions; /* number of partitions */ 94 | struct uuid d_stor_uuid; /* unique uuid for label */ 95 | 96 | u_int64_t d_total_size; /* total size incl everything (bytes) */ 97 | u_int64_t d_bbase; /* boot area base offset (bytes) */ 98 | /* boot area is pbase - bbase */ 99 | u_int64_t d_pbase; /* first allocatable offset (bytes) */ 100 | u_int64_t d_pstop; /* last allocatable offset+1 (bytes) */ 101 | u_int64_t d_abase; /* location of backup copy if not 0 */ 102 | 103 | u_char d_packname[64]; 104 | u_char d_reserved[64]; 105 | 106 | /* 107 | * Note: offsets are relative to the base of the slice, NOT to 108 | * d_pbase. Unlike 32 bit disklabels the on-disk format for 109 | * a 64 bit disklabel remains slice-relative. 110 | * 111 | * An uninitialized partition has a p_boffset and p_bsize of 0. 112 | * 113 | * If p_fstype is not supported for a live partition it is set 114 | * to FS_OTHER. This is typically the case when the filesystem 115 | * is identified by its uuid. 116 | */ 117 | struct partition64 { /* the partition table */ 118 | u_int64_t p_boffset; /* slice relative offset, in bytes */ 119 | u_int64_t p_bsize; /* size of partition, in bytes */ 120 | u_int8_t p_fstype; 121 | u_int8_t p_unused01; /* reserved, must be 0 */ 122 | u_int8_t p_unused02; /* reserved, must be 0 */ 123 | u_int8_t p_unused03; /* reserved, must be 0 */ 124 | u_int32_t p_unused04; /* reserved, must be 0 */ 125 | u_int32_t p_unused05; /* reserved, must be 0 */ 126 | u_int32_t p_unused06; /* reserved, must be 0 */ 127 | struct uuid p_type_uuid;/* mount type as UUID */ 128 | struct uuid p_stor_uuid;/* unique uuid for storage */ 129 | } d_partitions[MAXPARTITIONS64];/* actually may be more */ 130 | }; 131 | 132 | #define FS_UNUSED 0 133 | #define FS_SWAP 1 134 | #define FS_UFS 7 135 | #define FS_OTHER 10 136 | #define FS_HAMMER 22 137 | #define FS_HAMMER2 23 138 | #define FS_ZFS 27 139 | 140 | #endif /* !_SYS_DISKLABEL64_H_ */ 141 | -------------------------------------------------------------------------------- /util/label.c: -------------------------------------------------------------------------------- 1 | /* 2 | * label - read disklabel 64 3 | */ 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "edit.h" 9 | #include "disklabel64.h" 10 | 11 | enum { 12 | Maxpath = 128, 13 | }; 14 | 15 | static int blank; 16 | static int file; 17 | static int doautox; 18 | static int printflag; 19 | static Part **opart; 20 | static int nopart; 21 | static char *osecbuf; 22 | static char *secbuf; 23 | static int rdonly; 24 | static int dowrite; 25 | static int docache; 26 | static int donvram; 27 | 28 | static Part *mkpart(char*, vlong, vlong, int); 29 | static void rdpart(Edit*); 30 | 31 | static void cmdsum(Edit*, Part*, vlong, vlong); 32 | static char *cmdokname(Edit*, char*); 33 | static char *cmdctlprint(Edit*, int, char**); 34 | 35 | char *okname(Edit *edit, char*name); 36 | 37 | Edit edit = { 38 | .okname=cmdokname, 39 | .sum= cmdsum, 40 | 41 | .unit= "sector", 42 | }; 43 | 44 | 45 | #define TB (1024LL*GB) 46 | #define GB (1024*1024*1024) 47 | #define MB (1024*1024) 48 | #define KB (1024) 49 | 50 | void 51 | usage(void) 52 | { 53 | fprint(2, "usage: disk/label [-s sectorsize] /dev/sdC0/bsd386\n"); 54 | exits("usage"); 55 | } 56 | 57 | void 58 | main(int argc, char **argv) 59 | { 60 | int i; 61 | Disk *disk; 62 | vlong secsize; 63 | 64 | secsize = 0; 65 | printflag++; 66 | rdonly++; 67 | ARGBEGIN{ 68 | case 'f': 69 | file++; 70 | break; 71 | case 's': 72 | secsize = atoi(ARGF()); 73 | break; 74 | default: 75 | usage(); 76 | }ARGEND; 77 | 78 | if(argc != 1) 79 | usage(); 80 | 81 | disk = opendisk(argv[0], rdonly, file); 82 | if(disk == nil) 83 | sysfatal("cannot open disk: %r"); 84 | 85 | if(secsize != 0) { 86 | disk->secsize = secsize; 87 | disk->secs = disk->size / secsize; 88 | } 89 | edit.unitsz = disk->secsize; 90 | edit.end = disk->secs; 91 | 92 | secbuf = emalloc(disk->secsize+1); 93 | osecbuf = emalloc(disk->secsize+1); 94 | edit.disk = disk; 95 | 96 | if(blank == 0) 97 | rdpart(&edit); 98 | 99 | opart = emalloc(edit.npart*sizeof(opart[0])); 100 | 101 | /* save old partition table */ 102 | for(i=0; i>> "); 119 | runcmd(&edit, getline(&edit)); 120 | } 121 | } 122 | 123 | static void 124 | cmdsum(Edit *edit, Part *p, vlong a, vlong b) 125 | { 126 | vlong sz, div; 127 | char *suf, *name; 128 | char c; 129 | 130 | c = p && p->changed ? '\'' : ' '; 131 | name = p ? p->name : "empty"; 132 | 133 | sz = (b-a)*edit->disk->secsize; 134 | if(sz >= 1*TB){ 135 | suf = "TB"; 136 | div = TB; 137 | }else if(sz >= 1*GB){ 138 | suf = "GB"; 139 | div = GB; 140 | }else if(sz >= 1*MB){ 141 | suf = "MB"; 142 | div = MB; 143 | }else if(sz >= 1*KB){ 144 | suf = "KB"; 145 | div = KB; 146 | }else{ 147 | if (sz < 0) 148 | fprint(2, "%s: negative size!\n", argv0); 149 | suf = "B "; 150 | div = 1; 151 | } 152 | 153 | if(div == 1) 154 | print("%c %-12s %*lld %-*lld (%lld sectors, %lld %s)\n", c, name, 155 | edit->disk->width, a, edit->disk->width, b, b-a, sz, suf); 156 | else 157 | print("%c %-12s %*lld %-*lld (%lld sectors, %lld.%.2d %s)\n", c, name, 158 | edit->disk->width, a, edit->disk->width, b, b-a, 159 | sz/div, (int)(((sz%div)*100)/div), suf); 160 | } 161 | 162 | static char isfrog[256]={ 163 | /*NUL*/ 1, 1, 1, 1, 1, 1, 1, 1, 164 | /*BKS*/ 1, 1, 1, 1, 1, 1, 1, 1, 165 | /*DLE*/ 1, 1, 1, 1, 1, 1, 1, 1, 166 | /*CAN*/ 1, 1, 1, 1, 1, 1, 1, 1, 167 | [' '] 1, 168 | ['/'] 1, 169 | [0x7f] 1, 170 | }; 171 | 172 | static char* 173 | cmdokname(Edit*, char *elem) 174 | { 175 | for(; *elem; elem++) 176 | if(isfrog[*(uchar*)elem]) 177 | return "bad character in name"; 178 | return nil; 179 | } 180 | 181 | static Part* 182 | mkpart(char *name, vlong start, vlong end, int changed) 183 | { 184 | Part *p; 185 | 186 | p = emalloc(sizeof(*p)); 187 | p->name = estrdup(name); 188 | p->ctlname = estrdup(name); 189 | p->start = start; 190 | p->end = end; 191 | p->changed = changed; 192 | return p; 193 | } 194 | 195 | static char* 196 | rdbsdpart(Edit *edit); 197 | 198 | /* disklabel partition is first sector of the partition */ 199 | static void 200 | rdpart(Edit *edit) 201 | { 202 | int i, nline, nf, waserr; 203 | vlong a, b; 204 | char *line[128]; 205 | char *f[5]; 206 | char *err; 207 | Disk *disk; 208 | 209 | disk = edit->disk; 210 | seek(disk->fd, disk->secsize, 0); 211 | if(readn(disk->fd, osecbuf, disk->secsize) != disk->secsize) 212 | return; 213 | osecbuf[disk->secsize] = '\0'; 214 | memmove(secbuf, osecbuf, disk->secsize+1); 215 | 216 | if(strncmp(secbuf, "part", 4) != 0){ 217 | err = rdbsdpart(edit); 218 | if(err != nil){ 219 | fprint(2, "no plan9 partition table found\n"); 220 | exits(err); 221 | } 222 | return; 223 | } 224 | 225 | waserr = 0; 226 | nline = getfields(secbuf, line, nelem(line), 1, "\n"); 227 | for(i=0; i= b) 243 | goto Error; 244 | 245 | if(err = addpart(edit, mkpart(f[1], a, b, 0))) { 246 | fprint(2, "?%s: not continuing\n", err); 247 | exits("partition"); 248 | } 249 | 250 | } 251 | } 252 | 253 | static char* 254 | rdbsdpart(Edit *edit) 255 | { 256 | int i, j; 257 | char *name, *nname; 258 | Disk *disk; 259 | char *err; 260 | int start, end; 261 | struct disklabel64 dl; 262 | disk = edit->disk; 263 | 264 | seek(disk->fd, 0, 0); 265 | 266 | readn(disk->fd, (void *)&dl, sizeof(struct disklabel64)); 267 | if(dl.d_magic != DISKMAGIC64) 268 | return "bad magic"; 269 | 270 | for(i = 0; i < dl.d_npartitions; i++){ 271 | switch(dl.d_partitions[i].p_fstype){ 272 | case FS_UNUSED: continue; 273 | case FS_SWAP: 274 | name = "bsdswap"; 275 | break; 276 | case FS_UFS: 277 | name = "ufs"; 278 | break; 279 | case FS_HAMMER: 280 | name = "hammer"; 281 | break; 282 | case FS_HAMMER2: 283 | name = "hammer2"; 284 | break; 285 | case FS_ZFS: 286 | name = "zfs"; 287 | break; 288 | case FS_OTHER: 289 | default: 290 | name = "other"; 291 | break; 292 | } 293 | 294 | nname = name; 295 | for(j = 0; j < 16; j++){ 296 | if(!okname(edit, nname)) break; 297 | 298 | sprint(nname, "%s.%d", name,j+1); 299 | } 300 | start = dl.d_partitions[i].p_boffset / disk->secsize; 301 | end = start + (dl.d_partitions[i].p_bsize / disk->secsize); 302 | if(err = addpart(edit, mkpart(nname, start, end, 0))){ 303 | fprint(2, "?%s: not continuing\n", err); 304 | return err; 305 | } 306 | } 307 | return nil; 308 | } 309 | 310 | 311 | static vlong 312 | min(vlong a, vlong b) 313 | { 314 | if(a < b) 315 | return a; 316 | return b; 317 | } 318 | 319 | -------------------------------------------------------------------------------- /xxhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | xxHash - Extremely Fast Hash algorithm 3 | Header File 4 | Copyright (C) 2012-2016, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - xxHash source repository : https://github.com/Cyan4973/xxHash 33 | */ 34 | 35 | /* DRAGONFLY ADDITION - allows inclusion in conf/files */ 36 | #define XXH_NAMESPACE h2_ 37 | 38 | /* Notice extracted from xxHash homepage : 39 | 40 | xxHash is an extremely fast Hash algorithm, running at RAM speed limits. 41 | It also successfully passes all tests from the SMHasher suite. 42 | 43 | Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) 44 | 45 | Name Speed Q.Score Author 46 | xxHash 5.4 GB/s 10 47 | CrapWow 3.2 GB/s 2 Andrew 48 | MumurHash 3a 2.7 GB/s 10 Austin Appleby 49 | SpookyHash 2.0 GB/s 10 Bob Jenkins 50 | SBox 1.4 GB/s 9 Bret Mulvey 51 | Lookup3 1.2 GB/s 9 Bob Jenkins 52 | SuperFastHash 1.2 GB/s 1 Paul Hsieh 53 | CityHash64 1.05 GB/s 10 Pike & Alakuijala 54 | FNV 0.55 GB/s 5 Fowler, Noll, Vo 55 | CRC32 0.43 GB/s 9 56 | MD5-32 0.33 GB/s 10 Ronald L. Rivest 57 | SHA1-32 0.28 GB/s 10 58 | 59 | Q.Score is a measure of quality of the hash function. 60 | It depends on successfully passing SMHasher test set. 61 | 10 is a perfect score. 62 | 63 | A 64-bits version, named XXH64, is available since r35. 64 | It offers much better speed, but for 64-bits applications only. 65 | Name Speed on 64 bits Speed on 32 bits 66 | XXH64 13.8 GB/s 1.9 GB/s 67 | XXH32 6.8 GB/s 6.0 GB/s 68 | */ 69 | 70 | #ifndef XXHASH_H_5627135585666179 71 | #define XXHASH_H_5627135585666179 1 72 | #define XXH_CPU_LITTLE_ENDIAN 1 73 | /* **************************** 74 | * Definitions 75 | ******************************/ 76 | typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; 77 | 78 | 79 | /* **************************** 80 | * API modifier 81 | ******************************/ 82 | /*!XXH_PRIVATE_API 83 | * Transforms all publics symbols within `xxhash.c` into private ones. 84 | * Methodology : 85 | * instead of : #include "xxhash.h" 86 | * do : 87 | * #define XXH_PRIVATE_API 88 | * #include "xxhash.c" // note the .c , instead of .h 89 | * also : don't compile and link xxhash.c separately 90 | */ 91 | #ifdef XXH_PRIVATE_API 92 | # if defined(__GNUC__) 93 | # define XXH_PUBLIC_API static __attribute__((unused)) 94 | # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 95 | # define XXH_PUBLIC_API static inline 96 | # elif defined(_MSC_VER) 97 | # define XXH_PUBLIC_API static __inline 98 | # else 99 | # define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ 100 | # endif 101 | #else 102 | # define XXH_PUBLIC_API /* do nothing */ 103 | #endif 104 | 105 | /*!XXH_NAMESPACE, aka Namespace Emulation : 106 | 107 | If you want to include _and expose_ xxHash functions from within your own library, 108 | but also want to avoid symbol collisions with another library which also includes xxHash, 109 | 110 | you can use XXH_NAMESPACE, to automatically prefix any public symbol from `xxhash.c` 111 | with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). 112 | 113 | Note that no change is required within the calling program as long as it also includes `xxhash.h` : 114 | regular symbol name will be automatically translated by this header. 115 | */ 116 | #ifdef XXH_NAMESPACE 117 | # define XXH_CAT(A,B) A##B 118 | # define XXH_NAME2(A,B) XXH_CAT(A,B) 119 | # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) 120 | # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) 121 | # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) 122 | # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) 123 | # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) 124 | # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) 125 | # define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) 126 | # define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) 127 | # define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) 128 | # define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) 129 | # define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) 130 | # define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) 131 | # define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) 132 | #endif 133 | 134 | 135 | /* ************************************* 136 | * Version 137 | ***************************************/ 138 | #define XXH_VERSION_MAJOR 0 139 | #define XXH_VERSION_MINOR 6 140 | #define XXH_VERSION_RELEASE 0 141 | #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) 142 | XXH_PUBLIC_API unsigned XXH_versionNumber (void); 143 | 144 | 145 | /* **************************** 146 | * Simple Hash Functions 147 | ******************************/ 148 | typedef unsigned int XXH32_hash_t; 149 | typedef unsigned long long XXH64_hash_t; 150 | 151 | XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); 152 | XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); 153 | 154 | /*! 155 | XXH32() : 156 | Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". 157 | The memory between input & input+length must be valid (allocated and read-accessible). 158 | "seed" can be used to alter the result predictably. 159 | Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s 160 | XXH64() : 161 | Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". 162 | "seed" can be used to alter the result predictably. 163 | This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). 164 | */ 165 | 166 | 167 | /* **************************** 168 | * Streaming Hash Functions 169 | ******************************/ 170 | typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ 171 | typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ 172 | 173 | /*! Dynamic allocation of states 174 | Compatible with dynamic libraries */ 175 | 176 | XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); 177 | XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); 178 | 179 | XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); 180 | XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); 181 | 182 | 183 | /* hash streaming */ 184 | 185 | XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); 186 | XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); 187 | XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); 188 | 189 | XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); 190 | XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); 191 | XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); 192 | 193 | /*! 194 | These functions generate the xxHash of an input provided in multiple segments, 195 | as opposed to provided as a single block. 196 | 197 | XXH state must first be allocated, using either static or dynamic method provided above. 198 | 199 | Start a new hash by initializing state with a seed, using XXHnn_reset(). 200 | 201 | Then, feed the hash state by calling XXHnn_update() as many times as necessary. 202 | Obviously, input must be valid, hence allocated and read accessible. 203 | The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. 204 | 205 | Finally, a hash value can be produced anytime, by using XXHnn_digest(). 206 | This function returns the nn-bits hash as an int or long long. 207 | 208 | It's still possible to continue inserting input into the hash state after a digest, 209 | and later on generate some new hashes, by calling again XXHnn_digest(). 210 | 211 | When done, free XXH state space if it was allocated dynamically. 212 | */ 213 | 214 | 215 | /* ************************** 216 | * Canonical representation 217 | ****************************/ 218 | typedef struct { unsigned char digest[4]; } XXH32_canonical_t; 219 | typedef struct { unsigned char digest[8]; } XXH64_canonical_t; 220 | 221 | XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); 222 | XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); 223 | 224 | XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); 225 | XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); 226 | 227 | /*! Default result type for XXH functions are primitive unsigned 32 and 64 bits. 228 | * The canonical representation uses human-readable write convention, aka big-endian (large digits first). 229 | * These functions allow transformation of hash result into and from its canonical format. 230 | * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. 231 | */ 232 | 233 | 234 | #ifdef XXH_STATIC_LINKING_ONLY 235 | 236 | /* This part contains definition which shall only be used with static linking. 237 | The prototypes / types defined here are not guaranteed to remain stable. 238 | They could change in a future version, becoming incompatible with a different version of the library */ 239 | 240 | struct XXH32_state_s { 241 | unsigned long long total_len; 242 | unsigned seed; 243 | unsigned v1; 244 | unsigned v2; 245 | unsigned v3; 246 | unsigned v4; 247 | unsigned mem32[4]; /* buffer defined as U32 for alignment */ 248 | unsigned memsize; 249 | }; /* typedef'd to XXH32_state_t */ 250 | 251 | struct XXH64_state_s { 252 | unsigned long long total_len; 253 | unsigned long long seed; 254 | unsigned long long v1; 255 | unsigned long long v2; 256 | unsigned long long v3; 257 | unsigned long long v4; 258 | unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ 259 | unsigned memsize; 260 | }; /* typedef'd to XXH64_state_t */ 261 | 262 | 263 | #endif 264 | 265 | 266 | #if defined (__cplusplus) 267 | } 268 | #endif 269 | 270 | #endif /* XXHASH_H_5627135585666179 */ 271 | -------------------------------------------------------------------------------- /util/edit.c: -------------------------------------------------------------------------------- 1 | /* 2 | * disk partition editor 3 | */ 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "edit.h" 10 | 11 | char* 12 | getline(Edit *edit) 13 | { 14 | static int inited; 15 | static Biobuf bin; 16 | char *p; 17 | int n; 18 | 19 | if(!inited){ 20 | Binit(&bin, 0, OREAD); 21 | inited = 1; 22 | } 23 | p = Brdline(&bin, '\n'); 24 | n = Blinelen(&bin); 25 | if(p == nil || n < 1){ 26 | if(edit->changed) 27 | fprint(2, "?warning: changes not written\n"); 28 | exits(0); 29 | } 30 | p[n - 1] = '\0'; 31 | while(isspace(*p)) 32 | p++; 33 | return p; 34 | } 35 | 36 | Part* 37 | findpart(Edit *edit, char *name) 38 | { 39 | int i; 40 | 41 | for(i=0; inpart; i++) 42 | if(strcmp(edit->part[i]->name, name) == 0) 43 | return edit->part[i]; 44 | return nil; 45 | } 46 | 47 | //static char* 48 | char* 49 | okname(Edit *edit, char *name) 50 | { 51 | int i; 52 | static char msg[100]; 53 | 54 | if(name[0] == '\0') 55 | return "partition has no name"; 56 | 57 | // if(strlen(name) >= NAMELEN) 58 | // return "name too long"; 59 | // 60 | for(i=0; inpart; i++) { 61 | if(strcmp(name, edit->part[i]->name) == 0) { 62 | sprint(msg, "already have partition with name \"%s\"", name); 63 | return msg; 64 | } 65 | } 66 | return nil; 67 | } 68 | 69 | char* 70 | addpart(Edit *edit, Part *p) 71 | { 72 | int i; 73 | static char msg[100]; 74 | char *err; 75 | 76 | if(err = okname(edit, p->name)) 77 | return err; 78 | 79 | for(i=0; inpart; i++) { 80 | if(p->start < edit->part[i]->end && edit->part[i]->start < p->end) { 81 | sprint(msg, "\"%s\" %lld-%lld overlaps with \"%s\" %lld-%lld", 82 | p->name, p->start, p->end, 83 | edit->part[i]->name, edit->part[i]->start, edit->part[i]->end); 84 | // return msg; 85 | } 86 | } 87 | 88 | if(edit->npart >= nelem(edit->part)) 89 | return "too many partitions"; 90 | 91 | edit->part[i=edit->npart++] = p; 92 | for(; i > 0 && p->start < edit->part[i-1]->start; i--) { 93 | edit->part[i] = edit->part[i-1]; 94 | edit->part[i-1] = p; 95 | } 96 | 97 | if(p->changed) 98 | edit->changed = 1; 99 | return nil; 100 | } 101 | 102 | char* 103 | delpart(Edit *edit, Part *p) 104 | { 105 | int i; 106 | 107 | for(i=0; inpart; i++) 108 | if(edit->part[i] == p) 109 | break; 110 | assert(i < edit->npart); 111 | edit->npart--; 112 | for(; inpart; i++) 113 | edit->part[i] = edit->part[i+1]; 114 | 115 | edit->changed = 1; 116 | return nil; 117 | } 118 | 119 | static char* 120 | editdot(Edit *edit, int argc, char **argv) 121 | { 122 | char *err; 123 | vlong ndot; 124 | 125 | if(argc == 1) { 126 | print("\t. %lld\n", edit->dot); 127 | return nil; 128 | } 129 | 130 | if(argc > 2) 131 | return "args"; 132 | 133 | if(err = parseexpr(argv[1], edit->dot, edit->end, edit->end, edit->unitsz, &ndot)) 134 | return err; 135 | 136 | edit->dot = ndot; 137 | return nil; 138 | } 139 | 140 | static char* 141 | editadd(Edit *edit, int argc, char **argv) 142 | { 143 | char *name, *err, *q; 144 | static char msg[100]; 145 | vlong start, end, maxend; 146 | int i; 147 | 148 | if(argc < 2) 149 | return "args"; 150 | 151 | name = estrdup(argv[1]); 152 | if((err = okname(edit, name)) || (err = edit->okname(edit, name))) 153 | return err; 154 | 155 | if(argc >= 3) 156 | q = argv[2]; 157 | else { 158 | fprint(2, "start %s: ", edit->unit); 159 | q = getline(edit); 160 | } 161 | if(err = parseexpr(q, edit->dot, edit->end, edit->end, edit->unitsz, &start)) 162 | return err; 163 | 164 | if(start < 0 || start >= edit->end) 165 | return "start out of range"; 166 | 167 | for(i=0; i < edit->npart; i++) { 168 | if(edit->part[i]->start <= start && start < edit->part[i]->end) { 169 | sprint(msg, "start %s in partition \"%s\"", edit->unit, edit->part[i]->name); 170 | return msg; 171 | } 172 | } 173 | 174 | maxend = edit->end; 175 | for(i=0; i < edit->npart; i++) 176 | if(start < edit->part[i]->start && edit->part[i]->start < maxend) 177 | maxend = edit->part[i]->start; 178 | 179 | if(argc >= 4) 180 | q = argv[3]; 181 | else { 182 | fprint(2, "end [%lld..%lld] ", start, maxend); 183 | q = getline(edit); 184 | } 185 | if(err = parseexpr(q, edit->dot, maxend, edit->end, edit->unitsz, &end)) 186 | return err; 187 | 188 | if(start == end) 189 | return "size zero partition"; 190 | 191 | if(end <= start || end > maxend) 192 | return "end out of range"; 193 | 194 | if(argc > 4) 195 | return "args"; 196 | 197 | if(err = edit->add(edit, name, start, end)) 198 | return err; 199 | 200 | edit->dot = end; 201 | return nil; 202 | } 203 | 204 | static char* 205 | editdel(Edit *edit, int argc, char **argv) 206 | { 207 | Part *p; 208 | 209 | if(argc != 2) 210 | return "args"; 211 | 212 | if((p = findpart(edit, argv[1])) == nil) 213 | return "no such partition"; 214 | 215 | return edit->del(edit, p); 216 | } 217 | 218 | static char *helptext = 219 | ". [newdot] - display or set value of dot\n" 220 | "a name [start [end]] - add partition\n" 221 | "d name - delete partition\n" 222 | "h - print help message\n" 223 | "p - print partition table\n" 224 | "P - print commands to update sd(3) device\n" 225 | "w - write partition table\n" 226 | "q - quit\n"; 227 | 228 | static char* 229 | edithelp(Edit *edit, int, char**) 230 | { 231 | print("%s", helptext); 232 | if(edit->help) 233 | return edit->help(edit); 234 | return nil; 235 | } 236 | 237 | static char* 238 | editprint(Edit *edit, int argc, char**) 239 | { 240 | vlong lastend; 241 | int i; 242 | Part **part; 243 | 244 | if(argc != 1) 245 | return "args"; 246 | 247 | lastend = 0; 248 | part = edit->part; 249 | for(i=0; inpart; i++) { 250 | if(lastend < part[i]->start) 251 | edit->sum(edit, nil, lastend, part[i]->start); 252 | edit->sum(edit, part[i], part[i]->start, part[i]->end); 253 | lastend = part[i]->end; 254 | } 255 | if(lastend < edit->end) 256 | edit->sum(edit, nil, lastend, edit->end); 257 | return nil; 258 | } 259 | 260 | char* 261 | editwrite(Edit *edit, int argc, char**) 262 | { 263 | int i; 264 | char *err; 265 | 266 | if(argc != 1) 267 | return "args"; 268 | 269 | if(edit->disk->rdonly) 270 | return "read only"; 271 | 272 | err = edit->write(edit); 273 | if(err) 274 | return err; 275 | for(i=0; inpart; i++) 276 | edit->part[i]->changed = 0; 277 | edit->changed = 0; 278 | return nil; 279 | } 280 | 281 | static char* 282 | editquit(Edit *edit, int argc, char**) 283 | { 284 | static int warned; 285 | 286 | if(argc != 1) { 287 | warned = 0; 288 | return "args"; 289 | } 290 | 291 | if(edit->changed && (!edit->warned || edit->lastcmd != 'q')) { 292 | edit->warned = 1; 293 | return "changes unwritten"; 294 | } 295 | 296 | exits(0); 297 | return nil; /* not reached */ 298 | } 299 | 300 | char* 301 | editctlprint(Edit *edit, int argc, char **) 302 | { 303 | if(argc != 1) 304 | return "args"; 305 | 306 | if(edit->printctl) 307 | edit->printctl(edit, 1); 308 | else 309 | ctldiff(edit, 1); 310 | return nil; 311 | } 312 | 313 | typedef struct Cmd Cmd; 314 | struct Cmd { 315 | char c; 316 | char *(*fn)(Edit*, int ,char**); 317 | }; 318 | 319 | Cmd cmds[] = { 320 | '.', editdot, 321 | 'a', editadd, 322 | 'd', editdel, 323 | '?', edithelp, 324 | 'h', edithelp, 325 | 'P', editctlprint, 326 | 'p', editprint, 327 | 'w', editwrite, 328 | 'q', editquit, 329 | }; 330 | 331 | void 332 | runcmd(Edit *edit, char *cmd) 333 | { 334 | char *f[10], *err; 335 | int i, nf; 336 | 337 | while(*cmd && isspace(*cmd)) 338 | cmd++; 339 | 340 | nf = tokenize(cmd, f, nelem(f)); 341 | if(nf >= 10) { 342 | fprint(2, "?\n"); 343 | return; 344 | } 345 | 346 | if(nf < 1) 347 | return; 348 | if(strlen(f[0]) != 1) { 349 | fprint(2, "?\n"); 350 | return; 351 | } 352 | 353 | err = nil; 354 | for(i=0; iext) 362 | err = edit->ext(edit, nf, f); 363 | else 364 | err = "unknown command"; 365 | } 366 | if(err) 367 | fprint(2, "?%s\n", err); 368 | edit->lastcmd = f[0][0]; 369 | } 370 | 371 | static Part* 372 | ctlmkpart(char *name, vlong start, vlong end, int changed) 373 | { 374 | Part *p; 375 | 376 | p = emalloc(sizeof(*p)); 377 | p->name = estrdup(name); 378 | p->ctlname = estrdup(name); 379 | p->start = start; 380 | p->end = end; 381 | p->changed = changed; 382 | return p; 383 | } 384 | 385 | static void 386 | rdctlpart(Edit *edit) 387 | { 388 | int i, nline, nf; 389 | char *line[128]; 390 | char buf[4096]; 391 | vlong a, b; 392 | char *f[5]; 393 | Disk *disk; 394 | 395 | disk = edit->disk; 396 | edit->nctlpart = 0; 397 | seek(disk->ctlfd, 0, 0); 398 | if(readn(disk->ctlfd, buf, sizeof buf) <= 0) { 399 | return; 400 | } 401 | 402 | nline = getfields(buf, line, nelem(line), 1, "\n"); 403 | for(i=0; i= b) 415 | break; 416 | 417 | /* only gather partitions contained in the disk partition we are editing */ 418 | if(a < disk->offset || disk->offset+disk->secs < b) 419 | continue; 420 | 421 | a -= disk->offset; 422 | b -= disk->offset; 423 | 424 | /* the partition we are editing does not count */ 425 | if(strcmp(f[1], disk->part) == 0) 426 | continue; 427 | 428 | if(edit->nctlpart >= nelem(edit->ctlpart)) { 429 | fprint(2, "?too many partitions in ctl file\n"); 430 | exits("ctlpart"); 431 | } 432 | edit->ctlpart[edit->nctlpart++] = ctlmkpart(f[1], a, b, 0); 433 | } 434 | } 435 | 436 | static vlong 437 | ctlstart(Part *p) 438 | { 439 | if(p->ctlstart) 440 | return p->ctlstart; 441 | return p->start; 442 | } 443 | 444 | static vlong 445 | ctlend(Part *p) 446 | { 447 | if(p->ctlend) 448 | return p->ctlend; 449 | return p->end; 450 | } 451 | 452 | static int 453 | areequiv(Part *p, Part *q) 454 | { 455 | if(p->ctlname[0]=='\0' || q->ctlname[0]=='\0') 456 | return 0; 457 | 458 | return strcmp(p->ctlname, q->ctlname) == 0 459 | && ctlstart(p) == ctlstart(q) && ctlend(p) == ctlend(q); 460 | } 461 | 462 | static void 463 | unchange(Edit *edit, Part *p) 464 | { 465 | int i; 466 | Part *q; 467 | 468 | for(i=0; inctlpart; i++) { 469 | q = edit->ctlpart[i]; 470 | if(p->start <= q->start && q->end <= p->end) { 471 | q->changed = 0; 472 | } 473 | } 474 | assert(p->changed == 0); 475 | } 476 | 477 | int 478 | ctldiff(Edit *edit, int ctlfd) 479 | { 480 | int i, j, waserr; 481 | Part *p; 482 | vlong offset; 483 | 484 | rdctlpart(edit); 485 | 486 | /* everything is bogus until we prove otherwise */ 487 | for(i=0; inctlpart; i++) 488 | edit->ctlpart[i]->changed = 1; 489 | 490 | /* 491 | * partitions with same info have not changed, 492 | * and neither have partitions inside them. 493 | */ 494 | for(i=0; inctlpart; i++) 495 | for(j=0; jnpart; j++) 496 | if(areequiv(edit->ctlpart[i], edit->part[j])) { 497 | unchange(edit, edit->ctlpart[i]); 498 | break; 499 | } 500 | 501 | waserr = 0; 502 | /* 503 | * delete all the changed partitions except data (we'll add them back if necessary) 504 | */ 505 | for(i=0; inctlpart; i++) { 506 | p = edit->ctlpart[i]; 507 | if(p->changed) 508 | if(fprint(ctlfd, "delpart %s\n", p->ctlname)<0) { 509 | fprint(2, "delpart failed: %s: %r\n", p->ctlname); 510 | waserr = -1; 511 | } 512 | } 513 | 514 | /* 515 | * add all the partitions from the real list; 516 | * this is okay since adding a parition with 517 | * information identical to what is there is a no-op. 518 | */ 519 | offset = edit->disk->offset; 520 | for(i=0; inpart; i++) { 521 | p = edit->part[i]; 522 | if(p->ctlname[0]) { 523 | if(fprint(ctlfd, "part %s %lld %lld\n", p->ctlname, offset+ctlstart(p), offset+ctlend(p)) < 0) { 524 | fprint(2, "adding part failed: %s: %r\n", p->ctlname); 525 | waserr = -1; 526 | } 527 | } 528 | } 529 | return waserr; 530 | } 531 | 532 | void* 533 | emalloc(ulong sz) 534 | { 535 | void *v; 536 | 537 | v = malloc(sz); 538 | if(v == nil) 539 | sysfatal("malloc %lud fails", sz); 540 | memset(v, 0, sz); 541 | return v; 542 | } 543 | 544 | char* 545 | estrdup(char *s) 546 | { 547 | s = strdup(s); 548 | if(s == nil) 549 | sysfatal("strdup (%.10s) fails", s); 550 | return s; 551 | } 552 | 553 | -------------------------------------------------------------------------------- /hammer2.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011-2017 The DragonFly Project. All rights reserved. 3 | * 4 | * This code is derived from software contributed to The DragonFly Project 5 | * by Matthew Dillon 6 | * by Venkatesh Srinivas 7 | * 8 | * Redistribution and use in source and binary forms, with or without 9 | * modification, are permitted provided that the following conditions 10 | * are met: 11 | * 12 | * 1. Redistributions of source code must retain the above copyright 13 | * notice, this list of conditions and the following disclaimer. 14 | * 2. Redistributions in binary form must reproduce the above copyright 15 | * notice, this list of conditions and the following disclaimer in 16 | * the documentation and/or other materials provided with the 17 | * distribution. 18 | * 3. Neither the name of The DragonFly Project nor the names of its 19 | * contributors may be used to endorse or promote products derived 20 | * from this software without specific, prior written permission. 21 | * 22 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 25 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 26 | * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 27 | * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 28 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 30 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 31 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 32 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | /* 37 | * HAMMER2 IN-MEMORY CACHE OF MEDIA STRUCTURES 38 | * 39 | * This header file contains structures used internally by the HAMMER2 40 | * implementation. See hammer2_disk.h for on-disk structures. 41 | * 42 | * There is an in-memory representation of all on-media data structure. 43 | * Almost everything is represented by a hammer2_chain structure in-memory. 44 | * Other higher-level structures typically map to chains. 45 | * 46 | * A great deal of data is accessed simply via its buffer cache buffer, 47 | * which is mapped for the duration of the chain's lock. Hammer2 must 48 | * implement its own buffer cache layer on top of the system layer to 49 | * allow for different threads to lock different sub-block-sized buffers. 50 | * 51 | * When modifications are made to a chain a new filesystem block must be 52 | * allocated. Multiple modifications do not typically allocate new blocks 53 | * until the current block has been flushed. Flushes do not block the 54 | * front-end unless the front-end operation crosses the current inode being 55 | * flushed. 56 | * 57 | * The in-memory representation may remain cached (for example in order to 58 | * placemark clustering locks) even after the related data has been 59 | * detached. 60 | */ 61 | 62 | /* 63 | * The xid tracks internal transactional updates. 64 | * 65 | * XXX fix-me, really needs to be 64-bits 66 | */ 67 | 68 | #define HAMMER2_XID_MIN 0x00000000U 69 | #define HAMMER2_XID_MAX 0x7FFFFFFFU 70 | 71 | #define HAMMER2_LIMIT_DIRTY_CHAINS (65536) 72 | 73 | typedef struct hammer2_io hammer2_io_t; 74 | 75 | #define HAMMER2_DIO_INPROG 0x8000000000000000LLU /* bio in progress */ 76 | #define HAMMER2_DIO_GOOD 0x4000000000000000LLU /* dio->bp is stable */ 77 | #define HAMMER2_DIO_WAITING 0x2000000000000000LLU /* wait on INPROG */ 78 | #define HAMMER2_DIO_DIRTY 0x1000000000000000LLU /* flush last drop */ 79 | 80 | #define HAMMER2_DIO_MASK 0x00FFFFFFFFFFFFFFLLU 81 | 82 | /* 83 | * Special notes on flags: 84 | * 85 | * INITIAL - This flag allows a chain to be created and for storage to 86 | * be allocated without having to immediately instantiate the 87 | * related buffer. The data is assumed to be all-zeros. It 88 | * is primarily used for indirect blocks. 89 | * 90 | * MODIFIED - The chain's media data has been modified. Prevents chain 91 | * free on lastdrop if still in the topology. 92 | * 93 | * UPDATE - Chain might not be modified but parent blocktable needs 94 | * an update. Prevents chain free on lastdrop if still in 95 | * the topology. 96 | * 97 | * FICTITIOUS - Faked chain as a placeholder for an error condition. This 98 | * chain is unsuitable for I/O. 99 | * 100 | * BMAPPED - Indicates that the chain is present in the parent blockmap. 101 | * 102 | * BMAPUPD - Indicates that the chain is present but needs to be updated 103 | * in the parent blockmap. 104 | */ 105 | #define HAMMER2_CHAIN_MODIFIED 0x00000001 /* dirty chain data */ 106 | #define HAMMER2_CHAIN_ALLOCATED 0x00000002 /* kmalloc'd chain */ 107 | #define HAMMER2_CHAIN_DESTROY 0x00000004 108 | #define HAMMER2_CHAIN_DEDUPABLE 0x00000008 /* registered w/dedup */ 109 | #define HAMMER2_CHAIN_DELETED 0x00000010 /* deleted chain */ 110 | #define HAMMER2_CHAIN_INITIAL 0x00000020 /* initial create */ 111 | #define HAMMER2_CHAIN_UPDATE 0x00000040 /* need parent update */ 112 | #define HAMMER2_CHAIN_DEFERRED 0x00000080 /* flush depth defer */ 113 | #define HAMMER2_CHAIN_TESTEDGOOD 0x00000100 /* crc tested good */ 114 | #define HAMMER2_CHAIN_ONFLUSH 0x00000200 /* on a flush list */ 115 | #define HAMMER2_CHAIN_FICTITIOUS 0x00000400 /* unsuitable for I/O */ 116 | #define HAMMER2_CHAIN_VOLUMESYNC 0x00000800 /* needs volume sync */ 117 | #define HAMMER2_CHAIN_DELAYED 0x00001000 /* delayed flush */ 118 | #define HAMMER2_CHAIN_COUNTEDBREFS 0x00002000 /* block table stats */ 119 | #define HAMMER2_CHAIN_ONRBTREE 0x00004000 /* on parent RB tree */ 120 | #define HAMMER2_CHAIN_ONLRU 0x00008000 /* on LRU list */ 121 | #define HAMMER2_CHAIN_EMBEDDED 0x00010000 /* embedded data */ 122 | #define HAMMER2_CHAIN_RELEASE 0x00020000 /* don't keep around */ 123 | #define HAMMER2_CHAIN_BMAPPED 0x00040000 /* present in blkmap */ 124 | #define HAMMER2_CHAIN_BMAPUPD 0x00080000 /* +needs updating */ 125 | #define HAMMER2_CHAIN_IOINPROG 0x00100000 /* I/O interlock */ 126 | #define HAMMER2_CHAIN_IOSIGNAL 0x00200000 /* I/O interlock */ 127 | #define HAMMER2_CHAIN_PFSBOUNDARY 0x00400000 /* super->pfs inode */ 128 | #define HAMMER2_CHAIN_HINT_LEAF_COUNT 0x00800000 /* redo leaf count */ 129 | 130 | #define HAMMER2_CHAIN_FLUSH_MASK (HAMMER2_CHAIN_MODIFIED | \ 131 | HAMMER2_CHAIN_UPDATE | \ 132 | HAMMER2_CHAIN_ONFLUSH | \ 133 | HAMMER2_CHAIN_DESTROY) 134 | 135 | /* 136 | * Hammer2 error codes, used by chain->error and cluster->error. The error 137 | * code is typically set on-lock unless no I/O was requested, and set on 138 | * I/O otherwise. If set for a cluster it generally means that the cluster 139 | * code could not find a valid copy to present. 140 | * 141 | * All H2 error codes are flags and can be accumulated by ORing them 142 | * together. 143 | * 144 | * IO - An I/O error occurred 145 | * CHECK - I/O succeeded but did not match the check code 146 | * INCOMPLETE - A cluster is not complete enough to use, or 147 | * a chain cannot be loaded because its parent has an error. 148 | * 149 | * NOTE: API allows callers to check zero/non-zero to determine if an error 150 | * condition exists. 151 | * 152 | * NOTE: Chain's data field is usually NULL on an IO error but not necessarily 153 | * NULL on other errors. Check chain->error, not chain->data. 154 | */ 155 | #define HAMMER2_ERROR_NONE 0 /* no error (must be 0) */ 156 | #define HAMMER2_ERROR_EIO 0x00000001 /* device I/O error */ 157 | #define HAMMER2_ERROR_CHECK 0x00000002 /* check code error */ 158 | #define HAMMER2_ERROR_INCOMPLETE 0x00000004 /* incomplete cluster */ 159 | #define HAMMER2_ERROR_DEPTH 0x00000008 /* tmp depth limit */ 160 | #define HAMMER2_ERROR_BADBREF 0x00000010 /* illegal bref */ 161 | #define HAMMER2_ERROR_ENOSPC 0x00000020 /* allocation failure */ 162 | #define HAMMER2_ERROR_ENOENT 0x00000040 /* entry not found */ 163 | #define HAMMER2_ERROR_ENOTEMPTY 0x00000080 /* dir not empty */ 164 | #define HAMMER2_ERROR_EAGAIN 0x00000100 /* retry */ 165 | #define HAMMER2_ERROR_ENOTDIR 0x00000200 /* not directory */ 166 | #define HAMMER2_ERROR_EISDIR 0x00000400 /* is directory */ 167 | #define HAMMER2_ERROR_EINPROGRESS 0x00000800 /* already running */ 168 | #define HAMMER2_ERROR_ABORTED 0x00001000 /* aborted operation */ 169 | #define HAMMER2_ERROR_EOF 0x00002000 /* end of scan */ 170 | #define HAMMER2_ERROR_EINVAL 0x00004000 /* catch-all */ 171 | #define HAMMER2_ERROR_EEXIST 0x00008000 /* entry exists */ 172 | #define HAMMER2_ERROR_EDEADLK 0x00010000 173 | #define HAMMER2_ERROR_ESRCH 0x00020000 174 | #define HAMMER2_ERROR_ETIMEDOUT 0x00040000 175 | 176 | /* 177 | * Flags passed to hammer2_chain_lookup() and hammer2_chain_next() 178 | * 179 | * NOTES: 180 | * NOLOCK - Input and output chains are referenced only and not 181 | * locked. Output chain might be temporarily locked 182 | * internally. 183 | * 184 | * NODATA - Asks that the chain->data not be resolved in order 185 | * to avoid I/O. 186 | * 187 | * NODIRECT - Prevents a lookup of offset 0 in an inode from returning 188 | * the inode itself if the inode is in DIRECTDATA mode 189 | * (i.e. file is <= 512 bytes). Used by the synchronization 190 | * code to prevent confusion. 191 | * 192 | * SHARED - The input chain is expected to be locked shared, 193 | * and the output chain is locked shared. 194 | * 195 | * MATCHIND - Allows an indirect block / freemap node to be returned 196 | * when the passed key range matches the radix. Remember 197 | * that key_end is inclusive (e.g. {0x000,0xFFF}, 198 | * not {0x000,0x1000}). 199 | * 200 | * (Cannot be used for remote or cluster ops). 201 | * 202 | * ALLNODES - Allows NULL focus. 203 | * 204 | * ALWAYS - Always resolve the data. If ALWAYS and NODATA are both 205 | * missing, bulk file data is not resolved but inodes and 206 | * other meta-data will. 207 | * 208 | * NOUNLOCK - Used by hammer2_chain_next() to leave the lock on 209 | * the input chain intact. The chain is still dropped. 210 | * This allows the caller to add a reference to the chain 211 | * and retain it in a locked state (used by the 212 | * XOP/feed/collect code). 213 | */ 214 | #define HAMMER2_LOOKUP_NOLOCK 0x00000001 /* ref only */ 215 | #define HAMMER2_LOOKUP_NODATA 0x00000002 /* data left NULL */ 216 | #define HAMMER2_LOOKUP_NODIRECT 0x00000004 /* no offset=0 DD */ 217 | #define HAMMER2_LOOKUP_SHARED 0x00000100 218 | #define HAMMER2_LOOKUP_MATCHIND 0x00000200 /* return all chains */ 219 | #define HAMMER2_LOOKUP_ALLNODES 0x00000400 /* allow NULL focus */ 220 | #define HAMMER2_LOOKUP_ALWAYS 0x00000800 /* resolve data */ 221 | #define HAMMER2_LOOKUP_NOUNLOCK 0x00001000 /* leave lock intact */ 222 | 223 | /* 224 | * Flags passed to hammer2_chain_modify() and hammer2_chain_resize() 225 | * 226 | * NOTE: OPTDATA allows us to avoid instantiating buffers for INDIRECT 227 | * blocks in the INITIAL-create state. 228 | */ 229 | #define HAMMER2_MODIFY_OPTDATA 0x00000002 /* data can be NULL */ 230 | #define HAMMER2_MODIFY_NO_MODIFY_TID 0x00000004 231 | #define HAMMER2_MODIFY_UNUSED0008 0x00000008 232 | 233 | /* 234 | * Flags passed to hammer2_chain_lock() 235 | * 236 | * NOTE: RDONLY is set to optimize cluster operations when *no* modifications 237 | * will be made to either the cluster being locked or any underlying 238 | * cluster. It allows the cluster to lock and access data for a subset 239 | * of available nodes instead of all available nodes. 240 | */ 241 | #define HAMMER2_RESOLVE_NEVER 1 242 | #define HAMMER2_RESOLVE_MAYBE 2 243 | #define HAMMER2_RESOLVE_ALWAYS 3 244 | #define HAMMER2_RESOLVE_MASK 0x0F 245 | 246 | #define HAMMER2_RESOLVE_SHARED 0x10 /* request shared lock */ 247 | #define HAMMER2_RESOLVE_LOCKAGAIN 0x20 /* another shared lock */ 248 | #define HAMMER2_RESOLVE_RDONLY 0x40 /* higher level op flag */ 249 | 250 | /* 251 | * Flags passed to hammer2_chain_delete() 252 | */ 253 | #define HAMMER2_DELETE_PERMANENT 0x0001 254 | 255 | /* 256 | * Flags passed to hammer2_chain_insert() or hammer2_chain_rename() 257 | * or hammer2_chain_create(). 258 | */ 259 | #define HAMMER2_INSERT_PFSROOT 0x0004 260 | #define HAMMER2_INSERT_SAMEPARENT 0x0008 261 | 262 | /* 263 | * Flags passed to hammer2_chain_delete_duplicate() 264 | */ 265 | #define HAMMER2_DELDUP_RECORE 0x0001 266 | 267 | /* 268 | * Cluster different types of storage together for allocations 269 | */ 270 | #define HAMMER2_FREECACHE_INODE 0 271 | #define HAMMER2_FREECACHE_INDIR 1 272 | #define HAMMER2_FREECACHE_DATA 2 273 | #define HAMMER2_FREECACHE_UNUSED3 3 274 | #define HAMMER2_FREECACHE_TYPES 4 275 | 276 | /* 277 | * hammer2_freemap_alloc() block preference 278 | */ 279 | #define HAMMER2_OFF_NOPREF ((hammer2_off_t)-1) 280 | 281 | /* 282 | * BMAP read-ahead maximum parameters 283 | */ 284 | #define HAMMER2_BMAP_COUNT 16 /* max bmap read-ahead */ 285 | #define HAMMER2_BMAP_BYTES (HAMMER2_PBUFSIZE * HAMMER2_BMAP_COUNT) 286 | 287 | /* 288 | * hammer2_freemap_adjust() 289 | */ 290 | #define HAMMER2_FREEMAP_DORECOVER 1 291 | #define HAMMER2_FREEMAP_DOMAYFREE 2 292 | #define HAMMER2_FREEMAP_DOREALFREE 3 293 | 294 | /* 295 | * HAMMER2 cluster - A set of chains representing the same entity. 296 | * 297 | * hammer2_cluster typically represents a temporary set of representitive 298 | * chains. The one exception is that a hammer2_cluster is embedded in 299 | * hammer2_inode. This embedded cluster is ONLY used to track the 300 | * representitive chains and cannot be directly locked. 301 | * 302 | * A cluster is usually temporary (and thus per-thread) for locking purposes, 303 | * allowing us to embed the asynchronous storage required for cluster 304 | * operations in the cluster itself and adjust the state and status without 305 | * having to worry too much about SMP issues. 306 | * 307 | * The exception is the cluster embedded in the hammer2_inode structure. 308 | * This is used to cache the cluster state on an inode-by-inode basis. 309 | * Individual hammer2_chain structures not incorporated into clusters might 310 | * also stick around to cache miscellanious elements. 311 | * 312 | * Because the cluster is a 'working copy' and is usually subject to cluster 313 | * quorum rules, it is quite possible for us to end up with an insufficient 314 | * number of live chains to execute an operation. If an insufficient number 315 | * of chains remain in a working copy, the operation may have to be 316 | * downgraded, retried, stall until the requisit number of chains are 317 | * available, or possibly even error out depending on the mount type. 318 | * 319 | * A cluster's focus is set when it is locked. The focus can only be set 320 | * to a chain still part of the synchronized set. 321 | */ 322 | #define HAMMER2_XOPFIFO 16 323 | #define HAMMER2_XOPFIFO_MASK (HAMMER2_XOPFIFO - 1) 324 | #define HAMMER2_XOPGROUPS 32 325 | #define HAMMER2_XOPGROUPS_MASK (HAMMER2_XOPGROUPS - 1) 326 | 327 | #define HAMMER2_MAXCLUSTER 8 328 | #define HAMMER2_XOPMASK_CLUSTER (uint64_t)((1LLU << HAMMER2_MAXCLUSTER) - 1) 329 | #define HAMMER2_XOPMASK_VOP (uint64_t)0x0000000080000000LLU 330 | #define HAMMER2_XOPMASK_FIFOW (uint64_t)0x0000000040000000LLU 331 | #define HAMMER2_XOPMASK_WAIT (uint64_t)0x0000000020000000LLU 332 | #define HAMMER2_XOPMASK_FEED (uint64_t)0x0000000100000000LLU 333 | 334 | #define HAMMER2_XOPMASK_ALLDONE (HAMMER2_XOPMASK_VOP | HAMMER2_XOPMASK_CLUSTER) 335 | 336 | #define HAMMER2_SPECTHREADS 1 /* sync */ 337 | 338 | /* 339 | * Support structure for dedup heuristic. 340 | */ 341 | struct hammer2_dedup { 342 | hammer2_off_t data_off; 343 | u64int data_crc; 344 | u32int ticks; 345 | u32int unused03; 346 | }; 347 | 348 | typedef struct hammer2_dedup hammer2_dedup_t; 349 | 350 | struct hammer2_dev { 351 | int volhdrno; /* last volhdrno written */ 352 | hammer2_volume_data_t voldata; 353 | hammer2_volume_data_t volsync; /* synchronized voldata */ 354 | }; 355 | 356 | typedef struct hammer2_dev hammer2_dev_t; 357 | 358 | /* 359 | * Per-cluster management structure. This structure will be tied to a 360 | * system mount point if the system is mounting the PFS, but is also used 361 | * to manage clusters encountered during the super-root scan or received 362 | * via LNK_SPANs that might not be mounted. 363 | * 364 | * This structure is also used to represent the super-root that hangs off 365 | * of a hard mount point. The super-root is not really a cluster element. 366 | * In this case the spmp_hmp field will be non-NULL. It's just easier to do 367 | * this than to special case super-root manipulation in the hammer2_chain* 368 | * code as being only hammer2_dev-related. 369 | * 370 | * pfs_mode and pfs_nmasters are rollup fields which critically describes 371 | * how elements of the cluster act on the cluster. pfs_mode is only applicable 372 | * when a PFS is mounted by the system. pfs_nmasters is our best guess as to 373 | * how many masters have been configured for a cluster and is always 374 | * applicable. pfs_types[] is an array with 1:1 correspondance to the 375 | * iroot cluster and describes the PFS types of the nodes making up the 376 | * cluster. 377 | * 378 | * WARNING! Portions of this structure have deferred initialization. In 379 | * particular, if not mounted there will be no wthread. 380 | * umounted network PFSs will also be missing iroot and numerous 381 | * other fields will not be initialized prior to mount. 382 | * 383 | * Synchronization threads are chain-specific and only applicable 384 | * to local hard PFS entries. A hammer2_pfs structure may contain 385 | * more than one when multiple hard PFSs are present on the local 386 | * machine which require synchronization monitoring. Most PFSs 387 | * (such as snapshots) are 1xMASTER PFSs which do not need a 388 | * synchronization thread. 389 | * 390 | * WARNING! The chains making up pfs->iroot's cluster are accounted for in 391 | * hammer2_dev->mount_count when the pfs is associated with a mount 392 | * point. 393 | */ 394 | struct hammer2_pfs { 395 | struct mount *mp; 396 | }; 397 | 398 | typedef struct hammer2_pfs hammer2_pfs_t; 399 | 400 | TAILQ_HEAD(hammer2_pfslist, hammer2_pfs); 401 | 402 | #define HAMMER2_LRU_LIMIT 1024 /* per pmp lru_list */ 403 | 404 | #define HAMMER2_DIRTYCHAIN_WAITING 0x80000000 405 | #define HAMMER2_DIRTYCHAIN_MASK 0x7FFFFFFF 406 | 407 | #define HAMMER2_LWINPROG_WAITING 0x80000000 408 | #define HAMMER2_LWINPROG_WAITING0 0x40000000 409 | #define HAMMER2_LWINPROG_MASK 0x3FFFFFFF 410 | 411 | /* 412 | * hammer2_cluster_check 413 | */ 414 | #define HAMMER2_CHECK_NULL 0x00000001 415 | 416 | /* copied from hammer2_xxhash.h, we don't need the whole header */ 417 | #define XXH_HAMMER2_SEED 0x4d617474446c6c6eLLU 418 | 419 | -------------------------------------------------------------------------------- /9p.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include <9p.h> 8 | 9 | // for sha2_256 10 | #include 11 | #include 12 | 13 | #include "uuid.h" 14 | #include "hammer2_disk.h" 15 | #include "hammer2.h" 16 | #include "9phammer.h" 17 | #include "lz4.h" 18 | #include "xxhash.h" 19 | #include "faketypes.h" 20 | 21 | char *filename; 22 | int devfd; 23 | 24 | static ulong *crctab; 25 | void fileread(Req *r); 26 | 27 | void loadinodes(inode i, DirEnts *dirents); 28 | int verifycheck(hammer2_blockref_t *block, void *data); 29 | char* loadblock(hammer2_blockref_t *block, void *dst, int dstsize, int *rsize); 30 | 31 | root_t root; 32 | 33 | 34 | typedef struct{ 35 | hammer2_tid_t inum; 36 | hammer2_blockref_t block; 37 | } FEntry; 38 | 39 | typedef struct fileblocklist fileblocklist_t; 40 | struct fileblocklist { 41 | hammer2_key_t start; 42 | hammer2_key_t end; 43 | 44 | hammer2_blockref_t* datablock; 45 | fileblocklist_t *next; 46 | }; 47 | 48 | u64int maxinodes; 49 | FEntry *inodes; 50 | 51 | Qid makeqid(hammer2_tid_t inum, uchar type) { 52 | Qid r = { 53 | .path inum, 54 | .type type, 55 | }; 56 | return r; 57 | } 58 | 59 | 60 | // Gets a cached FEntry from the list, which must have already had its metadata 61 | // populated with mkinode 62 | FEntry* getfentry(Qid q) { 63 | if (q.path > maxinodes) { 64 | return nil; 65 | } 66 | return &inodes[q.path]; 67 | } 68 | 69 | // Creates an FEntry for an inode and adds it to the cached list. 70 | FEntry* mkinode(hammer2_tid_t inum, hammer2_blockref_t block){ 71 | if (inum >= maxinodes) { 72 | // FIXME: Be more intelligent about reallocs. We realloc twice 73 | // as many to minimize the number of reallocs needed. 74 | inodes = realloc(inodes, (inum*2)*sizeof(FEntry)); 75 | maxinodes = inum*2; 76 | } 77 | FEntry *dir = &inodes[inum]; 78 | dir->inum = inum; 79 | dir->block = block; 80 | return dir; 81 | } 82 | 83 | void fsstart(Srv *) { 84 | int i, j; 85 | hammer2_dev_t hddev; 86 | hammer2_inode_data_t suproot; 87 | // FIXME: don't hardcode this; 88 | devfd = open(filename, OREAD); 89 | readvolume(devfd, &hddev); 90 | hammer2_volume_data_t vol = hddev.voldata; 91 | for(i =0; i < HAMMER2_SET_COUNT; i++) { 92 | if (vol.sroot_blockset.blockref[i].type == HAMMER2_BREF_TYPE_INODE) { 93 | loadinode(&vol.sroot_blockset.blockref[i], &suproot); 94 | 95 | if (suproot.meta.pfs_type == HAMMER2_PFSTYPE_SUPROOT) { 96 | // found the superroot, now found the mountable root 97 | for(j = 0; j < 4; j++) { 98 | if (suproot.u.blockset.blockref[j].type == HAMMER2_BREF_TYPE_INODE) { 99 | loadinode(&suproot.u.blockset.blockref[j], &root.inode); 100 | if (strcmp(root.pfsname, (char *)root.inode.filename) == 0) { 101 | root.Qid = makeqid(root.meta.inum, QTDIR); 102 | maxinodes = suproot.u.blockset.blockref[j].embed.stats.inode_count; 103 | inodes = emalloc9p(maxinodes*sizeof(FEntry)); 104 | root.DirEnts.cap = 0; 105 | root.DirEnts.count = 0; 106 | mkinode(root.meta.inum, suproot.u.blockset.blockref[j]); 107 | loadinodes(root, &root.DirEnts); 108 | return; 109 | } 110 | } 111 | } 112 | } 113 | } 114 | } 115 | sysfatal("Could not find root %s", root.pfsname); 116 | } 117 | 118 | typedef struct Aux Aux; 119 | struct Aux { 120 | Qid; 121 | RWLock; 122 | // The currently walked to inode 123 | hammer2_inode_data_t *inode; 124 | 125 | // If loading an indirect block, the parent is an Aux structure 126 | // that caused this indirection. 127 | Aux *parent; 128 | 129 | // The start of this indirection layer 130 | hammer2_blockref_t *blocks; 131 | // The current offset and index into blocks 132 | int offset; 133 | int count; 134 | 135 | union { 136 | struct { 137 | uchar *lastbuf; 138 | vlong lastbufoffset; 139 | int lastbufcount; 140 | 141 | fileblocklist_t *datablocks; 142 | } file; 143 | 144 | DirEnts dir; 145 | } cache; 146 | }; 147 | 148 | void fsattach(Req *r) { 149 | Aux *a; 150 | a = emalloc9p(sizeof(Aux)); 151 | a->inode = &root.inode; 152 | a->parent = nil; 153 | a->blocks = &(root.u.blockset.blockref[0]); 154 | a->offset = 0; 155 | a->count = 4; 156 | 157 | a->cache.dir = root.DirEnts; 158 | 159 | a->Qid = root.Qid; 160 | r->fid->qid = root.Qid; 161 | r->ofcall.qid = root.Qid; 162 | r->fid->aux = a; 163 | respond(r, nil); 164 | } 165 | 166 | // Load all the inodes stored under i into the inodes array 167 | void loadinodes(inode i, DirEnts *dirents) { 168 | Aux orig; 169 | Aux *cur; 170 | 171 | orig.parent = nil; 172 | orig.inode = &i; 173 | orig.blocks = &i.u.blockset.blockref[0]; 174 | orig.offset = 0; 175 | orig.count = 4; 176 | cur = &orig; 177 | 178 | start: 179 | while(cur->offset >= cur->count) { 180 | if (cur->parent == nil) { 181 | return; 182 | } 183 | Aux *tmp = cur; 184 | cur = cur->parent; 185 | free(tmp->blocks); 186 | free(tmp); 187 | } 188 | 189 | hammer2_blockref_t *block = &cur->blocks[cur->offset]; 190 | 191 | // Things for indirect blocks. 192 | int radix; 193 | int dsize; 194 | int indblocks; 195 | Aux *tmp; 196 | char *err; 197 | switch (block->type) { 198 | case HAMMER2_BREF_TYPE_INODE: 199 | mkinode(block->key, *block); 200 | cur->offset++; 201 | goto start; 202 | case HAMMER2_BREF_TYPE_EMPTY: 203 | cur->offset++; 204 | goto start; 205 | case HAMMER2_BREF_TYPE_DIRENT: 206 | if (dirents != nil) { 207 | if (dirents->cap == 0) { 208 | // Arbitrarily start with enough space for 16 209 | // directories. 210 | dirents->cap = 16; 211 | dirents->entry = emalloc9p(sizeof(hammer2_blockref_t)*16); 212 | } else if (dirents->cap == dirents->count) { 213 | dirents->cap *= 2; 214 | dirents->entry = erealloc9p(dirents->entry, dirents->cap*sizeof(hammer2_blockref_t)); 215 | } 216 | dirents->entry[dirents->count++] = *block; 217 | } 218 | cur->offset++; 219 | goto start; 220 | case HAMMER2_BREF_TYPE_INDIRECT: 221 | radix = block->data_off & HAMMER2_OFF_MASK_RADIX; 222 | dsize = 1<offset++; 229 | tmp = cur; 230 | 231 | cur = emalloc9p(sizeof(Aux)); 232 | 233 | cur->inode = &i; 234 | cur->parent = tmp; 235 | cur->blocks = emalloc9p(HAMMER2_PBUFSIZE); 236 | assert(cur->blocks != nil); 237 | cur->offset = 0; 238 | cur->count = indblocks; 239 | 240 | // Load the data from disk and start again at the new level 241 | // of indirection. 242 | err = loadblock(block, cur->blocks, dsize, nil); 243 | if (err != nil) { 244 | fprint(2, "loading err: %s\n", err); 245 | } 246 | goto start; 247 | default: 248 | // For now assume if we got here something went wrong. 249 | printf("Type %d\n", block->type); 250 | sysfatal("Unhandled block type in directory"); 251 | } 252 | } 253 | 254 | int cacheddirread(int n, Dir *dir, void *aux) { 255 | Aux *a = aux; 256 | if (n < 0 || n >= a->cache.dir.count) { 257 | return -1; 258 | } 259 | 260 | hammer2_blockref_t block = a->cache.dir.entry[n]; 261 | 262 | dir->qid = makeqid(block.embed.dirent.inum, 0); 263 | FEntry *f = getfentry(dir->qid); 264 | if (f == nil) { 265 | sysfatal("could not load inode"); 266 | } 267 | 268 | inode i; 269 | loadinode(&f->block, &i); 270 | switch (i.meta.type) { 271 | case HAMMER2_OBJTYPE_DIRECTORY: 272 | dir->qid.type = QTDIR; 273 | break; 274 | case HAMMER2_OBJTYPE_REGFILE: 275 | case HAMMER2_OBJTYPE_SOFTLINK: 276 | dir->qid.type = QTFILE; 277 | break; 278 | default: 279 | printf("type %d\n", i.meta.type); 280 | sysfatal("Unhandled OBJTYPE"); 281 | } 282 | dir->mode = i.meta.mode; 283 | if (dir->qid.type == QTDIR){ 284 | dir->mode |= DMDIR; 285 | } 286 | dir->atime = i.meta.atime / 1000000; // unsupported. 287 | dir->mtime = i.meta.mtime / 1000000; // hammer2 is nanoseconds, 9p is seconds 288 | dir->length = i.meta.size; 289 | if (block.embed.dirent.namlen <= 64) 290 | dir->name = estrdup9p(block.check.buf); 291 | else { 292 | char *err; 293 | char data[HAMMER2_BLOCKREF_LEAF_MAX+1]; 294 | int size; 295 | err = loadblock(&block, data, HAMMER2_BLOCKREF_LEAF_MAX, &size); 296 | if (err != nil){ 297 | fprint(2, "%s\n", err); 298 | } 299 | dir->name = estrdup9p(data); 300 | } 301 | 302 | // FIXME: Get from uid/gid from inode and parse /etc/passwd ( 303 | // or add an /adm/users?) 304 | dir->uid = estrdup9p(getuser()); 305 | dir->gid = estrdup9p(getuser()); 306 | return 0; 307 | } 308 | 309 | Qid loadsubdir(Aux *a, char *name) { 310 | int i; 311 | Qid r; 312 | char namedata[HAMMER2_BLOCKREF_LEAF_MAX+1]; 313 | for(i=0; i < a->cache.dir.count; i++){ 314 | char *ename; 315 | hammer2_blockref_t block = a->cache.dir.entry[i]; 316 | if (block.embed.dirent.namlen > 64) { 317 | char *err; 318 | int size; 319 | err = loadblock(&block, namedata, HAMMER2_BLOCKREF_LEAF_MAX, &size); 320 | if (err != nil){ 321 | fprint(2, "%s\n", err); 322 | } 323 | ename = namedata; 324 | } else { 325 | ename = block.check.buf; 326 | } 327 | 328 | if (strcmp(name, ename) == 0) { 329 | inode in; 330 | FEntry *fe; 331 | r = makeqid(block.embed.dirent.inum, 0); 332 | fe = getfentry(r); 333 | if (fe == nil) { 334 | sysfatal("could not load inode"); 335 | } 336 | loadinode(&fe->block, &in); 337 | switch (in.meta.type) { 338 | case HAMMER2_OBJTYPE_DIRECTORY: 339 | r.type = QTDIR; 340 | return r; 341 | case HAMMER2_OBJTYPE_REGFILE: 342 | case HAMMER2_OBJTYPE_SOFTLINK: 343 | r.type = QTFILE; 344 | return r; 345 | default: 346 | printf("%s type %d\n", name, in.meta.type); 347 | sysfatal("Unhandled OBJTYPE"); 348 | } 349 | } 350 | } 351 | r.path = 0; 352 | r.vers = 0; 353 | r.type = 0; 354 | return r; 355 | } 356 | 357 | char* fswalk(Fid *fid, char *name, Qid *qid) { 358 | Aux *a; 359 | Qid q; 360 | a = fid->aux; 361 | if (strcmp(name, "/") == 0) { 362 | fid->qid = root.Qid; 363 | a->inode = &root.inode; 364 | a->parent = nil; 365 | a->blocks = &(root.u.blockset.blockref[0]); 366 | a->offset = 0; 367 | a->count = 4; 368 | a->cache.dir = root.DirEnts; 369 | *qid = root.Qid; 370 | return nil; 371 | } else if (strcmp(name, "..") == 0) { 372 | // The parent should always be a directory. 373 | q = makeqid(a->inode->meta.iparent, QTDIR); 374 | } else if (strcmp(name, ".") == 0) { 375 | *qid = a->Qid; 376 | fid->qid = a->Qid; 377 | return nil; 378 | } else { 379 | q = loadsubdir(a, name); 380 | if (q.path == 0) { 381 | return "not found"; 382 | } 383 | } 384 | FEntry *fe = getfentry(q); 385 | 386 | a->inode = emalloc9p(sizeof(inode)); 387 | 388 | loadinode(&fe->block, a->inode); 389 | 390 | a->parent = nil; 391 | a->blocks = &(a->inode->u.blockset.blockref[0]); 392 | a->offset = 0; 393 | a->count = 4; 394 | switch(q.type){ 395 | case QTDIR: 396 | a->cache.dir.cap = 0; 397 | a->cache.dir.count = 0; 398 | a->cache.dir.entry = nil; 399 | loadinodes(*a->inode, &a->cache.dir); 400 | break; 401 | case QTFILE: 402 | a->cache.file.lastbuf = nil; 403 | a->cache.file.lastbufcount = 0; 404 | break; 405 | default: 406 | return "unhandled qid type"; 407 | } 408 | 409 | fid->qid = q; 410 | *qid = q; 411 | return nil; 412 | } 413 | 414 | char* fswalkclone(Fid *old, Fid *new) { 415 | // FIXME: Fix memory leak by implementing destroy. 416 | Aux *oaux = old->aux; 417 | Aux *naux = emalloc9p(sizeof(Aux)); 418 | new->aux = naux; 419 | memcpy(naux, oaux, sizeof(Aux)); 420 | 421 | // Cache can be freed behind our back, so don't reuse it. 422 | int bsize; 423 | switch (old->qid.type){ 424 | case QTDIR: 425 | // We don't leave extra capacity since we know how many entries 426 | // are in the directory and it's unlikely to change. 427 | bsize = oaux->cache.dir.count*sizeof(hammer2_blockref_t); 428 | naux->cache.dir.cap = oaux->cache.dir.count; 429 | naux->cache.dir.count = oaux->cache.dir.count; 430 | memcpy(naux->cache.dir.entry, oaux->cache.dir.entry, bsize); 431 | break; 432 | case QTFILE: 433 | naux->cache.file.lastbuf = nil; 434 | naux->cache.file.lastbufcount = 0; 435 | break; 436 | } 437 | return nil; 438 | } 439 | 440 | typedef struct fileblocklist fileblocklist_t; 441 | 442 | fileblocklist_t* loaddatablocklist(inode *in) { 443 | fileblocklist_t* headn = nil; 444 | fileblocklist_t* curn = nil; 445 | if (in->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA) { 446 | return nil; 447 | } 448 | 449 | Aux *orig = emalloc9p(sizeof(Aux)); 450 | Aux *cur; 451 | orig->parent = nil; 452 | 453 | // Make a copy of the blockset so that we don't can just assume that 454 | // everything needs to be freed in the end. 455 | orig->blocks = emalloc9p(sizeof(hammer2_blockset_t)); 456 | memcpy(orig->blocks, &in->u.blockset, sizeof(hammer2_blockset_t)); 457 | orig->offset = 0; 458 | orig->count = 4; 459 | cur = orig; 460 | 461 | 462 | Aux *tmp; 463 | char *err; 464 | int nbytes; 465 | int radix; 466 | int chksize; 467 | 468 | while(cur != nil){ 469 | nextlevel: 470 | for(int i = cur->offset; i < cur->count; i++) { 471 | // save i at the next block for the next indirection level 472 | // to pick up where we left off when there's an indirect 473 | // block. 474 | // at the same place. 475 | cur->offset = i+1; 476 | 477 | hammer2_blockref_t *block = &cur->blocks[i]; 478 | 479 | switch(block->type){ 480 | case HAMMER2_BREF_TYPE_DATA: 481 | // add a new fileblocklist_t to the end of the 482 | // list with the current block. 483 | if(headn == nil) { 484 | headn = emalloc9p(sizeof(fileblocklist_t)); 485 | curn = headn; 486 | } else { 487 | curn->next = emalloc9p(sizeof(fileblocklist_t)); 488 | curn = curn->next; 489 | } 490 | curn->next = nil; 491 | curn->start = block->key; 492 | curn->end = block->key + (1<keybits); 493 | if(curn->end > in->meta.size){ 494 | curn->end = in->meta.size; 495 | } 496 | 497 | // Make a copy so that this indirection level (including 498 | // any empty or indirect ones that we don't care about 499 | // after our list is complete) can be freed when we. 500 | // exit. 501 | curn->datablock = emalloc9p(sizeof(hammer2_blockref_t)); 502 | memcpy(curn->datablock, block, sizeof(hammer2_blockref_t)); 503 | 504 | break; 505 | case HAMMER2_BREF_TYPE_INDIRECT: 506 | tmp = cur; 507 | cur = emalloc9p(sizeof(Aux)); 508 | cur->parent = tmp; 509 | 510 | cur->blocks = emalloc9p(HAMMER2_PBUFSIZE); 511 | cur->offset = 0; 512 | 513 | radix = block->data_off & HAMMER2_OFF_MASK_RADIX; 514 | chksize = 1<count = chksize / sizeof(hammer2_blockref_t); 516 | 517 | err = loadblock(block, cur->blocks, chksize, &nbytes); 518 | if(err != nil){ 519 | sysfatal(err); 520 | } 521 | goto nextlevel; 522 | case HAMMER2_BREF_TYPE_EMPTY: 523 | // don't care 524 | break; 525 | default: 526 | sysfatal("Unhandled BREF type while loading file"); 527 | } 528 | } 529 | 530 | tmp = cur; 531 | cur = cur->parent; 532 | 533 | free(tmp->blocks); 534 | free(tmp); 535 | } 536 | return headn; 537 | } 538 | 539 | void fsopen(Req *r) { 540 | Aux *a = r->fid->aux; 541 | if (r->fid->qid.path == root.Qid.path) { 542 | a->cache.dir = root.DirEnts; 543 | } 544 | if (r->fid->qid.type == QTFILE){ 545 | inode *i = a->inode; 546 | if (i->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA) { 547 | // Nothing, content is embedded in inode 548 | } else { 549 | a->cache.file.datablocks = loaddatablocklist(i); 550 | } 551 | 552 | } 553 | respond(r, nil); 554 | } 555 | 556 | void fsstat(Req *r) { 557 | if (r->fid->qid.path == root.Qid.path) { 558 | r->d.mode = DMREAD | DMEXEC | DMDIR; 559 | //r->d.name = estrdup9p(getuser()); 560 | r->d.uid = estrdup9p(getuser()); 561 | r->d.gid = estrdup9p(getuser()); 562 | r->d.qid = r->fid->qid; 563 | 564 | respond(r, nil); 565 | return; 566 | } 567 | FEntry *f = getfentry(r->fid->qid); 568 | if (f == nil) { 569 | respond(r, "not found"); 570 | return; 571 | } 572 | 573 | inode i; 574 | loadinode(&f->block, &i); 575 | r->d.qid = r->fid->qid; 576 | r->d.mode = i.meta.mode; 577 | if (r->fid->qid.type == QTDIR){ 578 | r->d.mode |= DMDIR; 579 | } 580 | r->d.atime = i.meta.atime / 1000000; 581 | r->d.mtime = i.meta.mtime / 1000000; 582 | r->d.length = i.meta.size; 583 | r->d.uid = estrdup9p(getuser()); 584 | r->d.gid = estrdup9p(getuser()); 585 | 586 | respond(r, nil); 587 | } 588 | 589 | void fsread(Req *r) { 590 | switch(r->fid->qid.type) { 591 | case QTDIR: 592 | // Walking to the dir should have cached the dirents in 593 | // fid->aux.cache.dir 594 | dirread9p(r, cacheddirread, r->fid->aux); 595 | break; 596 | case QTFILE: 597 | fileread(r); 598 | return; 599 | default: 600 | printf("Type: %d\n", r->fid->qid.type); 601 | sysfatal("Unhandled QID type."); 602 | } 603 | respond(r, nil); 604 | } 605 | 606 | void fileread(Req *r) { 607 | Aux *a = r->fid->aux; 608 | // Reading at/past EOF. 609 | if (r->ifcall.offset == a->inode->meta.size) { 610 | r->ofcall.count = 0; 611 | respond(r, nil); 612 | return; 613 | } else if(r->ifcall.offset > a->inode->meta.size) { 614 | r->ofcall.count = -1; 615 | respond(r, "read past end of file"); 616 | return; 617 | } 618 | 619 | // If data is stored in the inode, don't bother getting anything from 620 | // disk. 621 | if (a->inode->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA) { 622 | assert(a->inode->meta.size > r->ifcall.offset); 623 | assert(r->ifcall.offset < HAMMER2_EMBEDDED_BYTES); 624 | int size = a->inode->meta.size - r->ifcall.offset; 625 | memcpy(r->ofcall.data, (void *)&a->inode->u.data[r->ifcall.offset], size); 626 | r->ofcall.count = size; 627 | respond(r, nil); 628 | return; 629 | } 630 | 631 | // Check if it's in the data read from the last block before doing 632 | // anything. 633 | rlock(a); 634 | if (a->cache.file.lastbufcount > 0 635 | && r->ifcall.offset >= a->cache.file.lastbufoffset 636 | && r->ifcall.offset < a->cache.file.lastbufcount + a->cache.file.lastbufoffset 637 | ){ 638 | // It's still cached from the last read, so just copy it from memory. 639 | int count = r->ifcall.count; 640 | int offstart = r->ifcall.offset - a->cache.file.lastbufoffset; 641 | if (count + r->ifcall.offset > a->inode->meta.size) { 642 | // Ensure we don't send past EOF. 643 | count = a->inode->meta.size - r->ifcall.offset; 644 | } 645 | if (count + offstart > a->cache.file.lastbufcount) { 646 | // We can only send as many bytes as are in lastbuf. 647 | count = a->cache.file.lastbufcount - offstart; 648 | } 649 | assert(count > 0); 650 | assert(offstart + count <= a->cache.file.lastbufcount); 651 | 652 | memcpy(r->ofcall.data, (uchar *)&a->cache.file.lastbuf[offstart],count); 653 | runlock(a); 654 | r->ofcall.count = count; 655 | respond(r, nil); 656 | return; 657 | } 658 | runlock(a); 659 | 660 | hammer2_blockref_t *block = nil; 661 | 662 | // for autozero 663 | fileblocklist_t *cur; 664 | hammer2_key_t nearest = a->inode->meta.size; 665 | for(cur = a->cache.file.datablocks; cur != nil; cur = cur->next){ 666 | nearest = cur->start; 667 | assert(cur->end > cur->start); 668 | if(r->ifcall.offset >= cur->start && r->ifcall.offset < cur->end){ 669 | // Found the block. 670 | block = cur->datablock; 671 | break; 672 | } else if (cur->start > r->ifcall.offset) { 673 | // Passed the block and never found it. 674 | // FIXME: This assumes the list is sorted, but we haven't 675 | // done anything to guarantee that. (It seems to be the 676 | // case on disk from DragonFly, regardless. 677 | nearest = cur->start; 678 | break; 679 | } 680 | } 681 | if(block == nil){ 682 | // No block was found, so fill the zero hole. 683 | assert(nearest > r->ifcall.offset); 684 | int count = nearest-r->ifcall.offset; 685 | if(count > r->ifcall.count) { 686 | count = r->ifcall.count; 687 | } 688 | assert(count > 0); 689 | r->ofcall.count = count; 690 | memset(r->ofcall.data, 0, count); 691 | respond(r, nil); 692 | return; 693 | } 694 | 695 | wlock(a); 696 | a->cache.file.lastbuf = realloc(a->cache.file.lastbuf, HAMMER2_BLOCKREF_LEAF_MAX+1); 697 | 698 | char *err = loadblock(block, a->cache.file.lastbuf, HAMMER2_BLOCKREF_LEAF_MAX+1, &a->cache.file.lastbufcount); 699 | if (err != nil) { 700 | a->cache.file.lastbufcount = 0; 701 | 702 | respond(r, err); 703 | wunlock(a); 704 | return; 705 | } 706 | a->cache.file.lastbufoffset = block->key; 707 | assert(a->cache.file.lastbuf != nil); 708 | 709 | int roffset = r->ifcall.offset-block->key; 710 | int count = r->ifcall.count; 711 | if(r->ifcall.offset + count > cur->end){ 712 | count = cur->end-r->ifcall.offset; 713 | } 714 | assert(count > 0); 715 | memcpy(r->ofcall.data, &a->cache.file.lastbuf[roffset], count); 716 | r->ofcall.count = count; 717 | respond(r, nil); 718 | wunlock(a); 719 | return; 720 | } 721 | 722 | int verifycheck(hammer2_blockref_t *block, void *data) { 723 | int radix = block->data_off & HAMMER2_OFF_MASK_RADIX; 724 | int size = 1<methods)){ 728 | case HAMMER2_CHECK_NONE: 729 | case HAMMER2_CHECK_DISABLED: 730 | r = 1; 731 | break; 732 | case HAMMER2_CHECK_ISCSI32: 733 | r = (icrc32(data, size) == block->check.iscsi32.value); 734 | break; 735 | case HAMMER2_CHECK_SHA192: 736 | // I have no idea what the connection between this and sha192 737 | // is, but it's the algorithm that DragonFly uses to calculate 738 | // the sha192 hash in hammer2_chain.c. 739 | { 740 | union { 741 | uchar digest8[SHA2_256dlen]; 742 | uvlong digest64[SHA2_256dlen/8]; 743 | } digest; 744 | 745 | sha2_256(data, size, digest.digest8, nil); 746 | digest.digest64[2] ^= digest.digest64[3]; 747 | r = (memcmp(digest.digest8, block->check.sha192.data, 24) == 0); 748 | } 749 | break; 750 | case HAMMER2_CHECK_XXHASH64: 751 | r = (XXH64(data, size, XXH_HAMMER2_SEED) == block->check.xxhash64.value); 752 | break; 753 | case HAMMER2_CHECK_FREEMAP: 754 | // we shouldn't encounter a freemap while reading a file.. 755 | assert(0); 756 | break; 757 | default: 758 | r = 0; 759 | } 760 | return r; 761 | } 762 | 763 | /* Loads the block pointed to at data_off into dst (after decompression), and 764 | stores the size in rsize. 765 | Returns an error string if smething went wrong. 766 | Also validates check code */ 767 | char* loadblock(hammer2_blockref_t *block, void *dst, int dstsize, int *rsize) { 768 | uchar blockdata[HAMMER2_BLOCKREF_LEAF_MAX+1]; 769 | int dsize = 1<<(block->data_off & HAMMER2_OFF_MASK_RADIX); 770 | int off = block->data_off & HAMMER2_OFF_MASK_LO; 771 | int csize; 772 | int decsize; 773 | 774 | pread(devfd, blockdata, HAMMER2_BLOCKREF_LEAF_MAX+1, block->data_off & HAMMER2_OFF_MASK_HI); 775 | if (!verifycheck(block, &blockdata[off])) { 776 | return "invalid checksum"; 777 | } 778 | switch (HAMMER2_DEC_COMP(block->methods)){ 779 | case HAMMER2_COMP_AUTOZERO: 780 | case HAMMER2_COMP_NONE: 781 | if (rsize != nil) 782 | *rsize = dsize; 783 | memcpy(dst, &blockdata[off] , dsize); 784 | break; 785 | case HAMMER2_COMP_LZ4: 786 | csize = *(int*) &blockdata[off]; 787 | decsize = LZ4_decompress_safe((char *)blockdata+off+4, (char *)dst, csize, dstsize); 788 | if (decsize < 0) 789 | return "bad read"; 790 | if (rsize != nil) 791 | *rsize = decsize; 792 | break; 793 | case HAMMER2_COMP_ZLIB: 794 | inflateinit(); 795 | decsize = inflatezlibblock( 796 | dst, dstsize, 797 | &blockdata[off], HAMMER2_BLOCKREF_LEAF_MAX-off 798 | ); 799 | if (decsize < 0){ 800 | return flateerr(*rsize); 801 | } 802 | if (rsize != nil) 803 | *rsize = decsize; 804 | break; 805 | default: 806 | printf("Comp: %d\n", HAMMER2_DEC_COMP(block->methods)); 807 | return "Unhandled compression"; 808 | } 809 | return nil; 810 | } 811 | 812 | hammer2_crc32_t icrc32(void *data, int size) { 813 | if(crctab == nil) 814 | crctab = mkcrctab(0x82f63b78); 815 | return blockcrc(crctab, 0, data, size); 816 | } 817 | 818 | -------------------------------------------------------------------------------- /xxhash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xxHash - Fast Hash algorithm 3 | * Copyright (C) 2012-2016, Yann Collet 4 | * 5 | * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are 9 | * met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above 14 | * copyright notice, this list of conditions and the following disclaimer 15 | * in the documentation and/or other materials provided with the 16 | * distribution. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | * 30 | * You can contact the author at : 31 | * - xxHash homepage: http://www.xxhash.com 32 | * - xxHash source repository : https://github.com/Cyan4973/xxHash 33 | */ 34 | 35 | #include 36 | #include 37 | #include "faketypes.h" 38 | /* ************************************* 39 | * Tuning parameters 40 | ***************************************/ 41 | /*!XXH_FORCE_MEMORY_ACCESS : 42 | * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. 43 | * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. 44 | * The below switch allow to select different access method for improved performance. 45 | * Method 0 (default) : use `memcpy()`. Safe and portable. 46 | * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). 47 | * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. 48 | * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. 49 | * It can generate buggy code on targets which do not support unaligned memory accesses. 50 | * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) 51 | * See http://stackoverflow.com/a/32095106/646947 for details. 52 | * Prefer these methods in priority order (0 > 1 > 2) 53 | */ 54 | #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ 55 | # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) 56 | # define XXH_FORCE_MEMORY_ACCESS 2 57 | # elif defined(__INTEL_COMPILER) || \ 58 | (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) 59 | # define XXH_FORCE_MEMORY_ACCESS 1 60 | # endif 61 | #endif 62 | 63 | /*!XXH_ACCEPT_NULL_INPUT_POINTER : 64 | * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. 65 | * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. 66 | * By default, this option is disabled. To enable it, uncomment below define : 67 | */ 68 | /* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ 69 | 70 | /*!XXH_FORCE_NATIVE_FORMAT : 71 | * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. 72 | * Results are therefore identical for little-endian and big-endian CPU. 73 | * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. 74 | * Should endian-independance be of no importance for your application, you may set the #define below to 1, 75 | * to improve speed for Big-endian CPU. 76 | * This option has no impact on Little_Endian CPU. 77 | */ 78 | #ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ 79 | # define XXH_FORCE_NATIVE_FORMAT 0 80 | #endif 81 | 82 | /*!XXH_FORCE_ALIGN_CHECK : 83 | * This is a minor performance trick, only useful with lots of very small keys. 84 | * It means : check for aligned/unaligned input. 85 | * The check costs one initial branch per hash; set to 0 when the input data 86 | * is guaranteed to be aligned. 87 | */ 88 | #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ 89 | # if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) 90 | # define XXH_FORCE_ALIGN_CHECK 0 91 | # else 92 | # define XXH_FORCE_ALIGN_CHECK 1 93 | # endif 94 | #endif 95 | 96 | #if defined(_KERNEL) 97 | #include 98 | #include 99 | #else 100 | 101 | /* ************************************* 102 | * Includes & Memory related functions 103 | ***************************************/ 104 | /* Modify the local functions below should you wish to use some other memory routines */ 105 | /* for malloc(), free() */ 106 | static void* XXH_malloc(size_t s) { return malloc(s); } 107 | static void XXH_free (void* p) { free(p); } 108 | /* for memcpy() */ 109 | #endif 110 | 111 | static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } 112 | 113 | #define XXH_STATIC_LINKING_ONLY 114 | #include "xxhash.h" 115 | 116 | 117 | /* ************************************* 118 | * Compiler Specific Options 119 | ***************************************/ 120 | #ifdef _MSC_VER /* Visual Studio */ 121 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 122 | # define FORCE_INLINE static __forceinline 123 | #else 124 | # if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ 125 | # ifdef __GNUC__ 126 | # define FORCE_INLINE static inline __attribute__((always_inline)) 127 | # else 128 | # define FORCE_INLINE static inline 129 | # endif 130 | # else 131 | # define FORCE_INLINE static 132 | # endif /* __STDC_VERSION__ */ 133 | #endif 134 | 135 | 136 | /* ************************************* 137 | * Basic Types 138 | ***************************************/ 139 | #ifndef MEM_MODULE 140 | # define MEM_MODULE 141 | # if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ 142 | # include 143 | typedef uint8_t BYTE; 144 | typedef uint16_t U16; 145 | typedef uint32_t U32; 146 | typedef int32_t S32; 147 | typedef uint64_t U64; 148 | # else 149 | typedef unsigned char BYTE; 150 | typedef unsigned short U16; 151 | typedef unsigned int U32; 152 | typedef signed int S32; 153 | typedef unsigned long long U64; 154 | # endif 155 | #endif 156 | 157 | 158 | #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) 159 | 160 | /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ 161 | static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } 162 | static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } 163 | 164 | #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) 165 | 166 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ 167 | /* currently only defined for gcc and icc */ 168 | typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; 169 | 170 | static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } 171 | static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } 172 | 173 | #else 174 | 175 | /* portable and safe solution. Generally efficient. 176 | * see : http://stackoverflow.com/a/32095106/646947 177 | */ 178 | 179 | static U32 XXH_read32(const void* memPtr) 180 | { 181 | U32 val; 182 | memcpy(&val, memPtr, sizeof(val)); 183 | return val; 184 | } 185 | 186 | static U64 XXH_read64(const void* memPtr) 187 | { 188 | U64 val; 189 | memcpy(&val, memPtr, sizeof(val)); 190 | return val; 191 | } 192 | 193 | #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ 194 | 195 | 196 | /* **************************************** 197 | * Compiler-specific Functions and Macros 198 | ******************************************/ 199 | #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 200 | 201 | /* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ 202 | #if defined(_MSC_VER) 203 | # define XXH_rotl32(x,r) _rotl(x,r) 204 | # define XXH_rotl64(x,r) _rotl64(x,r) 205 | #else 206 | # define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) 207 | # define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) 208 | #endif 209 | 210 | #if defined(_MSC_VER) /* Visual Studio */ 211 | # define XXH_swap32 _byteswap_ulong 212 | # define XXH_swap64 _byteswap_uint64 213 | #elif GCC_VERSION >= 403 214 | # define XXH_swap32 __builtin_bswap32 215 | # define XXH_swap64 __builtin_bswap64 216 | #else 217 | static U32 XXH_swap32 (U32 x) 218 | { 219 | return ((x << 24) & 0xff000000 ) | 220 | ((x << 8) & 0x00ff0000 ) | 221 | ((x >> 8) & 0x0000ff00 ) | 222 | ((x >> 24) & 0x000000ff ); 223 | } 224 | static U64 XXH_swap64 (U64 x) 225 | { 226 | return ((x << 56) & 0xff00000000000000ULL) | 227 | ((x << 40) & 0x00ff000000000000ULL) | 228 | ((x << 24) & 0x0000ff0000000000ULL) | 229 | ((x << 8) & 0x000000ff00000000ULL) | 230 | ((x >> 8) & 0x00000000ff000000ULL) | 231 | ((x >> 24) & 0x0000000000ff0000ULL) | 232 | ((x >> 40) & 0x000000000000ff00ULL) | 233 | ((x >> 56) & 0x00000000000000ffULL); 234 | } 235 | #endif 236 | 237 | 238 | /* ************************************* 239 | * Architecture Macros 240 | ***************************************/ 241 | typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; 242 | 243 | /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ 244 | #ifndef XXH_CPU_LITTLE_ENDIAN 245 | static const int g_one = 1; 246 | # define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) 247 | #endif 248 | 249 | 250 | /* *************************** 251 | * Memory reads 252 | *****************************/ 253 | typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; 254 | 255 | FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) 256 | { 257 | if (align==XXH_unaligned) 258 | return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); 259 | else 260 | return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); 261 | } 262 | 263 | FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) 264 | { 265 | return XXH_readLE32_align(ptr, endian, XXH_unaligned); 266 | } 267 | 268 | static U32 XXH_readBE32(const void* ptr) 269 | { 270 | return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); 271 | } 272 | 273 | FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) 274 | { 275 | if (align==XXH_unaligned) 276 | return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); 277 | else 278 | return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); 279 | } 280 | 281 | FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) 282 | { 283 | return XXH_readLE64_align(ptr, endian, XXH_unaligned); 284 | } 285 | 286 | static U64 XXH_readBE64(const void* ptr) 287 | { 288 | return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); 289 | } 290 | 291 | 292 | /* ************************************* 293 | * Macros 294 | ***************************************/ 295 | #define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ 296 | 297 | 298 | /* ************************************* 299 | * Constants 300 | ***************************************/ 301 | static const U32 PRIME32_1 = 2654435761U; 302 | static const U32 PRIME32_2 = 2246822519U; 303 | static const U32 PRIME32_3 = 3266489917U; 304 | static const U32 PRIME32_4 = 668265263U; 305 | static const U32 PRIME32_5 = 374761393U; 306 | 307 | static const U64 PRIME64_1 = 11400714785074694791ULL; 308 | static const U64 PRIME64_2 = 14029467366897019727ULL; 309 | static const U64 PRIME64_3 = 1609587929392839161ULL; 310 | static const U64 PRIME64_4 = 9650029242287828579ULL; 311 | static const U64 PRIME64_5 = 2870177450012600261ULL; 312 | 313 | XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } 314 | 315 | 316 | /* *************************** 317 | * Simple Hash Functions 318 | *****************************/ 319 | 320 | static U32 XXH32_round(U32 seed, U32 input) 321 | { 322 | seed += input * PRIME32_2; 323 | seed = XXH_rotl32(seed, 13); 324 | seed *= PRIME32_1; 325 | return seed; 326 | } 327 | 328 | FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) 329 | { 330 | const BYTE* p = (const BYTE*)input; 331 | const BYTE* bEnd = p + len; 332 | U32 h32; 333 | #define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) 334 | 335 | #ifdef XXH_ACCEPT_NULL_INPUT_POINTER 336 | if (p==NULL) { 337 | len=0; 338 | bEnd=p=(const BYTE*)(size_t)16; 339 | } 340 | #endif 341 | 342 | if (len>=16) { 343 | const BYTE* const limit = bEnd - 16; 344 | U32 v1 = seed + PRIME32_1 + PRIME32_2; 345 | U32 v2 = seed + PRIME32_2; 346 | U32 v3 = seed + 0; 347 | U32 v4 = seed - PRIME32_1; 348 | 349 | do { 350 | v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; 351 | v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; 352 | v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; 353 | v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; 354 | } while (p<=limit); 355 | 356 | h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); 357 | } else { 358 | h32 = seed + PRIME32_5; 359 | } 360 | 361 | h32 += (U32) len; 362 | 363 | while (p+4<=bEnd) { 364 | h32 += XXH_get32bits(p) * PRIME32_3; 365 | h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; 366 | p+=4; 367 | } 368 | 369 | while (p> 15; 376 | h32 *= PRIME32_2; 377 | h32 ^= h32 >> 13; 378 | h32 *= PRIME32_3; 379 | h32 ^= h32 >> 16; 380 | 381 | return h32; 382 | } 383 | 384 | 385 | XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) 386 | { 387 | #if 0 388 | /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ 389 | XXH32_CREATESTATE_STATIC(state); 390 | XXH32_reset(state, seed); 391 | XXH32_update(state, input, len); 392 | return XXH32_digest(state); 393 | #else 394 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 395 | 396 | if (XXH_FORCE_ALIGN_CHECK) { 397 | if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ 398 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 399 | return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); 400 | else 401 | return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); 402 | } } 403 | 404 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 405 | return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); 406 | else 407 | return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); 408 | #endif 409 | } 410 | 411 | 412 | static U64 XXH64_round(U64 acc, U64 input) 413 | { 414 | acc += input * PRIME64_2; 415 | acc = XXH_rotl64(acc, 31); 416 | acc *= PRIME64_1; 417 | return acc; 418 | } 419 | 420 | static U64 XXH64_mergeRound(U64 acc, U64 val) 421 | { 422 | val = XXH64_round(0, val); 423 | acc ^= val; 424 | acc = acc * PRIME64_1 + PRIME64_4; 425 | return acc; 426 | } 427 | 428 | FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) 429 | { 430 | const BYTE* p = (const BYTE*)input; 431 | const BYTE* const bEnd = p + len; 432 | U64 h64; 433 | #define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) 434 | 435 | #ifdef XXH_ACCEPT_NULL_INPUT_POINTER 436 | if (p==NULL) { 437 | len=0; 438 | bEnd=p=(const BYTE*)(size_t)32; 439 | } 440 | #endif 441 | 442 | if (len>=32) { 443 | const BYTE* const limit = bEnd - 32; 444 | U64 v1 = seed + PRIME64_1 + PRIME64_2; 445 | U64 v2 = seed + PRIME64_2; 446 | U64 v3 = seed + 0; 447 | U64 v4 = seed - PRIME64_1; 448 | 449 | do { 450 | v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; 451 | v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; 452 | v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; 453 | v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; 454 | } while (p<=limit); 455 | 456 | h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); 457 | h64 = XXH64_mergeRound(h64, v1); 458 | h64 = XXH64_mergeRound(h64, v2); 459 | h64 = XXH64_mergeRound(h64, v3); 460 | h64 = XXH64_mergeRound(h64, v4); 461 | 462 | } else { 463 | h64 = seed + PRIME64_5; 464 | } 465 | 466 | h64 += (U64) len; 467 | 468 | while (p+8<=bEnd) { 469 | U64 const k1 = XXH64_round(0, XXH_get64bits(p)); 470 | h64 ^= k1; 471 | h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; 472 | p+=8; 473 | } 474 | 475 | if (p+4<=bEnd) { 476 | h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; 477 | h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; 478 | p+=4; 479 | } 480 | 481 | while (p> 33; 488 | h64 *= PRIME64_2; 489 | h64 ^= h64 >> 29; 490 | h64 *= PRIME64_3; 491 | h64 ^= h64 >> 32; 492 | 493 | return h64; 494 | } 495 | 496 | 497 | XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) 498 | { 499 | #if 0 500 | /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ 501 | XXH64_CREATESTATE_STATIC(state); 502 | XXH64_reset(state, seed); 503 | XXH64_update(state, input, len); 504 | return XXH64_digest(state); 505 | #else 506 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 507 | 508 | if (XXH_FORCE_ALIGN_CHECK) { 509 | if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ 510 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 511 | return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); 512 | else 513 | return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); 514 | } } 515 | 516 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 517 | return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); 518 | else 519 | return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); 520 | #endif 521 | } 522 | 523 | 524 | /* ************************************************** 525 | * Advanced Hash Functions 526 | ****************************************************/ 527 | 528 | #if !defined(_KERNEL) 529 | 530 | XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) 531 | { 532 | return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); 533 | } 534 | XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) 535 | { 536 | XXH_free(statePtr); 537 | return XXH_OK; 538 | } 539 | 540 | XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) 541 | { 542 | return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); 543 | } 544 | XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) 545 | { 546 | XXH_free(statePtr); 547 | return XXH_OK; 548 | } 549 | 550 | #endif 551 | 552 | /*** Hash feed ***/ 553 | 554 | XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) 555 | { 556 | XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ 557 | memset(&state, 0, sizeof(state)); 558 | state.seed = seed; 559 | state.v1 = seed + PRIME32_1 + PRIME32_2; 560 | state.v2 = seed + PRIME32_2; 561 | state.v3 = seed + 0; 562 | state.v4 = seed - PRIME32_1; 563 | memcpy(statePtr, &state, sizeof(state)); 564 | return XXH_OK; 565 | } 566 | 567 | 568 | XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) 569 | { 570 | XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ 571 | memset(&state, 0, sizeof(state)); 572 | state.seed = seed; 573 | state.v1 = seed + PRIME64_1 + PRIME64_2; 574 | state.v2 = seed + PRIME64_2; 575 | state.v3 = seed + 0; 576 | state.v4 = seed - PRIME64_1; 577 | memcpy(statePtr, &state, sizeof(state)); 578 | return XXH_OK; 579 | } 580 | 581 | 582 | FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) 583 | { 584 | const BYTE* p = (const BYTE*)input; 585 | const BYTE* const bEnd = p + len; 586 | 587 | #ifdef XXH_ACCEPT_NULL_INPUT_POINTER 588 | if (input==NULL) return XXH_ERROR; 589 | #endif 590 | 591 | state->total_len += len; 592 | 593 | if (state->memsize + len < 16) { /* fill in tmp buffer */ 594 | XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); 595 | state->memsize += (U32)len; 596 | return XXH_OK; 597 | } 598 | 599 | if (state->memsize) { /* some data left from previous update */ 600 | XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); 601 | { const U32* p32 = state->mem32; 602 | state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; 603 | state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; 604 | state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; 605 | state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; 606 | } 607 | p += 16-state->memsize; 608 | state->memsize = 0; 609 | } 610 | 611 | if (p <= bEnd-16) { 612 | const BYTE* const limit = bEnd - 16; 613 | U32 v1 = state->v1; 614 | U32 v2 = state->v2; 615 | U32 v3 = state->v3; 616 | U32 v4 = state->v4; 617 | 618 | do { 619 | v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; 620 | v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; 621 | v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; 622 | v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; 623 | } while (p<=limit); 624 | 625 | state->v1 = v1; 626 | state->v2 = v2; 627 | state->v3 = v3; 628 | state->v4 = v4; 629 | } 630 | 631 | if (p < bEnd) { 632 | XXH_memcpy(state->mem32, p, bEnd-p); 633 | state->memsize = (int)(bEnd-p); 634 | } 635 | 636 | return XXH_OK; 637 | } 638 | 639 | XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) 640 | { 641 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 642 | 643 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 644 | return XXH32_update_endian(state_in, input, len, XXH_littleEndian); 645 | else 646 | return XXH32_update_endian(state_in, input, len, XXH_bigEndian); 647 | } 648 | 649 | 650 | 651 | FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) 652 | { 653 | const BYTE * p = (const BYTE*)state->mem32; 654 | const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; 655 | U32 h32; 656 | 657 | if (state->total_len >= 16) { 658 | h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); 659 | } else { 660 | h32 = state->seed + PRIME32_5; 661 | } 662 | 663 | h32 += (U32) state->total_len; 664 | 665 | while (p+4<=bEnd) { 666 | h32 += XXH_readLE32(p, endian) * PRIME32_3; 667 | h32 = XXH_rotl32(h32, 17) * PRIME32_4; 668 | p+=4; 669 | } 670 | 671 | while (p> 15; 678 | h32 *= PRIME32_2; 679 | h32 ^= h32 >> 13; 680 | h32 *= PRIME32_3; 681 | h32 ^= h32 >> 16; 682 | 683 | return h32; 684 | } 685 | 686 | 687 | XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) 688 | { 689 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 690 | 691 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 692 | return XXH32_digest_endian(state_in, XXH_littleEndian); 693 | else 694 | return XXH32_digest_endian(state_in, XXH_bigEndian); 695 | } 696 | 697 | 698 | 699 | /* **** XXH64 **** */ 700 | 701 | FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) 702 | { 703 | const BYTE* p = (const BYTE*)input; 704 | const BYTE* const bEnd = p + len; 705 | 706 | #ifdef XXH_ACCEPT_NULL_INPUT_POINTER 707 | if (input==NULL) return XXH_ERROR; 708 | #endif 709 | 710 | state->total_len += len; 711 | 712 | if (state->memsize + len < 32) { /* fill in tmp buffer */ 713 | XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); 714 | state->memsize += (U32)len; 715 | return XXH_OK; 716 | } 717 | 718 | if (state->memsize) { /* tmp buffer is full */ 719 | XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); 720 | state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); 721 | state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); 722 | state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); 723 | state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); 724 | p += 32-state->memsize; 725 | state->memsize = 0; 726 | } 727 | 728 | if (p+32 <= bEnd) { 729 | const BYTE* const limit = bEnd - 32; 730 | U64 v1 = state->v1; 731 | U64 v2 = state->v2; 732 | U64 v3 = state->v3; 733 | U64 v4 = state->v4; 734 | 735 | do { 736 | v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; 737 | v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; 738 | v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; 739 | v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; 740 | } while (p<=limit); 741 | 742 | state->v1 = v1; 743 | state->v2 = v2; 744 | state->v3 = v3; 745 | state->v4 = v4; 746 | } 747 | 748 | if (p < bEnd) { 749 | XXH_memcpy(state->mem64, p, bEnd-p); 750 | state->memsize = (int)(bEnd-p); 751 | } 752 | 753 | return XXH_OK; 754 | } 755 | 756 | XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) 757 | { 758 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 759 | 760 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 761 | return XXH64_update_endian(state_in, input, len, XXH_littleEndian); 762 | else 763 | return XXH64_update_endian(state_in, input, len, XXH_bigEndian); 764 | } 765 | 766 | 767 | 768 | FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) 769 | { 770 | const BYTE * p = (const BYTE*)state->mem64; 771 | const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; 772 | U64 h64; 773 | 774 | if (state->total_len >= 32) { 775 | U64 const v1 = state->v1; 776 | U64 const v2 = state->v2; 777 | U64 const v3 = state->v3; 778 | U64 const v4 = state->v4; 779 | 780 | h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); 781 | h64 = XXH64_mergeRound(h64, v1); 782 | h64 = XXH64_mergeRound(h64, v2); 783 | h64 = XXH64_mergeRound(h64, v3); 784 | h64 = XXH64_mergeRound(h64, v4); 785 | } else { 786 | h64 = state->seed + PRIME64_5; 787 | } 788 | 789 | h64 += (U64) state->total_len; 790 | 791 | while (p+8<=bEnd) { 792 | U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); 793 | h64 ^= k1; 794 | h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; 795 | p+=8; 796 | } 797 | 798 | if (p+4<=bEnd) { 799 | h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; 800 | h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; 801 | p+=4; 802 | } 803 | 804 | while (p> 33; 811 | h64 *= PRIME64_2; 812 | h64 ^= h64 >> 29; 813 | h64 *= PRIME64_3; 814 | h64 ^= h64 >> 32; 815 | 816 | return h64; 817 | } 818 | 819 | 820 | XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) 821 | { 822 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 823 | 824 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 825 | return XXH64_digest_endian(state_in, XXH_littleEndian); 826 | else 827 | return XXH64_digest_endian(state_in, XXH_bigEndian); 828 | } 829 | 830 | 831 | /* ************************** 832 | * Canonical representation 833 | ****************************/ 834 | 835 | /*! Default XXH result types are basic unsigned 32 and 64 bits. 836 | * The canonical representation follows human-readable write convention, aka big-endian (large digits first). 837 | * These functions allow transformation of hash result into and from its canonical format. 838 | * This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. 839 | */ 840 | 841 | XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) 842 | { 843 | assert(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); 844 | if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); 845 | memcpy(dst, &hash, sizeof(*dst)); 846 | } 847 | 848 | XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) 849 | { 850 | XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); 851 | if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); 852 | memcpy(dst, &hash, sizeof(*dst)); 853 | } 854 | 855 | XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) 856 | { 857 | return XXH_readBE32(src); 858 | } 859 | 860 | XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) 861 | { 862 | return XXH_readBE64(src); 863 | } 864 | 865 | -------------------------------------------------------------------------------- /lz4.h: -------------------------------------------------------------------------------- 1 | /* 2 | * LZ4 - Fast LZ compression algorithm 3 | * Header File 4 | * Copyright (C) 2011-present, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 homepage : http://www.lz4.org 33 | - LZ4 source repository : https://github.com/lz4/lz4 34 | */ 35 | 36 | #ifndef LZ4_H_2983827168210 37 | #define LZ4_H_2983827168210 38 | 39 | /* --- Dependency --- */ 40 | /* Fake size_t; from /sys/include/ape/stddef.h */ 41 | typedef unsigned long size_t; 42 | /* Fake ptrdiff_t from the same place */ 43 | typedef long ptrdiff_t; 44 | 45 | 46 | /** 47 | Introduction 48 | 49 | LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core, 50 | scalable with multi-cores CPU. It features an extremely fast decoder, with speed in 51 | multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. 52 | 53 | The LZ4 compression library provides in-memory compression and decompression functions. 54 | Compression can be done in: 55 | - a single step (described as Simple Functions) 56 | - a single step, reusing a context (described in Advanced Functions) 57 | - unbounded multiple steps (described as Streaming compression) 58 | 59 | lz4.h provides block compression functions. It gives full buffer control to user. 60 | Decompressing an lz4-compressed block also requires metadata (such as compressed size). 61 | Each application is free to encode such metadata in whichever way it wants. 62 | 63 | An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), 64 | take care of encoding standard metadata alongside LZ4-compressed blocks. 65 | Frame format is required for interoperability. 66 | It is delivered through a companion API, declared in lz4frame.h. 67 | */ 68 | 69 | /*^*************************************************************** 70 | * Export parameters 71 | *****************************************************************/ 72 | /* 73 | * LZ4_DLL_EXPORT : 74 | * Enable exporting of functions when building a Windows DLL 75 | * LZ4LIB_VISIBILITY : 76 | * Control library symbols visibility. 77 | */ 78 | #ifndef LZ4LIB_VISIBILITY 79 | # define LZ4LIB_VISIBILITY 80 | #endif 81 | #define LZ4LIB_API LZ4LIB_VISIBILITY 82 | 83 | /*------ Version ------*/ 84 | #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ 85 | #define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ 86 | #define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ 87 | 88 | #define LZ4_VERSION_NUMBER (1 *100*100 + 8 *100 + 3) 89 | 90 | #define LZ4_LIB_VERSION 1.8.3 91 | #define LZ4_QUOTE(str) #str 92 | #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) 93 | #define LZ4_VERSION_STRING "1.8.3" 94 | 95 | LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */ 96 | LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; unseful to check dll version */ 97 | 98 | 99 | /*-************************************ 100 | * Tuning parameter 101 | **************************************/ 102 | /*! 103 | * LZ4_MEMORY_USAGE : 104 | * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) 105 | * Increasing memory usage improves compression ratio 106 | * Reduced memory usage may improve speed, thanks to cache effect 107 | * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache 108 | */ 109 | #ifndef LZ4_MEMORY_USAGE 110 | # define LZ4_MEMORY_USAGE 14 111 | #endif 112 | 113 | /*-************************************ 114 | * Simple Functions 115 | **************************************/ 116 | /*! LZ4_compress_default() : 117 | Compresses 'srcSize' bytes from buffer 'src' 118 | into already allocated 'dst' buffer of size 'dstCapacity'. 119 | Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). 120 | It also runs faster, so it's a recommended setting. 121 | If the function cannot compress 'src' into a more limited 'dst' budget, 122 | compression stops *immediately*, and the function result is zero. 123 | Note : as a consequence, 'dst' content is not valid. 124 | Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). 125 | srcSize : max supported value is LZ4_MAX_INPUT_SIZE. 126 | dstCapacity : size of buffer 'dst' (which must be already allocated) 127 | return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) 128 | or 0 if compression fails */ 129 | LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); 130 | 131 | /*! LZ4_decompress_safe() : 132 | compressedSize : is the exact complete size of the compressed block. 133 | dstCapacity : is the size of destination buffer, which must be already allocated. 134 | return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) 135 | If destination buffer is not large enough, decoding will stop and output an error code (negative value). 136 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 137 | This function is protected against malicious data packets. 138 | */ 139 | LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); 140 | 141 | 142 | /*-************************************ 143 | * Advanced Functions 144 | **************************************/ 145 | #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ 146 | #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) 147 | 148 | /*! 149 | LZ4_compressBound() : 150 | Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) 151 | This function is primarily useful for memory allocation purposes (destination buffer size). 152 | Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). 153 | Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) 154 | inputSize : max supported value is LZ4_MAX_INPUT_SIZE 155 | return : maximum output size in a "worst case" scenario 156 | or 0, if input size is incorrect (too large or negative) 157 | */ 158 | LZ4LIB_API int LZ4_compressBound(int inputSize); 159 | 160 | /*! 161 | LZ4_compress_fast() : 162 | Same as LZ4_compress_default(), but allows selection of "acceleration" factor. 163 | The larger the acceleration value, the faster the algorithm, but also the lesser the compression. 164 | It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. 165 | An acceleration value of "1" is the same as regular LZ4_compress_default() 166 | Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c). 167 | */ 168 | LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); 169 | 170 | 171 | /*! 172 | LZ4_compress_fast_extState() : 173 | Same compression function, just using an externally allocated memory space to store compression state. 174 | Use LZ4_sizeofState() to know how much memory must be allocated, 175 | and allocate it on 8-bytes boundaries (using malloc() typically). 176 | Then, provide this buffer as 'void* state' to compression function. 177 | */ 178 | LZ4LIB_API int LZ4_sizeofState(void); 179 | LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); 180 | 181 | 182 | /*! LZ4_compress_destSize() : 183 | * Reverse the logic : compresses as much data as possible from 'src' buffer 184 | * into already allocated buffer 'dst', of size >= 'targetDestSize'. 185 | * This function either compresses the entire 'src' content into 'dst' if it's large enough, 186 | * or fill 'dst' buffer completely with as much data as possible from 'src'. 187 | * note: acceleration parameter is fixed to "default". 188 | * 189 | * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. 190 | * New value is necessarily <= input value. 191 | * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) 192 | * or 0 if compression fails. 193 | */ 194 | LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); 195 | 196 | 197 | /*! LZ4_decompress_fast() : **unsafe!** 198 | * This function used to be a bit faster than LZ4_decompress_safe(), 199 | * though situation has changed in recent versions, 200 | * and now `LZ4_decompress_safe()` can be as fast and sometimes faster than `LZ4_decompress_fast()`. 201 | * Moreover, LZ4_decompress_fast() is not protected vs malformed input, as it doesn't perform full validation of compressed data. 202 | * As a consequence, this function is no longer recommended, and may be deprecated in future versions. 203 | * It's only remaining specificity is that it can decompress data without knowing its compressed size. 204 | * 205 | * originalSize : is the uncompressed size to regenerate. 206 | * `dst` must be already allocated, its size must be >= 'originalSize' bytes. 207 | * @return : number of bytes read from source buffer (== compressed size). 208 | * If the source stream is detected malformed, the function stops decoding and returns a negative result. 209 | * note : This function requires uncompressed originalSize to be known in advance. 210 | * The function never writes past the output buffer. 211 | * However, since it doesn't know its 'src' size, it may read past the intended input. 212 | * Also, because match offsets are not validated during decoding, 213 | * reads from 'src' may underflow. 214 | * Use this function in trusted environment **only**. 215 | */ 216 | LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); 217 | 218 | /*! LZ4_decompress_safe_partial() : 219 | * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', 220 | * into destination buffer 'dst' of size 'dstCapacity'. 221 | * Up to 'targetOutputSize' bytes will be decoded. 222 | * The function stops decoding on reaching this objective, 223 | * which can boost performance when only the beginning of a block is required. 224 | * 225 | * @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity) 226 | * If source stream is detected malformed, function returns a negative result. 227 | * 228 | * Note : @return can be < targetOutputSize, if compressed block contains less data. 229 | * 230 | * Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity, 231 | * and expects targetOutputSize <= dstCapacity. 232 | * It effectively stops decoding on reaching targetOutputSize, 233 | * so dstCapacity is kind of redundant. 234 | * This is because in a previous version of this function, 235 | * decoding operation would not "break" a sequence in the middle. 236 | * As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize, 237 | * it could write more bytes, though only up to dstCapacity. 238 | * Some "margin" used to be required for this operation to work properly. 239 | * This is no longer necessary. 240 | * The function nonetheless keeps its signature, in an effort to not break API. 241 | */ 242 | LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); 243 | 244 | 245 | /*-********************************************* 246 | * Streaming Compression Functions 247 | ***********************************************/ 248 | typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ 249 | 250 | /*! LZ4_createStream() and LZ4_freeStream() : 251 | * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. 252 | * LZ4_freeStream() releases its memory. 253 | */ 254 | LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); 255 | LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); 256 | 257 | /*! LZ4_resetStream() : 258 | * An LZ4_stream_t structure can be allocated once and re-used multiple times. 259 | * Use this function to start compressing a new stream. 260 | */ 261 | LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); 262 | 263 | /*! LZ4_loadDict() : 264 | * Use this function to load a static dictionary into LZ4_stream_t. 265 | * Any previous data will be forgotten, only 'dictionary' will remain in memory. 266 | * Loading a size of 0 is allowed, and is the same as reset. 267 | * @return : dictionary size, in bytes (necessarily <= 64 KB) 268 | */ 269 | LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); 270 | 271 | /*! LZ4_compress_fast_continue() : 272 | * Compress 'src' content using data from previously compressed blocks, for better compression ratio. 273 | * 'dst' buffer must be already allocated. 274 | * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. 275 | * 276 | * @return : size of compressed block 277 | * or 0 if there is an error (typically, cannot fit into 'dst'). 278 | * 279 | * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. 280 | * Each block has precise boundaries. 281 | * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. 282 | * Each block must be decompressed separately, calling LZ4_decompress_*() with associated metadata. 283 | * 284 | * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory! 285 | * 286 | * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. 287 | * Make sure that buffers are separated, by at least one byte. 288 | * This construction ensures that each block only depends on previous block. 289 | * 290 | * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. 291 | * 292 | * Note 5 : After an error, the stream status is invalid, it can only be reset or freed. 293 | */ 294 | LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); 295 | 296 | /*! LZ4_saveDict() : 297 | * If last 64KB data cannot be guaranteed to remain available at its current memory location, 298 | * save it into a safer place (char* safeBuffer). 299 | * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), 300 | * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. 301 | * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. 302 | */ 303 | LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); 304 | 305 | 306 | /*-********************************************** 307 | * Streaming Decompression Functions 308 | * Bufferless synchronous API 309 | ************************************************/ 310 | typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ 311 | 312 | /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : 313 | * creation / destruction of streaming decompression tracking context. 314 | * A tracking context can be re-used multiple times. 315 | */ 316 | LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); 317 | LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); 318 | 319 | /*! LZ4_setStreamDecode() : 320 | * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. 321 | * Use this function to start decompression of a new stream of blocks. 322 | * A dictionary can optionally be set. Use NULL or size 0 for a reset order. 323 | * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. 324 | * @return : 1 if OK, 0 if error 325 | */ 326 | LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); 327 | 328 | /*! LZ4_decoderRingBufferSize() : v1.8.2 329 | * Note : in a ring buffer scenario (optional), 330 | * blocks are presumed decompressed next to each other 331 | * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), 332 | * at which stage it resumes from beginning of ring buffer. 333 | * When setting such a ring buffer for streaming decompression, 334 | * provides the minimum size of this ring buffer 335 | * to be compatible with any source respecting maxBlockSize condition. 336 | * @return : minimum ring buffer size, 337 | * or 0 if there is an error (invalid maxBlockSize). 338 | */ 339 | LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); 340 | #define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs)) /* for static allocation; mbs presumed valid */ 341 | 342 | /*! LZ4_decompress_*_continue() : 343 | * These decoding functions allow decompression of consecutive blocks in "streaming" mode. 344 | * A block is an unsplittable entity, it must be presented entirely to a decompression function. 345 | * Decompression functions only accepts one block at a time. 346 | * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded. 347 | * If less than 64KB of data has been decoded, all the data must be present. 348 | * 349 | * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : 350 | * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). 351 | * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. 352 | * In which case, encoding and decoding buffers do not need to be synchronized. 353 | * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. 354 | * - Synchronized mode : 355 | * Decompression buffer size is _exactly_ the same as compression buffer size, 356 | * and follows exactly same update rule (block boundaries at same positions), 357 | * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), 358 | * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). 359 | * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. 360 | * In which case, encoding and decoding buffers do not need to be synchronized, 361 | * and encoding ring buffer can have any size, including small ones ( < 64 KB). 362 | * 363 | * Whenever these conditions are not possible, 364 | * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, 365 | * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. 366 | */ 367 | LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); 368 | LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); 369 | 370 | 371 | /*! LZ4_decompress_*_usingDict() : 372 | * These decoding functions work the same as 373 | * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() 374 | * They are stand-alone, and don't need an LZ4_streamDecode_t structure. 375 | * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. 376 | */ 377 | LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); 378 | LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); 379 | 380 | 381 | /*^********************************************** 382 | * !!!!!! STATIC LINKING ONLY !!!!!! 383 | ***********************************************/ 384 | 385 | /*-************************************ 386 | * Unstable declarations 387 | ************************************** 388 | * Declarations in this section should be considered unstable. 389 | * Use at your own peril, etc., etc. 390 | * They may be removed in the future. 391 | * Their signatures may change. 392 | **************************************/ 393 | 394 | #ifdef LZ4_STATIC_LINKING_ONLY 395 | 396 | /*! LZ4_resetStream_fast() : 397 | * Use this, like LZ4_resetStream(), to prepare a context for a new chain of 398 | * calls to a streaming API (e.g., LZ4_compress_fast_continue()). 399 | * 400 | * Note: 401 | * Using this in advance of a non- streaming-compression function is redundant, 402 | * and potentially bad for performance, since they all perform their own custom 403 | * reset internally. 404 | * 405 | * Differences from LZ4_resetStream(): 406 | * When an LZ4_stream_t is known to be in a internally coherent state, 407 | * it can often be prepared for a new compression with almost no work, only 408 | * sometimes falling back to the full, expensive reset that is always required 409 | * when the stream is in an indeterminate state (i.e., the reset performed by 410 | * LZ4_resetStream()). 411 | * 412 | * LZ4_streams are guaranteed to be in a valid state when: 413 | * - returned from LZ4_createStream() 414 | * - reset by LZ4_resetStream() 415 | * - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged 416 | * - the stream was in a valid state and was reset by LZ4_resetStream_fast() 417 | * - the stream was in a valid state and was then used in any compression call 418 | * that returned success 419 | * - the stream was in an indeterminate state and was used in a compression 420 | * call that fully reset the state (e.g., LZ4_compress_fast_extState()) and 421 | * that returned success 422 | * 423 | * When a stream isn't known to be in a valid state, it is not safe to pass to 424 | * any fastReset or streaming function. It must first be cleansed by the full 425 | * LZ4_resetStream(). 426 | */ 427 | LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); 428 | 429 | /*! LZ4_compress_fast_extState_fastReset() : 430 | * A variant of LZ4_compress_fast_extState(). 431 | * 432 | * Using this variant avoids an expensive initialization step. It is only safe 433 | * to call if the state buffer is known to be correctly initialized already 434 | * (see above comment on LZ4_resetStream_fast() for a definition of "correctly 435 | * initialized"). From a high level, the difference is that this function 436 | * initializes the provided state with a call to something like 437 | * LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a 438 | * call to LZ4_resetStream(). 439 | */ 440 | LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); 441 | 442 | /*! LZ4_attach_dictionary() : 443 | * This is an experimental API that allows for the efficient use of a 444 | * static dictionary many times. 445 | * 446 | * Rather than re-loading the dictionary buffer into a working context before 447 | * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a 448 | * working LZ4_stream_t, this function introduces a no-copy setup mechanism, 449 | * in which the working stream references the dictionary stream in-place. 450 | * 451 | * Several assumptions are made about the state of the dictionary stream. 452 | * Currently, only streams which have been prepared by LZ4_loadDict() should 453 | * be expected to work. 454 | * 455 | * Alternatively, the provided dictionary stream pointer may be NULL, in which 456 | * case any existing dictionary stream is unset. 457 | * 458 | * If a dictionary is provided, it replaces any pre-existing stream history. 459 | * The dictionary contents are the only history that can be referenced and 460 | * logically immediately precede the data compressed in the first subsequent 461 | * compression call. 462 | * 463 | * The dictionary will only remain attached to the working stream through the 464 | * first compression call, at the end of which it is cleared. The dictionary 465 | * stream (and source buffer) must remain in-place / accessible / unchanged 466 | * through the completion of the first compression call on the stream. 467 | */ 468 | LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream); 469 | 470 | #endif 471 | 472 | /*-************************************ 473 | * Private definitions 474 | ************************************** 475 | * Do not use these definitions. 476 | * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. 477 | * Using these definitions will expose code to API and/or ABI break in future versions of the library. 478 | **************************************/ 479 | #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) 480 | #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) 481 | #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ 482 | 483 | typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; 484 | struct LZ4_stream_t_internal { 485 | unsigned int hashTable[LZ4_HASH_SIZE_U32]; 486 | unsigned int currentOffset; 487 | unsigned short initCheck; 488 | unsigned short tableType; 489 | const unsigned char* dictionary; 490 | const LZ4_stream_t_internal* dictCtx; 491 | unsigned int dictSize; 492 | }; 493 | 494 | typedef struct { 495 | const unsigned char* externalDict; 496 | size_t extDictSize; 497 | const unsigned char* prefixEnd; 498 | size_t prefixSize; 499 | } LZ4_streamDecode_t_internal; 500 | 501 | /*! 502 | * LZ4_stream_t : 503 | * information structure to track an LZ4 stream. 504 | * init this structure before first use. 505 | * note : only use in association with static linking ! 506 | * this definition is not API/ABI safe, 507 | * it may change in a future version ! 508 | */ 509 | #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) 510 | #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) 511 | union LZ4_stream_u { 512 | unsigned long long table[LZ4_STREAMSIZE_U64]; 513 | LZ4_stream_t_internal internal_donotuse; 514 | } ; /* previously typedef'd to LZ4_stream_t */ 515 | 516 | 517 | /*! 518 | * LZ4_streamDecode_t : 519 | * information structure to track an LZ4 stream during decompression. 520 | * init this structure using LZ4_setStreamDecode (or memset()) before first use 521 | * note : only use in association with static linking ! 522 | * this definition is not API/ABI safe, 523 | * and may change in a future version ! 524 | */ 525 | #define LZ4_STREAMDECODESIZE_U64 4 526 | #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) 527 | union LZ4_streamDecode_u { 528 | unsigned long long table[LZ4_STREAMDECODESIZE_U64]; 529 | LZ4_streamDecode_t_internal internal_donotuse; 530 | } ; /* previously typedef'd to LZ4_streamDecode_t */ 531 | 532 | 533 | /*-************************************ 534 | * Obsolete Functions 535 | **************************************/ 536 | 537 | /*! Deprecation warnings 538 | Should deprecation warnings be a problem, 539 | it is generally possible to disable them, 540 | typically with -Wno-deprecated-declarations for gcc 541 | or _CRT_SECURE_NO_WARNINGS in Visual. 542 | Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ 543 | #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS 544 | # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ 545 | #else 546 | # pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") 547 | # define LZ4_DEPRECATED(message) 548 | #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ 549 | 550 | /* Obsolete compression functions */ 551 | LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); 552 | LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); 553 | LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); 554 | LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 555 | LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); 556 | LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); 557 | 558 | /* Obsolete decompression functions */ 559 | LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); 560 | LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); 561 | 562 | /* Obsolete streaming functions; degraded functionality; do not use! 563 | * 564 | * In order to perform streaming compression, these functions depended on data 565 | * that is no longer tracked in the state. They have been preserved as well as 566 | * possible: using them will still produce a correct output. However, they don't 567 | * actually retain any history between compression calls. The compression ratio 568 | * achieved will therefore be no better than compressing each chunk 569 | * independently. 570 | */ 571 | LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); 572 | LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); 573 | LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); 574 | LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); 575 | 576 | /* Obsolete streaming decoding functions */ 577 | LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); 578 | LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); 579 | 580 | #endif /* LZ4_H_2983827168210 */ 581 | 582 | -------------------------------------------------------------------------------- /hammer2_disk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011-2014 The DragonFly Project. All rights reserved. 3 | * 4 | * This code is derived from software contributed to The DragonFly Project 5 | * by Matthew Dillon 6 | * by Venkatesh Srinivas 7 | * 8 | * Redistribution and use in source and binary forms, with or without 9 | * modification, are permitted provided that the following conditions 10 | * are met: 11 | * 12 | * 1. Redistributions of source code must retain the above copyright 13 | * notice, this list of conditions and the following disclaimer. 14 | * 2. Redistributions in binary form must reproduce the above copyright 15 | * notice, this list of conditions and the following disclaimer in 16 | * the documentation and/or other materials provided with the 17 | * distribution. 18 | * 3. Neither the name of The DragonFly Project nor the names of its 19 | * contributors may be used to endorse or promote products derived 20 | * from this software without specific, prior written permission. 21 | * 22 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 25 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 26 | * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 27 | * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 28 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 30 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 31 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 32 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | /* 37 | * The structures below represent the on-disk media structures for the HAMMER2 38 | * filesystem. Note that all fields for on-disk structures are naturally 39 | * aligned. The host endian format is typically used - compatibility is 40 | * possible if the implementation detects reversed endian and adjusts accesses 41 | * accordingly. 42 | * 43 | * HAMMER2 primarily revolves around the directory topology: inodes, 44 | * directory entries, and block tables. Block device buffer cache buffers 45 | * are always 64KB. Logical file buffers are typically 16KB. All data 46 | * references utilize 64-bit byte offsets. 47 | * 48 | * Free block management is handled independently using blocks reserved by 49 | * the media topology. 50 | */ 51 | 52 | /* 53 | * The data at the end of a file or directory may be a fragment in order 54 | * to optimize storage efficiency. The minimum fragment size is 1KB. 55 | * Since allocations are in powers of 2 fragments must also be sized in 56 | * powers of 2 (1024, 2048, ... 65536). 57 | * 58 | * For the moment the maximum allocation size is HAMMER2_PBUFSIZE (64K), 59 | * which is 2^16. Larger extents may be supported in the future. Smaller 60 | * fragments might be supported in the future (down to 64 bytes is possible), 61 | * but probably will not be. 62 | * 63 | * A full indirect block use supports 512 x 128-byte blockrefs in a 64KB 64 | * buffer. Indirect blocks down to 1KB are supported to keep small 65 | * directories small. 66 | * 67 | * A maximally sized file (2^64-1 bytes) requires ~6 indirect block levels 68 | * using 64KB indirect blocks (128 byte refs, 512 or radix 9 per indblk). 69 | * 70 | * 16(datablk) + 9 + 9 + 9 + 9 + 9 + 9 = ~70. 71 | * 16(datablk) + 7 + 9 + 9 + 9 + 9 + 9 = ~68. (smaller top level indblk) 72 | * 73 | * The actual depth depends on copies redundancy and whether the filesystem 74 | * has chosen to use a smaller indirect block size at the top level or not. 75 | */ 76 | #define HAMMER2_ALLOC_MIN 1024 /* minimum allocation size */ 77 | #define HAMMER2_RADIX_MIN 10 /* minimum allocation size 2^N */ 78 | #define HAMMER2_ALLOC_MAX 65536 /* maximum allocation size */ 79 | #define HAMMER2_RADIX_MAX 16 /* maximum allocation size 2^N */ 80 | #define HAMMER2_RADIX_KEY 64 /* number of bits in key */ 81 | 82 | /* 83 | * MINALLOCSIZE - The minimum allocation size. This can be smaller 84 | * or larger than the minimum physical IO size. 85 | * 86 | * NOTE: Should not be larger than 1K since inodes 87 | * are 1K. 88 | * 89 | * MINIOSIZE - The minimum IO size. This must be less than 90 | * or equal to HAMMER2_LBUFSIZE. 91 | * 92 | * HAMMER2_LBUFSIZE - Nominal buffer size for I/O rollups. 93 | * 94 | * HAMMER2_PBUFSIZE - Topological block size used by files for all 95 | * blocks except the block straddling EOF. 96 | * 97 | * HAMMER2_SEGSIZE - Allocation map segment size, typically 2MB 98 | * (space represented by a level0 bitmap). 99 | */ 100 | 101 | #define HAMMER2_SEGSIZE (1 << HAMMER2_FREEMAP_LEVEL0_RADIX) 102 | #define HAMMER2_SEGRADIX HAMMER2_FREEMAP_LEVEL0_RADIX 103 | 104 | #define HAMMER2_PBUFRADIX 16 /* physical buf (1<<16) bytes */ 105 | #define HAMMER2_PBUFSIZE 65536 106 | #define HAMMER2_LBUFRADIX 14 /* logical buf (1<<14) bytes */ 107 | #define HAMMER2_LBUFSIZE 16384 108 | 109 | /* 110 | * Generally speaking we want to use 16K and 64K I/Os 111 | */ 112 | #define HAMMER2_MINIORADIX HAMMER2_LBUFRADIX 113 | #define HAMMER2_MINIOSIZE HAMMER2_LBUFSIZE 114 | 115 | #define HAMMER2_IND_BYTES_MIN 4096 116 | #define HAMMER2_IND_BYTES_NOM HAMMER2_LBUFSIZE 117 | #define HAMMER2_IND_BYTES_MAX HAMMER2_PBUFSIZE 118 | #define HAMMER2_IND_RADIX_MIN 12 119 | #define HAMMER2_IND_RADIX_NOM HAMMER2_LBUFRADIX 120 | #define HAMMER2_IND_RADIX_MAX HAMMER2_PBUFRADIX 121 | #define HAMMER2_IND_COUNT_MIN (HAMMER2_IND_BYTES_MIN / \ 122 | sizeof(hammer2_blockref_t)) 123 | #define HAMMER2_IND_COUNT_MAX (HAMMER2_IND_BYTES_MAX / \ 124 | sizeof(hammer2_blockref_t)) 125 | 126 | /* 127 | * In HAMMER2, arrays of blockrefs are fully set-associative, meaning that 128 | * any element can occur at any index and holes can be anywhere. As a 129 | * future optimization we will be able to flag that such arrays are sorted 130 | * and thus optimize lookups, but for now we don't. 131 | * 132 | * Inodes embed either 512 bytes of direct data or an array of 8 blockrefs, 133 | * resulting in highly efficient storage for files <= 512 bytes and for files 134 | * <= 512KB. Up to 8 directory entries can be referenced from a directory 135 | * without requiring an indirect block. 136 | * 137 | * Indirect blocks are typically either 4KB (64 blockrefs / ~4MB represented), 138 | * or 64KB (1024 blockrefs / ~64MB represented). 139 | */ 140 | #define HAMMER2_SET_RADIX 2 /* radix 2 = 4 entries */ 141 | #define HAMMER2_SET_COUNT (1 << HAMMER2_SET_RADIX) 142 | #define HAMMER2_EMBEDDED_BYTES 512 /* inode blockset/dd size */ 143 | #define HAMMER2_EMBEDDED_RADIX 9 144 | 145 | #define HAMMER2_PBUFMASK (HAMMER2_PBUFSIZE - 1) 146 | #define HAMMER2_LBUFMASK (HAMMER2_LBUFSIZE - 1) 147 | #define HAMMER2_SEGMASK (HAMMER2_SEGSIZE - 1) 148 | 149 | #define HAMMER2_LBUFMASK64 ((hammer2_off_t)HAMMER2_LBUFMASK) 150 | #define HAMMER2_PBUFSIZE64 ((hammer2_off_t)HAMMER2_PBUFSIZE) 151 | #define HAMMER2_PBUFMASK64 ((hammer2_off_t)HAMMER2_PBUFMASK) 152 | #define HAMMER2_SEGSIZE64 ((hammer2_off_t)HAMMER2_SEGSIZE) 153 | #define HAMMER2_SEGMASK64 ((hammer2_off_t)HAMMER2_SEGMASK) 154 | 155 | #define HAMMER2_UUID_STRING "5cbb9ad1-862d-11dc-a94d-01301bb8a9f5" 156 | 157 | /* 158 | * A HAMMER2 filesystem is always sized in multiples of 8MB. 159 | * 160 | * A 4MB segment is reserved at the beginning of each 2GB zone. This segment 161 | * contains the volume header (or backup volume header), the free block 162 | * table, and possibly other information in the future. 163 | * 164 | * 4MB = 64 x 64K blocks. Each 4MB segment is broken down as follows: 165 | * 166 | * +-----------------------+ 167 | * | Volume Hdr | block 0 volume header & alternates 168 | * +-----------------------+ (first four zones only) 169 | * | FreeBlk Section A | block 1-4 170 | * +-----------------------+ 171 | * | FreeBlk Section B | block 5-8 172 | * +-----------------------+ 173 | * | FreeBlk Section C | block 9-12 174 | * +-----------------------+ 175 | * | FreeBlk Section D | block 13-16 176 | * +-----------------------+ 177 | * | | block 17...63 178 | * | reserved | 179 | * | | 180 | * +-----------------------+ 181 | * 182 | * The first few 2GB zones contain volume headers and volume header backups. 183 | * After that the volume header block# is reserved for future use. Similarly, 184 | * there are many blocks related to various Freemap levels which are not 185 | * used in every segment and those are also reserved for future use. 186 | * 187 | * Freemap (see the FREEMAP document) 188 | * 189 | * The freemap utilizes blocks #1-16 in 8 sets of 4 blocks. Each block in 190 | * a set represents a level of depth in the freemap topology. Eight sets 191 | * exist to prevent live updates from disturbing the state of the freemap 192 | * were a crash/reboot to occur. That is, a live update is not committed 193 | * until the update's flush reaches the volume root. There are FOUR volume 194 | * roots representing the last four synchronization points, so the freemap 195 | * must be consistent no matter which volume root is chosen by the mount 196 | * code. 197 | * 198 | * Each freemap set is 4 x 64K blocks and represents the 2GB, 2TB, 2PB, 199 | * and 2EB indirect map. The volume header itself has a set of 8 freemap 200 | * blockrefs representing another 3 bits, giving us a total 64 bits of 201 | * representable address space. 202 | * 203 | * The Level 0 64KB block represents 2GB of storage represented by 204 | * (64 x struct hammer2_bmap_data). Each structure represents 2MB of storage 205 | * and has a 256 bit bitmap, using 2 bits to represent a 16KB chunk of 206 | * storage. These 2 bits represent the following states: 207 | * 208 | * 00 Free 209 | * 01 (reserved) (Possibly partially allocated) 210 | * 10 Possibly free 211 | * 11 Allocated 212 | * 213 | * One important thing to note here is that the freemap resolution is 16KB, 214 | * but the minimum storage allocation size is 1KB. The hammer2 vfs keeps 215 | * track of sub-allocations in memory, which means that on a unmount or reboot 216 | * the entire 16KB of a partially allocated block will be considered fully 217 | * allocated. It is possible for fragmentation to build up over time, but 218 | * defragmentation is fairly easy to accomplish since all modifications 219 | * allocate a new block. 220 | * 221 | * The Second thing to note is that due to the way snapshots and inode 222 | * replication works, deleting a file cannot immediately free the related 223 | * space. Furthermore, deletions often do not bother to traverse the 224 | * block subhierarchy being deleted. And to go even further, whole 225 | * sub-directory trees can be deleted simply by deleting the directory inode 226 | * at the top. So even though we have a symbol to represent a 'possibly free' 227 | * block (binary 10), only the bulk free scanning code can actually use it. 228 | * Normal 'rm's or other deletions do not. 229 | * 230 | * WARNING! ZONE_SEG and VOLUME_ALIGN must be a multiple of 1<= ZONE_SEG. 232 | * 233 | * In Summary: 234 | * 235 | * (1) Modifications to freemap blocks 'allocate' a new copy (aka use a block 236 | * from the next set). The new copy is reused until a flush occurs at 237 | * which point the next modification will then rotate to the next set. 238 | * 239 | * (2) A total of 10 freemap sets is required. 240 | * 241 | * - 8 sets - 2 sets per volume header backup x 4 volume header backups 242 | * - 2 sets used as backing store for the bulk freemap scan. 243 | * - The freemap recovery scan which runs on-mount just uses the inactive 244 | * set for whichever volume header was selected by the mount code. 245 | * 246 | */ 247 | #define HAMMER2_VOLUME_ALIGN (8 * 1024 * 1024) 248 | #define HAMMER2_VOLUME_ALIGN64 ((hammer2_off_t)HAMMER2_VOLUME_ALIGN) 249 | #define HAMMER2_VOLUME_ALIGNMASK (HAMMER2_VOLUME_ALIGN - 1) 250 | #define HAMMER2_VOLUME_ALIGNMASK64 ((hammer2_off_t)HAMMER2_VOLUME_ALIGNMASK) 251 | 252 | #define HAMMER2_NEWFS_ALIGN (HAMMER2_VOLUME_ALIGN) 253 | #define HAMMER2_NEWFS_ALIGN64 ((hammer2_off_t)HAMMER2_VOLUME_ALIGN) 254 | #define HAMMER2_NEWFS_ALIGNMASK (HAMMER2_VOLUME_ALIGN - 1) 255 | #define HAMMER2_NEWFS_ALIGNMASK64 ((hammer2_off_t)HAMMER2_NEWFS_ALIGNMASK) 256 | 257 | #define HAMMER2_ZONE_BYTES64 (2LLU * 1024 * 1024 * 1024) 258 | #define HAMMER2_ZONE_MASK64 (HAMMER2_ZONE_BYTES64 - 1) 259 | #define HAMMER2_ZONE_SEG (4 * 1024 * 1024) 260 | #define HAMMER2_ZONE_SEG64 ((hammer2_off_t)HAMMER2_ZONE_SEG) 261 | #define HAMMER2_ZONE_BLOCKS_SEG (HAMMER2_ZONE_SEG / HAMMER2_PBUFSIZE) 262 | 263 | #define HAMMER2_ZONE_FREEMAP_INC 5 /* 5 deep */ 264 | 265 | #define HAMMER2_ZONE_VOLHDR 0 /* volume header or backup */ 266 | #define HAMMER2_ZONE_FREEMAP_00 1 /* normal freemap rotation */ 267 | #define HAMMER2_ZONE_FREEMAP_01 6 /* normal freemap rotation */ 268 | #define HAMMER2_ZONE_FREEMAP_02 11 /* normal freemap rotation */ 269 | #define HAMMER2_ZONE_FREEMAP_03 16 /* normal freemap rotation */ 270 | #define HAMMER2_ZONE_FREEMAP_04 21 /* normal freemap rotation */ 271 | #define HAMMER2_ZONE_FREEMAP_05 26 /* normal freemap rotation */ 272 | #define HAMMER2_ZONE_FREEMAP_06 31 /* normal freemap rotation */ 273 | #define HAMMER2_ZONE_FREEMAP_07 36 /* normal freemap rotation */ 274 | #define HAMMER2_ZONE_FREEMAP_END 41 /* (non-inclusive) */ 275 | 276 | #define HAMMER2_ZONE_UNUSED41 41 277 | #define HAMMER2_ZONE_UNUSED42 42 278 | #define HAMMER2_ZONE_UNUSED43 43 279 | #define HAMMER2_ZONE_UNUSED44 44 280 | #define HAMMER2_ZONE_UNUSED45 45 281 | #define HAMMER2_ZONE_UNUSED46 46 282 | #define HAMMER2_ZONE_UNUSED47 47 283 | #define HAMMER2_ZONE_UNUSED48 48 284 | #define HAMMER2_ZONE_UNUSED49 49 285 | #define HAMMER2_ZONE_UNUSED50 50 286 | #define HAMMER2_ZONE_UNUSED51 51 287 | #define HAMMER2_ZONE_UNUSED52 52 288 | #define HAMMER2_ZONE_UNUSED53 53 289 | #define HAMMER2_ZONE_UNUSED54 54 290 | #define HAMMER2_ZONE_UNUSED55 55 291 | #define HAMMER2_ZONE_UNUSED56 56 292 | #define HAMMER2_ZONE_UNUSED57 57 293 | #define HAMMER2_ZONE_UNUSED58 58 294 | #define HAMMER2_ZONE_UNUSED59 59 295 | #define HAMMER2_ZONE_UNUSED60 60 296 | #define HAMMER2_ZONE_UNUSED61 61 297 | #define HAMMER2_ZONE_UNUSED62 62 298 | #define HAMMER2_ZONE_UNUSED63 63 299 | #define HAMMER2_ZONE_END 64 /* non-inclusive */ 300 | 301 | #define HAMMER2_NFREEMAPS 8 /* FREEMAP_00 - FREEMAP_07 */ 302 | 303 | /* relative to FREEMAP_x */ 304 | #define HAMMER2_ZONEFM_LEVEL1 0 /* 1GB leafmap */ 305 | #define HAMMER2_ZONEFM_LEVEL2 1 /* 256GB indmap */ 306 | #define HAMMER2_ZONEFM_LEVEL3 2 /* 64TB indmap */ 307 | #define HAMMER2_ZONEFM_LEVEL4 3 /* 16PB indmap */ 308 | #define HAMMER2_ZONEFM_LEVEL5 4 /* 4EB indmap */ 309 | /* LEVEL6 is a set of 4 blockrefs in the volume header 16EB */ 310 | 311 | /* 312 | * Freemap radix. Assumes a set-count of 4, 128-byte blockrefs, 313 | * 32KB indirect block for freemap (LEVELN_PSIZE below). 314 | * 315 | * Leaf entry represents 4MB of storage broken down into a 512-bit 316 | * bitmap, 2-bits per entry. So course bitmap item represents 16KB. 317 | */ 318 | #define HAMMER2_FREEMAP_LEVEL6_RADIX 64 /* 16EB (end) */ 319 | #define HAMMER2_FREEMAP_LEVEL5_RADIX 62 /* 4EB */ 320 | #define HAMMER2_FREEMAP_LEVEL4_RADIX 54 /* 16PB */ 321 | #define HAMMER2_FREEMAP_LEVEL3_RADIX 46 /* 64TB */ 322 | #define HAMMER2_FREEMAP_LEVEL2_RADIX 38 /* 256GB */ 323 | #define HAMMER2_FREEMAP_LEVEL1_RADIX 30 /* 1GB */ 324 | #define HAMMER2_FREEMAP_LEVEL0_RADIX 22 /* 4MB (128by in l-1 leaf) */ 325 | 326 | #define HAMMER2_FREEMAP_LEVELN_PSIZE 32768 /* physical bytes */ 327 | 328 | #define HAMMER2_FREEMAP_LEVEL5_SIZE ((hammer2_off_t)1 << \ 329 | HAMMER2_FREEMAP_LEVEL5_RADIX) 330 | #define HAMMER2_FREEMAP_LEVEL4_SIZE ((hammer2_off_t)1 << \ 331 | HAMMER2_FREEMAP_LEVEL4_RADIX) 332 | #define HAMMER2_FREEMAP_LEVEL3_SIZE ((hammer2_off_t)1 << \ 333 | HAMMER2_FREEMAP_LEVEL3_RADIX) 334 | #define HAMMER2_FREEMAP_LEVEL2_SIZE ((hammer2_off_t)1 << \ 335 | HAMMER2_FREEMAP_LEVEL2_RADIX) 336 | #define HAMMER2_FREEMAP_LEVEL1_SIZE ((hammer2_off_t)1 << \ 337 | HAMMER2_FREEMAP_LEVEL1_RADIX) 338 | #define HAMMER2_FREEMAP_LEVEL0_SIZE ((hammer2_off_t)1 << \ 339 | HAMMER2_FREEMAP_LEVEL0_RADIX) 340 | 341 | #define HAMMER2_FREEMAP_LEVEL5_MASK (HAMMER2_FREEMAP_LEVEL5_SIZE - 1) 342 | #define HAMMER2_FREEMAP_LEVEL4_MASK (HAMMER2_FREEMAP_LEVEL4_SIZE - 1) 343 | #define HAMMER2_FREEMAP_LEVEL3_MASK (HAMMER2_FREEMAP_LEVEL3_SIZE - 1) 344 | #define HAMMER2_FREEMAP_LEVEL2_MASK (HAMMER2_FREEMAP_LEVEL2_SIZE - 1) 345 | #define HAMMER2_FREEMAP_LEVEL1_MASK (HAMMER2_FREEMAP_LEVEL1_SIZE - 1) 346 | #define HAMMER2_FREEMAP_LEVEL0_MASK (HAMMER2_FREEMAP_LEVEL0_SIZE - 1) 347 | 348 | #define HAMMER2_FREEMAP_COUNT (int)(HAMMER2_FREEMAP_LEVELN_PSIZE / \ 349 | sizeof(hammer2_bmap_data_t)) 350 | 351 | /* 352 | * 16KB bitmap granularity (x2 bits per entry). 353 | */ 354 | #define HAMMER2_FREEMAP_BLOCK_RADIX 14 355 | #define HAMMER2_FREEMAP_BLOCK_SIZE (1 << HAMMER2_FREEMAP_BLOCK_RADIX) 356 | #define HAMMER2_FREEMAP_BLOCK_MASK (HAMMER2_FREEMAP_BLOCK_SIZE - 1) 357 | 358 | /* 359 | * bitmap[] structure. 2 bits per HAMMER2_FREEMAP_BLOCK_SIZE. 360 | * 361 | * 8 x 64-bit elements, 2 bits per block. 362 | * 32 blocks (radix 5) per element. 363 | * representing INDEX_SIZE bytes worth of storage per element. 364 | */ 365 | 366 | typedef u64int hammer2_bitmap_t; 367 | 368 | #define HAMMER2_BMAP_ALLONES ((hammer2_bitmap_t)-1) 369 | #define HAMMER2_BMAP_ELEMENTS 8 370 | #define HAMMER2_BMAP_BITS_PER_ELEMENT 64 371 | #define HAMMER2_BMAP_INDEX_RADIX 5 /* 32 blocks per element */ 372 | #define HAMMER2_BMAP_BLOCKS_PER_ELEMENT (1 << HAMMER2_BMAP_INDEX_RADIX) 373 | 374 | #define HAMMER2_BMAP_INDEX_SIZE (HAMMER2_FREEMAP_BLOCK_SIZE * \ 375 | HAMMER2_BMAP_BLOCKS_PER_ELEMENT) 376 | #define HAMMER2_BMAP_INDEX_MASK (HAMMER2_BMAP_INDEX_SIZE - 1) 377 | 378 | #define HAMMER2_BMAP_SIZE (HAMMER2_BMAP_INDEX_SIZE * \ 379 | HAMMER2_BMAP_ELEMENTS) 380 | #define HAMMER2_BMAP_MASK (HAMMER2_BMAP_SIZE - 1) 381 | 382 | /* 383 | * Two linear areas can be reserved after the initial 2MB segment in the base 384 | * zone (the one starting at offset 0). These areas are NOT managed by the 385 | * block allocator and do not fall under HAMMER2 crc checking rules based 386 | * at the volume header (but can be self-CRCd internally, depending). 387 | */ 388 | #define HAMMER2_BOOT_MIN_BYTES HAMMER2_VOLUME_ALIGN 389 | #define HAMMER2_BOOT_NOM_BYTES (64*1024*1024) 390 | #define HAMMER2_BOOT_MAX_BYTES (256*1024*1024) 391 | 392 | #define HAMMER2_REDO_MIN_BYTES HAMMER2_VOLUME_ALIGN 393 | #define HAMMER2_REDO_NOM_BYTES (256*1024*1024) 394 | #define HAMMER2_REDO_MAX_BYTES (1024*1024*1024) 395 | 396 | /* 397 | * Most HAMMER2 types are implemented as unsigned 64-bit integers. 398 | * Transaction ids are monotonic. 399 | * 400 | * We utilize 32-bit iSCSI CRCs. 401 | */ 402 | typedef u64int hammer2_tid_t; 403 | typedef u64int hammer2_off_t; 404 | typedef u64int hammer2_key_t; 405 | typedef u32int hammer2_crc32_t; 406 | 407 | /* 408 | * Miscellanious ranges (all are unsigned). 409 | */ 410 | #define HAMMER2_TID_MIN 1ULL 411 | #define HAMMER2_TID_MAX 0xFFFFFFFFFFFFFFFFULL 412 | #define HAMMER2_KEY_MIN 0ULL 413 | #define HAMMER2_KEY_MAX 0xFFFFFFFFFFFFFFFFULL 414 | #define HAMMER2_OFFSET_MIN 0ULL 415 | #define HAMMER2_OFFSET_MAX 0xFFFFFFFFFFFFFFFFULL 416 | 417 | /* 418 | * HAMMER2 data offset special cases and masking. 419 | * 420 | * All HAMMER2 data offsets have to be broken down into a 64K buffer base 421 | * offset (HAMMER2_OFF_MASK_HI) and a 64K buffer index (HAMMER2_OFF_MASK_LO). 422 | * 423 | * Indexes into physical buffers are always 64-byte aligned. The low 6 bits 424 | * of the data offset field specifies how large the data chunk being pointed 425 | * to as a power of 2. The theoretical minimum radix is thus 6 (The space 426 | * needed in the low bits of the data offset field). However, the practical 427 | * minimum allocation chunk size is 1KB (a radix of 10), so HAMMER2 sets 428 | * HAMMER2_RADIX_MIN to 10. The maximum radix is currently 16 (64KB), but 429 | * we fully intend to support larger extents in the future. 430 | * 431 | * WARNING! A radix of 0 (such as when data_off is all 0's) is a special 432 | * case which means no data associated with the blockref, and 433 | * not the '1 byte' it would otherwise calculate to. 434 | */ 435 | #define HAMMER2_OFF_BAD ((hammer2_off_t)-1) 436 | #define HAMMER2_OFF_MASK 0xFFFFFFFFFFFFFFC0ULL 437 | #define HAMMER2_OFF_MASK_LO (HAMMER2_OFF_MASK & HAMMER2_PBUFMASK64) 438 | #define HAMMER2_OFF_MASK_HI (~HAMMER2_PBUFMASK64) 439 | #define HAMMER2_OFF_MASK_RADIX 0x000000000000003FULL 440 | #define HAMMER2_MAX_COPIES 6 441 | 442 | /* 443 | * HAMMER2 directory support and pre-defined keys 444 | */ 445 | #define HAMMER2_DIRHASH_VISIBLE 0x8000000000000000ULL 446 | #define HAMMER2_DIRHASH_USERMSK 0x7FFFFFFFFFFFFFFFULL 447 | #define HAMMER2_DIRHASH_LOMASK 0x0000000000007FFFULL 448 | #define HAMMER2_DIRHASH_HIMASK 0xFFFFFFFFFFFF0000ULL 449 | #define HAMMER2_DIRHASH_FORCED 0x0000000000008000ULL /* bit forced on */ 450 | 451 | #define HAMMER2_SROOT_KEY 0x0000000000000000ULL /* volume to sroot */ 452 | #define HAMMER2_BOOT_KEY 0xd9b36ce135528000ULL /* sroot to BOOT PFS */ 453 | 454 | /************************************************************************ 455 | * DMSG SUPPORT * 456 | ************************************************************************ 457 | * LNK_VOLCONF 458 | * 459 | * All HAMMER2 directories directly under the super-root on your local 460 | * media can be mounted separately, even if they share the same physical 461 | * device. 462 | * 463 | * When you do a HAMMER2 mount you are effectively tying into a HAMMER2 464 | * cluster via local media. The local media does not have to participate 465 | * in the cluster, other than to provide the hammer2_volconf[] array and 466 | * root inode for the mount. 467 | * 468 | * This is important: The mount device path you specify serves to bootstrap 469 | * your entry into the cluster, but your mount will make active connections 470 | * to ALL copy elements in the hammer2_volconf[] array which match the 471 | * PFSID of the directory in the super-root that you specified. The local 472 | * media path does not have to be mentioned in this array but becomes part 473 | * of the cluster based on its type and access rights. ALL ELEMENTS ARE 474 | * TREATED ACCORDING TO TYPE NO MATTER WHICH ONE YOU MOUNT FROM. 475 | * 476 | * The actual cluster may be far larger than the elements you list in the 477 | * hammer2_volconf[] array. You list only the elements you wish to 478 | * directly connect to and you are able to access the rest of the cluster 479 | * indirectly through those connections. 480 | * 481 | * WARNING! This structure must be exactly 128 bytes long for its config 482 | * array to fit in the volume header. 483 | */ 484 | 485 | #pragma packed on 486 | 487 | struct hammer2_volconf { 488 | u8int copyid; /* 00 copyid 0-255 (must match slot) */ 489 | u8int inprog; /* 01 operation in progress, or 0 */ 490 | u8int chain_to; /* 02 operation chaining to, or 0 */ 491 | u8int chain_from; /* 03 operation chaining from, or 0 */ 492 | u16int flags; /* 04-05 flags field */ 493 | u8int error; /* 06 last operational error */ 494 | u8int priority; /* 07 priority and round-robin flag */ 495 | u8int remote_pfs_type;/* 08 probed direct remote PFS type */ 496 | u8int reserved08[23]; /* 09-1F */ 497 | uuid_t pfs_clid; /* 20-2F copy target must match this uuid */ 498 | u8int label[16]; /* 30-3F import/export label */ 499 | u8int path[64]; /* 40-7F target specification string or key */ 500 | }; 501 | 502 | typedef struct hammer2_volconf hammer2_volconf_t; 503 | 504 | /* 505 | * HAMMER2 directory entry header (embedded in blockref) exactly 16 bytes 506 | */ 507 | struct hammer2_dirent_head { 508 | hammer2_tid_t inum; /* inode number */ 509 | u16int namlen; /* name length */ 510 | u8int type; /* OBJTYPE_* */ 511 | u8int unused0B; 512 | u8int unused0C[4]; 513 | }; 514 | 515 | typedef struct hammer2_dirent_head hammer2_dirent_head_t; 516 | 517 | /* 518 | * The media block reference structure. This forms the core of the HAMMER2 519 | * media topology recursion. This 128-byte data structure is embedded in the 520 | * volume header, in inodes (which are also directory entries), and in 521 | * indirect blocks. 522 | * 523 | * A blockref references a single media item, which typically can be a 524 | * directory entry (aka inode), indirect block, or data block. 525 | * 526 | * The primary feature a blockref represents is the ability to validate 527 | * the entire tree underneath it via its check code. Any modification to 528 | * anything propagates up the blockref tree all the way to the root, replacing 529 | * the related blocks and compounding the generated check code. 530 | * 531 | * The check code can be a simple 32-bit iscsi code, a 64-bit crc, or as 532 | * complex as a 512 bit cryptographic hash. I originally used a 64-byte 533 | * blockref but later expanded it to 128 bytes to be able to support the 534 | * larger check code as well as to embed statistics for quota operation. 535 | * 536 | * Simple check codes are not sufficient for unverified dedup. Even with 537 | * a maximally-sized check code unverified dedup should only be used in 538 | * in subdirectory trees where you do not need 100% data integrity. 539 | * 540 | * Unverified dedup is deduping based on meta-data only without verifying 541 | * that the data blocks are actually identical. Verified dedup guarantees 542 | * integrity but is a far more I/O-expensive operation. 543 | * 544 | * -- 545 | * 546 | * mirror_tid - per cluster node modified (propagated upward by flush) 547 | * modify_tid - clc record modified (not propagated). 548 | * update_tid - clc record updated (propagated upward on verification) 549 | * 550 | * CLC - Stands for 'Cluster Level Change', identifiers which are identical 551 | * within the topology across all cluster nodes (when fully 552 | * synchronized). 553 | * 554 | * NOTE: The range of keys represented by the blockref is (key) to 555 | * ((key) + (1LL << keybits) - 1). HAMMER2 usually populates 556 | * blocks bottom-up, inserting a new root when radix expansion 557 | * is required. 558 | * 559 | * leaf_count - Helps manage leaf collapse calculations when indirect 560 | * blocks become mostly empty. This value caps out at 561 | * HAMMER2_BLOCKREF_LEAF_MAX (65535). 562 | * 563 | * Used by the chain code to determine when to pull leafs up 564 | * from nearly empty indirect blocks. For the purposes of this 565 | * calculation, BREF_TYPE_INODE is considered a leaf, along 566 | * with DIRENT and DATA. 567 | * 568 | * RESERVED FIELDS 569 | * 570 | * A number of blockref fields are reserved and should generally be set to 571 | * 0 for future compatibility. 572 | * 573 | * FUTURE BLOCKREF EXPANSION 574 | * 575 | * CONTENT ADDRESSABLE INDEXING (future) - Using a 256 or 512-bit check code. 576 | */ 577 | struct hammer2_blockref { /* MUST BE EXACTLY 64 BYTES */ 578 | u8int type; /* type of underlying item */ 579 | u8int methods; /* check method & compression method */ 580 | u8int copyid; /* specify which copy this is */ 581 | u8int keybits; /* #of keybits masked off 0=leaf */ 582 | u8int vradix; /* virtual data/meta-data size */ 583 | u8int flags; /* blockref flags */ 584 | u16int leaf_count; /* leaf aggregation count */ 585 | hammer2_key_t key; /* key specification */ 586 | hammer2_tid_t mirror_tid; /* media flush topology & freemap */ 587 | hammer2_tid_t modify_tid; /* clc modify (not propagated) */ 588 | hammer2_off_t data_off; /* low 6 bits is phys size (radix)*/ 589 | hammer2_tid_t update_tid; /* clc modify (propagated upward) */ 590 | union { 591 | char buf[16]; 592 | 593 | /* 594 | * Directory entry header (BREF_TYPE_DIRENT) 595 | * 596 | * NOTE: check.buf contains filename if <= 64 bytes. Longer 597 | * filenames are stored in a data reference of size 598 | * HAMMER2_ALLOC_MIN (at least 256, typically 1024). 599 | * 600 | * NOTE: inode structure may contain a copy of a recently 601 | * associated filename, for recovery purposes. 602 | * 603 | * NOTE: Superroot entries are INODEs, not DIRENTs. Code 604 | * allows both cases. 605 | */ 606 | hammer2_dirent_head_t dirent; 607 | 608 | /* 609 | * Statistics aggregation (BREF_TYPE_INODE, BREF_TYPE_INDIRECT) 610 | */ 611 | struct { 612 | hammer2_key_t data_count; 613 | hammer2_key_t inode_count; 614 | } stats; 615 | } embed; 616 | union { /* check info */ 617 | char buf[64]; 618 | struct { 619 | u32int value; 620 | u32int reserved[15]; 621 | } iscsi32; 622 | struct { 623 | u64int value; 624 | u64int reserved[7]; 625 | } xxhash64; 626 | struct { 627 | char data[24]; 628 | char reserved[40]; 629 | } sha192; 630 | struct { 631 | char data[32]; 632 | char reserved[32]; 633 | } sha256; 634 | struct { 635 | char data[64]; 636 | } sha512; 637 | 638 | /* 639 | * Freemap hints are embedded in addition to the icrc32. 640 | * 641 | * bigmask - Radixes available for allocation (0-31). 642 | * Heuristical (may be permissive but not 643 | * restrictive). Typically only radix values 644 | * 10-16 are used (i.e. (1<<10) through (1<<16)). 645 | * 646 | * avail - Total available space remaining, in bytes 647 | */ 648 | struct { 649 | u32int icrc32; 650 | u32int bigmask; /* available radixes */ 651 | u64int avail; /* total available bytes */ 652 | char reserved[48]; 653 | } freemap; 654 | } check; 655 | }; 656 | 657 | typedef struct hammer2_blockref hammer2_blockref_t; 658 | #pragma packed off 659 | 660 | #define HAMMER2_BLOCKREF_BYTES 128 /* blockref struct in bytes */ 661 | #define HAMMER2_BLOCKREF_RADIX 7 662 | 663 | #define HAMMER2_BLOCKREF_LEAF_MAX 65535 664 | 665 | /* 666 | * On-media and off-media blockref types. 667 | * 668 | * types >= 128 are pseudo values that should never be present on-media. 669 | */ 670 | #define HAMMER2_BREF_TYPE_EMPTY 0 671 | #define HAMMER2_BREF_TYPE_INODE 1 672 | #define HAMMER2_BREF_TYPE_INDIRECT 2 673 | #define HAMMER2_BREF_TYPE_DATA 3 674 | #define HAMMER2_BREF_TYPE_DIRENT 4 675 | #define HAMMER2_BREF_TYPE_FREEMAP_NODE 5 676 | #define HAMMER2_BREF_TYPE_FREEMAP_LEAF 6 677 | #define HAMMER2_BREF_TYPE_FREEMAP 254 /* pseudo-type */ 678 | #define HAMMER2_BREF_TYPE_VOLUME 255 /* pseudo-type */ 679 | 680 | #define HAMMER2_BREF_FLAG_PFSROOT 0x01 /* see also related opflag */ 681 | #define HAMMER2_BREF_FLAG_ZERO 0x02 682 | 683 | /* 684 | * Encode/decode check mode and compression mode for 685 | * bref.methods. The compression level is not encoded in 686 | * bref.methods. 687 | */ 688 | #define HAMMER2_ENC_CHECK(n) (((n) & 15) << 4) 689 | #define HAMMER2_DEC_CHECK(n) (((n) >> 4) & 15) 690 | #define HAMMER2_ENC_COMP(n) ((n) & 15) 691 | #define HAMMER2_DEC_COMP(n) ((n) & 15) 692 | 693 | #define HAMMER2_CHECK_NONE 0 694 | #define HAMMER2_CHECK_DISABLED 1 695 | #define HAMMER2_CHECK_ISCSI32 2 696 | #define HAMMER2_CHECK_XXHASH64 3 697 | #define HAMMER2_CHECK_SHA192 4 698 | #define HAMMER2_CHECK_FREEMAP 5 699 | 700 | #define HAMMER2_CHECK_DEFAULT HAMMER2_CHECK_XXHASH64 701 | 702 | /* user-specifiable check modes only */ 703 | #define HAMMER2_CHECK_STRINGS { "none", "disabled", "crc32", \ 704 | "xxhash64", "sha192" } 705 | #define HAMMER2_CHECK_STRINGS_COUNT 5 706 | 707 | /* 708 | * Encode/decode check or compression algorithm request in 709 | * ipdata->meta.check_algo and ipdata->meta.comp_algo. 710 | */ 711 | #define HAMMER2_ENC_ALGO(n) (n) 712 | #define HAMMER2_DEC_ALGO(n) ((n) & 15) 713 | #define HAMMER2_ENC_LEVEL(n) ((n) << 4) 714 | #define HAMMER2_DEC_LEVEL(n) (((n) >> 4) & 15) 715 | 716 | #define HAMMER2_COMP_NONE 0 717 | #define HAMMER2_COMP_AUTOZERO 1 718 | #define HAMMER2_COMP_LZ4 2 719 | #define HAMMER2_COMP_ZLIB 3 720 | 721 | #define HAMMER2_COMP_NEWFS_DEFAULT HAMMER2_COMP_LZ4 722 | #define HAMMER2_COMP_STRINGS { "none", "autozero", "lz4", "zlib" } 723 | #define HAMMER2_COMP_STRINGS_COUNT 4 724 | 725 | /* 726 | * Passed to hammer2_chain_create(), causes methods to be inherited from 727 | * parent. 728 | */ 729 | #define HAMMER2_METH_DEFAULT -1 730 | 731 | /* 732 | * HAMMER2 block references are collected into sets of 4 blockrefs. These 733 | * sets are fully associative, meaning the elements making up a set are 734 | * not sorted in any way and may contain duplicate entries, holes, or 735 | * entries which shortcut multiple levels of indirection. Sets are used 736 | * in various ways: 737 | * 738 | * (1) When redundancy is desired a set may contain several duplicate 739 | * entries pointing to different copies of the same data. Up to 4 copies 740 | * are supported. 741 | * 742 | * (2) The blockrefs in a set can shortcut multiple levels of indirections 743 | * within the bounds imposed by the parent of set. 744 | * 745 | * When a set fills up another level of indirection is inserted, moving 746 | * some or all of the set's contents into indirect blocks placed under the 747 | * set. This is a top-down approach in that indirect blocks are not created 748 | * until the set actually becomes full (that is, the entries in the set can 749 | * shortcut the indirect blocks when the set is not full). Depending on how 750 | * things are filled multiple indirect blocks will eventually be created. 751 | * 752 | * Indirect blocks are typically 4KB (64 entres) or 64KB (1024 entries) and 753 | * are also treated as fully set-associative. 754 | */ 755 | struct hammer2_blockset { 756 | hammer2_blockref_t blockref[HAMMER2_SET_COUNT]; 757 | }; 758 | 759 | typedef struct hammer2_blockset hammer2_blockset_t; 760 | 761 | /* 762 | * hammer2_bmap_data - A freemap entry in the LEVEL1 block. 763 | * 764 | * Each 128-byte entry contains the bitmap and meta-data required to manage 765 | * a LEVEL0 (128KB) block of storage. The storage is managed in 128 x 1KB 766 | * chunks. 767 | * 768 | * A smaller allocation granularity is supported via a linear iterator and/or 769 | * must otherwise be tracked in ram. 770 | * 771 | * (data structure must be 128 bytes exactly) 772 | * 773 | * linear - A BYTE linear allocation offset used for sub-16KB allocations 774 | * only. May contain values between 0 and 2MB. Must be ignored 775 | * if 16KB-aligned (i.e. force bitmap scan), otherwise may be 776 | * used to sub-allocate within the 16KB block (which is already 777 | * marked as allocated in the bitmap). 778 | * 779 | * Sub-allocations need only be 1KB-aligned and do not have to be 780 | * size-aligned, and 16KB or larger allocations do not update this 781 | * field, resulting in pretty good packing. 782 | * 783 | * Please note that file data granularity may be limited by 784 | * other issues such as buffer cache direct-mapping and the 785 | * desire to support sector sizes up to 16KB (so H2 only issues 786 | * I/O's in multiples of 16KB anyway). 787 | * 788 | * class - Clustering class. Cleared to 0 only if the entire leaf becomes 789 | * free. Used to cluster device buffers so all elements must have 790 | * the same device block size, but may mix logical sizes. 791 | * 792 | * Typically integrated with the blockref type in the upper 8 bits 793 | * to localize inodes and indrect blocks, improving bulk free scans 794 | * and directory scans. 795 | * 796 | * bitmap - Two bits per 16KB allocation block arranged in arrays of 797 | * 32-bit elements, 256x2 bits representing ~4MB worth of media 798 | * storage. Bit patterns are as follows: 799 | * 800 | * 00 Unallocated 801 | * 01 (reserved) 802 | * 10 Possibly free 803 | * 11 Allocated 804 | */ 805 | #pragma packed on 806 | struct hammer2_bmap_data { 807 | s32int linear; /* 00 linear sub-granular allocation offset */ 808 | u16int class; /* 04-05 clustering class ((type<<8)|radix) */ 809 | u8int reserved06; /* 06 */ 810 | u8int reserved07; /* 07 */ 811 | u32int reserved08; /* 08 */ 812 | u32int reserved0C; /* 0C */ 813 | u32int reserved10; /* 10 */ 814 | u32int reserved14; /* 14 */ 815 | u32int reserved18; /* 18 */ 816 | u32int avail; /* 1C */ 817 | u32int reserved20[8]; /* 20-3F 256 bits manages 128K/1KB/2-bits */ 818 | /* 40-7F 512 bits manages 4MB of storage */ 819 | hammer2_bitmap_t bitmapq[HAMMER2_BMAP_ELEMENTS]; 820 | }; 821 | 822 | typedef struct hammer2_bmap_data hammer2_bmap_data_t; 823 | 824 | /* 825 | * In HAMMER2 inodes ARE directory entries, with a special exception for 826 | * hardlinks. The inode number is stored in the inode rather than being 827 | * based on the location of the inode (since the location moves every time 828 | * the inode or anything underneath the inode is modified). 829 | * 830 | * The inode is 1024 bytes, made up of 256 bytes of meta-data, 256 bytes 831 | * for the filename, and 512 bytes worth of direct file data OR an embedded 832 | * blockset. The in-memory hammer2_inode structure contains only the mostly- 833 | * node-independent meta-data portion (some flags are node-specific and will 834 | * not be synchronized). The rest of the inode is node-specific and chain I/O 835 | * is required to obtain it. 836 | * 837 | * Directories represent one inode per blockref. Inodes are not laid out 838 | * as a file but instead are represented by the related blockrefs. The 839 | * blockrefs, in turn, are indexed by the 64-bit directory hash key. Remember 840 | * that blocksets are fully associative, so a certain degree efficiency is 841 | * achieved just from that. 842 | * 843 | * Up to 512 bytes of direct data can be embedded in an inode, and since 844 | * inodes are essentially directory entries this also means that small data 845 | * files end up simply being laid out linearly in the directory, resulting 846 | * in fewer seeks and highly optimal access. 847 | * 848 | * The compression mode can be changed at any time in the inode and is 849 | * recorded on a blockref-by-blockref basis. 850 | * 851 | * Hardlinks are supported via the inode map. Essentially the way a hardlink 852 | * works is that all individual directory entries representing the same file 853 | * are special cased and specify the same inode number. The actual file 854 | * is placed in the nearest parent directory that is parent to all instances 855 | * of the hardlink. If all hardlinks to a file are in the same directory 856 | * the actual file will also be placed in that directory. This file uses 857 | * the inode number as the directory entry key and is invisible to normal 858 | * directory scans. Real directory entry keys are differentiated from the 859 | * inode number key via bit 63. Access to the hardlink silently looks up 860 | * the real file and forwards all operations to that file. Removal of the 861 | * last hardlink also removes the real file. 862 | * 863 | * (attr_tid) is only updated when the inode's specific attributes or regular 864 | * file size has changed, and affects path lookups and stat. (attr_tid) 865 | * represents a special cache coherency lock under the inode. The inode 866 | * blockref's modify_tid will always cover it. 867 | * 868 | * (dirent_tid) is only updated when an entry under a directory inode has 869 | * been created, deleted, renamed, or had its attributes change, and affects 870 | * directory lookups and scans. (dirent_tid) represents another special cache 871 | * coherency lock under the inode. The inode blockref's modify_tid will 872 | * always cover it. 873 | */ 874 | #define HAMMER2_INODE_BYTES 1024 /* (asserted by code) */ 875 | #define HAMMER2_INODE_MAXNAME 256 /* maximum name in bytes */ 876 | #define HAMMER2_INODE_VERSION_ONE 1 877 | 878 | #define HAMMER2_INODE_START 1024 /* dynamically allocated */ 879 | 880 | struct hammer2_inode_meta { 881 | u16int version; /* 0000 inode data version */ 882 | u8int reserved02; /* 0002 */ 883 | u8int pfs_subtype; /* 0003 pfs sub-type */ 884 | 885 | /* 886 | * core inode attributes, inode type, misc flags 887 | */ 888 | u32int uflags; /* 0004 chflags */ 889 | u32int rmajor; /* 0008 available for device nodes */ 890 | u32int rminor; /* 000C available for device nodes */ 891 | u64int ctime; /* 0010 inode change time */ 892 | u64int mtime; /* 0018 modified time */ 893 | u64int atime; /* 0020 access time (unsupported) */ 894 | u64int btime; /* 0028 birth time */ 895 | uuid_t uid; /* 0030 uid / degenerate unix uid */ 896 | uuid_t gid; /* 0040 gid / degenerate unix gid */ 897 | 898 | u8int type; /* 0050 object type */ 899 | u8int op_flags; /* 0051 operational flags */ 900 | u16int cap_flags; /* 0052 capability flags */ 901 | u32int mode; /* 0054 unix modes (typ low 16 bits) */ 902 | 903 | /* 904 | * inode size, identification, localized recursive configuration 905 | * for compression and backup copies. 906 | * 907 | * NOTE: Nominal parent inode number (iparent) is only applicable 908 | * for directories but can also help for files during 909 | * catastrophic recovery. 910 | */ 911 | hammer2_tid_t inum; /* 0058 inode number */ 912 | hammer2_off_t size; /* 0060 size of file */ 913 | u64int nlinks; /* 0068 hard links (typ only dirs) */ 914 | hammer2_tid_t iparent; /* 0070 nominal parent inum */ 915 | hammer2_key_t name_key; /* 0078 full filename key */ 916 | u16int name_len; /* 0080 filename length */ 917 | u8int ncopies; /* 0082 ncopies to local media */ 918 | u8int comp_algo; /* 0083 compression request & algo */ 919 | 920 | /* 921 | * These fields are currently only applicable to PFSROOTs. 922 | * 923 | * NOTE: We can't use {volume_data->fsid, pfs_clid} to uniquely 924 | * identify an instance of a PFS in the cluster because 925 | * a mount may contain more than one copy of the PFS as 926 | * a separate node. {pfs_clid, pfs_fsid} must be used for 927 | * registration in the cluster. 928 | */ 929 | u8int target_type; /* 0084 hardlink target type */ 930 | u8int check_algo; /* 0085 check code request & algo */ 931 | u8int pfs_nmasters; /* 0086 (if PFSROOT) if multi-master */ 932 | u8int pfs_type; /* 0087 (if PFSROOT) node type */ 933 | u64int pfs_inum; /* 0088 (if PFSROOT) inum allocator */ 934 | uuid_t pfs_clid; /* 0090 (if PFSROOT) cluster uuid */ 935 | uuid_t pfs_fsid; /* 00A0 (if PFSROOT) unique uuid */ 936 | 937 | /* 938 | * Quotas and aggregate sub-tree inode and data counters. Note that 939 | * quotas are not replicated downward, they are explicitly set by 940 | * the sysop and in-memory structures keep track of inheritence. 941 | */ 942 | hammer2_key_t data_quota; /* 00B0 subtree quota in bytes */ 943 | hammer2_key_t unusedB8; /* 00B8 subtree byte count */ 944 | hammer2_key_t inode_quota; /* 00C0 subtree quota inode count */ 945 | hammer2_key_t unusedC8; /* 00C8 subtree inode count */ 946 | 947 | /* 948 | * The last snapshot tid is tested against modify_tid to determine 949 | * when a copy must be made of a data block whos check mode has been 950 | * disabled (a disabled check mode allows data blocks to be updated 951 | * in place instead of copy-on-write). 952 | */ 953 | hammer2_tid_t pfs_lsnap_tid; /* 00D0 last snapshot tid */ 954 | hammer2_tid_t reservedD8; /* 00D8 (avail) */ 955 | 956 | /* 957 | * Tracks (possibly degenerate) free areas covering all sub-tree 958 | * allocations under inode, not counting the inode itself. 959 | * 0/0 indicates empty entry. fully set-associative. 960 | * 961 | * (not yet implemented) 962 | */ 963 | u64int decrypt_check; /* 00E0 decryption validator */ 964 | hammer2_off_t reservedE0[3]; /* 00E8/F0/F8 */ 965 | }; 966 | 967 | typedef struct hammer2_inode_meta hammer2_inode_meta_t; 968 | 969 | struct hammer2_inode_data { 970 | hammer2_inode_meta_t meta; /* 0000-00FF */ 971 | unsigned char filename[HAMMER2_INODE_MAXNAME]; 972 | /* 0100-01FF (256 char, unterminated) */ 973 | union { /* 0200-03FF (64x8 = 512 bytes) */ 974 | struct hammer2_blockset blockset; 975 | char data[HAMMER2_EMBEDDED_BYTES]; 976 | } u; 977 | }; 978 | 979 | typedef struct hammer2_inode_data hammer2_inode_data_t; 980 | 981 | #define HAMMER2_OPFLAG_DIRECTDATA 0x01 982 | #define HAMMER2_OPFLAG_PFSROOT 0x02 /* (see also bref flag) */ 983 | #define HAMMER2_OPFLAG_COPYIDS 0x04 /* copyids override parent */ 984 | 985 | #define HAMMER2_OBJTYPE_UNKNOWN 0 986 | #define HAMMER2_OBJTYPE_DIRECTORY 1 987 | #define HAMMER2_OBJTYPE_REGFILE 2 988 | #define HAMMER2_OBJTYPE_FIFO 4 989 | #define HAMMER2_OBJTYPE_CDEV 5 990 | #define HAMMER2_OBJTYPE_BDEV 6 991 | #define HAMMER2_OBJTYPE_SOFTLINK 7 992 | #define HAMMER2_OBJTYPE_UNUSED08 8 993 | #define HAMMER2_OBJTYPE_SOCKET 9 994 | #define HAMMER2_OBJTYPE_WHITEOUT 10 995 | 996 | #define HAMMER2_COPYID_NONE 0 997 | #define HAMMER2_COPYID_LOCAL ((u8int)-1) 998 | 999 | #define HAMMER2_COPYID_COUNT 256 1000 | 1001 | /* 1002 | * PFS types identify the role of a PFS within a cluster. The PFS types 1003 | * is stored on media and in LNK_SPAN messages and used in other places. 1004 | * 1005 | * The low 4 bits specify the current active type while the high 4 bits 1006 | * specify the transition target if the PFS is being upgraded or downgraded, 1007 | * If the upper 4 bits are not zero it may effect how a PFS is used during 1008 | * the transition. 1009 | * 1010 | * Generally speaking, downgrading a MASTER to a SLAVE cannot complete until 1011 | * at least all MASTERs have updated their pfs_nmasters field. And upgrading 1012 | * a SLAVE to a MASTER cannot complete until the new prospective master has 1013 | * been fully synchronized (though theoretically full synchronization is 1014 | * not required if a (new) quorum of other masters are fully synchronized). 1015 | * 1016 | * It generally does not matter which PFS element you actually mount, you 1017 | * are mounting 'the cluster'. So, for example, a network mount will mount 1018 | * a DUMMY PFS type on a memory filesystem. However, there are two exceptions. 1019 | * In order to gain the benefits of a SOFT_MASTER or SOFT_SLAVE, those PFSs 1020 | * must be directly mounted. 1021 | */ 1022 | #define HAMMER2_PFSTYPE_NONE 0x00 1023 | #define HAMMER2_PFSTYPE_CACHE 0x01 1024 | #define HAMMER2_PFSTYPE_UNUSED02 0x02 1025 | #define HAMMER2_PFSTYPE_SLAVE 0x03 1026 | #define HAMMER2_PFSTYPE_SOFT_SLAVE 0x04 1027 | #define HAMMER2_PFSTYPE_SOFT_MASTER 0x05 1028 | #define HAMMER2_PFSTYPE_MASTER 0x06 1029 | #define HAMMER2_PFSTYPE_UNUSED07 0x07 1030 | #define HAMMER2_PFSTYPE_SUPROOT 0x08 1031 | #define HAMMER2_PFSTYPE_DUMMY 0x09 1032 | #define HAMMER2_PFSTYPE_MAX 16 1033 | 1034 | #define HAMMER2_PFSTRAN_NONE 0x00 /* no transition in progress */ 1035 | #define HAMMER2_PFSTRAN_CACHE 0x10 1036 | #define HAMMER2_PFSTRAN_UNMUSED20 0x20 1037 | #define HAMMER2_PFSTRAN_SLAVE 0x30 1038 | #define HAMMER2_PFSTRAN_SOFT_SLAVE 0x40 1039 | #define HAMMER2_PFSTRAN_SOFT_MASTER 0x50 1040 | #define HAMMER2_PFSTRAN_MASTER 0x60 1041 | #define HAMMER2_PFSTRAN_UNUSED70 0x70 1042 | #define HAMMER2_PFSTRAN_SUPROOT 0x80 1043 | #define HAMMER2_PFSTRAN_DUMMY 0x90 1044 | 1045 | #define HAMMER2_PFS_DEC(n) ((n) & 0x0F) 1046 | #define HAMMER2_PFS_DEC_TRANSITION(n) (((n) >> 4) & 0x0F) 1047 | #define HAMMER2_PFS_ENC_TRANSITION(n) (((n) & 0x0F) << 4) 1048 | 1049 | #define HAMMER2_PFSSUBTYPE_NONE 0 1050 | #define HAMMER2_PFSSUBTYPE_SNAPSHOT 1 /* manual/managed snapshot */ 1051 | #define HAMMER2_PFSSUBTYPE_AUTOSNAP 2 /* automatic snapshot */ 1052 | 1053 | /* 1054 | * PFS mode of operation is a bitmask. This is typically not stored 1055 | * on-media, but defined here because the field may be used in dmsgs. 1056 | */ 1057 | #define HAMMER2_PFSMODE_QUORUM 0x01 1058 | #define HAMMER2_PFSMODE_RW 0x02 1059 | 1060 | /* 1061 | * Allocation Table 1062 | * 1063 | */ 1064 | 1065 | 1066 | /* 1067 | * Flags (8 bits) - blockref, for freemap only 1068 | * 1069 | * Note that the minimum chunk size is 1KB so we could theoretically have 1070 | * 10 bits here, but we might have some future extension that allows a 1071 | * chunk size down to 256 bytes and if so we will need bits 8 and 9. 1072 | */ 1073 | #define HAMMER2_AVF_SELMASK 0x03 /* select group */ 1074 | #define HAMMER2_AVF_ALL_ALLOC 0x04 /* indicate all allocated */ 1075 | #define HAMMER2_AVF_ALL_FREE 0x08 /* indicate all free */ 1076 | #define HAMMER2_AVF_RESERVED10 0x10 1077 | #define HAMMER2_AVF_RESERVED20 0x20 1078 | #define HAMMER2_AVF_RESERVED40 0x40 1079 | #define HAMMER2_AVF_RESERVED80 0x80 1080 | #define HAMMER2_AVF_AVMASK32 ((u32int)0xFFFFFF00LU) 1081 | #define HAMMER2_AVF_AVMASK64 ((u64int)0xFFFFFFFFFFFFFF00LLU) 1082 | 1083 | #define HAMMER2_AV_SELECT_A 0x00 1084 | #define HAMMER2_AV_SELECT_B 0x01 1085 | #define HAMMER2_AV_SELECT_C 0x02 1086 | #define HAMMER2_AV_SELECT_D 0x03 1087 | 1088 | /* 1089 | * The volume header eats a 64K block. There is currently an issue where 1090 | * we want to try to fit all nominal filesystem updates in a 512-byte section 1091 | * but it may be a lost cause due to the need for a blockset. 1092 | * 1093 | * All information is stored in host byte order. The volume header's magic 1094 | * number may be checked to determine the byte order. If you wish to mount 1095 | * between machines w/ different endian modes you'll need filesystem code 1096 | * which acts on the media data consistently (either all one way or all the 1097 | * other). Our code currently does not do that. 1098 | * 1099 | * A read-write mount may have to recover missing allocations by doing an 1100 | * incremental mirror scan looking for modifications made after alloc_tid. 1101 | * If alloc_tid == last_tid then no recovery operation is needed. Recovery 1102 | * operations are usually very, very fast. 1103 | * 1104 | * Read-only mounts do not need to do any recovery, access to the filesystem 1105 | * topology is always consistent after a crash (is always consistent, period). 1106 | * However, there may be shortcutted blockref updates present from deep in 1107 | * the tree which are stored in the volumeh eader and must be tracked on 1108 | * the fly. 1109 | * 1110 | * NOTE: The copyinfo[] array contains the configuration for both the 1111 | * cluster connections and any local media copies. The volume 1112 | * header will be replicated for each local media copy. 1113 | * 1114 | * The mount command may specify multiple medias or just one and 1115 | * allow HAMMER2 to pick up the others when it checks the copyinfo[] 1116 | * array on mount. 1117 | * 1118 | * NOTE: root_blockref points to the super-root directory, not the root 1119 | * directory. The root directory will be a subdirectory under the 1120 | * super-root. 1121 | * 1122 | * The super-root directory contains all root directories and all 1123 | * snapshots (readonly or writable). It is possible to do a 1124 | * null-mount of the super-root using special path constructions 1125 | * relative to your mounted root. 1126 | * 1127 | * NOTE: HAMMER2 allows any subdirectory tree to be managed as if it were 1128 | * a PFS, including mirroring and storage quota operations, and this is 1129 | * prefered over creating discrete PFSs in the super-root. Instead 1130 | * the super-root is most typically used to create writable snapshots, 1131 | * alternative roots, and so forth. The super-root is also used by 1132 | * the automatic snapshotting mechanism. 1133 | */ 1134 | #define HAMMER2_VOLUME_ID_HBO 0x48414d3205172011LLU 1135 | #define HAMMER2_VOLUME_ID_ABO 0x11201705324d4148LLU 1136 | 1137 | struct hammer2_volume_data { 1138 | /* 1139 | * sector #0 - 512 bytes 1140 | */ 1141 | u64int magic; /* 0000 Signature */ 1142 | hammer2_off_t boot_beg; /* 0008 Boot area (future) */ 1143 | hammer2_off_t boot_end; /* 0010 (size = end - beg) */ 1144 | hammer2_off_t aux_beg; /* 0018 Aux area (future) */ 1145 | hammer2_off_t aux_end; /* 0020 (size = end - beg) */ 1146 | hammer2_off_t volu_size; /* 0028 Volume size, bytes */ 1147 | 1148 | u32int version; /* 0030 */ 1149 | u32int flags; /* 0034 */ 1150 | u8int copyid; /* 0038 copyid of phys vol */ 1151 | u8int freemap_version; /* 0039 freemap algorithm */ 1152 | u8int peer_type; /* 003A HAMMER2_PEER_xxx */ 1153 | u8int reserved003B; /* 003B */ 1154 | u32int reserved003C; /* 003C */ 1155 | 1156 | uuid_t fsid; /* 0040 */ 1157 | uuid_t fstype; /* 0050 */ 1158 | 1159 | /* 1160 | * allocator_size is precalculated at newfs time and does not include 1161 | * reserved blocks, boot, or redo areas. 1162 | * 1163 | * Initial non-reserved-area allocations do not use the freemap 1164 | * but instead adjust alloc_iterator. Dynamic allocations take 1165 | * over starting at (allocator_beg). This makes newfs_hammer2's 1166 | * job a lot easier and can also serve as a testing jig. 1167 | */ 1168 | hammer2_off_t allocator_size; /* 0060 Total data space */ 1169 | hammer2_off_t allocator_free; /* 0068 Free space */ 1170 | hammer2_off_t allocator_beg; /* 0070 Initial allocations */ 1171 | 1172 | /* 1173 | * mirror_tid reflects the highest committed change for this 1174 | * block device regardless of whether it is to the super-root 1175 | * or to a PFS or whatever. 1176 | * 1177 | * freemap_tid reflects the highest committed freemap change for 1178 | * this block device. 1179 | */ 1180 | hammer2_tid_t mirror_tid; /* 0078 committed tid (vol) */ 1181 | hammer2_tid_t reserved0080; /* 0080 */ 1182 | hammer2_tid_t reserved0088; /* 0088 */ 1183 | hammer2_tid_t freemap_tid; /* 0090 committed tid (fmap) */ 1184 | hammer2_tid_t bulkfree_tid; /* 0098 bulkfree incremental */ 1185 | hammer2_tid_t reserved00A0[5]; /* 00A0-00C7 */ 1186 | 1187 | /* 1188 | * Copyids are allocated dynamically from the copyexists bitmap. 1189 | * An id from the active copies set (up to 8, see copyinfo later on) 1190 | * may still exist after the copy set has been removed from the 1191 | * volume header and its bit will remain active in the bitmap and 1192 | * cannot be reused until it is 100% removed from the hierarchy. 1193 | */ 1194 | u32int copyexists[8]; /* 00C8-00E7 copy exists bmap */ 1195 | char reserved0140[248]; /* 00E8-01DF */ 1196 | 1197 | /* 1198 | * 32 bit CRC array at the end of the first 512 byte sector. 1199 | * 1200 | * icrc_sects[7] - First 512-4 bytes of volume header (including all 1201 | * the other icrc's except this one). 1202 | * 1203 | * icrc_sects[6] - Sector 1 (512 bytes) of volume header, which is 1204 | * the blockset for the root. 1205 | * 1206 | * icrc_sects[5] - Sector 2 1207 | * icrc_sects[4] - Sector 3 1208 | * icrc_sects[3] - Sector 4 (the freemap blockset) 1209 | */ 1210 | hammer2_crc32_t icrc_sects[8]; /* 01E0-01FF */ 1211 | 1212 | /* 1213 | * sector #1 - 512 bytes 1214 | * 1215 | * The entire sector is used by a blockset. 1216 | */ 1217 | hammer2_blockset_t sroot_blockset; /* 0200-03FF Superroot dir */ 1218 | 1219 | /* 1220 | * sector #2-7 1221 | */ 1222 | char sector2[512]; /* 0400-05FF reserved */ 1223 | char sector3[512]; /* 0600-07FF reserved */ 1224 | hammer2_blockset_t freemap_blockset; /* 0800-09FF freemap */ 1225 | char sector5[512]; /* 0A00-0BFF reserved */ 1226 | char sector6[512]; /* 0C00-0DFF reserved */ 1227 | char sector7[512]; /* 0E00-0FFF reserved */ 1228 | 1229 | /* 1230 | * sector #8-71 - 32768 bytes 1231 | * 1232 | * Contains the configuration for up to 256 copyinfo targets. These 1233 | * specify local and remote copies operating as masters or slaves. 1234 | * copyid's 0 and 255 are reserved (0 indicates an empty slot and 255 1235 | * indicates the local media). 1236 | * 1237 | * Each inode contains a set of up to 8 copyids, either inherited 1238 | * from its parent or explicitly specified in the inode, which 1239 | * indexes into this array. 1240 | */ 1241 | /* 1000-8FFF copyinfo config */ 1242 | hammer2_volconf_t copyinfo[HAMMER2_COPYID_COUNT]; 1243 | 1244 | /* 1245 | * Remaining sections are reserved for future use. 1246 | */ 1247 | char reserved0400[0x6FFC]; /* 9000-FFFB reserved */ 1248 | 1249 | /* 1250 | * icrc on entire volume header 1251 | */ 1252 | hammer2_crc32_t icrc_volheader; /* FFFC-FFFF full volume icrc*/ 1253 | }; 1254 | 1255 | typedef struct hammer2_volume_data hammer2_volume_data_t; 1256 | 1257 | /* 1258 | * Various parts of the volume header have their own iCRCs. 1259 | * 1260 | * The first 512 bytes has its own iCRC stored at the end of the 512 bytes 1261 | * and not included the icrc calculation. 1262 | * 1263 | * The second 512 bytes also has its own iCRC but it is stored in the first 1264 | * 512 bytes so it covers the entire second 512 bytes. 1265 | * 1266 | * The whole volume block (64KB) has an iCRC covering all but the last 4 bytes, 1267 | * which is where the iCRC for the whole volume is stored. This is currently 1268 | * a catch-all for anything not individually iCRCd. 1269 | */ 1270 | #define HAMMER2_VOL_ICRC_SECT0 7 1271 | #define HAMMER2_VOL_ICRC_SECT1 6 1272 | 1273 | #define HAMMER2_VOLUME_BYTES 65536 1274 | 1275 | #define HAMMER2_VOLUME_ICRC0_OFF 0 1276 | #define HAMMER2_VOLUME_ICRC1_OFF 512 1277 | #define HAMMER2_VOLUME_ICRCVH_OFF 0 1278 | 1279 | #define HAMMER2_VOLUME_ICRC0_SIZE (512 - 4) 1280 | #define HAMMER2_VOLUME_ICRC1_SIZE (512) 1281 | #define HAMMER2_VOLUME_ICRCVH_SIZE (65536 - 4) 1282 | 1283 | #define HAMMER2_VOL_VERSION_MIN 1 1284 | #define HAMMER2_VOL_VERSION_DEFAULT 1 1285 | #define HAMMER2_VOL_VERSION_WIP 2 1286 | 1287 | #define HAMMER2_NUM_VOLHDRS 4 1288 | 1289 | union hammer2_media_data { 1290 | hammer2_volume_data_t voldata; 1291 | hammer2_inode_data_t ipdata; 1292 | hammer2_blockset_t blkset; 1293 | hammer2_blockref_t npdata[HAMMER2_IND_COUNT_MAX]; 1294 | hammer2_bmap_data_t bmdata[HAMMER2_FREEMAP_COUNT]; 1295 | char buf[HAMMER2_PBUFSIZE]; 1296 | }; 1297 | 1298 | typedef union hammer2_media_data hammer2_media_data_t; 1299 | #pragma packed off 1300 | 1301 | --------------------------------------------------------------------------------