├── BenchRedisHLL.c ├── Benchwyhll.c ├── LICENSE ├── README.md ├── hyperloglog.c ├── hyperloglog.h ├── makefile ├── wyhash.h ├── wyhll.h └── wyhll.png /BenchRedisHLL.c: -------------------------------------------------------------------------------- 1 | #include "hyperloglog.h" 2 | #include "wyhash.h" 3 | #include 4 | const size_t N=10000; 5 | 6 | int main(void){ 7 | FILE *f=fopen("RedisHLL.xls","wt"); 8 | uint64_t seed=0, r; 9 | for(uint32_t n=1; n<=15000000; n+=(n/2)+1){ 10 | double rmse=0; 11 | for(size_t it=0; it 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # wyHLL: WangYi's 3-bits HyperLogLog 2 | 3 | ![](wyhll.png) 4 | -------------------------------------------------------------------------------- /hyperloglog.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "hyperloglog.h" 10 | 11 | /* 12 | * 一些工具函数 13 | */ 14 | 15 | /* debug宏 */ 16 | #ifdef DEBUG 17 | #define xprintf(fmt, ...) printf("FILE:<%s>,FUN:<%s>,LINE:<%d>,"fmt"\n", __FILE__, __FUNCTION__, __LINE__, ##__VA_ARGS__) 18 | #else 19 | #define xprintf(fmt, ...) 20 | #endif 21 | 22 | /* assert宏, assert失败时打印表达式 */ 23 | #define xassert(X) if(!(X)){ \ 24 | xprintf(" (%s) assert fail!", #X); \ 25 | assert(0); \ 26 | } 27 | 28 | 29 | enum 30 | { 31 | _Little_endian = 0, 32 | _Big_endian 33 | }; 34 | 35 | /*判断大小端的代码*/ 36 | inline static int byte_order() 37 | { 38 | union 39 | { 40 | short a; 41 | char b; 42 | }w; 43 | 44 | w.a = 1; 45 | 46 | return (w.b == 1) ? _Little_endian : _Big_endian; 47 | } 48 | 49 | 50 | const uint16_t hll_sparse_max_bytes = 3000; /*sparse编码方式最大占用空间*/ 51 | 52 | /* 53 | * 关于sds内存分配的一些简单代码 54 | */ 55 | typedef struct sdshdr sdshdr; 56 | typedef char *sds; 57 | 58 | struct __attribute__ ((__packed__)) sdshdr 59 | { 60 | uint16_t len; /* 已使用的空间 */ 61 | uint16_t alloc; /* 分配的总空间 */ 62 | char buf[]; 63 | }; 64 | 65 | #define sdsHdrSize sizeof(sdshdr) 66 | 67 | static inline size_t sdsavail(const sds s) 68 | { 69 | sdshdr *sh = (sdshdr*)(s - sdsHdrSize); 70 | return (sh->alloc - sh->len); 71 | } 72 | 73 | static inline void sdssetlen(void *ptr, size_t len) 74 | { 75 | sds s = (sds)ptr; 76 | sdshdr *sh = (sdshdr*)(s - sdsHdrSize); 77 | sh->len = len; 78 | } 79 | 80 | static inline void sdssetalloc(void *ptr, size_t len) 81 | { 82 | sds s = (sds)ptr; 83 | sdshdr *sh = (sdshdr*)(s - sdsHdrSize); 84 | sh->alloc = len; 85 | } 86 | 87 | static inline size_t sdslen(const void *ptr) 88 | { 89 | sds s = (sds)ptr; 90 | sdshdr *sh = (sdshdr*)(s - sdsHdrSize); 91 | return sh->len; 92 | } 93 | 94 | static inline void sdsIncrLen(void *ptr, int inc) 95 | { 96 | sds s = (sds)ptr; 97 | sdshdr *sh = (sdshdr*)(s - sdsHdrSize); 98 | sh->len += inc; 99 | } 100 | 101 | sds sdsnewlen(const void *init, size_t initlen) 102 | { 103 | 104 | sdshdr *sh = (sdshdr*)calloc(sdsHdrSize + initlen + 1, 1); 105 | sds s = (char*)sh + sdsHdrSize; 106 | if(init) 107 | { 108 | memcpy(s, init, initlen); 109 | } 110 | sh->alloc = initlen; 111 | sh->len = initlen; 112 | s[initlen] = '\0'; 113 | return s; 114 | } 115 | 116 | void sdsfree(void *s) 117 | { 118 | if (s == NULL) return; 119 | free(((char*)s - sdsHdrSize)); 120 | } 121 | 122 | /* 123 | * 小于3/4直接翻倍, 超过直接等于最大值, 124 | * 达到最大值不再分配内存 125 | */ 126 | sds sdsMakeRoomFor(void *ptr, size_t addlen) 127 | { 128 | void *sh, *newsh; 129 | sds s = (sds)ptr; 130 | size_t avail = sdsavail(s); 131 | size_t len = sdslen(s); 132 | size_t newlen = len + addlen; 133 | 134 | if(avail >= addlen || len >= hll_sparse_max_bytes) 135 | { 136 | return s; 137 | } 138 | 139 | if(newlen * 2 < (hll_sparse_max_bytes / 4 * 3)) 140 | { 141 | newlen *= 2; 142 | } 143 | else 144 | { 145 | newlen = hll_sparse_max_bytes; 146 | } 147 | 148 | sh = (char*)s - sdsHdrSize; 149 | newsh = realloc(sh, sdsHdrSize + newlen + 1); 150 | 151 | if(!newsh) 152 | { 153 | return NULL; 154 | } 155 | 156 | s = (char*)newsh + sdsHdrSize; 157 | sdssetlen(s, len); 158 | sdssetalloc(s, newlen); 159 | 160 | return ((char*)newsh + sdsHdrSize); 161 | 162 | } 163 | 164 | /* 165 | * hyperloglog的核心函数 166 | */ 167 | 168 | /*除去特别明显说明的函数, 比如: is_xxx 169 | * 其他均返回 0 表示正确, 其余表示错误 170 | */ 171 | 172 | 173 | struct hllhdr 174 | { 175 | char magic[4]; 176 | /* 字符串"HYLL",表示这是一个HLL对象, 因为可以对普通字符串进行操作, 用于校验 */ 177 | uint8_t encoding; /* 存储格式 */ 178 | uint8_t notused[3]; /* 保留字段,对齐 */ 179 | uint8_t card[8]; /* 基数的缓存结果 */ 180 | uint8_t registers[]; /* Data bytes. 视存储方式的不同有所不同*/ 181 | }; 182 | 183 | 184 | #define HLL_INVALIDATE_CACHE(hdr) (hdr)->card[7] |= (1<<7) 185 | #define HLL_VALID_CACHE(hdr) (((hdr)->card[7] & (1<<7)) == 0) 186 | /*card最高位(共64)表明数据是否可用, 如果进行了数据修改,将card[7]第七位设置为1,表示不可用; 187 | 如果经过查询后设置为0,下次再取时可用, 桶的计数值发生变化, 并不会立即触发刷新缓存 188 | */ 189 | 190 | #define HLL_P 14 /* 14位作为桶 */ 191 | #define HLL_Q (64-HLL_P) /* 用来实际计数的位数(50位) */ 192 | #define HLL_REGISTERS (1< 50 */ 195 | #define HLL_REGISTER_MAX ((1<> _fb) | (b1 << _fb8)) & HLL_REGISTER_MAX; \ 214 | } while(0) 215 | 216 | /*Dense编码方式下,取第regnum个桶值*/ 217 | 218 | #define HLL_DENSE_SET_REGISTER(p,regnum,val) do { \ 219 | uint8_t *_p = (uint8_t*) p; \ 220 | unsigned long _byte = regnum*HLL_BITS/8; \ 221 | unsigned long _fb = regnum*HLL_BITS&7; \ 222 | unsigned long _fb8 = 8 - _fb; \ 223 | unsigned long _v = val; \ 224 | _p[_byte] &= ~(HLL_REGISTER_MAX << _fb); \ 225 | _p[_byte] |= _v << _fb; \ 226 | _p[_byte+1] &= ~(HLL_REGISTER_MAX >> _fb8); \ 227 | _p[_byte+1] |= _v >> _fb8; \ 228 | } while(0) 229 | /*Dense编码方式下, 存储第regnum个桶的值 */ 230 | 231 | 232 | /***************************************sparse的一些宏定义***************************/ 233 | 234 | /* val 1vvvvvxx 表示连续xx个桶的基数为(vvvvv + 1) 235 | * zero 00xxxxxx 连续xxxxxx + 1个桶的值为0 236 | * xzero 01xxxxxx yyyyyyyy 后(14位+1)表示连续多少个桶位0 237 | */ 238 | 239 | #define HLL_SPARSE_XZERO_BIT 0x40 /* 01xxxxxx xzero类型*/ 240 | #define HLL_SPARSE_VAL_BIT 0x80 /* 1vvvvvxx val类型*/ 241 | 242 | #define HLL_SPARSE_IS_ZERO(p) (((*(p)) & 0xc0) == 0) 243 | /*oxc0=0x11000000 判断是否时zero类型 */ 244 | #define HLL_SPARSE_IS_XZERO(p) (((*(p)) & 0xc0) == HLL_SPARSE_XZERO_BIT) 245 | /*是否时xzero类型*/ 246 | #define HLL_SPARSE_IS_VAL(p) ((*(p)) & HLL_SPARSE_VAL_BIT) 247 | /*判断是否是val类型*/ 248 | 249 | #define HLL_SPARSE_ZERO_LEN(p) (((*(p)) & 0x3f)+1) 250 | /*00xxxxxx & 0x00111111 获得后6位的值,即zero的长度*/ 251 | #define HLL_SPARSE_XZERO_LEN(p) (((((*(p)) & 0x3f) << 8) | (*((p)+1)))+1) 252 | /* 01xxxxxx yyyyyyy 获取xzero的长度 */ 253 | #define HLL_SPARSE_VAL_VALUE(p) ((((*(p)) >> 2) & 0x1f)+1) 254 | /*001vvvvv & 值0x00011111 获得中间5位的值即*/ 255 | #define HLL_SPARSE_VAL_LEN(p) (((*(p)) & 0x3)+1) 256 | /*获得VAL类型长度*/ 257 | #define HLL_SPARSE_VAL_MAX_VALUE 32 258 | /*spase值5bit最大32*/ 259 | #define HLL_SPARSE_VAL_MAX_LEN 4 260 | /*长度2bit 最大4*/ 261 | #define HLL_SPARSE_ZERO_MAX_LEN 64 262 | /*zero类型6位表示长度, 64*/ 263 | #define HLL_SPARSE_XZERO_MAX_LEN 16384 264 | /*xzero类型14bit, 最大16384*/ 265 | #define HLL_SPARSE_VAL_SET(p,val,len) do { \ 266 | *(p) = (((val)-1)<<2|((len)-1))|HLL_SPARSE_VAL_BIT; \ 267 | } while(0) 268 | /*sparse编码 设置val的值, 同时传入len和val*/ 269 | 270 | #define HLL_SPARSE_ZERO_SET(p,len) do { \ 271 | *(p) = (len)-1; \ 272 | } while(0) 273 | /*设置zero的长度, 由于是再后6位, 头两位为0*/ 274 | #define HLL_SPARSE_XZERO_SET(p,len) do { \ 275 | int _l = (len)-1; \ 276 | *(p) = (_l>>8) | HLL_SPARSE_XZERO_BIT; \ 277 | *((p)+1) = (_l&0xff); \ 278 | } while(0) 279 | /*sparse编码设置xzero的长度*/ 280 | 281 | #define HLL_ALPHA_INF 0.721347520444481703680 /* constant for 0.5/ln(2) 用来修正的一个宏 */ 282 | 283 | 284 | 285 | /*hash算法*/ 286 | uint64_t MurmurHash64A (const void * key, int len, unsigned int seed) 287 | { 288 | const uint64_t m = 0xc6a4a7935bd1e995; 289 | const int r = 47; 290 | uint64_t h = seed ^ (len * m); 291 | const uint8_t *data = (const uint8_t *)key; 292 | const uint8_t *end = data + (len-(len&7)); 293 | 294 | while(data != end) { 295 | uint64_t k; 296 | 297 | if(byte_order() == _Little_endian) 298 | { 299 | k = *((uint64_t*)data); 300 | } 301 | else 302 | { 303 | k = (uint64_t) data[0]; 304 | k |= (uint64_t) data[1] << 8; 305 | k |= (uint64_t) data[2] << 16; 306 | k |= (uint64_t) data[3] << 24; 307 | k |= (uint64_t) data[4] << 32; 308 | k |= (uint64_t) data[5] << 40; 309 | k |= (uint64_t) data[6] << 48; 310 | k |= (uint64_t) data[7] << 56; 311 | } 312 | 313 | k *= m; 314 | k ^= k >> r; 315 | k *= m; 316 | h ^= k; 317 | h *= m; 318 | data += 8; 319 | } 320 | 321 | switch(len & 7) { 322 | case 7: h ^= (uint64_t)data[6] << 48; /* fall-thru */ 323 | case 6: h ^= (uint64_t)data[5] << 40; /* fall-thru */ 324 | case 5: h ^= (uint64_t)data[4] << 32; /* fall-thru */ 325 | case 4: h ^= (uint64_t)data[3] << 24; /* fall-thru */ 326 | case 3: h ^= (uint64_t)data[2] << 16; /* fall-thru */ 327 | case 2: h ^= (uint64_t)data[1] << 8; /* fall-thru */ 328 | case 1: h ^= (uint64_t)data[0]; 329 | h *= m; /* fall-thru */ 330 | }; 331 | 332 | h ^= h >> r; 333 | h *= m; 334 | h ^= h >> r; 335 | return h; 336 | } 337 | 338 | /*根据hash算法,计算出index(桶的下标), count(后50的计数值)*/ 339 | int hllPatLen(const unsigned char *ele, size_t elesize, long *regp) 340 | { 341 | uint64_t hash, bit, index; 342 | int count; 343 | 344 | hash = MurmurHash64A(ele,elesize,0xadc83b19ULL); 345 | index = hash & HLL_P_MASK; /* Register index. */ 346 | hash >>= HLL_P; /* Remove bits used to address the register. */ 347 | hash |= ((uint64_t)1< oldcount) 365 | { 366 | HLL_DENSE_SET_REGISTER(registers,index,count); 367 | return 1; 368 | } 369 | else 370 | { 371 | return 0; 372 | } 373 | } 374 | 375 | /*往data数据中放入一个新的值*/ 376 | int hllDenseAdd(uint8_t *registers, const unsigned char *ele, size_t elesize) 377 | { 378 | long index; 379 | uint8_t count = hllPatLen(ele,elesize,&index); 380 | return hllDenseSet(registers,index,count); 381 | } 382 | 383 | 384 | /*统计基数值的函数,传入reghisto, 每个位置存储index对应计数出现的次数*/ 385 | void hllDenseRegHisto(uint8_t *registers, int* reghisto) 386 | { 387 | int j; 388 | 389 | if (HLL_REGISTERS == 16384 && HLL_BITS == 6) 390 | { 391 | uint8_t *r = registers; 392 | unsigned long r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, 393 | r10, r11, r12, r13, r14, r15; 394 | for (j = 0; j < 1024; j++) 395 | { 396 | r0 = r[0] & 63; 397 | r1 = (r[0] >> 6 | r[1] << 2) & 63; 398 | r2 = (r[1] >> 4 | r[2] << 4) & 63; 399 | r3 = (r[2] >> 2) & 63; 400 | r4 = r[3] & 63; 401 | r5 = (r[3] >> 6 | r[4] << 2) & 63; 402 | r6 = (r[4] >> 4 | r[5] << 4) & 63; 403 | r7 = (r[5] >> 2) & 63; 404 | r8 = r[6] & 63; 405 | r9 = (r[6] >> 6 | r[7] << 2) & 63; 406 | r10 = (r[7] >> 4 | r[8] << 4) & 63; 407 | r11 = (r[8] >> 2) & 63; 408 | r12 = r[9] & 63; 409 | r13 = (r[9] >> 6 | r[10] << 2) & 63; 410 | r14 = (r[10] >> 4 | r[11] << 4) & 63; 411 | r15 = (r[11] >> 2) & 63; 412 | 413 | reghisto[r0]++; 414 | reghisto[r1]++; 415 | reghisto[r2]++; 416 | reghisto[r3]++; 417 | reghisto[r4]++; 418 | reghisto[r5]++; 419 | reghisto[r6]++; 420 | reghisto[r7]++; 421 | reghisto[r8]++; 422 | reghisto[r9]++; 423 | reghisto[r10]++; 424 | reghisto[r11]++; 425 | reghisto[r12]++; 426 | reghisto[r13]++; 427 | reghisto[r14]++; 428 | reghisto[r15]++; 429 | 430 | r += 12; 431 | } 432 | /*对于我们自定义的分桶方法, 采用这种方法更高效*/ 433 | } 434 | else 435 | { 436 | for(j = 0; j < HLL_REGISTERS; j++) { 437 | unsigned long reg; 438 | HLL_DENSE_GET_REGISTER(reg,registers,j); 439 | reghisto[reg]++; 440 | } 441 | } 442 | } 443 | 444 | 445 | /*进行编码转换*/ 446 | hllhdr* hllSparseToDense(hllhdr *ptr) 447 | { 448 | if(!ptr) 449 | { 450 | return NULL; 451 | } 452 | 453 | sds sparse = (sds)ptr; 454 | hllhdr *hdr = NULL; 455 | int idx = 0, runlen, regval; 456 | uint8_t *p = (uint8_t*)sparse, *end = p + sdslen(sparse); 457 | 458 | if (ptr->encoding == HLL_DENSE) return ptr; 459 | 460 | hdr = (hllhdr*)sdsnewlen(NULL,HLL_DENSE_SIZE); 461 | /*dense方式占用的空间是固定的*/ 462 | *hdr = *ptr; /*结构体赋值*/ 463 | hdr->encoding = HLL_DENSE; 464 | 465 | p += HLL_HDR_SIZE; 466 | while(p < end) 467 | { 468 | if (HLL_SPARSE_IS_ZERO(p)) 469 | { 470 | runlen = HLL_SPARSE_ZERO_LEN(p); 471 | idx += runlen; 472 | p++; 473 | } 474 | else if (HLL_SPARSE_IS_XZERO(p)) 475 | { 476 | runlen = HLL_SPARSE_XZERO_LEN(p); 477 | idx += runlen; 478 | p += 2; 479 | } 480 | /*对于xzero和zero直接就是0, 不需要对data进行操作*/ 481 | else 482 | { 483 | runlen = HLL_SPARSE_VAL_LEN(p); 484 | regval = HLL_SPARSE_VAL_VALUE(p); 485 | if ((runlen + idx) > HLL_REGISTERS) break; /* Overflow. */ 486 | while(runlen--) 487 | { 488 | HLL_DENSE_SET_REGISTER(hdr->registers, idx, regval); 489 | idx++; 490 | } 491 | p++; 492 | } 493 | } 494 | if (idx != HLL_REGISTERS) 495 | { 496 | sdsfree(hdr); 497 | return NULL; 498 | } 499 | 500 | sdsfree(ptr); 501 | return hdr; 502 | } 503 | 504 | /* 对于sparse编码方式赋值, 当数据超过3000或者一个VAL超过32 505 | * 由于会进行扩展空间, 所以需要传入指向指针的指针 506 | * 507 | */ 508 | int hllSparseSet(hllhdr **ptr, long index, uint8_t count) 509 | { 510 | hllhdr *hdr = *ptr; 511 | uint8_t oldcount, *sparse, *end, *p, *prev, *next; 512 | long first, span; 513 | long is_zero = 0, is_xzero = 0, is_val = 0, runlen = 0; 514 | 515 | if (count > HLL_SPARSE_VAL_MAX_VALUE) goto promote; 516 | 517 | hdr = (hllhdr*)sdsMakeRoomFor(hdr, 3); 518 | /* 一次最多可能扩展3个字节, 先一步分配空间 */ 519 | if(!hdr) 520 | { 521 | xprintf("memory rellac error: %s!", sterrror(errno)); 522 | return -1; 523 | } 524 | 525 | 526 | /* 第一步需要找到将要更改的那个数据(zero xzero val) */ 527 | sparse = p = ((uint8_t*)hdr) + HLL_HDR_SIZE; 528 | end = p + sdslen(hdr) - HLL_HDR_SIZE; 529 | 530 | first = 0; 531 | prev = NULL; 532 | next = NULL; 533 | span = 0; 534 | while(p < end) 535 | { 536 | long oplen; 537 | 538 | oplen = 1; 539 | if (HLL_SPARSE_IS_ZERO(p)) 540 | { 541 | span = HLL_SPARSE_ZERO_LEN(p); 542 | } 543 | else if (HLL_SPARSE_IS_VAL(p)) 544 | { 545 | span = HLL_SPARSE_VAL_LEN(p); 546 | } 547 | else 548 | { 549 | span = HLL_SPARSE_XZERO_LEN(p); 550 | oplen = 2; 551 | } 552 | 553 | if (index <= first + span - 1) break; 554 | prev = p; 555 | p += oplen; 556 | first += span; 557 | } 558 | if (span == 0) return -1; 559 | 560 | next = HLL_SPARSE_IS_XZERO(p) ? (p + 2) : (p + 1); 561 | if (next >= end) next = NULL; 562 | 563 | if (HLL_SPARSE_IS_ZERO(p)) 564 | { 565 | is_zero = 1; 566 | runlen = HLL_SPARSE_ZERO_LEN(p); 567 | } 568 | else if (HLL_SPARSE_IS_XZERO(p)) 569 | { 570 | is_xzero = 1; 571 | runlen = HLL_SPARSE_XZERO_LEN(p); 572 | } 573 | else 574 | { 575 | is_val = 1; 576 | runlen = HLL_SPARSE_VAL_LEN(p); 577 | } 578 | 579 | /*first和p指向的是将要更改的前一个数据,*/ 580 | 581 | /*先处理几种简单的情况*/ 582 | if (is_val) 583 | { 584 | oldcount = HLL_SPARSE_VAL_VALUE(p); 585 | if (oldcount >= count) return 0; 586 | 587 | if (runlen == 1) 588 | { 589 | HLL_SPARSE_VAL_SET(p,count,1); 590 | goto updated; 591 | } 592 | } 593 | /*对于VAL数据, 新的计数值更小, 直接返回 594 | * 如果长度为1, 表示之前是没有值, 直接设置值更新数据即可 595 | */ 596 | 597 | 598 | if (is_zero && runlen == 1) 599 | { 600 | HLL_SPARSE_VAL_SET(p,count,1); 601 | goto updated; 602 | } 603 | 604 | 605 | /*更可能的情况,需要拆分移动数据*/ 606 | uint8_t seq[5], *n = seq; 607 | int last = first+span-1; 608 | int len; 609 | 610 | if (is_zero || is_xzero) 611 | { 612 | /*对于XZERO和ZERO,最大可能XZERO-VAL-XZERO*/ 613 | if (index != first) 614 | { 615 | len = index-first; 616 | if (len > HLL_SPARSE_ZERO_MAX_LEN) 617 | { 618 | HLL_SPARSE_XZERO_SET(n,len); 619 | n += 2; 620 | } 621 | else 622 | { 623 | HLL_SPARSE_ZERO_SET(n,len); 624 | n++; 625 | } 626 | } 627 | HLL_SPARSE_VAL_SET(n,count,1); 628 | n++; 629 | if (index != last) 630 | { 631 | len = last-index; 632 | if (len > HLL_SPARSE_ZERO_MAX_LEN) 633 | { 634 | HLL_SPARSE_XZERO_SET(n,len); 635 | n += 2; 636 | } 637 | else 638 | { 639 | HLL_SPARSE_ZERO_SET(n,len); 640 | n++; 641 | } 642 | } 643 | } 644 | else 645 | { 646 | /*对于一个VAL, 可能拆分为最大VAL-VAL-VAL*/ 647 | int curval = HLL_SPARSE_VAL_VALUE(p); 648 | 649 | if (index != first) 650 | { 651 | len = index-first; 652 | HLL_SPARSE_VAL_SET(n,curval,len); 653 | n++; 654 | } 655 | HLL_SPARSE_VAL_SET(n, count, 1); 656 | n++; 657 | if (index != last) 658 | { 659 | len = last-index; 660 | HLL_SPARSE_VAL_SET(n,curval,len); 661 | n++; 662 | } 663 | } 664 | 665 | /* 修改数据,移动内容 */ 666 | int seqlen = n-seq; 667 | int oldlen = is_xzero ? 2 : 1; 668 | int deltalen = seqlen-oldlen; 669 | 670 | if (deltalen > 0 && 671 | sdslen(hdr)+deltalen > hll_sparse_max_bytes) goto promote; 672 | if (deltalen && next) memmove(next+deltalen,next,end-next); 673 | sdsIncrLen(hdr,deltalen); 674 | memcpy(p, seq, seqlen); 675 | end += deltalen; 676 | 677 | updated: 678 | /*在更新后,有可能左右可以合并*/ 679 | p = prev ? prev : sparse; 680 | int scanlen = 5; /* Scan up to 5 upcodes starting from prev. */ 681 | while (p < end && scanlen--) 682 | { 683 | if (HLL_SPARSE_IS_XZERO(p)) 684 | { 685 | p += 2; 686 | continue; 687 | } 688 | else if (HLL_SPARSE_IS_ZERO(p)) 689 | { 690 | p++; 691 | continue; 692 | } 693 | if (p + 1 < end && HLL_SPARSE_IS_VAL(p+1)) 694 | { 695 | int v1 = HLL_SPARSE_VAL_VALUE(p); 696 | int v2 = HLL_SPARSE_VAL_VALUE(p+1); 697 | if (v1 == v2) 698 | { 699 | int len = HLL_SPARSE_VAL_LEN(p)+HLL_SPARSE_VAL_LEN(p+1); 700 | if (len <= HLL_SPARSE_VAL_MAX_LEN) 701 | { 702 | HLL_SPARSE_VAL_SET(p+1,v1,len); 703 | memmove(p, p+1, end-p); 704 | sdsIncrLen(hdr, -1); 705 | end--; 706 | continue; 707 | } 708 | } 709 | } 710 | p++; 711 | } 712 | 713 | HLL_INVALIDATE_CACHE(hdr); 714 | *ptr = hdr; 715 | return 1; 716 | 717 | promote: 718 | if((hdr = hllSparseToDense(hdr)) == NULL) return -1; /* 更新编码 */ 719 | 720 | int dense_retval = hllDenseSet(hdr->registers, index, count); 721 | assert(dense_retval == 1); 722 | *ptr = hdr; 723 | return dense_retval; 724 | } 725 | 726 | 727 | int hllSparseAdd(hllhdr **ptr, const unsigned char *ele, size_t elesize) 728 | { 729 | long index; 730 | uint8_t count = hllPatLen(ele, elesize, &index); 731 | return hllSparseSet(ptr,index,count); 732 | } 733 | 734 | /*sparse方式count*/ 735 | void hllSparseRegHisto(uint8_t *sparse, int sparselen, int *invalid, int* reghisto) 736 | { 737 | int idx = 0, runlen, regval; 738 | uint8_t *end = sparse+sparselen, *p = sparse; 739 | 740 | while(p < end) 741 | { 742 | if (HLL_SPARSE_IS_ZERO(p)) 743 | { 744 | runlen = HLL_SPARSE_ZERO_LEN(p); 745 | idx += runlen; 746 | reghisto[0] += runlen; 747 | p++; 748 | } 749 | else if (HLL_SPARSE_IS_XZERO(p)) 750 | { 751 | runlen = HLL_SPARSE_XZERO_LEN(p); 752 | idx += runlen; 753 | reghisto[0] += runlen; 754 | p += 2; 755 | } 756 | else 757 | { 758 | runlen = HLL_SPARSE_VAL_LEN(p); 759 | regval = HLL_SPARSE_VAL_VALUE(p); 760 | idx += runlen; 761 | reghisto[regval] += runlen; 762 | p++; 763 | } 764 | } 765 | if (idx != HLL_REGISTERS && invalid) *invalid = 1; 766 | } 767 | 768 | /*Raw编码方式获取count*/ 769 | void hllRawRegHisto(uint8_t *registers, int* reghisto) 770 | { 771 | uint64_t *word = (uint64_t*) registers; 772 | uint8_t *bytes; 773 | int j; 774 | 775 | for (j = 0; j < HLL_REGISTERS/8; j++) 776 | { 777 | if (*word == 0) 778 | { 779 | reghisto[0] += 8; 780 | } 781 | else 782 | { 783 | bytes = (uint8_t*) word; 784 | reghisto[bytes[0]]++; 785 | reghisto[bytes[1]]++; 786 | reghisto[bytes[2]]++; 787 | reghisto[bytes[3]]++; 788 | reghisto[bytes[4]]++; 789 | reghisto[bytes[5]]++; 790 | reghisto[bytes[6]]++; 791 | reghisto[bytes[7]]++; 792 | } 793 | word++; 794 | } 795 | } 796 | 797 | /* Helper function sigma as defined in 798 | * "New cardinality estimation algorithms for HyperLogLog sketches" 799 | * Otmar Ertl, arXiv:1702.01284 */ 800 | double hllSigma(double x) 801 | { 802 | if (x == 1.) return INFINITY; 803 | double zPrime; 804 | double y = 1; 805 | double z = x; 806 | do 807 | { 808 | x *= x; 809 | zPrime = z; 810 | z += x * y; 811 | y += y; 812 | } while(zPrime != z); 813 | return z; 814 | } 815 | 816 | /* Helper function tau as defined in 817 | * "New cardinality estimation algorithms for HyperLogLog sketches" 818 | * Otmar Ertl, arXiv:1702.01284 */ 819 | double hllTau(double x) 820 | { 821 | if (x == 0. || x == 1.) return 0.; 822 | double zPrime; 823 | double y = 1.0; 824 | double z = 1 - x; 825 | do 826 | { 827 | x = sqrt(x); 828 | zPrime = z; 829 | y *= 0.5; 830 | z -= pow(1 - x, 2)*y; 831 | } while(zPrime != z); 832 | return z / 3; 833 | } 834 | 835 | 836 | /*hllhdr 获取count值*/ 837 | uint64_t hllCount(struct hllhdr *hdr, int *invalid) { 838 | double m = HLL_REGISTERS; 839 | double E; 840 | int j; 841 | /* 最大是 HLL_Q + 1*/ 842 | int reghisto[64] = {0}; 843 | 844 | /*根据编码方式进行编码*/ 845 | if (hdr->encoding == HLL_DENSE) 846 | { 847 | hllDenseRegHisto(hdr->registers,reghisto); 848 | } 849 | else if (hdr->encoding == HLL_SPARSE) 850 | { 851 | hllSparseRegHisto(hdr->registers, 852 | sdslen((sds)hdr)-HLL_HDR_SIZE,invalid,reghisto); 853 | } 854 | else if (hdr->encoding == HLL_RAW) 855 | { 856 | hllRawRegHisto(hdr->registers,reghisto); 857 | } 858 | else 859 | { 860 | *invalid = 1; 861 | return 0; 862 | } 863 | 864 | /* Estimate cardinality form register histogram. See: 865 | * "New cardinality estimation algorithms for HyperLogLog sketches" 866 | * Otmar Ertl, arXiv:1702.01284 */ 867 | double z = m * hllTau((m-reghisto[HLL_Q+1])/(double)m); 868 | for (j = HLL_Q; j >= 1; --j) { 869 | z += reghisto[j]; 870 | z *= 0.5; 871 | } 872 | z += m * hllSigma(reghisto[0]/(double)m); 873 | E = llroundl(HLL_ALPHA_INF*m*m/z); 874 | 875 | /*进行计算和修正, 在这里无法说明*/ 876 | 877 | return (uint64_t) E; 878 | } 879 | 880 | /*根据编码方式进行添加 881 | * @return 1 表示进行了修改 882 | * @return 0 表示未进行修改 883 | * @return -1 表示发生错误 884 | */ 885 | int hllAdd(hllhdr **ptr, const unsigned char *ele, size_t elesize) 886 | { 887 | switch((*ptr)->encoding) 888 | { 889 | case HLL_DENSE: 890 | { 891 | if(hllDenseAdd((*ptr)->registers,ele,elesize)) 892 | { 893 | HLL_INVALIDATE_CACHE(*ptr); 894 | return 1; 895 | } 896 | return 0; 897 | } 898 | case HLL_SPARSE: 899 | { 900 | return hllSparseAdd(ptr,ele,elesize); 901 | } 902 | default: return -1; /* Invalid representation. */ 903 | } 904 | } 905 | 906 | 907 | /*这里的max就是一个简单的数组, 不需要考虑什么编码*/ 908 | int hllMerge(uint8_t *max, hllhdr *ptr) 909 | { 910 | int i; 911 | 912 | if (ptr->encoding == HLL_DENSE) 913 | { 914 | uint8_t val; 915 | 916 | for (i = 0; i < HLL_REGISTERS; i++) 917 | { 918 | HLL_DENSE_GET_REGISTER(val, ptr->registers,i); 919 | if (val > max[i]) max[i] = val; 920 | } 921 | } 922 | else 923 | { 924 | uint8_t *p = (uint8_t*)ptr, *end = p + sdslen(ptr); 925 | long runlen, regval; 926 | 927 | p += HLL_HDR_SIZE; 928 | i = 0; 929 | while(p < end) 930 | { 931 | if (HLL_SPARSE_IS_ZERO(p)) 932 | { 933 | runlen = HLL_SPARSE_ZERO_LEN(p); 934 | i += runlen; 935 | p++; 936 | } else if (HLL_SPARSE_IS_XZERO(p)) { 937 | runlen = HLL_SPARSE_XZERO_LEN(p); 938 | i += runlen; 939 | p += 2; 940 | } else { 941 | runlen = HLL_SPARSE_VAL_LEN(p); 942 | regval = HLL_SPARSE_VAL_VALUE(p); 943 | if ((runlen + i) > HLL_REGISTERS) break; /* Overflow. */ 944 | while(runlen--) { 945 | if (regval > max[i]) max[i] = regval; 946 | i++; 947 | } 948 | p++; 949 | } 950 | } 951 | if (i != HLL_REGISTERS) return -1; 952 | } 953 | return 0; 954 | } 955 | 956 | 957 | /* 958 | * 对外部提供的接口 959 | */ 960 | 961 | void deleteHLLObject(hllhdr* hdr) 962 | { 963 | sdsfree(hdr); 964 | } 965 | 966 | /*创建结构体*/ 967 | hllhdr *createHLLObject(void) 968 | { 969 | struct hllhdr *hdr; 970 | sds s; 971 | uint8_t *p; 972 | int sparselen = HLL_HDR_SIZE + 973 | (((HLL_REGISTERS+(HLL_SPARSE_XZERO_MAX_LEN-1)) / 974 | HLL_SPARSE_XZERO_MAX_LEN)*2); 975 | /* HLL_SPARSE_XZERO_MAX_LEN 个值需要2字节, 哪怕超过1个也需要2字节 */ 976 | 977 | int aux; 978 | 979 | aux = HLL_REGISTERS; 980 | s = sdsnewlen(NULL,sparselen); 981 | hdr = (hllhdr*)s; 982 | p = (uint8_t*)s + HLL_HDR_SIZE; 983 | while(aux) 984 | { 985 | int xzero = HLL_SPARSE_XZERO_MAX_LEN; 986 | if (xzero > aux) xzero = aux; 987 | HLL_SPARSE_XZERO_SET(p,xzero); 988 | p += 2; 989 | aux -= xzero; 990 | } 991 | assert((p-(uint8_t*)s) == sparselen); 992 | 993 | 994 | memcpy(hdr->magic, hll_magic_char, 4); 995 | hdr->encoding = HLL_SPARSE; 996 | return hdr; 997 | } 998 | 999 | /* 确认字符串是否是有效的hll字符串,还记得那个HYLL魔数吗 */ 1000 | int isHLLObjectOrReply(void *ptr) { 1001 | struct hllhdr *hdr; 1002 | 1003 | hdr = ptr; 1004 | 1005 | if (hdr->magic[0] != 'H' || hdr->magic[1] != 'Y' || 1006 | hdr->magic[2] != 'L' || hdr->magic[3] != 'L') goto invalid; 1007 | 1008 | if (hdr->encoding > HLL_MAX_ENCODING) goto invalid; 1009 | 1010 | if (hdr->encoding == HLL_DENSE && 1011 | sdslen(ptr) != HLL_DENSE_SIZE) goto invalid; 1012 | 1013 | return 0; 1014 | 1015 | invalid: 1016 | return -1; 1017 | } 1018 | 1019 | /*add命令*/ 1020 | int pfaddCommand(hllhdr **ptr, const unsigned char *key, int len) 1021 | { 1022 | if(!ptr || !*ptr || !key) 1023 | { 1024 | return -1; 1025 | } 1026 | 1027 | len = (len > 0) ? len : strlen(key); 1028 | return hllAdd(ptr, key, len); 1029 | 1030 | } 1031 | 1032 | 1033 | /* 统计基数的命令接口 */ 1034 | int pfcountCommand(hllhdr *hdr, uint64_t *ret) 1035 | { 1036 | uint64_t card; 1037 | if (HLL_VALID_CACHE(hdr)) 1038 | { 1039 | card = (uint64_t)hdr->card[0]; 1040 | card |= (uint64_t)hdr->card[1] << 8; 1041 | card |= (uint64_t)hdr->card[2] << 16; 1042 | card |= (uint64_t)hdr->card[3] << 24; 1043 | card |= (uint64_t)hdr->card[4] << 32; 1044 | card |= (uint64_t)hdr->card[5] << 40; 1045 | card |= (uint64_t)hdr->card[6] << 48; 1046 | card |= (uint64_t)hdr->card[7] << 56; 1047 | } 1048 | else { 1049 | int invalid = 0; 1050 | card = hllCount(hdr,&invalid); 1051 | if (invalid) 1052 | { 1053 | return -1; 1054 | } 1055 | hdr->card[0] = card & 0xff; 1056 | hdr->card[1] = (card >> 8) & 0xff; 1057 | hdr->card[2] = (card >> 16) & 0xff; 1058 | hdr->card[3] = (card >> 24) & 0xff; 1059 | hdr->card[4] = (card >> 32) & 0xff; 1060 | hdr->card[5] = (card >> 40) & 0xff; 1061 | hdr->card[6] = (card >> 48) & 0xff; 1062 | hdr->card[7] = (card >> 56) & 0xff; 1063 | } 1064 | 1065 | *ret = card; 1066 | return 0; 1067 | } 1068 | 1069 | 1070 | /* 合并两个hllhdr结构, 合并多个的情况应该很少见 */ 1071 | int pfmergeCommand(hllhdr **hdr, hllhdr *src) 1072 | { 1073 | uint8_t max[HLL_REGISTERS]; 1074 | int j; 1075 | int use_dense = 0; 1076 | 1077 | if(!hdr || !*hdr || !src) 1078 | { 1079 | return -1; 1080 | } 1081 | 1082 | memset(max,0,sizeof(max)); 1083 | 1084 | if ((*hdr)->encoding == HLL_DENSE) use_dense = 1; 1085 | 1086 | if(hllMerge(max, src)) 1087 | { 1088 | return -1; 1089 | } 1090 | 1091 | for(j = 0; j < HLL_REGISTERS; j++) 1092 | { 1093 | if(use_dense) 1094 | { 1095 | hllDenseSet((*hdr)->registers, j, max[j]); 1096 | } 1097 | else 1098 | { 1099 | hllSparseSet(hdr, j, max[j]); 1100 | } 1101 | } 1102 | 1103 | HLL_INVALIDATE_CACHE(*hdr); 1104 | return 0; 1105 | } 1106 | 1107 | 1108 | 1109 | 1110 | -------------------------------------------------------------------------------- /hyperloglog.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef _HYPERLOGLOG_H 3 | #define _HYPERLOGLOG_H 4 | 5 | #include 6 | static const char *hll_magic_char = "HYLL"; 7 | 8 | typedef struct hllhdr hllhdr; 9 | 10 | hllhdr *createHLLObject(); 11 | int pfaddCommand(hllhdr **ptr, const unsigned char *key, int len); 12 | int pfcountCommand(hllhdr *hdr, uint64_t *ret); 13 | int pfmergeCommand(hllhdr **dest, hllhdr *src); 14 | hllhdr* hllSparseToDense(hllhdr *ptr); 15 | void deleteHLLObject(hllhdr* hdr); 16 | 17 | #endif -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | all: BenchRedisHLL Benchwyhll 2 | BenchRedisHLL: BenchRedisHLL.c 3 | gcc BenchRedisHLL.c hyperloglog.c -o BenchRedisHLL -Ofast -fpermissive -w -lm -march=native 4 | Benchwyhll: Benchwyhll.c wyhll.h 5 | gcc Benchwyhll.c -o Benchwyhll -Ofast -Wall -lm -march=native 6 | 7 | -------------------------------------------------------------------------------- /wyhash.h: -------------------------------------------------------------------------------- 1 | // This is free and unencumbered software released into the public domain under The Unlicense (http://unlicense.org/) 2 | // main repo: https://github.com/wangyi-fudan/wyhash 3 | // author: 王一 Wang Yi 4 | // contributors: Reini Urban, Dietrich Epp, Joshua Haberman, Tommy Ettinger, Daniel Lemire, Otmar Ertl, cocowalla, leo-yuriev, Diego Barrios Romero, paulie-g, dumblob, Yann Collet, ivte-ms, hyb, James Z.M. Gao, easyaspi314 (Devin), TheOneric 5 | 6 | /* quick example: 7 | string s="fjsakfdsjkf"; 8 | uint64_t hash=wyhash(s.c_str(), s.size(), 0, _wyp); 9 | */ 10 | 11 | #ifndef wyhash_final_version_2 12 | #define wyhash_final_version_2 13 | 14 | #ifndef WYHASH_CONDOM 15 | //pretections that produces different results 16 | //0: read 8 bytes before and after boundaries, dangerous but fastest. 17 | //1: normal valid behavior 18 | //2: extra protection against entropy loss (probability=2^-63), aka. "blind multiplication" 19 | #define WYHASH_CONDOM 1 20 | #endif 21 | 22 | #ifndef WYHASH_32BIT_MUM 23 | //0: normal version, slow on 32 bit system 24 | //1: faster on 32 bit system but produces different results 25 | #define WYHASH_32BIT_MUM 0 26 | #endif 27 | 28 | //includes 29 | #include 30 | #include 31 | #if defined(_MSC_VER) && defined(_M_X64) 32 | #include 33 | #pragma intrinsic(_umul128) 34 | #endif 35 | 36 | //likely and unlikely macros 37 | #if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) 38 | #define _likely_(x) __builtin_expect(x,1) 39 | #define _unlikely_(x) __builtin_expect(x,0) 40 | #else 41 | #define _likely_(x) (x) 42 | #define _unlikely_(x) (x) 43 | #endif 44 | 45 | //128bit multiply function 46 | static inline uint64_t _wyrot(uint64_t x) { return (x>>32)|(x<<32); } 47 | static inline void _wymum(uint64_t *A, uint64_t *B){ 48 | #if(WYHASH_32BIT_MUM) 49 | uint64_t hh=(*A>>32)*(*B>>32), hl=(*A>>32)*(uint32_t)*B, lh=(uint32_t)*A*(*B>>32), ll=(uint64_t)(uint32_t)*A*(uint32_t)*B; 50 | #if(WYHASH_CONDOM>1) 51 | *A^=_wyrot(hl)^hh; *B^=_wyrot(lh)^ll; 52 | #else 53 | *A=_wyrot(hl)^hh; *B=_wyrot(lh)^ll; 54 | #endif 55 | #elif defined(__SIZEOF_INT128__) 56 | __uint128_t r=*A; r*=*B; 57 | #if(WYHASH_CONDOM>1) 58 | *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); 59 | #else 60 | *A=(uint64_t)r; *B=(uint64_t)(r>>64); 61 | #endif 62 | #elif defined(_MSC_VER) && defined(_M_X64) 63 | #if(WYHASH_CONDOM>1) 64 | uint64_t a, b; 65 | a=_umul128(*A,*B,&b); 66 | *A^=a; *B^=b; 67 | #else 68 | *A=_umul128(*A,*B,B); 69 | #endif 70 | #else 71 | uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B, hi, lo; 72 | uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; 74 | #if(WYHASH_CONDOM>1) 75 | *A^=lo; *B^=hi; 76 | #else 77 | *A=lo; *B=hi; 78 | #endif 79 | #endif 80 | } 81 | 82 | //multiply and xor mix function, aka MUM 83 | static inline uint64_t _wymix(uint64_t A, uint64_t B){ _wymum(&A,&B); return A^B; } 84 | 85 | //endian macros 86 | #ifndef WYHASH_LITTLE_ENDIAN 87 | #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) 88 | #define WYHASH_LITTLE_ENDIAN 1 89 | #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) 90 | #define WYHASH_LITTLE_ENDIAN 0 91 | #else 92 | #warning could not determine endianess! Falling back to little endian. 93 | #define WYHASH_LITTLE_ENDIAN 1 94 | #endif 95 | #endif 96 | 97 | //read functions 98 | #if (WYHASH_LITTLE_ENDIAN) 99 | static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v;} 100 | static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return v;} 101 | #elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) 102 | static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return __builtin_bswap64(v);} 103 | static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return __builtin_bswap32(v);} 104 | #elif defined(_MSC_VER) 105 | static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return _byteswap_uint64(v);} 106 | static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return _byteswap_ulong(v);} 107 | #else 108 | static inline uint64_t _wyr8(const uint8_t *p) { 109 | uint64_t v; memcpy(&v, p, 8); 110 | return (((v >> 56) & 0xff)| ((v >> 40) & 0xff00)| ((v >> 24) & 0xff0000)| ((v >> 8) & 0xff000000)| ((v << 8) & 0xff00000000)| ((v << 24) & 0xff0000000000)| ((v << 40) & 0xff000000000000)| ((v << 56) & 0xff00000000000000)); 111 | } 112 | static inline uint64_t _wyr4(const uint8_t *p) { 113 | uint32_t v; memcpy(&v, p, 4); 114 | return (((v >> 24) & 0xff)| ((v >> 8) & 0xff00)| ((v << 8) & 0xff0000)| ((v << 24) & 0xff000000)); 115 | } 116 | #endif 117 | static inline uint64_t _wyr3(const uint8_t *p, uint64_t k) { return (((uint64_t)p[0])<<16)|(((uint64_t)p[k>>1])<<8)|p[k-1];} 118 | 119 | //wyhash main function 120 | static inline uint64_t wyhash(const void *key, uint64_t len, uint64_t seed, const uint64_t *secret){ 121 | const uint8_t *p=(const uint8_t *)key; uint64_t a,b; seed^=*secret; 122 | if(_likely_(len<=16)){ 123 | #if(WYHASH_CONDOM>0) 124 | if(_likely_(len<=8)){ 125 | if(_likely_(len>=4)){ a=_wyr4(p); b=_wyr4(p+len-4); } 126 | else if (_likely_(len)){ a=_wyr3(p,len); b=0; } 127 | else a=b=0; 128 | } 129 | else{ a=_wyr8(p); b=_wyr8(p+len-8); } 130 | #else 131 | uint64_t s=(len<8)*((8-len)<<3); 132 | a=_wyr8(p)<>s; 133 | #endif 134 | } 135 | else{ 136 | uint64_t i=len; 137 | if(_unlikely_(i>48)){ 138 | uint64_t see1=seed, see2=seed; 139 | do{ 140 | seed=_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed); 141 | see1=_wymix(_wyr8(p+16)^secret[2],_wyr8(p+24)^see1); 142 | see2=_wymix(_wyr8(p+32)^secret[3],_wyr8(p+40)^see2); 143 | p+=48; i-=48; 144 | }while(i>48); 145 | seed^=see1^see2; 146 | } 147 | while(_unlikely_(i>16)){ seed=_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed); i-=16; p+=16; } 148 | a=_wyr8(p+i-16); b=_wyr8(p+i-8); 149 | } 150 | return _wymix(secret[1]^len,_wymix(a^secret[1], b^seed)); 151 | } 152 | 153 | //the default secret parameters 154 | static const uint64_t _wyp[5] = {0xa0761d6478bd642full, 0xe7037ed1a0b428dbull, 0x8ebc6af09c88c6e3ull, 0x589965cc75374cc3ull, 0x1d8e4e27c47d124full}; 155 | 156 | //a useful 64bit-64bit mix function to produce determinstic psudeo random numbers that can pass BigCrush and PractRand 157 | static inline uint64_t wyhash64(uint64_t A, uint64_t B){ A^=_wyp[0]; B^=_wyp[1]; _wymum(&A,&B); return _wymix(A^_wyp[0],B^_wyp[1]);} 158 | 159 | //The wyrand PRNG that pass BigCrush and PractRand 160 | static inline uint64_t wyrand(uint64_t *seed){ *seed+=_wyp[0]; return _wymix(*seed,*seed^_wyp[1]);} 161 | 162 | // convert any 64 bit psudeo random numbers to uniform distribution [0,1). It can be combined with wyrand, wyhash64 or wyhash. 163 | static inline double wy2u01(uint64_t r){ const double _wynorm=1.0/(1ull<<52); return (r>>12)*_wynorm;} 164 | 165 | // convert any 64 bit psudeo random numbers to APPROXIMATE Gaussian distribution. It can be combined with wyrand, wyhash64 or wyhash. 166 | static inline double wy2gau(uint64_t r){ const double _wynorm=1.0/(1ull<<20); return ((r&0x1fffff)+((r>>21)&0x1fffff)+((r>>42)&0x1fffff))*_wynorm-3.0;} 167 | 168 | // fast range integer random number generation on [0,k) credit to Daniel Lemire. May not work when WYHASH_32BIT_MUM=1. It can be combined with wyrand, wyhash64 or wyhash. 169 | static inline uint64_t wy2u0k(uint64_t r, uint64_t k){ _wymum(&r,&k); return k; } 170 | 171 | //make your own secret 172 | static inline void make_secret(uint64_t seed, uint64_t *secret){ 173 | uint8_t c[] = {15, 23, 27, 29, 30, 39, 43, 45, 46, 51, 53, 54, 57, 58, 60, 71, 75, 77, 78, 83, 85, 86, 89, 90, 92, 99, 101, 102, 105, 106, 108, 113, 114, 116, 120, 135, 139, 141, 142, 147, 149, 150, 153, 154, 156, 163, 165, 166, 169, 170, 172, 177, 178, 180, 184, 195, 197, 198, 201, 202, 204, 209, 210, 212, 216, 225, 226, 228, 232, 240 }; 174 | for(size_t i=0;i<5;i++){ 175 | uint8_t ok; 176 | do{ 177 | ok=1; secret[i]=0; 178 | for(size_t j=0;j<64;j+=8) secret[i]|=((uint64_t)c[wyrand(&seed)%sizeof(c)])<> 1) & 0x5555555555555555; 189 | x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333); 190 | x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f; 191 | x = (x * 0x0101010101010101) >> 56; 192 | if(x!=32){ ok=0; break; } 193 | #endif 194 | } 195 | if(!ok)continue; 196 | for(uint64_t j=3;j<0x100000000ull;j+=2) if(secret[i]%j==0){ ok=0; break; } 197 | }while(!ok); 198 | } 199 | } 200 | #endif 201 | 202 | /* test vectors for portability test 203 | wyhash("",0,_wyp)=42bc986dc5eec4d3 204 | wyhash("a",1,_wyp)=84508dc903c31551 205 | wyhash("abc",2,_wyp)=bc54887cfc9ecb1 206 | wyhash("message digest",3,_wyp)=adc146444841c430 207 | wyhash("abcdefghijklmnopqrstuvwxyz",4,_wyp)=9a64e42e897195b9 208 | wyhash("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",5,_wyp)=9199383239c32554 209 | wyhash("12345678901234567890123456789012345678901234567890123456789012345678901234567890",6,_wyp)=7c1ccf6bba30f5a5 210 | */ 211 | 212 | /* The Unlisence 213 | This is free and unencumbered software released into the public domain. 214 | 215 | Anyone is free to copy, modify, publish, use, compile, sell, or 216 | distribute this software, either in source code form or as a compiled 217 | binary, for any purpose, commercial or non-commercial, and by any 218 | means. 219 | 220 | In jurisdictions that recognize copyright laws, the author or authors 221 | of this software dedicate any and all copyright interest in the 222 | software to the public domain. We make this dedication for the benefit 223 | of the public at large and to the detriment of our heirs and 224 | successors. We intend this dedication to be an overt act of 225 | relinquishment in perpetuity of all present and future rights to this 226 | software under copyright law. 227 | 228 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 229 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 230 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 231 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 232 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 233 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 234 | OTHER DEALINGS IN THE SOFTWARE. 235 | 236 | For more information, please refer to 237 | */ 238 | -------------------------------------------------------------------------------- /wyhll.h: -------------------------------------------------------------------------------- 1 | /* This is free and unencumbered software released into the public domain under The Unlicense (http://unlicense.org/) 2 | main repo: https://github.com/wangyi-fudan/wyhll 3 | author: 王一 Wang Yi 4 | 5 | simple example: 6 | wyhll s; 7 | wyhll_configure(&s,12<<10,1000000); // configure 12kb data to count at most 1 million objects 8 | uint8_t *data; 9 | wyhll_alloc(s,&data); 10 | wyhll_add(s, data, item, item_len); // insert item 11 | cout< 17 | #include 18 | #include "wyhash.h" 19 | #include 20 | #include 21 | 22 | typedef struct wyhll_t{ uint64_t bins, p[6]; }wyhll; 23 | 24 | static inline void wyhll_configure(wyhll *s, uint64_t bytes, uint64_t capacity){ 25 | s->bins=(bytes-1)*8/3; 26 | double p=(double)s->bins/capacity; 27 | if(p>0.5) p=0.5; 28 | for(uint32_t i=0; i<6; i++) s->p[i]=pow(p,(i+1)/6.0)*~0ull; 29 | } 30 | 31 | static inline uint64_t wyhll_bytes(wyhll s){ return s.bins*3/8+1; } 32 | 33 | static inline uint64_t wyhll_alloc(wyhll s, uint8_t **data){ *data=(uint8_t*)calloc(wyhll_bytes(s),1); return data!=NULL; } 34 | 35 | static inline void wyhll_clear(wyhll s, uint8_t *data){ memset(data,0,wyhll_bytes(s)); } 36 | 37 | static inline uint8_t wyhll_get(uint8_t *data, uint64_t i){ 38 | uint64_t b=(i*3)>>3, f=(i*3)&7,f8=8-f; 39 | return ((data[b]>>f)|(data[b+1]<>3, f=(i*3)&7,f8=8-f; 44 | data[b]&=~(7<>f8); data[b+1]|=v>>f8; 45 | } 46 | 47 | static inline void wyhll_add(wyhll s, uint8_t *data, void *item, uint64_t item_size){ 48 | uint64_t h=wyhash(item,item_size,0,_wyp); 49 | uint64_t b=wy2u0k(wyhash64(h,h),s.bins); 50 | uint8_t v; 51 | if(h>=s.p[0]) v=1; 52 | else if(h>=s.p[1]) v=2; 53 | else if(h>=s.p[2]) v=3; 54 | else if(h>=s.p[3]) v=4; 55 | else if(h>=s.p[4]) v=5; 56 | else if(h>=s.p[5]) v=6; 57 | else v=7; 58 | if(v>wyhll_get(data,b)) wyhll_set(data,b,v); 59 | } 60 | 61 | static inline double wyhll_loglike(wyhll s, uint32_t *m, double N){ 62 | double sum=0; 63 | sum-=m[0]*N/s.bins; 64 | sum+=m[1]*log(exp(-N/s.bins*s.p[0]/~0ull)-exp(-N/s.bins)); 65 | sum+=m[2]*log(exp(-N/s.bins*s.p[1]/~0ull)-exp(-N/s.bins*s.p[0]/~0ull)); 66 | sum+=m[3]*log(exp(-N/s.bins*s.p[2]/~0ull)-exp(-N/s.bins*s.p[1]/~0ull)); 67 | sum+=m[4]*log(exp(-N/s.bins*s.p[3]/~0ull)-exp(-N/s.bins*s.p[2]/~0ull)); 68 | sum+=m[5]*log(exp(-N/s.bins*s.p[4]/~0ull)-exp(-N/s.bins*s.p[3]/~0ull)); 69 | sum+=m[6]*log(exp(-N/s.bins*s.p[5]/~0ull)-exp(-N/s.bins*s.p[4]/~0ull)); 70 | sum+=m[7]*log1p(-exp(-N/s.bins*s.p[5]/~0ull)); 71 | return sum; 72 | } 73 | 74 | static inline uint64_t wyhll_solve(wyhll s, uint32_t *m){ 75 | if(m[0]==s.bins) return 0; 76 | double r=(sqrt(5.0)+1)/2, a=0, b=log(s.bins*((double)~0ull/s.p[5])), c=b-(b-a)/r, d=a+(b-a)/r; 77 | while(fabs(a-b)>1e-5){ 78 | if(wyhll_loglike(s,m,exp(c))>wyhll_loglike(s,m,exp(d))) b=d; 79 | else a=c; 80 | c=b-(b-a)/r; d=a+(b-a)/r; 81 | } 82 | return exp((a+b)/2)+0.5; 83 | } 84 | 85 | static inline uint64_t wyhll_cardinality(wyhll s, uint8_t *data){ 86 | uint32_t m[8]={}; 87 | for(uint64_t i=0; ivb?va:vb]++; 96 | } 97 | return wyhll_solve(s,m); 98 | } 99 | #endif 100 | -------------------------------------------------------------------------------- /wyhll.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyi-fudan/wyHLL/fc19e8d22ee190e7fa9c8a7b60b7e08eaca6b673/wyhll.png --------------------------------------------------------------------------------