├── LICENSE ├── README ├── allten └── allten.go ├── beggar └── main.go ├── bigintfuzz ├── grab.sh └── main.go ├── bluniq └── main.go ├── branchfree ├── select.go └── select_test.go ├── bucket └── main.go ├── cachetest ├── README.md ├── bloom.go ├── clock │ └── clock.go ├── data │ └── generate.go ├── main.go ├── prob.go ├── randmark │ └── randmark.go ├── random │ └── random.go ├── slru │ └── slru.go └── tworand │ └── two.go ├── cstbucket ├── main.go ├── parse.go ├── parse.rl └── parse_test.go ├── ddmin └── main.go ├── entropy └── main.go ├── ewmaest ├── demo │ └── main.go └── ewmaest.go ├── fastpprof └── main.go ├── fastrand ├── fastrand.go └── fastrand_test.go ├── fastsearch ├── fastsearch.go └── fastsearch_test.go ├── fsmdemo ├── fishasm │ ├── fishasm.s │ └── fishasm_stub.go ├── fishfsm │ └── fishfsm.go ├── fishvmc.c ├── main.go └── sotest │ ├── fsmso │ └── main.go │ ├── run.sh │ └── runner.c ├── fuzzprot ├── fuzzprot.go └── fuzzprot_test.go ├── gcwatch └── main.go ├── gddo ├── README └── gddo.go ├── gfmt └── main.go ├── glj ├── README ├── main.go └── setup.sh ├── go-wtflog └── wtflog.go ├── grinderplot └── main.go ├── hashbench ├── README └── bench_test.go ├── hist ├── hist.pl └── main.go ├── hllbench └── bench_test.go ├── interp ├── interp.go └── interp_test.go ├── inthash ├── cycles │ ├── main.go │ └── output.txt ├── inthash.go └── inthash_test.go ├── intset ├── intset.go ├── nlz.go └── nlz_amd64.s ├── jolly └── main.go ├── jumpreplica └── main.go ├── leven └── leven.go ├── lz └── lz.go ├── lzpack ├── lz4pack.go ├── lz4unpack.go ├── pack.py └── unpack.py ├── matcher ├── README ├── aho.go ├── bloom.go ├── bsearch.go ├── dmph.go ├── domains.txt ├── gen-mafsa.go ├── lexer.go ├── lexer.rl ├── mafsa.go ├── main.go ├── map.go ├── mph.go └── radix.go ├── maze └── maze.py ├── median └── median.go ├── morse └── main.go ├── mpush └── main.go ├── msgrpc ├── cli.go ├── cli.py ├── srv.go └── srv.py ├── mtest └── main.go ├── nlz ├── bench_test.go ├── main.go ├── nlz.go ├── nlz_amd64.s ├── popcount.go └── popcount_amd64.s ├── numerical ├── integrate.go ├── integrate_test.go ├── rdtsc.go ├── rdtsc_amd64.s ├── root.go └── root_test.go ├── oma └── main.go ├── photocmp └── main.go ├── pphrase ├── main.go └── words.txt ├── prior └── main.go ├── qrshow └── main.go ├── quantiles └── main.go ├── range2cidr └── main.go ├── rangebench └── bench_test.go ├── repl └── repl.go ├── rndsample └── main.go ├── rndtxt └── randomtext.go ├── servedir └── main.go ├── shellsort ├── shell.go └── shell_test.go ├── shlines └── main.go ├── shufsecs └── shufsecs.go ├── simhash ├── scanners.go ├── simhash.go └── simhash_test.go ├── sipsum └── main.go ├── skvchk └── main.go ├── skvdist ├── main.go └── skvdist.pl ├── sshdregex ├── asm │ ├── fsmasm_amd64.s │ ├── fsmasm_stub.go │ └── fsmasm_tinygo.go ├── bench_test.go ├── c │ ├── c.go │ └── fsmc.c ├── fsm.go ├── fsmunsafe.go ├── gen.sh ├── ragel.go └── ragel.rl ├── stablepart ├── bench │ ├── bench_test.go │ └── part.go └── stablepart.go ├── staticset └── staticset.go ├── strtable ├── strtable.go └── strtable_test.go ├── superbat ├── README ├── index.html ├── main.go └── public ├── testlibfsm ├── fishasm │ └── fishasm_stub.go ├── fuzz.sh └── main.go ├── threadtree └── threadthree.go ├── tinyjsonbench ├── github.go ├── main.go ├── repos.json └── run.sh ├── tinymalloc └── malloc.c ├── toepoch └── main.go ├── toms └── main.go ├── topkerr └── main.go ├── trepl └── main.go ├── tsip ├── Makefile ├── bench.txt ├── go │ ├── asm.go │ ├── cmd │ │ └── mapdist │ │ │ └── main.go │ ├── testdata │ │ └── tsip.txt │ ├── tsip.go │ ├── tsip_amd64.s │ ├── tsip_stub.go │ └── tsip_test.go ├── main.cc ├── oaath.c ├── oaathu.c ├── tsip.c ├── tsip.h └── tsip_test.c ├── u64cmp ├── bench_test.go └── cmp.go ├── udprelay ├── main.go └── udpclient.py ├── urlq └── main.go ├── uuid └── uuid.go ├── wasmfuzz └── main.go ├── wasmlink ├── Makefile ├── add.wat └── main.go ├── worker └── main.go └── wscat └── main.go /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 Damian Gryski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | This repo contains things not interesting or complete enough to have their own. 2 | 3 | * intset -- compress a stream of integers 4 | 5 | * lz -- lempel-ziv compression 6 | 7 | * threadtree -- a threaded binary-tree 8 | 9 | * numerical -- numerical integration and root finding 10 | 11 | * maze -- maze generation stuff 12 | 13 | * hist -- simple command-line histogramming tool 14 | 15 | * simhash -- trivial simhash implementation 16 | 17 | * wscat -- trivial websocket netcat 18 | 19 | * servedir -- trivial http fileserver 20 | 21 | * rndtxt -- generate random text strings 22 | 23 | * mpush -- push to multiple notification services (pushbullet, nma, pushover) 24 | 25 | * superbat -- batmanjs and go-restful playground 26 | 27 | * msgrpc -- msgpack rpc python/go interop samples 28 | 29 | * quantile -- testing different streaming quantile estimators 30 | 31 | * wtflog -- logging package with some renamed log levels 32 | 33 | * qrshow -- display QR codes in a terminal 34 | 35 | * nlz -- asm code to find number of leading zeros 36 | 37 | * httpecho -- server to dump information about an http request 38 | 39 | * lzpack -- trivial packed format for lz4 compression 40 | 41 | * grinderplot -- generate a flot chart from grinder logs 42 | 43 | * worker -- framework for spawning concurrent workers 44 | 45 | * gddo -- search godoc.org from the command line 46 | 47 | * uuid -- generate random UUIDs 48 | 49 | * entropy -- reducer to compute entropy per epoch for a set of values in a category 50 | 51 | * inthash -- integer hashing functions 52 | 53 | * udprelay -- simple udp-to-tcp multiplexing relay 54 | 55 | * shufsecs -- shuffle sorted epoch-data within epochs 56 | 57 | * strtable -- dumb string->uint32 hash table for profiling vs native maps 58 | 59 | * cachetest -- playing with different caching algorithms (clock, lru, lfu, random) 60 | 61 | * glj -- passing data from go to lua with msgpack 62 | 63 | * toms -- text filter for time.Duration to milliseconds 64 | 65 | * bluniq -- bloom-filter based unique filter 66 | 67 | * skvdist -- check distribution of shardedkv choosers 68 | 69 | * gcwatch -- print out garbage collection stats from /debug/vars 70 | 71 | * toepoch -- convert time fields to epochs 72 | 73 | * repl -- framework for making dumb repls for testing 74 | 75 | * skvchk -- tool for checking shardedkv distributions 76 | 77 | * interp -- interpolation search 78 | 79 | * oma -- simulation of the Dutch children's board game "Met de bus naar Oma" 80 | 81 | * rndsample - uniform random sample from stdin 82 | 83 | * pphrase -- simple passphrase generator 84 | 85 | * fastrand -- fast xorshift rng with bias-free [0..n) 86 | 87 | * range2cird -- turn IP ranges into CIDR 88 | 89 | * gfmt -- trivial filter wrapping go-linebreak 90 | 91 | * mtest -- port of libtommath test program 92 | 93 | * leven -- fastest levenshtein distance algorithm I could find 94 | 95 | * matcher -- test different methods of testing string set membership 96 | 97 | * cstbucket -- crunch carbonserver logs for time-ranges of queries 98 | 99 | * stablepart -- stable partition a sort.Interface on a boolean predicate 100 | 101 | * jumpreplica -- tool for playing with replica choices for jump-hash 102 | 103 | * sshdregex -- demo using ragel for optimized regexp matching 104 | 105 | * hllbench -- benchmark different hyperloglog implementations 106 | 107 | * shlines, sipsum -- tools for siphashing things 108 | 109 | * urlq -- extract query parameters from a list of URLs 110 | 111 | * median -- compute the median of 5 numbers with a sorting network 112 | 113 | * hashbench -- benchmark different hashing functions 114 | 115 | * fastpprof -- how to use pprof with fasthttp 116 | 117 | * ewmaest -- progress logging with ewma-based ETA estimation 118 | -------------------------------------------------------------------------------- /allten/allten.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "math" 7 | "math/rand" 8 | "strconv" 9 | "time" 10 | ) 11 | 12 | type op int 13 | 14 | const ( 15 | opAdd op = iota 16 | opSub 17 | opDiv 18 | opMul 19 | opPush 20 | ) 21 | 22 | func randop() op { 23 | return op(rand.Intn(int(opPush))) 24 | } 25 | 26 | type operation struct { 27 | n int 28 | o op 29 | } 30 | 31 | var opStrs = []string{"+", "-", "/", "*"} 32 | 33 | func (o operation) String() string { 34 | if o.o == opPush { 35 | return strconv.Itoa(o.n) 36 | } 37 | 38 | return opStrs[o.o] 39 | } 40 | 41 | type stack[T any] []T 42 | 43 | func (s *stack[T]) push(t T) { 44 | *s = append(*s, t) 45 | } 46 | 47 | func (s *stack[T]) pop() T { 48 | t := (*s)[len(*s)-1] 49 | *s = (*s)[:len(*s)-1] 50 | return t 51 | } 52 | 53 | type expr []operation 54 | 55 | func (ops expr) String() string { 56 | var s stack[string] 57 | 58 | for _, op := range ops { 59 | if op.o == opPush { 60 | s.push(op.String()) 61 | continue 62 | } 63 | a := s.pop() 64 | b := s.pop() 65 | s.push("(" + b + op.String() + a + ")") 66 | } 67 | 68 | return s.pop() 69 | } 70 | 71 | func (ops expr) eval() float64 { 72 | var s stack[float64] 73 | 74 | for _, op := range ops { 75 | if op.o == opPush { 76 | s.push(float64(op.n)) 77 | continue 78 | } 79 | a := s.pop() 80 | b := s.pop() 81 | switch op.o { 82 | case opAdd: 83 | s.push(b + a) 84 | case opSub: 85 | s.push(b - a) 86 | case opMul: 87 | s.push(b * a) 88 | case opDiv: 89 | s.push(b / a) 90 | } 91 | } 92 | 93 | return s.pop() 94 | } 95 | 96 | const eps = 0.0000001 97 | 98 | func f64eq(a, b float64) bool { 99 | return math.Abs(a-b) < eps 100 | } 101 | 102 | func randelt(in []int) (int, []int) { 103 | idx := rand.Intn(len(in)) 104 | n := in[idx] 105 | in[idx] = in[len(in)-1] 106 | in = in[:len(in)-1] 107 | return n, in 108 | } 109 | 110 | func solve(numbers []int, target float64) expr { 111 | in := make([]int, 0, len(numbers)) 112 | 113 | for i := 0; i < 10000; i++ { 114 | // copy in our numbers 115 | var n int 116 | in = append(in, numbers...) 117 | 118 | var ops expr 119 | var nnum, nops int 120 | 121 | for len(in) > 1 { 122 | // choose a number 123 | n, in = randelt(in) 124 | ops = append(ops, operation{n, opPush}) 125 | nnum++ 126 | 127 | // plus maybe some operations 128 | for n, i := rand.Intn(nnum-nops), 0; i < n; i++ { 129 | ops = append(ops, operation{0, randop()}) 130 | nops++ 131 | } 132 | } 133 | 134 | // last number 135 | n, in = randelt(in) 136 | ops = append(ops, operation{n, opPush}) 137 | nnum++ 138 | 139 | // remaining operations 140 | for nops < (nnum - 1) { 141 | ops = append(ops, operation{n, randop()}) 142 | nops++ 143 | } 144 | 145 | if v := ops.eval(); f64eq(v, target) { 146 | return ops 147 | } 148 | } 149 | 150 | return nil 151 | } 152 | 153 | func init() { 154 | rand.Seed(time.Now().UnixNano()) 155 | } 156 | 157 | func main() { 158 | target := flag.Int("t", 0, "target value") 159 | solns := flag.Int("c", 1, "number of solutions") 160 | flag.Parse() 161 | 162 | var numbers []int 163 | for _, arg := range flag.Args() { 164 | n, err := strconv.Atoi(arg) 165 | if err != nil { 166 | fmt.Printf("unable to parse number %q: %v", arg, err) 167 | return 168 | } 169 | 170 | numbers = append(numbers, int(n)) 171 | } 172 | 173 | targets := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} 174 | if *target != 0 { 175 | targets = []int{*target} 176 | } 177 | 178 | for _, t := range targets { 179 | var solnFound bool 180 | for i := 0; i < *solns; i++ { 181 | ops := solve(numbers, float64(t)) 182 | if ops != nil { 183 | solnFound = true 184 | fmt.Println(ops, "=", t) 185 | } 186 | } 187 | if !solnFound { 188 | fmt.Println(";", t, ": no solution found") 189 | } 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /beggar/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "math/rand" 8 | "os" 9 | "os/signal" 10 | "runtime" 11 | ) 12 | 13 | func shuffle(b []byte) []byte { 14 | d := make([]byte, len(b)) 15 | p := rand.Perm(len(b)) 16 | for i, v := range p { 17 | d[i] = b[v] 18 | } 19 | return d 20 | } 21 | 22 | type deck []byte 23 | 24 | type table struct { 25 | turn int 26 | players []deck 27 | stack deck 28 | 29 | turns int 30 | tricks int 31 | swap int 32 | tlens []int 33 | cards int 34 | } 35 | 36 | func (t *table) deal() { 37 | t.cards++ 38 | t.stack = append(t.stack, t.players[t.turn][0]) 39 | t.players[t.turn] = t.players[t.turn][1:] 40 | } 41 | 42 | func (t *table) done() bool { 43 | return len(t.players[0]) == 0 || len(t.players[1]) == 0 44 | } 45 | 46 | func (t *table) swapPlayer() { 47 | t.turn = 1 - t.turn 48 | t.turns++ 49 | fmt.Printf("t.turns = %+v\n", t.turns) 50 | } 51 | 52 | var needToPlay = [256]int{ 53 | 'A': 4, 54 | 'J': 1, 55 | 'K': 3, 56 | 'Q': 2, 57 | } 58 | 59 | func (t *table) trick(remain int) { 60 | 61 | // player `turn` just dealt a card 62 | // so player `1 - turn` has `remain` chances to cover it 63 | 64 | t.tricks++ 65 | t.swapPlayer() 66 | 67 | for ; remain > 0 && !t.done(); remain-- { 68 | t.deal() 69 | 70 | need := needToPlay[t.stack[len(t.stack)-1]] 71 | if need != 0 { 72 | // remain count is +1 because of -- at the bottom 73 | remain = need + 1 74 | t.swapPlayer() 75 | } 76 | } 77 | 78 | if remain == 0 { 79 | t.swapPlayer() 80 | 81 | // player `turn` didn't cover in time 82 | // so player 1-turn gets all the cards and gets to deal next 83 | t.players[t.turn] = append(t.players[t.turn], t.stack...) 84 | // t.tlens = append(t.tlens, len(t.stack)) 85 | fmt.Printf("t.players = %+v\n", t.players) 86 | t.stack = nil 87 | } 88 | } 89 | 90 | func (t *table) play() { 91 | 92 | for len(t.players[0]) > 0 && len(t.players[1]) > 0 { 93 | 94 | t.deal() 95 | 96 | need := needToPlay[t.stack[len(t.stack)-1]] 97 | if need != 0 { 98 | t.trick(need) 99 | } else { 100 | t.swapPlayer() 101 | } 102 | } 103 | } 104 | 105 | func oneGame() (d deck, turns int, tricks int) { 106 | 107 | for i := 0; i < 4; i++ { 108 | for j := 0; j < 9; j++ { 109 | d = append(d, '.') 110 | } 111 | 112 | d = append(d, 'K', 'Q', 'J', 'A') 113 | } 114 | 115 | d = shuffle(d) 116 | 117 | orig := make(deck, len(d)) 118 | copy(orig, d) 119 | 120 | t := table{ 121 | players: []deck{d[:26], d[26:]}, 122 | } 123 | 124 | t.play() 125 | 126 | return orig, t.turns, t.tricks 127 | } 128 | 129 | type game struct { 130 | d deck 131 | turns int 132 | tricks int 133 | } 134 | 135 | func playGames(games int, quitch chan struct{}, gamech chan game) { 136 | 137 | var played int 138 | 139 | var maxTurns int 140 | var maxGame deck 141 | var maxTricks int 142 | 143 | run: 144 | for { 145 | d, turns, tricks := oneGame() 146 | 147 | if turns > maxTurns { 148 | log.Printf("turns=%d deck=%s\n", turns, d) 149 | maxTurns = turns 150 | maxGame = d 151 | maxTricks = tricks 152 | } 153 | 154 | select { 155 | case <-quitch: 156 | break run 157 | default: 158 | 159 | } 160 | 161 | played++ 162 | 163 | if games > 0 && played > games { 164 | break run 165 | } 166 | } 167 | 168 | gamech <- game{maxGame, maxTurns, maxTricks} 169 | 170 | } 171 | 172 | func main() { 173 | 174 | games := flag.Int("g", 10, "number of games") 175 | p := flag.Int("p", runtime.NumCPU(), "number of parallel games") 176 | 177 | flag.Parse() 178 | 179 | quitch := make(chan struct{}) 180 | 181 | gamech := make(chan game) 182 | 183 | for i := 0; i < *p; i++ { 184 | go playGames(*games, quitch, gamech) 185 | } 186 | 187 | go func() { 188 | c := make(chan os.Signal, 1) 189 | signal.Notify(c, os.Interrupt) 190 | <-c 191 | fmt.Println("quitting...") 192 | close(quitch) 193 | }() 194 | 195 | for i := 0; i < *p; i++ { 196 | g := <-gamech 197 | fmt.Printf("turns=%d tricks=%d deck=%s\n", g.turns, g.tricks, g.d) 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /bigintfuzz/grab.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | curl -O https://raw.githubusercontent.com/gonzus/bigint/master/bigint.c 3 | curl -O https://raw.githubusercontent.com/gonzus/bigint/master/bigint.h 4 | -------------------------------------------------------------------------------- /bigintfuzz/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | /* 4 | 5 | #include 6 | #include 7 | #include "bigint.h" 8 | 9 | */ 10 | import "C" 11 | 12 | import ( 13 | "bytes" 14 | "fmt" 15 | "math/big" 16 | "math/rand" 17 | "strings" 18 | "os" 19 | "time" 20 | "unsafe" 21 | ) 22 | 23 | func main() { 24 | 25 | var a, b, c big.Int 26 | 27 | ca := bigintCreate(); 28 | cb := bigintCreate(); 29 | cc := bigintCreate(); 30 | 31 | 32 | rnd := rand.New(rand.NewSource(time.Now().UnixNano())) 33 | 34 | for i := 0; i < 1000000; i++ { 35 | if i % 10000 == 0 { 36 | fmt.Println("iterations", i) 37 | } 38 | nsize := rnd.Intn(200) + 1 39 | n := new(big.Int).Lsh(big.NewInt(1), uint(nsize)) 40 | a.Rand(rnd, n) 41 | 42 | 43 | nsize = rnd.Intn(200) + 1 44 | n = new(big.Int).Lsh(big.NewInt(1), uint(nsize)) 45 | b.Rand(rnd, n) 46 | 47 | ca = bigintAssignString(ca, a.String()) 48 | cb = bigintAssignString(cb, b.String()) 49 | 50 | switch rnd.Intn(3) { 51 | 52 | case 0: 53 | bigintAdd(ca, cb, cc) 54 | s := bigintFormat(cc) 55 | 56 | c.Add(&a, &b) 57 | if cstr := (&c).String(); s != cstr { 58 | log("+", &a, &b, &c, s) 59 | } 60 | 61 | case 1: 62 | r := bigintSub(ca, cb, cc) 63 | s := bigintFormat(r) 64 | 65 | c.Sub(&a, &b) 66 | if cstr := (&c).String(); s != cstr { 67 | log("-", &a, &b, &c, s) 68 | } 69 | 70 | case 2: 71 | r := bigintMul(ca, cb, cc) 72 | s := bigintFormat(r) 73 | 74 | c.Mul(&a, &b) 75 | if cstr := (&c).String(); s != cstr { 76 | log("*", &a, &b, &c, s) 77 | } 78 | } 79 | } 80 | } 81 | 82 | func compare(a, b string) string { 83 | 84 | var sb strings.Builder 85 | 86 | if len(a) != len(b) { 87 | return fmt.Sprintf("lengths differ: %v <=> %v", a, b) 88 | } 89 | 90 | for i := 0; i < len(a); i++ { 91 | if a[i] == b[i] { 92 | sb.WriteByte(a[i]) 93 | } else { 94 | sb.WriteString("**>") 95 | sb.WriteByte(a[i]) 96 | sb.WriteByte(':') 97 | sb.WriteByte(b[i]) 98 | sb.WriteString("<**") 99 | } 100 | } 101 | 102 | return sb.String() 103 | } 104 | 105 | func log(op string, a, b, c *big.Int, s string) { 106 | fmt.Printf("%v %s %v = %v\n", a, op, b, c) 107 | fmt.Printf("fail: got %q want %q\n", s, c.String()) 108 | fmt.Printf("diff: %s\n", compare(s, c.String())) 109 | os.Exit(1) 110 | } 111 | 112 | func bigintCreate() *C.bigint { 113 | return C.bigint_create() 114 | } 115 | 116 | func bigintAssignString(a *C.bigint, s string) *C.bigint { 117 | cs := C.CString(s) 118 | defer C.free(unsafe.Pointer(cs)) 119 | return C.bigint_assign_string(a, cs) 120 | } 121 | 122 | func bigintAdd(a, b, c *C.bigint) *C.bigint { 123 | return C.bigint_add(a, b, c) 124 | } 125 | 126 | func bigintSub(a, b, c *C.bigint) *C.bigint { 127 | return C.bigint_sub(a, b, c) 128 | } 129 | 130 | func bigintMul(a, b, c *C.bigint) *C.bigint { 131 | return C.bigint_mul(a, b, c) 132 | } 133 | 134 | func bigintFormat(a *C.bigint) string { 135 | var s [1000]byte 136 | C.bigint_format(a, (*C.char)(unsafe.Pointer(&s[0]))) 137 | i := bytes.IndexByte(s[:], 0) 138 | return string(s[:i]) 139 | } 140 | -------------------------------------------------------------------------------- /bluniq/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "log" 7 | "os" 8 | 9 | "github.com/dchest/siphash" 10 | "github.com/dgryski/go-bloomf" 11 | ) 12 | 13 | func main() { 14 | 15 | f := flag.String("f", "/dev/stdin", "input file") 16 | n := flag.Int("n", 10e6, "cardinality estimate of set") 17 | q := flag.Bool("q", false, "quiet") 18 | fprate := flag.Float64("fp", 0.00000001, "false positive rate") 19 | 20 | flag.Parse() 21 | 22 | file, err := os.Open(*f) 23 | if err != nil { 24 | log.Fatal(err) 25 | } 26 | 27 | var lines int 28 | b := bloomf.New(*n, *fprate, func(b []byte) uint64 { return siphash.Hash(0, 0, b) }) 29 | scanner := bufio.NewScanner(file) 30 | 31 | var total int 32 | 33 | for scanner.Scan() { 34 | lines++ 35 | l := scanner.Bytes() 36 | if !b.Lookup(l) { 37 | total++ 38 | if !*q { 39 | os.Stdout.Write(l) 40 | os.Stdout.Write([]byte("\n")) 41 | } 42 | } 43 | b.Insert(l) 44 | if lines%(1<<20) == 0 { 45 | log.Println(lines) 46 | } 47 | } 48 | log.Printf("unique %d\n", total) 49 | } 50 | -------------------------------------------------------------------------------- /branchfree/select.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | ) 6 | 7 | func inlineMin(x, y uint64) uint64 { 8 | return (uint64((int64(x)-int64(y))>>63) & (x ^ y)) ^ y 9 | } 10 | 11 | func fastMin(x, y uint64) uint64 { 12 | diff := int64(x) - int64(y) 13 | 14 | // sets bit63 to 0xFFFFFFFF if x> 63) 16 | 17 | // return ifXisSmaller if x is smaller than y, else y 18 | return (bit63 & (x ^ y)) ^ y 19 | } 20 | 21 | func min(x, y uint64) uint64 { 22 | if x < y { 23 | return x 24 | } 25 | 26 | return y 27 | } 28 | 29 | func main() { 30 | log.Printf("min(1,20)=%+v\n", fastMin(1, 20)) 31 | } 32 | -------------------------------------------------------------------------------- /branchfree/select_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | "testing/quick" 6 | ) 7 | 8 | func TestFastMin(t *testing.T) { 9 | 10 | f := func(x, y uint64) bool { 11 | // must be positive 12 | x &^= 1 << 63 13 | y &^= 1 << 63 14 | m := fastMin(x, y) 15 | n := min(x, y) 16 | o := inlineMin(x, y) 17 | return m == n && n == o 18 | } 19 | 20 | if err := quick.Check(f, nil); err != nil { 21 | t.Error(err) 22 | } 23 | } 24 | 25 | var total uint64 26 | 27 | func BenchmarkMin(b *testing.B) { 28 | x := uint64(1) 29 | y := XorshiftMult64(x) 30 | 31 | for i := 0; i < b.N; i++ { 32 | 33 | xp := x &^ 1 << 63 34 | yp := y &^ 1 << 63 35 | 36 | total += min(xp, yp) 37 | x, y = y, XorshiftMult64(y) 38 | } 39 | } 40 | 41 | func BenchmarkFastMin(b *testing.B) { 42 | x := uint64(1) 43 | y := XorshiftMult64(x) 44 | 45 | total = 0 46 | 47 | for i := 0; i < b.N; i++ { 48 | 49 | xp := x &^ 1 << 63 50 | yp := y &^ 1 << 63 51 | 52 | total += fastMin(xp, yp) 53 | x, y = y, XorshiftMult64(y) 54 | } 55 | } 56 | 57 | func BenchmarkInlineMin(b *testing.B) { 58 | x := uint64(1) 59 | y := XorshiftMult64(x) 60 | 61 | total = 0 62 | 63 | for i := 0; i < b.N; i++ { 64 | 65 | xp := x &^ 1 << 63 66 | yp := y &^ 1 << 63 67 | 68 | total += inlineMin(xp, yp) 69 | x, y = y, XorshiftMult64(y) 70 | } 71 | } 72 | 73 | // 64-bit xorshift multiply rng from http://vigna.di.unimi.it/ftp/papers/xorshift.pdf 74 | func XorshiftMult64(x uint64) uint64 { 75 | x ^= x >> 12 // a "te 76 | x ^= x << 25 // b 77 | x ^= x >> 27 // c 78 | return x * 2685821657736338717 79 | } 80 | -------------------------------------------------------------------------------- /bucket/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | ) 11 | 12 | func main() { 13 | 14 | bucketSize := flag.Int("n", 500, "bucket size") 15 | f := flag.String("f", "", "file to read") 16 | flag.Parse() 17 | 18 | var r io.Reader 19 | 20 | if *f == "" { 21 | r = os.Stdin 22 | } else { 23 | var err error 24 | r, err = os.Open(*f) 25 | if err != nil { 26 | log.Fatal(err) 27 | } 28 | } 29 | 30 | sc := bufio.NewScanner(r) 31 | 32 | counts := make(map[string]int) 33 | 34 | var bucket int 35 | 36 | for sc.Scan() { 37 | line := sc.Text() 38 | 39 | counts[line]++ 40 | 41 | if bucket++; bucket == *bucketSize { 42 | for k, v := range counts { 43 | fmt.Printf("%s\t%d\n", k, v) 44 | } 45 | bucket = 0 46 | counts = map[string]int{} 47 | } 48 | } 49 | 50 | if err := sc.Err(); err != nil { 51 | log.Fatal(err) 52 | } 53 | 54 | for k, v := range counts { 55 | fmt.Printf("%s\t%d\n", k, v) 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /cachetest/README.md: -------------------------------------------------------------------------------- 1 | Test different caching algorithms. 2 | 3 | Currently implemented: 4 | 5 | * random eviction 6 | * lru (external, from https://github.com/golang/groupcache ) 7 | * clock 8 | * slru 9 | * s4lru (external, from http://github.com/dgryski/go-s4lru ) 10 | * clock-pro (external, from https://github.com/dgryski/go-clockpro ) 11 | * arc (external, from https://github.com/dgryski/go-arc ) 12 | * tinylfu (external, from https://github.com/dgryski/go-tinylfu ) 13 | * tworand -- power of two-random-choices 14 | * randmark -- randomized marking 15 | 16 | usage: 17 | 18 | $ ./cachetest -n=5000 -alg=s4lru b.bf.Cap() { 21 | b.bf.Reset() 22 | } 23 | return alreadyPresent 24 | } 25 | -------------------------------------------------------------------------------- /cachetest/clock/clock.go: -------------------------------------------------------------------------------- 1 | package clock 2 | 3 | type entry struct { 4 | key string 5 | value interface{} 6 | used bool 7 | } 8 | 9 | type Cache struct { 10 | capacity int 11 | data map[string]int 12 | keys []entry 13 | hand int 14 | } 15 | 16 | func New(capacity int) *Cache { 17 | return &Cache{ 18 | capacity: capacity, 19 | data: make(map[string]int), 20 | keys: make([]entry, capacity), 21 | } 22 | } 23 | 24 | func (c *Cache) Get(key string) interface{} { 25 | idx, ok := c.data[key] 26 | if !ok { 27 | return nil 28 | } 29 | c.keys[idx].used = true 30 | return c.keys[idx].value 31 | } 32 | 33 | func (c *Cache) Set(key string, value interface{}) { 34 | 35 | slot := len(c.data) 36 | 37 | if slot == c.capacity { 38 | 39 | for c.keys[c.hand].used { 40 | c.keys[c.hand].used = false 41 | c.hand++ 42 | if c.hand == c.capacity { 43 | c.hand = 0 44 | } 45 | } 46 | 47 | slot = c.hand 48 | 49 | delete(c.data, c.keys[slot].key) 50 | 51 | } 52 | 53 | c.keys[slot].key = key 54 | c.keys[slot].value = value 55 | c.keys[slot].used = true 56 | c.data[key] = slot 57 | } 58 | -------------------------------------------------------------------------------- /cachetest/data/generate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/pingcap/go-ycsb/pkg/generator" 11 | ) 12 | 13 | func main() { 14 | if len(os.Args) < 4 { 15 | fmt.Println("error: invalid command") 16 | fmt.Println("Usage: generate.go ") 17 | return 18 | } 19 | 20 | fileName := os.Args[1] 21 | maxKeys, err := strconv.ParseInt(os.Args[2], 10, 64) 22 | if err != nil { 23 | panic(err) 24 | } 25 | workloadSize, err := strconv.ParseInt(os.Args[3], 10, 64) 26 | if err != nil { 27 | panic(err) 28 | } 29 | 30 | file, err := os.Create(fileName) 31 | if err != nil { 32 | panic(err) 33 | } 34 | defer file.Close() 35 | 36 | if err := genZipfWorkload(file, maxKeys, workloadSize); err != nil { 37 | panic(err) 38 | } 39 | } 40 | 41 | func genZipfWorkload(file *os.File, maxKeys, workloadSize int64) error { 42 | z := generator.NewScrambledZipfian(0, maxKeys, generator.ZipfianConstant) 43 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 44 | 45 | for i := int64(0); i < workloadSize; i++ { 46 | if _, err := fmt.Fprintln(file, z.Next(r)); err != nil { 47 | return err 48 | } 49 | } 50 | 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /cachetest/prob.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/dgryski/go-pcgr" 7 | ) 8 | 9 | type probkeeper struct { 10 | r pcgr.Rand 11 | f float32 12 | } 13 | 14 | func newProbkeeper(f float32) *probkeeper { 15 | return &probkeeper{ 16 | f:f, 17 | r:pcgr.New(rand.Int63(), rand.Int63()), 18 | } 19 | } 20 | 21 | func (p *probkeeper) allow(s string) bool { 22 | return p.r.Float32() < p.f 23 | } 24 | -------------------------------------------------------------------------------- /cachetest/randmark/randmark.go: -------------------------------------------------------------------------------- 1 | package randmark 2 | 3 | import ( 4 | "math/rand" 5 | ) 6 | 7 | type element struct { 8 | value interface{} 9 | slot int 10 | marked bool 11 | } 12 | 13 | type Cache struct { 14 | capacity int 15 | data map[string]element 16 | marked []string 17 | unmarked []string 18 | mark bool 19 | } 20 | 21 | func New(capacity int) *Cache { 22 | return &Cache{ 23 | mark:true, 24 | capacity: capacity, 25 | data: make(map[string]element), 26 | marked: make([]string, 0, capacity), 27 | unmarked: make([]string, 0, capacity), 28 | } 29 | } 30 | 31 | func (c *Cache) Get(key string) interface{} { 32 | e, ok := c.data[key] 33 | if !ok { 34 | // no key 35 | return nil 36 | } 37 | 38 | if e.marked == !c.mark { 39 | // key was not marked; we need to mark it 40 | 41 | // remove from unmarked list 42 | c.removeUnmarked(e.slot) 43 | 44 | // add to marked list 45 | slot := len(c.marked) 46 | c.marked = append(c.marked, key) 47 | 48 | // update element 49 | c.data[key] = element{value: e.value, marked: c.mark, slot: slot} 50 | } 51 | 52 | return e.value 53 | } 54 | 55 | func (c *Cache) removeUnmarked(slot int) { 56 | last := len(c.unmarked) - 1 57 | 58 | if slot == last { 59 | c.unmarked = c.unmarked[:last] 60 | return 61 | } 62 | 63 | // overwrite slot with last key 64 | key := c.unmarked[last] 65 | c.unmarked[slot] = key 66 | c.unmarked[last] = "" 67 | c.unmarked = c.unmarked[:last] 68 | 69 | // update slot of moved key 70 | e := c.data[key] 71 | e.slot = slot 72 | c.data[key] = e 73 | } 74 | 75 | func (c *Cache) Set(key string, value interface{}) { 76 | 77 | if len(c.data) == c.capacity { 78 | // need to evict 79 | 80 | // every page is marked 81 | if len(c.marked) == c.capacity { 82 | // unmark all the pages 83 | c.mark = !c.mark 84 | c.marked, c.unmarked = c.unmarked, c.marked 85 | } 86 | 87 | // evict unmarked page at random 88 | slot := rand.Intn(len(c.unmarked)) 89 | delete(c.data, c.unmarked[slot]) 90 | c.removeUnmarked(slot) 91 | } 92 | 93 | // drop in new key/value as marked 94 | slot := len(c.marked) 95 | c.marked = append(c.marked, key) 96 | c.data[key] = element{value: value, marked: c.mark, slot: slot} 97 | } 98 | -------------------------------------------------------------------------------- /cachetest/random/random.go: -------------------------------------------------------------------------------- 1 | package random 2 | 3 | // from https://github.com/cloudflare/jgc-talks/tree/master/GoSF/Go_Profiling 4 | 5 | import "math/rand" 6 | 7 | type Cache struct { 8 | capacity int 9 | data map[string]interface{} 10 | keys []string 11 | } 12 | 13 | func New(capacity int) *Cache { 14 | return &Cache{ 15 | capacity: capacity, 16 | data: make(map[string]interface{}), 17 | keys: make([]string, capacity), 18 | } 19 | } 20 | 21 | func (c *Cache) Get(key string) interface{} { 22 | return c.data[key] 23 | } 24 | 25 | func (c *Cache) Set(key string, value interface{}) { 26 | 27 | slot := len(c.data) 28 | 29 | if len(c.data) == c.capacity { 30 | 31 | slot = rand.Intn(c.capacity) 32 | 33 | delete(c.data, c.keys[slot]) 34 | 35 | } 36 | 37 | c.keys[slot] = key 38 | c.data[key] = value 39 | } 40 | -------------------------------------------------------------------------------- /cachetest/slru/slru.go: -------------------------------------------------------------------------------- 1 | package slru 2 | 3 | import "github.com/golang/groupcache/lru" 4 | 5 | type Cache struct { 6 | once *lru.Cache 7 | twice *lru.Cache 8 | } 9 | 10 | func New(oncecap, twicecap int) *Cache { 11 | c := &Cache{ 12 | once: lru.New(oncecap), 13 | twice: lru.New(twicecap), 14 | } 15 | // make sure keys evicted from two make it to the head of one 16 | c.twice.OnEvicted = func(k lru.Key, v interface{}) { 17 | c.once.Add(k, v) 18 | } 19 | return c 20 | } 21 | 22 | func (c *Cache) Get(key string) interface{} { 23 | 24 | val, ok := c.once.Get(key) 25 | 26 | if !ok { 27 | val, _ = c.twice.Get(key) 28 | return val 29 | } 30 | 31 | c.once.Remove(key) 32 | c.twice.Add(key, val) 33 | return val 34 | } 35 | 36 | func (c *Cache) Set(key string, value interface{}) { 37 | c.once.Add(key, value) 38 | } 39 | -------------------------------------------------------------------------------- /cachetest/tworand/two.go: -------------------------------------------------------------------------------- 1 | package tworand 2 | 3 | // "power of two random choices" -- pick 2 random elements, remove the oldest 4 | 5 | import "math/rand" 6 | 7 | type entry struct { 8 | key string 9 | val interface{} 10 | t int64 11 | } 12 | 13 | type Cache struct { 14 | capacity int 15 | keys map[string]int 16 | data []entry 17 | t int64 18 | } 19 | 20 | func New(capacity int) *Cache { 21 | return &Cache{ 22 | capacity: capacity, 23 | keys: make(map[string]int), 24 | data: make([]entry, capacity), 25 | } 26 | } 27 | 28 | func (c *Cache) Get(key string) interface{} { 29 | if v, ok := c.keys[key]; ok { 30 | c.data[v].t = c.t 31 | return c.data[v].val 32 | } 33 | 34 | return nil 35 | } 36 | 37 | func (c *Cache) Set(key string, value interface{}) { 38 | 39 | slot := len(c.data) 40 | 41 | if len(c.data) == c.capacity { 42 | 43 | slot = rand.Intn(c.capacity) 44 | if slot2 := rand.Intn(c.capacity); c.data[slot2].t < c.data[slot].t { 45 | slot = slot2 46 | } 47 | 48 | delete(c.keys, c.data[slot].key) 49 | } 50 | 51 | c.keys[key] = slot 52 | c.data[slot].key = key 53 | c.data[slot].val = value 54 | c.data[slot].t = c.t 55 | c.t++ 56 | } 57 | -------------------------------------------------------------------------------- /cstbucket/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //go:generate ragel -Z -G2 parse.rl 4 | 5 | import ( 6 | "bufio" 7 | "bytes" 8 | "flag" 9 | "fmt" 10 | "log" 11 | "os" 12 | "runtime/pprof" 13 | "time" 14 | 15 | "github.com/dchest/siphash" 16 | "github.com/lytics/hll" 17 | ) 18 | 19 | const ( 20 | lastMinute = 60 21 | last5m = 5 * 60 22 | last10m = 10 * 60 23 | last15m = 15 * 60 24 | last30m = 30 * 60 25 | last60m = 60 * 60 26 | last2h = 2 * 60 * 60 27 | last4h = 4 * 60 * 60 28 | last8h = 8 * 60 * 60 29 | last12h = 12 * 60 * 60 30 | last24h = 24 * 60 * 60 31 | last2d = 2 * 24 * 60 * 60 32 | last5d = 5 * 24 * 60 * 60 33 | last7d = 7 * 24 * 60 * 60 34 | last14d = 14 * 24 * 60 * 60 35 | last1month = 30 * 24 * 60 * 60 36 | last6month = 180 * 24 * 60 * 60 37 | last1y = 365 * 24 * 60 * 60 38 | allTime = 5 * 365 * 24 * 60 * 60 39 | ) 40 | 41 | var seen = []struct { 42 | epoch int64 43 | count uint64 44 | }{ 45 | {lastMinute, 0}, 46 | {last5m, 0}, 47 | {last10m, 0}, 48 | {last15m, 0}, 49 | {last30m, 0}, 50 | {last60m, 0}, 51 | {last2h, 0}, 52 | {last4h, 0}, 53 | {last8h, 0}, 54 | {last12h, 0}, 55 | {last24h, 0}, 56 | {last2d, 0}, 57 | {last5d, 0}, 58 | {last7d, 0}, 59 | {last14d, 0}, 60 | {last1month, 0}, 61 | {last6month, 0}, 62 | {last1y, 0}, 63 | {allTime, 0}, 64 | } 65 | 66 | const logFormat = "2006/01/02 15:04:05" 67 | 68 | func main() { 69 | 70 | cpuprofile := flag.String("cpuprofile", "", "pprof output file") 71 | 72 | tzoffset := flag.Int64("tz", 0, "time zone offset between log and query") 73 | 74 | flag.Parse() 75 | 76 | if *cpuprofile != "" { 77 | f, err := os.Create(*cpuprofile) 78 | if err != nil { 79 | log.Fatalf("%v", err) 80 | } 81 | if err := pprof.StartCPUProfile(f); err != nil { 82 | log.Fatalf("%v", err) 83 | } 84 | defer pprof.StopCPUProfile() 85 | } 86 | 87 | scanner := bufio.NewScanner(os.Stdin) 88 | 89 | h := hll.NewHll(14, 20) 90 | 91 | var t0 int64 92 | var prevt []byte 93 | 94 | for scanner.Scan() { 95 | t, m, e1, _, err := parseLine(scanner.Bytes()) 96 | if err != nil { 97 | continue 98 | } 99 | 100 | h.Add(siphash.Hash(0, 0, m)) 101 | 102 | if !bytes.Equal(prevt, t) { 103 | prevt = append(prevt[:0], t...) 104 | tm, _ := time.Parse(logFormat, string(prevt)) 105 | t0 = tm.Unix() 106 | } 107 | diff := t0 - e1 - *tzoffset 108 | 109 | var i int 110 | for i < len(seen) && seen[i].epoch < diff { 111 | i++ 112 | } 113 | if i == len(seen) { 114 | i = len(seen) - 1 115 | } 116 | seen[i].count++ 117 | } 118 | 119 | fmt.Printf("unique metrics %8d\n", h.Cardinality()) 120 | 121 | var maxcount uint64 122 | var total uint64 123 | for i := range seen { 124 | if seen[i].count > maxcount { 125 | maxcount = seen[i].count 126 | } 127 | total += seen[i].count 128 | } 129 | 130 | const starcols = 70 131 | stars := dupRune('*', starcols) 132 | var cumulative uint64 133 | for i := range seen { 134 | d := time.Duration(seen[i].epoch) * time.Second 135 | s := stars[:int(float64(float64(starcols)*(float64(seen[i].count)/float64(maxcount))))] 136 | cumulative += seen[i].count 137 | fmt.Printf("%12s %10d (%0.2f, cum %0.2f) %s\n", d.String(), seen[i].count, float64(seen[i].count)/float64(total), float64(cumulative)/float64(total), s) 138 | } 139 | } 140 | 141 | func dupRune(r rune, n int) string { 142 | s := make([]rune, n) 143 | for i := range s { 144 | s[i] = r 145 | } 146 | return string(s) 147 | } 148 | -------------------------------------------------------------------------------- /cstbucket/parse.rl: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | ) 7 | 8 | func parseLine(data []byte) ([]byte, []byte, int64, int64, error) { 9 | 10 | %% machine scanner; 11 | %% write data; 12 | 13 | cs, p, pe, eof := 0, 0, len(data), len(data) 14 | 15 | var n int 16 | 17 | var t0 []byte 18 | var m []byte 19 | var e1 int64 20 | var e2 int64 21 | 22 | var parsed bool 23 | 24 | // 2015/09/15 16:49:17 fetch: served "metric.name.with.lots.__of_dots__.and.underscores" from 1442324820 to 1442328420 25 | 26 | %%{ 27 | datetime = digit{4} '/' digit{2} '/' digit{2} ' ' digit{2} ':' digit{2} ':' digit{2} ; 28 | epoch = digit{10} ; 29 | quoted = '"' [^"]+ '"' ; 30 | 31 | action start { n = fpc } 32 | action save_datetime { t0 = data[n:fpc] } 33 | action save_metric { m = data[n:fpc] } 34 | action save_epoch1 { e1, _ = strconv.ParseInt(string(data[n:fpc]), 10, 64) } 35 | action save_epoch2 { e2, _ = strconv.ParseInt(string(data[n:fpc]), 10, 64) } 36 | 37 | main := (datetime >start %save_datetime) ' fetch: served ' ( quoted >start %save_metric ) ' from ' ( epoch >start %save_epoch1 ) ' to ' ( epoch >start %save_epoch2 ) @{ parsed = true }; 38 | 39 | write init; 40 | write exec; 41 | }%% 42 | 43 | if !parsed { 44 | return nil, nil, 0, 0, errors.New("parse error") 45 | } 46 | 47 | return t0, m, e1, e2, nil 48 | } 49 | -------------------------------------------------------------------------------- /cstbucket/parse_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func TestParseLine(t *testing.T) { 9 | 10 | q := []byte(`2015/09/15 16:49:17 fetch: served "metric.name.with.lots.__of_dots__.and.underscores" from 1442324820 to 1442328420`) 11 | 12 | t0, m, e1, e2, err := parseLine(q) 13 | 14 | var want = struct { 15 | t0 []byte 16 | m []byte 17 | e1 int64 18 | e2 int64 19 | }{ 20 | []byte("2015/09/15 16:49:17"), 21 | []byte(`"metric.name.with.lots.__of_dots__.and.underscores"`), 22 | 1442324820, 23 | 1442328420, 24 | } 25 | 26 | if !bytes.Equal(t0, want.t0) || !bytes.Equal(m, want.m) || e1 != want.e1 || e2 != want.e2 || err != nil { 27 | t.Errorf("parseLine(q)=(%v,%v,%v,%v,%v), want (%v,%v,%v,%v,%v)", string(t0), string(m), e1, e2, err, string(want.t0), string(want.m), want.e1, want.e2, nil) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /ddmin/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "os/exec" 10 | 11 | ddmin "github.com/dgryski/go-ddmin" 12 | ) 13 | 14 | func run(prog string, data []byte) ([]byte, error) { 15 | cmd := exec.Command(prog) 16 | cmd.Stdin = bytes.NewReader(data) 17 | var out bytes.Buffer 18 | cmd.Stdout = &out 19 | if err := cmd.Run(); err != nil { 20 | return nil, err 21 | } 22 | return out.Bytes(), nil 23 | } 24 | 25 | func main() { 26 | good := flag.String("good", "", "known good command") 27 | bad := flag.String("bad", "", "known bad command") 28 | flag.Parse() 29 | 30 | input, err := ioutil.ReadAll(os.Stdin) 31 | if err != nil { 32 | log.Fatalf("unable to read stdin: %v", err) 33 | } 34 | 35 | // cache of responses 36 | m := make(map[string]ddmin.Result) 37 | 38 | os.Stdout.Write(ddmin.Minimize(input, func(d []byte) ddmin.Result { 39 | if r, ok := m[string(d)]; ok { 40 | return r 41 | } 42 | 43 | g, err := run(*good, d) 44 | if err != nil { 45 | m[string(d)] = ddmin.Unresolved 46 | return ddmin.Unresolved 47 | } 48 | 49 | b, err := run(*bad, d) 50 | if err != nil { 51 | m[string(d)] = ddmin.Unresolved 52 | return ddmin.Unresolved 53 | } 54 | 55 | if bytes.Equal(b, g) { 56 | m[string(d)] = ddmin.Pass 57 | return ddmin.Pass 58 | } 59 | 60 | m[string(d)] = ddmin.Fail 61 | return ddmin.Fail 62 | })) 63 | } 64 | -------------------------------------------------------------------------------- /entropy/main.go: -------------------------------------------------------------------------------- 1 | // Package main is a reducer for calculating entropy for values for various categories 2 | /* 3 | Input format: 4 | epoch value category 5 | 6 | The file must be sorted by epoch. 7 | 8 | */ 9 | package main 10 | 11 | import ( 12 | "bufio" 13 | "bytes" 14 | "fmt" 15 | "os" 16 | "strconv" 17 | 18 | "github.com/dgryski/go-entropy" 19 | ) 20 | 21 | func main() { 22 | 23 | ex := make(map[string]entropy.Exact) 24 | 25 | scanner := bufio.NewScanner(os.Stdin) 26 | 27 | var epoch int 28 | 29 | for scanner.Scan() { 30 | fields := bytes.Fields(scanner.Bytes()) 31 | 32 | e, _ := strconv.Atoi(string(fields[0])) 33 | if e != epoch { 34 | for k, v := range ex { 35 | fmt.Println(epoch, k, v.Entropy()) 36 | } 37 | 38 | ex = make(map[string]entropy.Exact) 39 | epoch = e 40 | } 41 | 42 | m, ok := ex[string(fields[2])] 43 | if !ok { 44 | m = entropy.NewExact() 45 | ex[string(fields[2])] = m 46 | } 47 | 48 | m.Push(fields[1], 1) 49 | } 50 | 51 | for k, v := range ex { 52 | fmt.Println(epoch, k, v.Entropy()) 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /ewmaest/demo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/dgryski/trifles/ewmaest" 5 | "log" 6 | "math/rand" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | 12 | const ( 13 | totalItem = 1000 14 | maxBlockSize = 20 15 | ) 16 | 17 | est := ewmaest.New(totalItem) 18 | 19 | est.StartBlock() 20 | blockSize := rand.Intn(maxBlockSize) + 1 21 | for i := 0; i < totalItem; i += blockSize { 22 | time.Sleep(time.Duration(blockSize) * (50*time.Millisecond + time.Duration(rand.Intn(5*int(time.Millisecond))))) 23 | eta := est.CompleteItems(blockSize) 24 | log.Println(est.Progress(eta)) 25 | est.StartBlock() 26 | blockSize = rand.Intn(maxBlockSize) + 1 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /ewmaest/ewmaest.go: -------------------------------------------------------------------------------- 1 | package ewmaest 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type EWMAEstimate struct { 9 | TotalItems int 10 | Alpha float64 11 | 12 | completedItems int 13 | 14 | timePerItemEstimate time.Duration 15 | blockStartTime time.Time 16 | } 17 | 18 | func New(total int) *EWMAEstimate { 19 | return &EWMAEstimate{ 20 | TotalItems: total, 21 | Alpha: 0.25, 22 | } 23 | } 24 | 25 | func (e *EWMAEstimate) StartBlock() { 26 | e.blockStartTime = time.Now() 27 | } 28 | 29 | func (e *EWMAEstimate) CompleteItems(items int) time.Duration { 30 | lastBlockTime := time.Since(e.blockStartTime) 31 | lastItemEstimate := float64(lastBlockTime) / float64(items) 32 | e.timePerItemEstimate = time.Duration((e.Alpha * lastItemEstimate) + (1-e.Alpha)*float64(e.timePerItemEstimate)) 33 | 34 | e.completedItems += items 35 | 36 | remainingItems := e.TotalItems - e.completedItems 37 | remainingTime := time.Duration(remainingItems) * e.timePerItemEstimate 38 | 39 | return remainingTime 40 | } 41 | 42 | func (e *EWMAEstimate) Progress(remainingTime time.Duration) string { 43 | return fmt.Sprintf("Completed %d/%d (%02d %%), ETA %v (%v)", e.completedItems, e.TotalItems, int(100*float64(e.completedItems)/float64(e.TotalItems)), time.Duration(remainingTime.Seconds())*time.Second, time.Now().Add(remainingTime).Format(time.Stamp)) 44 | } 45 | -------------------------------------------------------------------------------- /fastpprof/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "log" 7 | "net/http" 8 | "net/http/pprof" 9 | "strconv" 10 | 11 | "github.com/valyala/fasthttp" 12 | "github.com/valyala/fasthttp/fasthttpadaptor" 13 | ) 14 | 15 | func registerPProf(h func(string, func(http.ResponseWriter, *http.Request))) { 16 | h("/debug/pprof/", func(w http.ResponseWriter, r *http.Request) { 17 | w.Header().Set("Content-Type", "text/html; charset=utf-8") 18 | pprof.Index(w, r) 19 | }) 20 | h("/debug/pprof/cmdline", pprof.Cmdline) 21 | h("/debug/pprof/profile", pprof.Profile) 22 | h("/debug/pprof/symbol", pprof.Symbol) 23 | h("/debug/pprof/block", pprof.Handler("block").ServeHTTP) 24 | h("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) 25 | h("/debug/pprof/goroutine", pprof.Handler("goroutine").ServeHTTP) 26 | h("/debug/pprof/threadcreate", pprof.Handler("threadcreate").ServeHTTP) 27 | } 28 | 29 | func main() { 30 | 31 | port := flag.Int("p", 8080, "http port") 32 | flag.Parse() 33 | 34 | pprofMux := http.NewServeMux() 35 | registerPProf(pprofMux.HandleFunc) 36 | fastpprof := fasthttpadaptor.NewFastHTTPHandler(pprofMux) 37 | 38 | requestHandler := func(ctx *fasthttp.RequestCtx) { 39 | switch { 40 | case bytes.Equal(ctx.Path(), []byte("/debug/pprof")): 41 | ctx.Redirect("/debug/pprof/", 301) 42 | case bytes.HasPrefix(ctx.Path(), []byte("/debug/pprof/")): 43 | fastpprof(ctx) 44 | default: 45 | ctx.NotFound() 46 | } 47 | } 48 | 49 | log.Println("listening on port", *port) 50 | 51 | err := fasthttp.ListenAndServe(":"+strconv.Itoa(*port), requestHandler) 52 | if err != nil { 53 | log.Println("error during fasthttp.ListenAndServe(): ", err) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /fastrand/fastrand.go: -------------------------------------------------------------------------------- 1 | // Package fastrand provides a fast random number generator based on xorshift 2 | /* 3 | This sort of thing is frequently needed during benchmarks, and using math/rand 4 | will make create a single-threaded bottleneck. Using rand.New() is expensive 5 | so always creating multiple sources isn't always an answer either. 6 | */ 7 | package fastrand 8 | 9 | type Rng uint64 10 | 11 | // Next returns the next random number 12 | func (r *Rng) Next() uint64 { 13 | *r ^= *r >> 12 // a 14 | *r ^= *r << 25 // b 15 | *r ^= *r >> 27 // c 16 | return uint64(*r * 2685821657736338717) 17 | } 18 | 19 | // Intn returns a uniform random integer [0,bound) 20 | func (r *Rng) Intn(bound uint64) uint64 { 21 | threshold := -bound % bound 22 | n := r.Next() 23 | for n < threshold { 24 | n = r.Next() 25 | } 26 | return n % bound 27 | } 28 | 29 | func (r *Rng) Int32n(n uint32) uint32 { 30 | 31 | bits := r.Next() 32 | 33 | random32bit := uint32(bits) 34 | bits >>= 32 35 | full := false 36 | 37 | multiresult := uint64(random32bit) * uint64(n) 38 | leftover := uint32(multiresult) 39 | if leftover < n { 40 | threshold := uint32(-n) % n 41 | for leftover < threshold { 42 | random32bit = uint32(bits) 43 | bits >>= 32 44 | if !full { 45 | bits = r.Next() 46 | } 47 | full = !full 48 | multiresult = uint64(random32bit) * uint64(n) 49 | leftover = uint32(multiresult) 50 | } 51 | } 52 | return uint32(multiresult >> 32) // [0, n) 53 | } 54 | -------------------------------------------------------------------------------- /fastrand/fastrand_test.go: -------------------------------------------------------------------------------- 1 | package fastrand 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func BenchmarkIntn(b *testing.B) { 8 | 9 | r := Rng(0xdeadbeef) 10 | 11 | for i := 0; i < b.N; i++ { 12 | r.Intn(1000) 13 | } 14 | } 15 | 16 | func BenchmarkInt32n(b *testing.B) { 17 | r := Rng(0xdeadbeef) 18 | for i := 0; i < b.N; i++ { 19 | r.Int32n(1000) 20 | } 21 | } 22 | 23 | func TestIntn(t *testing.T) { 24 | 25 | var counts [1000]int 26 | 27 | r := Rng(0xdeadbeef) 28 | for i := 0; i < 100e6; i++ { 29 | counts[r.Intn(1000)]++ 30 | } 31 | 32 | t.Logf("counts = %+v\n", counts) 33 | } 34 | 35 | func TestInt32n(t *testing.T) { 36 | 37 | var counts [1000]int 38 | 39 | r := Rng(0xdeadbeef) 40 | for i := 0; i < 100e6; i++ { 41 | counts[r.Int32n(1000)]++ 42 | } 43 | 44 | t.Logf("counts = %+v\n", counts) 45 | } 46 | -------------------------------------------------------------------------------- /fastsearch/fastsearch.go: -------------------------------------------------------------------------------- 1 | package fastsearch 2 | 3 | // https://github.com/python/cpython/blob/master/Objects/stringlib/fastsearch.h 4 | 5 | type bloom uint64 6 | 7 | func (b *bloom) add(ch byte) { 8 | *b |= 1 << (ch & 63) 9 | } 10 | 11 | func (b bloom) get(ch byte) bool { 12 | return (b & (1 << (ch & 63))) != 0 13 | } 14 | 15 | func fastsearch(s, p []byte) int { 16 | 17 | n := len(s) 18 | m := len(p) 19 | 20 | w := n - m 21 | 22 | if w < 0 { 23 | return -1 24 | } 25 | 26 | var mask bloom 27 | 28 | mlast := m - 1 29 | skip := mlast - 1 30 | 31 | ss := s[m-1:] 32 | pp := p[m-1:] 33 | 34 | /* create compressed boyer-moore delta 1 table */ 35 | 36 | /* process pattern[:-1] */ 37 | for i, v := range p[:mlast] { 38 | mask.add(v) 39 | if v == p[mlast] { 40 | skip = mlast - i - 1 41 | } 42 | } 43 | /* process pattern[-1] outside the loop */ 44 | mask.add(p[mlast]) 45 | 46 | for i := 0; i <= w; i++ { 47 | /* note: using mlast in the skip path slows things down on x86 */ 48 | if ss[i] == pp[0] { 49 | /* candidate match */ 50 | var j int 51 | for j = 0; j < mlast; j++ { 52 | if s[i+j] != p[j] { 53 | break 54 | } 55 | } 56 | if j == mlast { 57 | return i 58 | } 59 | /* miss: check if next character is part of pattern */ 60 | if !mask.get(ss[i+1]) { 61 | i = i + m 62 | } else { 63 | i = i + skip 64 | } 65 | } else { 66 | /* skip: check if next character is part of pattern */ 67 | if i+1 >= len(ss) { 68 | continue 69 | } 70 | if !mask.get(ss[i+1]) { 71 | i = i + m 72 | } 73 | } 74 | } 75 | 76 | return -1 77 | } 78 | -------------------------------------------------------------------------------- /fastsearch/fastsearch_test.go: -------------------------------------------------------------------------------- 1 | package fastsearch 2 | 3 | import "testing" 4 | 5 | func TestSearch(t *testing.T) { 6 | 7 | var tests = []struct { 8 | needle string 9 | haystack string 10 | i int 11 | }{ 12 | {"hello", "hello", 0}, 13 | {"hello", " hello", 1}, 14 | {"hello", " hell", -1}, 15 | {"ell", "hello", 1}, 16 | {"hellp", " hellohellohello", -1}, 17 | } 18 | 19 | for _, tt := range tests { 20 | if i := fastsearch([]byte(tt.haystack), []byte(tt.needle)); i != tt.i { 21 | t.Errorf("fastsearch(%q,%q)=%v, want %v", tt.haystack, tt.needle, i, tt.i) 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /fsmdemo/fishasm/fishasm.s: -------------------------------------------------------------------------------- 1 | #include "textflag.h" 2 | 3 | // func Match(data string) int 4 | TEXT ·Match(SB), NOSPLIT, $0-24 5 | MOVQ data_base+0(FP), DI 6 | MOVQ data_len+8(FP), SI 7 | ADDQ DI, SI 8 | 9 | state_0: 10 | MOVQ $-1, AX 11 | CMPQ DI,SI 12 | JE finish 13 | MOVBLZX (DI), DX 14 | ADDQ $1, DI 15 | 16 | CMPL DX, $0x66 17 | JNE state_0 18 | 19 | state_1: 20 | MOVQ $-1, AX 21 | CMPQ DI,SI 22 | JE finish 23 | MOVBLZX (DI), DX 24 | ADDQ $1, DI 25 | 26 | CMPL DX, $0x66 27 | JE state_1 28 | 29 | CMPL DX, $0x69 30 | JNE state_0 31 | 32 | state_2: 33 | MOVQ $-1, AX 34 | CMPQ DI,SI 35 | JE finish 36 | MOVBLZX (DI), DX 37 | ADDQ $1, DI 38 | 39 | CMPL DX, $0x66 40 | JE state_1 41 | 42 | CMPL DX, $0x69 43 | JE state_2 44 | 45 | CMPL DX, $0x73 46 | JNE state_0 47 | 48 | // state 3 49 | MOVQ $-1, AX 50 | CMPQ DI,SI 51 | JE finish 52 | MOVBLZX (DI), DX 53 | ADDQ $1, DI 54 | 55 | CMPL DX, $0x66 56 | JE state_1 57 | 58 | CMPL DX, $0x68 59 | JNE state_0 60 | 61 | // state 4 62 | MOVQ $0x04, AX 63 | // elided jmp to finish 64 | 65 | finish: 66 | MOVQ AX, ret+16(FP) 67 | RET 68 | -------------------------------------------------------------------------------- /fsmdemo/fishasm/fishasm_stub.go: -------------------------------------------------------------------------------- 1 | package fishasm 2 | 3 | //go:noescape 4 | func Match(data string) int 5 | -------------------------------------------------------------------------------- /fsmdemo/fishfsm/fishfsm.go: -------------------------------------------------------------------------------- 1 | package fishfsm 2 | 3 | func Match(data string) int { 4 | var idx = ^uint(0) 5 | 6 | l0: 7 | if idx++; idx >= uint(len(data)) { 8 | return -1 9 | } 10 | 11 | if data[idx] != 'f' { 12 | goto l0 13 | } 14 | 15 | l1: // e.g. "f" 16 | if idx++; idx >= uint(len(data)) { 17 | return -1 18 | } 19 | 20 | if data[idx] == 'f' { 21 | goto l1 22 | } 23 | 24 | if data[idx] != 'i' { 25 | goto l0 26 | } 27 | 28 | l2: // e.g. "fi" 29 | if idx++; idx >= uint(len(data)) { 30 | return -1 31 | } 32 | 33 | if data[idx] == 'f' { 34 | goto l1 35 | } 36 | 37 | if data[idx] == 'i' { 38 | goto l2 39 | } 40 | 41 | if data[idx] != 's' { 42 | goto l0 43 | } 44 | 45 | if idx++; idx >= uint(len(data)) { 46 | return -1 47 | } 48 | 49 | if data[idx] == 'f' { 50 | goto l1 51 | } 52 | 53 | if data[idx] != 'h' { 54 | goto l0 55 | } 56 | 57 | { 58 | return 4 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /fsmdemo/fishvmc.c: -------------------------------------------------------------------------------- 1 | 2 | int 3 | fishmain(const char *b, const char *e) 4 | { 5 | const char *p; 6 | int c; 7 | 8 | p = b; 9 | 10 | 11 | l0: 12 | if (p == e) return -1; 13 | 14 | c = (unsigned char) *p++; 15 | if (c != 'f') goto l0; 16 | 17 | l1: /* e.g. "f" */ 18 | if (p == e) return -1; 19 | 20 | c = (unsigned char) *p++; 21 | if (c == 'f') goto l1; 22 | if (c != 'i') goto l0; 23 | 24 | l2: /* e.g. "fi" */ 25 | if (p == e) return -1; 26 | 27 | c = (unsigned char) *p++; 28 | if (c == 'f') goto l1; 29 | if (c == 'i') goto l2; 30 | if (c != 's') goto l0; 31 | if (p == e) return -1; 32 | 33 | c = (unsigned char) *p++; 34 | if (c == 'f') goto l1; 35 | if (c != 'h') goto l0; 36 | return 0x1; /* "fi+sh" */ 37 | } 38 | 39 | -------------------------------------------------------------------------------- /fsmdemo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //go:generate sh -c "re -r pcre -k str -e fish -pl go 'fi+sh' >fishfsm/fishfsm.go" 4 | //go:generate sh -c "re -r pcre -k str -e fish -pl amd64_go 'fi+sh' >fishasm/fishasm.s" 5 | //go:generate sh -c "re -r pcre -k pair -e fish -pl vmc 'fi+sh' >fishvmc.c" 6 | 7 | import ( 8 | "fmt" 9 | "unsafe" 10 | 11 | "github.com/dgryski/trifles/fsmdemo/fishasm" 12 | "github.com/dgryski/trifles/fsmdemo/fishfsm" 13 | ) 14 | 15 | /* 16 | 17 | #include 18 | 19 | int fishmain(const char *b, const char *e); 20 | 21 | */ 22 | import "C" 23 | 24 | func main() { 25 | goFishing("fsm", fishfsm.Match) 26 | goFishing("asm", fishasm.Match) 27 | goFishing("C", vmcMatch) 28 | } 29 | 30 | func goFishing(what string, match func(s string) int) { 31 | 32 | lake := []string{"fsh", "catfishes", "dogo", "fiiish"} 33 | 34 | for _, caught := range lake { 35 | if match(caught) != -1 { 36 | fmt.Println("caught fish with", what, ":", caught) 37 | } 38 | } 39 | } 40 | 41 | func vmcMatch(s string) int { 42 | cstr := C.CString(s) 43 | C.free(unsafe.Pointer(cstr)) 44 | return int(C.fishmain(cstr, (*C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(cstr))+uintptr(len(s)))))) 45 | } 46 | -------------------------------------------------------------------------------- /fsmdemo/sotest/fsmso/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/dgryski/trifles/fsmdemo/fishasm" 4 | 5 | import "C" 6 | 7 | //export fishasmMatch 8 | func fishasmMatch(base *C.char, len C.int) C.int { 9 | return C.int(fishasm.Match(C.GoStringN(base, len))) 10 | } 11 | 12 | func main() {} 13 | -------------------------------------------------------------------------------- /fsmdemo/sotest/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | go build -buildmode=c-shared -o fsmso.so github.com/dgryski/trifles/fsmdemo/sotest/fsmso 4 | make runner 5 | ./runner 6 | -------------------------------------------------------------------------------- /fsmdemo/sotest/runner.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | int 6 | main(int argc, char **argv) 7 | { 8 | void *handle; 9 | int (*match)(char *s, int len); 10 | char *error; 11 | 12 | handle = dlopen("fsmso.so", RTLD_LAZY); 13 | if (!handle) { 14 | fprintf(stderr, "%s\n", dlerror()); 15 | exit(EXIT_FAILURE); 16 | } 17 | 18 | dlerror(); /* Clear any existing error */ 19 | 20 | *(void **) (&match) = dlsym(handle, "fishasmMatch"); 21 | 22 | if ((error = dlerror()) != NULL) { 23 | fprintf(stderr, "%s\n", error); 24 | exit(EXIT_FAILURE); 25 | } 26 | 27 | printf("%d\n", match("catfishies", 10)); 28 | printf("%d\n", match("dogo", 4)); 29 | dlclose(handle); 30 | exit(EXIT_SUCCESS); 31 | } 32 | -------------------------------------------------------------------------------- /fuzzprot/fuzzprot.go: -------------------------------------------------------------------------------- 1 | package fuzzprot 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | ) 7 | 8 | type user struct { 9 | Type string 10 | Name string 11 | Age int 12 | } 13 | 14 | func Unpack(data []byte) ([]user, error) { 15 | 16 | count := data[0] 17 | users := make([]user, count) 18 | 19 | data = data[1:] 20 | 21 | var uidx int 22 | for len(data) > 0 { 23 | switch data[0] { 24 | case 0: 25 | uidx++ 26 | data = data[1:] 27 | case 1: 28 | users[uidx].Type, data = grabString(data[1:]) 29 | case 2: 30 | users[uidx].Name, data = grabString(data[1:]) 31 | case 3: 32 | var err error 33 | data = data[1:] 34 | users[uidx].Age, err = strconv.Atoi(string(data[:2])) 35 | if err != nil { 36 | return nil, err 37 | } 38 | data = data[2:] 39 | default: 40 | return nil, errors.New("unknown field type") 41 | } 42 | } 43 | 44 | return users, nil 45 | } 46 | 47 | func grabString(data []byte) (string, []byte) { 48 | l, data := data[0], data[1:] 49 | return string(data[:l]), data[l:] 50 | } 51 | -------------------------------------------------------------------------------- /fuzzprot/fuzzprot_test.go: -------------------------------------------------------------------------------- 1 | package fuzzprot 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | // TestUnpack tests the unpacking routine 9 | func TestUnpack(t *testing.T) { 10 | var tests = []struct { 11 | input []byte 12 | users []user 13 | }{ 14 | {[]byte("\x01\x0346\x01\x03ADM\x02\x04Bill\x00"), []user{{Age: 46, Type: "ADM", Name: "Bill"}}}, 15 | } 16 | 17 | for i, tt := range tests { 18 | if u, _ := Unpack(tt.input); !reflect.DeepEqual(tt.users, u) { 19 | t.Errorf("%d: Unpack(...)=%v want %v\n", i, u, tt.users) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /gcwatch/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "runtime" 11 | "time" 12 | ) 13 | 14 | func getMemStats(url string) (*runtime.MemStats, error) { 15 | 16 | r, err := http.Get(url) 17 | if err != nil { 18 | return nil, err 19 | } 20 | defer r.Body.Close() 21 | 22 | if r.StatusCode >= 400 { 23 | return nil, errors.New(r.Status) 24 | } 25 | 26 | var expvars struct { 27 | Memstats runtime.MemStats `json:"memstats"` 28 | } 29 | 30 | json.NewDecoder(r.Body).Decode(&expvars) 31 | 32 | return &expvars.Memstats, nil 33 | 34 | } 35 | 36 | func main() { 37 | 38 | delay := flag.Duration("d", 10*time.Second, "time between fetches") 39 | units := flag.Duration("u", time.Millisecond, "gc time units") 40 | 41 | flag.Parse() 42 | 43 | host := "http://localhost:8080/debug/vars" 44 | 45 | if flag.NArg() >= 1 { 46 | host = flag.Arg(0) 47 | } 48 | 49 | log.Printf("fetching %s every %s", host, *delay) 50 | 51 | m, err := getMemStats(host) 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | ngcs := m.NumGC 57 | 58 | for range time.Tick(*delay) { 59 | m, err := getMemStats(host) 60 | if err != nil { 61 | log.Println(err) 62 | continue 63 | } 64 | 65 | var times []uint64 66 | for i := ngcs + 1; i <= m.NumGC; i++ { 67 | times = append(times, m.PauseNs[(i+255)%256]/uint64(*units)) 68 | } 69 | 70 | fmt.Println(time.Now().Format(time.Stamp), ": ", m.NumGC-ngcs, times) 71 | ngcs = m.NumGC 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /gddo/README: -------------------------------------------------------------------------------- 1 | gddo: search Go packages from the command line 2 | 3 | This programs merges results from go-search.org and godoc.org 4 | 5 | To install: go get github.com/dgryski/trifles/gddo 6 | 7 | Usage: gddo search-term 8 | 9 | Example: 10 | 11 | gddo minhash 12 | github.com/dgryski/go-minhash Package minhash implements the bottom-k sketch for streaming set similarity. 13 | github.com/y-matsuwitter/minhash minhash b-bit minhash implementation for golang Requirements golang Installation Usage Author Yuki Matsumoto (@y_matsuwitter) 14 | github.com/AlexeyKuklin/bbitminhash bbitminhash b-bit minhash implementation for golang See: Ping Li, Christian Konig. 15 | github.com/cevian/disttopk Count-Min Sketch, an approximate counting data structure for summarizing data streams 16 | -------------------------------------------------------------------------------- /gddo/gddo.go: -------------------------------------------------------------------------------- 1 | // Package main is a command-line client search for godoc.org 2 | package main 3 | 4 | import ( 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "sort" 13 | "strings" 14 | "text/tabwriter" 15 | ) 16 | 17 | type Hit struct { 18 | Rank int 19 | Path string 20 | Synopsis string 21 | } 22 | 23 | type ResultSet []Hit 24 | 25 | func (r ResultSet) Len() int { return len(r) } 26 | func (r ResultSet) Swap(i, j int) { r[i], r[j] = r[j], r[i] } 27 | func (r ResultSet) Less(i, j int) bool { return r[i].Rank < r[j].Rank } 28 | 29 | func get(url string, jsr interface{}) { 30 | r, err := http.Get(url) 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | defer r.Body.Close() 35 | 36 | json.NewDecoder(r.Body).Decode(&jsr) 37 | } 38 | 39 | type GddoResults struct { 40 | Results []struct { 41 | Path string 42 | Synopsis string 43 | } 44 | } 45 | 46 | func getGddo(q string) map[string]Hit { 47 | u := fmt.Sprintf("http://api.godoc.org/search?q=%s", url.QueryEscape(q)) 48 | var results GddoResults 49 | get(u, &results) 50 | 51 | hits := make(map[string]Hit) 52 | for i, pkg := range results.Results { 53 | // log.Println("gddo", i, pkg.Path) 54 | hits[pkg.Path] = Hit{i, pkg.Path, pkg.Synopsis} 55 | } 56 | return hits 57 | } 58 | 59 | type GcseResults struct { 60 | Hits []struct { 61 | Package string 62 | Synopsis string 63 | } 64 | } 65 | 66 | func getGcse(q string) map[string]Hit { 67 | u := fmt.Sprintf("http://go-search.org/api?action=search&q=%s", url.QueryEscape(q)) 68 | var results GcseResults 69 | get(u, &results) 70 | 71 | hits := make(map[string]Hit) 72 | for i, pkg := range results.Hits { 73 | // log.Println("gcse", i, pkg.Package) 74 | hits[pkg.Package] = Hit{i, pkg.Package, pkg.Synopsis} 75 | } 76 | return hits 77 | } 78 | 79 | type WalkResults struct { 80 | Packages []struct { 81 | ImportPath string `json:"import_path"` 82 | Synopsis string `json:"synopsis"` 83 | } `json:"packages"` 84 | } 85 | 86 | func getWalk(q string) map[string]Hit { 87 | u := fmt.Sprintf("https://gowalker.org/api/v1/search?key=%s", url.QueryEscape(q)) 88 | var results WalkResults 89 | get(u, &results) 90 | 91 | hits := make(map[string]Hit) 92 | for i, pkg := range results.Packages { 93 | // log.Println("walk", i, pkg.Package) 94 | hits[pkg.ImportPath] = Hit{i, pkg.ImportPath, pkg.Synopsis} 95 | } 96 | return hits 97 | } 98 | 99 | type GithubResults struct { 100 | Repositories []struct { 101 | HtmlUrl string `json:"html_url"` 102 | Description string `json:"description"` 103 | } `json:"items"` 104 | } 105 | 106 | func getGithub(q string) map[string]Hit { 107 | u := fmt.Sprintf("https://api.github.com/search/repositories?q=%s+language:go", url.QueryEscape(q)) 108 | var results GithubResults 109 | get(u, &results) 110 | 111 | hits := make(map[string]Hit) 112 | for i, pkg := range results.Repositories { 113 | path := strings.TrimPrefix(pkg.HtmlUrl, "https://") 114 | hits[path] = Hit{i, path, pkg.Description} 115 | } 116 | return hits 117 | } 118 | 119 | func main() { 120 | 121 | flag.Parse() 122 | 123 | ch := make(chan map[string]Hit) 124 | 125 | query := flag.Arg(0) 126 | 127 | searchers := []func(string) map[string]Hit{ 128 | getGddo, 129 | getGcse, 130 | getWalk, 131 | getGithub, 132 | } 133 | 134 | for _, s := range searchers { 135 | go func(s func(string) map[string]Hit) { ch <- s(query) }(s) 136 | } 137 | 138 | var rmaps []map[string]Hit 139 | 140 | for i := 0; i < len(searchers); i++ { 141 | rmaps = append(rmaps, <-ch) 142 | } 143 | 144 | r := make(map[string]Hit) 145 | 146 | for _, m := range rmaps { 147 | for p, h := range m { 148 | if rh, ok := r[p]; ok { 149 | if rh.Synopsis == "" { 150 | rh.Synopsis = h.Synopsis 151 | r[p] = rh 152 | } 153 | } else { 154 | r[p] = h 155 | } 156 | } 157 | } 158 | 159 | // wipe the votes 160 | for p, h := range r { 161 | h.Rank = 0 162 | r[p] = h 163 | } 164 | 165 | for _, m := range rmaps { 166 | merge(r, m) 167 | } 168 | 169 | var rs ResultSet 170 | for _, v := range r { 171 | rs = append(rs, v) 172 | } 173 | 174 | sort.Sort(rs) 175 | 176 | w := new(tabwriter.Writer) 177 | w.Init(os.Stdout, 0, 8, 0, '\t', 0) 178 | 179 | end := 20 180 | if len(rs) < end { 181 | end = len(rs) 182 | } 183 | 184 | for _, pkg := range rs[:end] { 185 | fmt.Fprintf(w, "%s \t%s\n", pkg.Path, pkg.Synopsis) 186 | } 187 | w.Flush() 188 | } 189 | 190 | func merge(r, r1 map[string]Hit) { 191 | 192 | // Add scores for all candidates in r1 193 | for p, h := range r { 194 | if r, ok := r1[h.Path]; ok { 195 | h.Rank += r.Rank 196 | } else { 197 | h.Rank += len(r1) 198 | } 199 | r[p] = h 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /gfmt/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "strings" 9 | 10 | "github.com/dgryski/go-linebreak" 11 | ) 12 | 13 | func main() { 14 | 15 | var wrap func(s string, w, g int) string 16 | 17 | width := flag.Int("w", 75, "line width") 18 | goal := flag.Int("g", 72, "line goal") 19 | fast := flag.Bool("f", false, "fast") 20 | 21 | flag.Parse() 22 | 23 | wrap = linebreak.Wrap 24 | if *fast { 25 | wrap = linebreak.Greedy 26 | } 27 | 28 | scanner := bufio.NewScanner(os.Stdin) 29 | 30 | var line []string 31 | for scanner.Scan() { 32 | t := scanner.Text() 33 | if t == "" { 34 | text := strings.Join(line, " ") 35 | fmt.Println(wrap(text, *goal, *width)) 36 | fmt.Println() 37 | line = line[:0] 38 | } 39 | line = append(line, t) 40 | } 41 | 42 | text := strings.Join(line, " ") 43 | fmt.Println(wrap(text, *goal, *width)) 44 | } 45 | -------------------------------------------------------------------------------- /glj/README: -------------------------------------------------------------------------------- 1 | A quick demo for passing data from Go to lua via msgpack. 2 | 3 | This prevents having to send each individual array value into lua separately, 4 | which is a massive speedup for large arrays. 5 | 6 | Sorry about the hard-coded paths for CGO. 7 | 8 | Run ./setup.sh to grab the lua_cmsgpack.c source code from 9 | github.com/antirez/lua-cmsgpack . I didn't want to commit it into my repo. 10 | 11 | Demo: 12 | 13 | $ ./setup.sh && go build . && ./glj 14 | 21 15 | -------------------------------------------------------------------------------- /glj/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | /* 4 | #cgo pkg-config: luajit 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | static int luaExecute(lua_State *L) { return lua_pcall(L, 0, LUA_MULTRET, 0); } 12 | static void luaNewtable(lua_State *L) { lua_createtable(L, 0, 0); } 13 | static void luaSetglobal(lua_State *L,char *s) { lua_setfield(L, LUA_GLOBALSINDEX, (s)); } 14 | 15 | int luaopen_cmsgpack (lua_State *L); 16 | 17 | */ 18 | import "C" 19 | 20 | import ( 21 | "fmt" 22 | "unsafe" 23 | 24 | "gopkg.in/vmihailenco/msgpack.v2" 25 | ) 26 | 27 | var sum = ` 28 | sum = function(t) 29 | total = 0 30 | for i, v in ipairs(t) do total = total + v end 31 | return total 32 | end 33 | 34 | 35 | data = cmsgpack.unpack(mpdata) 36 | return sum(data) 37 | ` 38 | 39 | func cptr(a []byte) *C.char { return (*C.char)(unsafe.Pointer(&a[0])) } 40 | 41 | func main() { 42 | 43 | L := C.luaL_newstate() 44 | defer C.lua_close(L) 45 | 46 | C.luaL_openlibs(L) /* Load Lua libraries */ 47 | C.luaopen_cmsgpack(L) 48 | 49 | sumS := C.CString(sum) 50 | defer C.free(unsafe.Pointer(sumS)) 51 | 52 | C.luaL_loadstring(L, sumS) 53 | 54 | msg, _ := msgpack.Marshal([]int{1, 2, 3, 4, 5, 6}) 55 | C.lua_pushstring(L, cptr(msg)) 56 | 57 | dstr := C.CString("mpdata") 58 | defer C.free(unsafe.Pointer(dstr)) 59 | C.luaSetglobal(L, dstr) 60 | 61 | C.luaExecute(L) 62 | 63 | sum := C.lua_tonumber(L, -1) 64 | 65 | fmt.Println(sum) // Output: 21 66 | } 67 | -------------------------------------------------------------------------------- /glj/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | curl -s -O https://raw.githubusercontent.com/antirez/lua-cmsgpack/master/lua_cmsgpack.c 3 | -------------------------------------------------------------------------------- /go-wtflog/wtflog.go: -------------------------------------------------------------------------------- 1 | // Package wtflog provides more intuitive logging levels 2 | /* 3 | Original proposal https://twitter.com/jbarnette/status/144791640589078530 4 | */ 5 | package wtflog 6 | 7 | import ( 8 | "log" 9 | "os" 10 | ) 11 | 12 | var ( 13 | Lwtf = log.New(os.Stderr, "WTF ", log.LstdFlags) 14 | Lomg = log.New(os.Stderr, "OMG ", log.LstdFlags) 15 | Lfyi = log.New(os.Stderr, "FYI ", log.LstdFlags) 16 | Lbtw = log.New(os.Stderr, "BTW ", log.LstdFlags) 17 | ) 18 | 19 | func WTF(args ...interface{}) { Lwtf.Println(args...) } 20 | func OMG(args ...interface{}) { Lomg.Println(args...) } 21 | func FYI(args ...interface{}) { Lfyi.Println(args...) } 22 | func BTW(args ...interface{}) { Lbtw.Println(args...) } 23 | -------------------------------------------------------------------------------- /grinderplot/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "flag" 7 | htemplate "html/template" 8 | "os" 9 | "sort" 10 | "strconv" 11 | ttemplate "text/template" 12 | ) 13 | 14 | func main() { 15 | 16 | outputCSV := flag.Bool("csv", false, "output CSV instead of HTML") 17 | 18 | flag.Parse() 19 | 20 | var data = make(map[int][]int) 21 | for _, fname := range flag.Args() { 22 | f, _ := os.Open(fname) 23 | b := bufio.NewReader(f) 24 | 25 | scanner := bufio.NewScanner(b) 26 | 27 | // skip first line with the header 28 | scanner.Scan() 29 | 30 | for scanner.Scan() { 31 | line := scanner.Bytes() 32 | parts := bytes.Split(line, []byte{','}) 33 | epochms, reqms := parts[3], parts[4] 34 | epoch, _ := strconv.Atoi(string(bytes.TrimSpace(epochms))) 35 | epoch /= 1000 * 60 36 | latency, _ := strconv.Atoi(string(bytes.TrimSpace(reqms))) 37 | data[epoch] = append(data[epoch], latency) 38 | } 39 | } 40 | 41 | var epochs []int 42 | for k, v := range data { 43 | epochs = append(epochs, k) 44 | sort.Ints(v) 45 | } 46 | 47 | sort.Ints(epochs) 48 | 49 | var graphdata [][]int 50 | for _, epoch := range epochs { 51 | l := len(data[epoch]) 52 | graphdata = append(graphdata, []int{epoch * 1000 * 60, data[epoch][(50*l)/100], data[epoch][(75*l)/100], data[epoch][(95*l)/100], data[epoch][(99*l)/100], l}) 53 | } 54 | 55 | if *outputCSV { 56 | csvTmpl.Execute(os.Stdout, graphdata) 57 | return 58 | } 59 | 60 | reportTmpl.Execute(os.Stdout, graphdata) 61 | } 62 | 63 | func joinInts(ints []int) string { 64 | 65 | b := make([]byte, 0, len(ints)*21) 66 | 67 | needComma := false 68 | for _, v := range ints { 69 | if needComma { 70 | b = append(b, ',') 71 | } 72 | b = strconv.AppendInt(b, int64(v), 10) 73 | needComma = true 74 | } 75 | 76 | return string(b) 77 | } 78 | 79 | var csvTmpl = ttemplate.Must(ttemplate.New("report").Funcs(ttemplate.FuncMap{"joinInts": joinInts}).Parse("" + 80 | `epoch,p50,p75,p95,p99,requests 81 | {{ range . }}{{ joinInts . }} 82 | {{ end }}`)) 83 | 84 | var reportTmpl = htemplate.Must(htemplate.New("report").Parse(` 85 | 86 | 87 | 88 | 89 | 90 | 107 | 108 | 109 | 110 |
111 | 112 | 113 | 114 | `)) 115 | -------------------------------------------------------------------------------- /hashbench/README: -------------------------------------------------------------------------------- 1 | See the benchstat output for current numbers. 2 | -------------------------------------------------------------------------------- /hist/hist.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # assume 80 col screen 4 | $keycols = 8; 5 | $starcols = 50; 6 | $cntcols = 8; 7 | 8 | # keep track of the values 9 | while (<>) { chomp; push @values, $_; ++$hist{$_}; $n++} 10 | 11 | # the most common value 12 | $maxcnt = 0; $mode = 0; 13 | foreach (keys %hist) { 14 | if ($maxcnt < $hist{$_}) { 15 | $mode = $_; 16 | $maxcnt = $hist{$_}; 17 | } 18 | } 19 | 20 | # mean 21 | $sum = 0; 22 | for (@values) { $sum += $_; } 23 | $avg = $sum / $n; 24 | 25 | # standard deviation 26 | $sdev = 0; 27 | for (@values) { $sdev += ($_-$avg)**2 / $n; } 28 | $sdev = sqrt($sdev); 29 | 30 | # print historgram 31 | 32 | # header 33 | $fmt = "%${keycols}s %-${starcols}s %${cntcols}s\n"; 34 | printf ($fmt, "Key", "", "Count"); 35 | 36 | # separator 37 | print "-" x ($keycols + $starcols + $cntcols + 4), "\n"; 38 | 39 | # heuristic -- sort numerically or not? 40 | if (grep (!/\d/, keys %hist)) { 41 | # yup, some text strings 42 | @hkeys = sort { $a cmp $b } keys %hist; 43 | } else { 44 | # nope, digits all the way 45 | @hkeys = sort { $a <=> $b } keys %hist; 46 | } 47 | 48 | # produce the graph 49 | # and find the mode 50 | foreach (@hkeys) { 51 | $ast = '*' x ($starcols * ($hist{$_} / $maxcnt)); 52 | # $ast = '*' x ($starcols * $hist{$_} / $n); 53 | printf $fmt, $_, $ast, $hist{$_}; 54 | } 55 | 56 | # find the median, which we can do now 'cause the keys are sorted 57 | $median = $n/2; 58 | foreach (@hkeys) { 59 | $median -= $hist{$_}; 60 | if ($median <= 0) { 61 | $median = $_; 62 | last ; 63 | } 64 | } 65 | 66 | # and summarize 67 | print "\n"; 68 | printf "items: %i\n", $n; 69 | printf "mean: %g\n", $avg; 70 | printf "mode: %s\n", $mode; 71 | printf "median: %g\n", $median; 72 | printf "stddev: %g\n", $sdev; 73 | -------------------------------------------------------------------------------- /hist/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "math" 9 | "os" 10 | "sort" 11 | "strconv" 12 | "strings" 13 | 14 | "github.com/dgryski/go-linlog" 15 | "github.com/dgryski/go-onlinestats" 16 | ) 17 | 18 | func main() { 19 | 20 | width := flag.Int("w", 100, "display width") 21 | llog := flag.Bool("l", false, "linear-log bucketing") 22 | percentiles := flag.String("p", "0.5,0.75,0.95,0.99,0.999", "percentile values to print") 23 | flag.Parse() 24 | 25 | hist := make(map[int]int) 26 | 27 | stats := onlinestats.NewRunning() 28 | 29 | min := int(math.MaxInt32) 30 | max := 0 31 | 32 | maxcount := 0 33 | 34 | sc := bufio.NewScanner(os.Stdin) 35 | for sc.Scan() { 36 | v, err := strconv.Atoi(sc.Text()) 37 | if err != nil { 38 | log.Fatal(err) 39 | } 40 | 41 | if *llog { 42 | r, _ := linlog.BinOf(uint64(v), 4, 2) 43 | v = int(r) 44 | } 45 | 46 | if v < min { 47 | min = v 48 | } 49 | 50 | if v > max { 51 | max = v 52 | } 53 | c := hist[v] + 1 54 | if c > maxcount { 55 | maxcount = c 56 | } 57 | 58 | hist[v] = c 59 | 60 | stats.Push(float64(v)) 61 | 62 | } 63 | if sc.Err() != nil { 64 | log.Fatal(sc.Err()) 65 | } 66 | 67 | printHistogram(hist, *width, maxcount) 68 | 69 | fmt.Println("items: ", stats.Len()) 70 | fmt.Println("min: ", min) 71 | 72 | keys := make([]int, 0, len(hist)) 73 | 74 | for k := range hist { 75 | keys = append(keys, k) 76 | } 77 | 78 | sort.Ints(keys) 79 | 80 | var quantiles []float64 81 | 82 | if *percentiles == "all" { 83 | for i := 0; i < 100; i++ { 84 | quantiles = append(quantiles, float64(i)/100.0) 85 | } 86 | } else { 87 | pfloats := strings.Split(*percentiles, ",") 88 | for _, p := range pfloats { 89 | f, err := strconv.ParseFloat(p, 64) 90 | if err != nil { 91 | log.Printf("unable to parse float %s: %s\n", p, err) 92 | continue 93 | } 94 | quantiles = append(quantiles, f) 95 | } 96 | } 97 | 98 | pidx := 0 99 | total := 0 100 | KEYS: 101 | for _, k := range keys { 102 | total += hist[k] 103 | p := float64(total) / float64(stats.Len()) 104 | for quantiles[pidx] <= p { 105 | fmt.Printf("% 4.1f: %d\n", quantiles[pidx]*100, k) 106 | pidx++ 107 | if pidx == len(quantiles) { 108 | break KEYS 109 | } 110 | } 111 | } 112 | 113 | fmt.Println("max: ", max) 114 | fmt.Println("stdev: ", stats.Stddev()) 115 | } 116 | 117 | func dupRune(r rune, n int) string { 118 | s := make([]rune, n) 119 | for i := range s { 120 | s[i] = r 121 | } 122 | return string(s) 123 | } 124 | 125 | func printHistogram(hist map[int]int, width, maxcount int) { 126 | 127 | const keycols = 8 128 | var starcols = width 129 | const countcols = 8 130 | 131 | var headerFmt = "%8s %-" + strconv.Itoa(width) + "s %8s\n" 132 | var rowFmt = "%8d %-" + strconv.Itoa(width) + "s %8d\n" 133 | 134 | fmt.Printf(headerFmt, "Key", "", "Count") 135 | 136 | fmt.Println(dupRune('-', keycols+starcols+countcols+4)) 137 | 138 | keys := make([]int, 0, len(hist)) 139 | 140 | for k := range hist { 141 | keys = append(keys, k) 142 | } 143 | 144 | sort.Ints(keys) 145 | 146 | for _, k := range keys { 147 | stars := dupRune('*', int(float64(float64(starcols)*(float64(hist[k])/float64(maxcount))))) 148 | fmt.Printf(rowFmt, k, stars, hist[k]) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /hllbench/bench_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | eclesh "github.com/eclesh/hyperloglog" 7 | lytics "github.com/lytics/hll" 8 | ) 9 | 10 | func BenchmarkLyticsPP(b *testing.B) { 11 | 12 | h := lytics.NewHll(14, 20) 13 | 14 | for i := 0; i < b.N; i++ { 15 | h.Add(uint64(i)) 16 | } 17 | } 18 | 19 | func BenchmarkEclsh(b *testing.B) { 20 | 21 | h, _ := eclesh.New(1 << 14) 22 | 23 | for i := 0; i < b.N; i++ { 24 | h.Add(uint32(i)) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /interp/interp.go: -------------------------------------------------------------------------------- 1 | // Package interp is an interpolation search 2 | // Moved to https://github.com/dgryski/go-interp 3 | package interp 4 | 5 | // modified from http://data.linkedin.com/blog/2010/06/beating-binary-search 6 | func Search(array []int, key int) int { 7 | 8 | min, max := array[0], array[len(array)-1] 9 | 10 | low, high := 0, len(array)-1 11 | 12 | for { 13 | if key < min { 14 | return low 15 | } 16 | 17 | if key > max { 18 | return high + 1 19 | } 20 | 21 | // make a guess of the location 22 | var guess int 23 | if high == low { 24 | guess = high 25 | } else { 26 | size := high - low 27 | offset := int(float64(size-1) * (float64(key-min) / float64(max-min))) 28 | guess = low + offset 29 | } 30 | 31 | // maybe we found it? 32 | if array[guess] == key { 33 | // scan backwards for start of value range 34 | for guess > 0 && array[guess-1] == key { 35 | guess-- 36 | } 37 | return guess 38 | } 39 | 40 | // if we guessed to high, guess lower or vice versa 41 | if array[guess] > key { 42 | high = guess - 1 43 | max = array[high] 44 | } else { 45 | low = guess + 1 46 | min = array[low] 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /interp/interp_test.go: -------------------------------------------------------------------------------- 1 | package interp 2 | 3 | import ( 4 | "math/rand" 5 | "sort" 6 | "testing" 7 | ) 8 | 9 | const maxLimit = 4e7 10 | 11 | var Ints []int 12 | 13 | func fillInts() { 14 | rand.Seed(0) 15 | 16 | Ints = make([]int, maxLimit) 17 | 18 | for i := 0; i < maxLimit; i++ { 19 | Ints[i] = rand.Int() 20 | } 21 | 22 | sort.Ints(Ints) 23 | } 24 | 25 | func benchmarkSearch(b *testing.B, limit int, search func([]int, int) int) { 26 | 27 | if Ints == nil { 28 | fillInts() 29 | } 30 | 31 | ints := Ints[:limit] 32 | 33 | b.ResetTimer() 34 | 35 | for i := 0; i < b.N; i++ { 36 | elt := ints[rand.Intn(limit)] 37 | search(ints, elt) 38 | } 39 | } 40 | 41 | func BenchmarkInterp100(b *testing.B) { benchmarkSearch(b, 100, Search) } 42 | func BenchmarkInterp1000(b *testing.B) { benchmarkSearch(b, 1000, Search) } 43 | func BenchmarkInterp10000(b *testing.B) { benchmarkSearch(b, 10000, Search) } 44 | func BenchmarkInterp1e5(b *testing.B) { benchmarkSearch(b, 1e5, Search) } 45 | func BenchmarkInterp1e6(b *testing.B) { benchmarkSearch(b, 1e6, Search) } 46 | func BenchmarkInterp1e7(b *testing.B) { benchmarkSearch(b, 1e7, Search) } 47 | func BenchmarkInterp4e7(b *testing.B) { benchmarkSearch(b, 4e7, Search) } 48 | 49 | func BenchmarkBin100(b *testing.B) { benchmarkSearch(b, 100, binsearch) } 50 | func BenchmarkBin1000(b *testing.B) { benchmarkSearch(b, 1000, binsearch) } 51 | func BenchmarkBin10000(b *testing.B) { benchmarkSearch(b, 10000, binsearch) } 52 | func BenchmarkBin1e5(b *testing.B) { benchmarkSearch(b, 1e5, binsearch) } 53 | func BenchmarkBin1e6(b *testing.B) { benchmarkSearch(b, 1e6, binsearch) } 54 | func BenchmarkBin1e7(b *testing.B) { benchmarkSearch(b, 1e7, binsearch) } 55 | func BenchmarkBin4e7(b *testing.B) { benchmarkSearch(b, 4e7, binsearch) } 56 | 57 | func BenchmarkStd100(b *testing.B) { benchmarkSearch(b, 100, sort.SearchInts) } 58 | func BenchmarkStd1000(b *testing.B) { benchmarkSearch(b, 1000, sort.SearchInts) } 59 | func BenchmarkStd10000(b *testing.B) { benchmarkSearch(b, 10000, sort.SearchInts) } 60 | func BenchmarkStd1e5(b *testing.B) { benchmarkSearch(b, 1e5, sort.SearchInts) } 61 | func BenchmarkStd1e6(b *testing.B) { benchmarkSearch(b, 1e6, sort.SearchInts) } 62 | func BenchmarkStd1e7(b *testing.B) { benchmarkSearch(b, 1e7, sort.SearchInts) } 63 | func BenchmarkStd4e7(b *testing.B) { benchmarkSearch(b, 4e7, sort.SearchInts) } 64 | 65 | func TestSearchSmall(t *testing.T) { 66 | 67 | rand.Seed(0) 68 | 69 | const limit = 10000 70 | 71 | var ints []int 72 | 73 | for i := 0; i < limit; i++ { 74 | ints = append(ints, rand.Int()) 75 | } 76 | 77 | sort.Ints(ints) 78 | 79 | for want, q := range ints { 80 | if idx := Search(ints, q); idx != want { 81 | t.Errorf("Search(ints, %v)=%v, want %v", q, idx, want) 82 | } 83 | } 84 | 85 | for i := 0; i < 10000; i++ { 86 | 87 | q := rand.Int() 88 | 89 | want := sort.SearchInts(ints, q) 90 | 91 | if idx := Search(ints, q); idx != want { 92 | t.Errorf("Search(ints, %v)=%v, want %v", q, idx, want) 93 | } 94 | } 95 | } 96 | 97 | func TestSearchBig(t *testing.T) { 98 | 99 | if testing.Short() { 100 | t.Skip("Skipping big test search") 101 | } 102 | 103 | rand.Seed(0) 104 | 105 | const maxLimit = 100e6 106 | 107 | ints := make([]int, maxLimit) 108 | 109 | for i := 0; i < maxLimit; i++ { 110 | ints[i] = rand.Int() 111 | } 112 | 113 | sort.Ints(ints) 114 | 115 | for i := 0; i < 100000000; i++ { 116 | elt := rand.Int() 117 | vidx := Search(ints, elt) 118 | idx := sort.SearchInts(ints, elt) 119 | if vidx != idx { 120 | t.Fatalf("Search failed for elt=%d: got %d want %d\n", elt, vidx, idx) 121 | } 122 | } 123 | } 124 | 125 | func binsearch(sortedArray []int, toFind int) int { 126 | 127 | low, high := 0, len(sortedArray) 128 | 129 | for low < high { 130 | mid := low + (high-low)/2 131 | if sortedArray[mid] > toFind { 132 | high = mid 133 | } else { 134 | low = mid + 1 135 | } 136 | } 137 | 138 | if low >= len(sortedArray) { 139 | low = -1 140 | } 141 | return low 142 | } 143 | -------------------------------------------------------------------------------- /inthash/cycles/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "math" 6 | 7 | "github.com/dgryski/trifles/inthash" 8 | "github.com/willf/bitset" 9 | ) 10 | 11 | func main() { 12 | 13 | b := bitset.New(math.MaxUint32) 14 | 15 | start := uint32(0) 16 | i32 := start 17 | steps := 0 18 | 19 | for { 20 | if b.Test(uint(i32)) && i32 != start { 21 | log.Println("mismatch for start=", start, "at i32=", i32) 22 | break 23 | } 24 | b.Set(uint(i32)) 25 | i32 = inthash.Jenkins32(i32) 26 | 27 | if i32 == start { 28 | log.Println("cycle for", start, "len=", steps) 29 | steps = 0 30 | for start < math.MaxUint32 && b.Test(uint(start)) { 31 | start++ 32 | } 33 | if start == math.MaxUint32 { 34 | log.Println("all done") 35 | break 36 | } 37 | log.Println("start=", start) 38 | i32 = start 39 | } 40 | 41 | steps++ 42 | if steps%(32*1024*1024) == 0 { 43 | log.Println("start=", start, "steps=", steps) 44 | } 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /inthash/inthash_test.go: -------------------------------------------------------------------------------- 1 | package inthash 2 | 3 | import ( 4 | "testing" 5 | "testing/quick" 6 | ) 7 | 8 | func TestHash64Inv(t *testing.T) { 9 | 10 | f := func(x uint64) bool { 11 | y := Hash64(uint64(x)) 12 | return Hash64Inv(y) == x 13 | } 14 | if err := quick.Check(f, nil); err != nil { 15 | t.Error(err) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /intset/nlz.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func nlz(x uint32) uint32 4 | -------------------------------------------------------------------------------- /intset/nlz_amd64.s: -------------------------------------------------------------------------------- 1 | // func nlz(x uint32) uint32 2 | TEXT ·nlz(SB),4,$0-12 3 | MOVL x+0(FP), AX 4 | TESTL AX, AX 5 | JZ zero 6 | BSRL AX, AX 7 | SUBL $31, AX 8 | NEGL AX 9 | MOVL AX, ret+8(FP) 10 | RET 11 | zero: 12 | MOVL $32, ret+8(FP) 13 | RET 14 | -------------------------------------------------------------------------------- /jolly/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "math/rand" 7 | ) 8 | 9 | func main() { 10 | n := flag.Int("n", 1, "games to simulate") 11 | 12 | flag.Parse() 13 | 14 | for i := 0; i < *n; i++ { 15 | t := oneGame() 16 | fmt.Println("turns", t) 17 | } 18 | } 19 | 20 | func oneGame() int { 21 | var turns int 22 | var square = 1 23 | 24 | var missturns int 25 | 26 | for square < 46 { 27 | turns++ 28 | 29 | var turnsmissed bool 30 | if missturns > 0 { 31 | turns += missturns 32 | missturns = 0 33 | turnsmissed = true 34 | } 35 | 36 | switch square { 37 | case 1, 2, 4, 5, 8, 9, 11, 12, 14, 16, 17, 18, 20, 22, 24, 26, 28, 29, 30, 32, 33, 36, 38, 39, 41, 42, 44, 45: 38 | square += roll() 39 | 40 | case 3: 41 | square = 1 42 | 43 | case 6: 44 | square = 18 45 | 46 | case 7, 19, 27, 35, 43: 47 | if turnsmissed { 48 | square += roll() 49 | } else { 50 | missturns = 1 51 | } 52 | 53 | case 10, 23: 54 | if d := roll(); d == 6 { 55 | square += d 56 | } 57 | 58 | case 13: 59 | if turnsmissed { 60 | square += roll() 61 | } else { 62 | missturns = 2 63 | } 64 | 65 | case 15: 66 | square = 18 67 | case 21: 68 | square = 24 69 | case 25: 70 | square += 4 71 | case 31: 72 | square -= 3 73 | case 34: 74 | square = 36 75 | case 37: 76 | square = 41 77 | case 40: 78 | square = 17 79 | } 80 | } 81 | 82 | return turns 83 | } 84 | 85 | func roll() int { 86 | return rand.Intn(6) + 1 87 | } 88 | -------------------------------------------------------------------------------- /jumpreplica/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "hash/fnv" 8 | "log" 9 | "os" 10 | "runtime/pprof" 11 | "time" 12 | 13 | "github.com/dgryski/go-onlinestats" 14 | "github.com/dgryski/go-shardedkv" 15 | "github.com/dgryski/go-shardedkv/choosers/chash" 16 | "github.com/dgryski/go-shardedkv/choosers/jump" 17 | "github.com/dgryski/go-shardedkv/choosers/ketama" 18 | ) 19 | 20 | type replichooser interface { 21 | shardedkv.Chooser 22 | ChooseReplicas(key string, n int) []string 23 | } 24 | 25 | func main() { 26 | 27 | chooser := flag.String("chooser", "jump", "chooser to use") 28 | numBuckets := flag.Int("b", 8, "buckets") 29 | replicas := flag.Int("r", 1, "replicas") 30 | cpuprofile := flag.String("cpuprofile", "", "cpu profile") 31 | 32 | flag.Parse() 33 | 34 | if *cpuprofile != "" { 35 | f, err := os.Create(*cpuprofile) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | pprof.StartCPUProfile(f) 40 | defer pprof.StopCPUProfile() 41 | } 42 | 43 | var buckets []string 44 | for i := 0; i < *numBuckets; i++ { 45 | buckets = append(buckets, fmt.Sprintf("192.168.%d.%d", i, 10+2*i)) 46 | } 47 | results := make(map[string]int) 48 | 49 | scan := bufio.NewScanner(os.Stdin) 50 | 51 | hasher := func(b []byte) uint64 { h := fnv.New64a(); h.Write(b); return h.Sum64() } 52 | 53 | var j replichooser 54 | 55 | switch *chooser { 56 | case "ketama": 57 | j = ketama.New() 58 | case "chash": 59 | j = chash.New() 60 | case "jump": 61 | j = jump.New(hasher) 62 | default: 63 | log.Fatal("unknown chooser", *chooser) 64 | } 65 | 66 | j.SetBuckets(buckets) 67 | 68 | var totalMetrics int 69 | t0 := time.Now() 70 | for scan.Scan() { 71 | key := scan.Text() 72 | totalMetrics++ 73 | 74 | r := j.ChooseReplicas(key, *replicas) 75 | 76 | for _, rr := range r { 77 | results[rr]++ 78 | } 79 | } 80 | t1 := time.Since(t0) 81 | 82 | stats := onlinestats.NewRunning() 83 | 84 | fmt.Printf("total metrics processed: %d, time spent ~%ds (%d/s)\n", totalMetrics, int(t1.Seconds()), int(float64(totalMetrics)/float64(t1.Seconds()))) 85 | fmt.Printf("replication count: %d, server count: %d\n", *replicas, *numBuckets) 86 | fmt.Println("server distribution:") 87 | 88 | min := int(^uint(0) / 2) 89 | max := 0 90 | for _, b := range buckets { 91 | v := results[b] 92 | if v < min { 93 | min = v 94 | } 95 | if v > max { 96 | max = v 97 | } 98 | fmt.Printf("- server %15s: %6d (%.2f%%)\n", b, v, 100*float64(v)/float64(totalMetrics)) 99 | stats.Push(float64(v)) 100 | } 101 | 102 | fmt.Printf("band: %d - %d (diff %d, %.1f%%), mean: %.2f, stddev: %.2f\n", min, max, (max - min), 100*float64(max-min)/float64(totalMetrics), stats.Mean(), stats.Stddev()) 103 | } 104 | -------------------------------------------------------------------------------- /leven/leven.go: -------------------------------------------------------------------------------- 1 | package leven 2 | 3 | // This implementation translated from the optimized C code at 4 | // https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#C 5 | func Levenshtein(s1, s2 []rune) int { 6 | s1len := len(s1) 7 | s2len := len(s2) 8 | column := make([]int, len(s1)+1) 9 | 10 | for y := 1; y <= s1len; y++ { 11 | column[y] = y 12 | } 13 | for x := 1; x <= s2len; x++ { 14 | column[0] = x 15 | lastdiag := x - 1 16 | for y := 1; y <= s1len; y++ { 17 | olddiag := column[y] 18 | var incr int 19 | if s1[y-1] != s2[x-1] { 20 | incr = 1 21 | } 22 | 23 | column[y] = min3(column[y]+1, column[y-1]+1, lastdiag+incr) 24 | lastdiag = olddiag 25 | } 26 | } 27 | return column[s1len] 28 | } 29 | 30 | func min3(a, b, c int) int { 31 | if a < b { 32 | if a < c { 33 | return a 34 | } 35 | } else { 36 | if b < c { 37 | return b 38 | } 39 | } 40 | return c 41 | } 42 | -------------------------------------------------------------------------------- /lzpack/lz4pack.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "flag" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | 11 | lz4 "github.com/bkaradzic/go-lz4" 12 | "github.com/dchest/siphash" 13 | binpack "github.com/dgryski/go-binpack" 14 | ) 15 | 16 | func main() { 17 | 18 | verbose := flag.Bool("v", false, "verbose") 19 | 20 | flag.Parse() 21 | 22 | scanner := bufio.NewScanner(os.Stdin) 23 | 24 | w := bufio.NewWriter(os.Stdout) 25 | defer w.Flush() 26 | 27 | for scanner.Scan() { 28 | 29 | fname := scanner.Text() 30 | 31 | b, err := ioutil.ReadFile(fname) 32 | if err != nil { 33 | log.Println(err) 34 | continue 35 | } 36 | 37 | sip := siphash.Hash(0, 0, b) 38 | 39 | data, err := lz4.Encode(nil, b) 40 | if err != nil { 41 | log.Println("error packing: ", err) 42 | continue 43 | } 44 | 45 | const headerSize = 4 + 8 + 2 + 4 // + len(fname) 46 | 47 | h := struct { 48 | PacketSize uint32 49 | SipHash uint64 50 | Fname []byte `binpack:"lenprefix=uint16"` 51 | UncompressedSize uint32 52 | }{ 53 | PacketSize: uint32(headerSize + len(fname) + len(data)), 54 | SipHash: sip, 55 | Fname: []byte(fname), 56 | UncompressedSize: uint32(len(b)), 57 | } 58 | 59 | if *verbose { 60 | log.Printf("%s: %d -> %d\n", fname, len(b), len(data)) 61 | } 62 | 63 | err = binpack.Write(w, binary.LittleEndian, h) 64 | if err != nil { 65 | log.Printf("error writing header %+v\n", err) 66 | break 67 | } 68 | 69 | _, err = w.Write(data) 70 | if err != nil { 71 | log.Printf("error writing %+v\n", err) 72 | break 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /lzpack/lz4unpack.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "flag" 7 | "io" 8 | "log" 9 | "os" 10 | 11 | lz4 "github.com/bkaradzic/go-lz4" 12 | "github.com/dchest/siphash" 13 | binpack "github.com/dgryski/go-binpack" 14 | ) 15 | 16 | func main() { 17 | 18 | verbose := flag.Bool("v", false, "verbose") 19 | 20 | flag.Parse() 21 | 22 | r := bufio.NewReader(os.Stdin) 23 | 24 | for { 25 | 26 | var header struct { 27 | PacketSize uint32 28 | SipHash uint64 29 | Fname []byte `binpack:"lenprefix=uint16"` 30 | UncompressedSize uint32 31 | } 32 | 33 | err := binpack.Read(r, binary.LittleEndian, &header) 34 | if err == io.EOF { 35 | break 36 | } 37 | 38 | compressedSize := header.PacketSize - 16 - 2 - uint32(len(header.Fname)) 39 | 40 | if *verbose { 41 | log.Printf("%s: %d -> %d\n", string(header.Fname), compressedSize, header.UncompressedSize) 42 | } 43 | 44 | data := make([]byte, compressedSize) 45 | 46 | n, err := io.ReadFull(r, data) 47 | if n != len(data) || err != nil { 48 | log.Fatalf("bad read: n=%d len(data)=%d err=%s\n", n, len(data), err) 49 | } 50 | 51 | data, err = lz4.Decode(nil, data) 52 | 53 | h := siphash.Hash(0, 0, data) 54 | 55 | if err != nil || h != header.SipHash { 56 | log.Println("FAIL: err=", err) 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /lzpack/pack.py: -------------------------------------------------------------------------------- 1 | import lz4 2 | import siphashc 3 | import sys 4 | import struct 5 | 6 | total = 0 7 | compressed = 0 8 | 9 | 10 | def pack(fname): 11 | 12 | global total, compressed 13 | 14 | 15 | f = open(fname) 16 | data = f.read() 17 | f.close() 18 | checksum = siphashc.siphash('\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0', data) 19 | fsize = len(data) 20 | data = lz4.compress(data) 21 | 22 | # size of packet(4), checksum(8), fnamelen(2)+fname, uncompressed size(4), compressed data 23 | l = len(data) 24 | pktlen = 4 + 8 + 2 + len(fname) + 4 + l 25 | 26 | total += fsize 27 | compressed += len(data) 28 | 29 | sys.stderr.write("%s: %d -> %d\n" %(fname, fsize, len(data))) 30 | sys.stdout.write( struct.pack(' %d (%f %%)\n" % (total, compressed, 100-(100.0*+compressed)/total)) 40 | -------------------------------------------------------------------------------- /lzpack/unpack.py: -------------------------------------------------------------------------------- 1 | import lz4 2 | import siphashc 3 | import sys 4 | import struct 5 | 6 | def unpack(stream): 7 | 8 | while True: 9 | data = stream.read(4) 10 | if len(data) != 4: 11 | break 12 | (pktSize,) = struct.unpack(' %d\n' % (fname, compressedSize, fsize)) 37 | else: 38 | sys.stderr.write('%s: checksum fail: got %d, want %d\n' % (fname, got, checksum)) 39 | 40 | unpack(sys.stdin) 41 | -------------------------------------------------------------------------------- /matcher/README: -------------------------------------------------------------------------------- 1 | 2 | After a tweet ( https://twitter.com/veorq/status/637616767058223104 ), I wanted 3 | to see if my suggestion if generating a finite state machine with ragel was 4 | actually faster. 5 | 6 | Turns out it wasn't. 7 | 8 | At least not for only 1000 keys anyway. Ragel segfaulted when trying to create 9 | a machine for 200,000 keys. :( 10 | 11 | I also tested armon/go-radix and cloudflare/ahocorasick. 12 | 13 | My test keys were the top 1000 email domains from the Linkedin leak a while ago. 14 | I've been using it as a reasonable real-world test set every since I watched 15 | John Graham-Cumming's talk https://www.youtube.com/watch?v=_41bkNr7eik 16 | 17 | Anyways, searching 200000 inputs for the thousand keys 10 times; we see the ranking 18 | 19 | (This version built with 1.8beta2) 20 | 21 | for i in radix aho bsearch mafsa mph ragel map; do ./matcher -f ./alldomains.txt -which $i; done 22 | 2016/12/28 11:37:25 using matcher=radix 23 | 2016/12/28 11:37:30 time.Since(t0)=4.863440361s 24 | 2016/12/28 11:37:30 found=15965200 25 | 2016/12/28 11:37:32 using matcher=aho 26 | 2016/12/28 11:37:36 time.Since(t0)=4.603819772s 27 | 2016/12/28 11:37:36 found=16274730 28 | 2016/12/28 11:37:38 using matcher=bsearch 29 | 2016/12/28 11:37:41 time.Since(t0)=3.437610561s 30 | 2016/12/28 11:37:41 found=15965200 31 | 2016/12/28 11:37:42 using matcher=mafsa 32 | 2016/12/28 11:37:46 time.Since(t0)=3.183283031s 33 | 2016/12/28 11:37:46 found=15965200 34 | 2016/12/28 11:37:47 using matcher=mph 35 | 2016/12/28 11:37:48 time.Since(t0)=1.060979543s 36 | 2016/12/28 11:37:48 found=15965200 37 | 2016/12/28 11:37:50 using matcher=ragel 38 | 2016/12/28 11:37:51 time.Since(t0)=959.092265ms 39 | 2016/12/28 11:37:51 found=15965200 40 | 2016/12/28 11:37:52 using matcher=map 41 | 2016/12/28 11:37:53 time.Since(t0)=698.283411ms 42 | 2016/12/28 11:37:53 found=15965200 43 | 44 | -------------------------------------------------------------------------------- /matcher/aho.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "log" 6 | "os" 7 | 8 | "github.com/cloudflare/ahocorasick" 9 | ) 10 | 11 | var lookupAho *ahocorasick.Matcher 12 | 13 | func initAho() { 14 | 15 | f, err := os.Open("domains.txt") 16 | if err != nil { 17 | log.Fatalf("err=%v", err) 18 | } 19 | 20 | scanner := bufio.NewScanner(f) 21 | var arr [][]byte 22 | for scanner.Scan() { 23 | b := append([]byte(nil), scanner.Bytes()...) 24 | arr = append(arr, b) 25 | } 26 | 27 | lookupAho = ahocorasick.NewMatcher(arr) 28 | 29 | } 30 | 31 | func MatchAho(b []byte) bool { 32 | return lookupAho.Match(b) != nil 33 | } 34 | -------------------------------------------------------------------------------- /matcher/bloom.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "log" 7 | "os" 8 | ) 9 | 10 | var lookupBloom bitvector 11 | 12 | const bloomK = 2 13 | const bloomM = 1 << 14 14 | 15 | func bkeys(b []byte) (uint32, uint32) { 16 | h1 := binary.LittleEndian.Uint32(b) 17 | i := 4 18 | if i > len(b)-4 { 19 | i = len(b) - 4 20 | } 21 | h2 := binary.LittleEndian.Uint32(b[i:]) 22 | return Xorshift32(h1), Xorshift32(h2) 23 | } 24 | 25 | func initBloom() { 26 | 27 | lookupBloom = newbv(bloomM) 28 | 29 | f, err := os.Open("domains.txt") 30 | if err != nil { 31 | log.Fatalf("err=%v", err) 32 | } 33 | 34 | scanner := bufio.NewScanner(f) 35 | for scanner.Scan() { 36 | b := scanner.Bytes() 37 | 38 | h1, h2 := bkeys(b) 39 | 40 | for i := uint32(0); i < bloomK; i++ { 41 | lookupBloom.set((h1 + (i * h2)) & (bloomM - 1)) 42 | } 43 | } 44 | } 45 | 46 | var filtered int 47 | 48 | // Lookup checks the bloom filter for the byte array b 49 | func MatchBloom(b []byte) bool { 50 | 51 | if lookupBloom == nil { 52 | return true 53 | } 54 | 55 | if len(b) < 4 { 56 | return true 57 | } 58 | 59 | h1, h2 := bkeys(b) 60 | 61 | for i := uint32(0); i < bloomK; i++ { 62 | if lookupBloom.get((h1+(i*h2))&(bloomM-1)) == 0 { 63 | filtered++ 64 | return false 65 | } 66 | } 67 | 68 | return true 69 | } 70 | 71 | // Internal routines for the bit vector 72 | type bitvector []uint64 73 | 74 | func newbv(size uint32) bitvector { 75 | return make([]uint64, uint(size+63)/64) 76 | } 77 | 78 | // get bit 'bit' in the bitvector d 79 | func (b bitvector) get(bit uint32) uint { 80 | 81 | shift := bit % 64 82 | bb := b[bit/64] 83 | bb &= (1 << shift) 84 | 85 | return uint(bb >> shift) 86 | } 87 | 88 | // set bit 'bit' in the bitvector d 89 | func (b bitvector) set(bit uint32) { 90 | b[bit/64] |= (1 << (bit % 64)) 91 | } 92 | 93 | // Xorshift32 is an xorshift RNG 94 | func Xorshift32(y uint32) uint32 { 95 | 96 | // http://www.jstatsoft.org/v08/i14/paper 97 | // Marasaglia's "favourite" 98 | 99 | y ^= (y << 13) 100 | y ^= (y >> 17) 101 | y ^= (y << 5) 102 | return y 103 | } 104 | -------------------------------------------------------------------------------- /matcher/bsearch.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "log" 6 | "os" 7 | "sort" 8 | ) 9 | 10 | var lookupBsearch []string 11 | 12 | func initBsearch() { 13 | 14 | f, err := os.Open("domains.txt") 15 | if err != nil { 16 | log.Fatalf("err=%v", err) 17 | } 18 | 19 | scanner := bufio.NewScanner(f) 20 | for scanner.Scan() { 21 | lookupBsearch = append(lookupBsearch, scanner.Text()) 22 | } 23 | sort.Strings(lookupBsearch) 24 | } 25 | 26 | func MatchBsearch(s string) bool { 27 | i := sort.SearchStrings(lookupBsearch, s) 28 | // the search fn gives the pos where it should be (if it's not there, where to insert it) 29 | // so we still need to verify 30 | return i < len(lookupBsearch) && lookupBsearch[i] == s 31 | } 32 | -------------------------------------------------------------------------------- /matcher/dmph.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "log" 6 | "os" 7 | 8 | "github.com/dgryski/go-mph" 9 | ) 10 | 11 | var lookupDMPH *mph.Table 12 | 13 | var domains []string 14 | 15 | func initDMPH() { 16 | 17 | f, err := os.Open("domains.txt") 18 | if err != nil { 19 | log.Fatalf("err=%v", err) 20 | } 21 | 22 | scanner := bufio.NewScanner(f) 23 | for scanner.Scan() { 24 | domains = append(domains, scanner.Text()) 25 | } 26 | 27 | lookupDMPH = mph.New(domains) 28 | 29 | } 30 | 31 | func MatchDMPH(s string) bool { 32 | i := lookupDMPH.Query(s) 33 | return domains[i] == s 34 | } 35 | -------------------------------------------------------------------------------- /matcher/gen-mafsa.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "bufio" 7 | "log" 8 | "os" 9 | "sort" 10 | 11 | "github.com/smartystreets/mafsa" 12 | ) 13 | 14 | func main() { 15 | 16 | f, err := os.Open("domains.txt") 17 | if err != nil { 18 | log.Fatalf("err=%v", err) 19 | } 20 | 21 | scanner := bufio.NewScanner(f) 22 | var arr []string 23 | for scanner.Scan() { 24 | arr = append(arr, scanner.Text()) 25 | } 26 | 27 | sort.Strings(arr) 28 | 29 | bt := mafsa.New() 30 | 31 | for _, s := range arr { 32 | if err := bt.Insert(s); err != nil { 33 | log.Fatalf("error inserting %q: %v", s, err) 34 | } 35 | } 36 | 37 | bt.Finish() 38 | 39 | if err := bt.Save("mafsa.out"); err != nil { 40 | log.Fatalf("error saving mafsa: %v", err) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /matcher/mafsa.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/smartystreets/mafsa" 7 | ) 8 | 9 | var lookupMAFSA *mafsa.MinTree 10 | 11 | func initMAFSA() { 12 | 13 | var err error 14 | 15 | lookupMAFSA, err = mafsa.Load("mafsa.out") 16 | if err != nil { 17 | log.Fatalf("error loading mafsa.out: %v", err) 18 | } 19 | 20 | } 21 | 22 | func MatchMAFSA(r []rune) bool { 23 | result := lookupMAFSA.Traverse(r) 24 | return result != nil && result.Final 25 | } 26 | -------------------------------------------------------------------------------- /matcher/map.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "log" 6 | "os" 7 | ) 8 | 9 | var lookupMap = make(map[string]struct{}) 10 | 11 | func initMap() { 12 | 13 | f, err := os.Open("domains.txt") 14 | if err != nil { 15 | log.Fatalf("err=%v", err) 16 | } 17 | 18 | scanner := bufio.NewScanner(f) 19 | for scanner.Scan() { 20 | lookupMap[scanner.Text()] = struct{}{} 21 | } 22 | } 23 | 24 | func MatchMap(s string) bool { 25 | _, ok := lookupMap[s] 26 | return ok 27 | } 28 | -------------------------------------------------------------------------------- /matcher/mph.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "log" 6 | "os" 7 | 8 | "github.com/cespare/mph" 9 | ) 10 | 11 | var lookupMPH *mph.Table 12 | 13 | func initMPH() { 14 | 15 | f, err := os.Open("domains.txt") 16 | if err != nil { 17 | log.Fatalf("err=%v", err) 18 | } 19 | 20 | scanner := bufio.NewScanner(f) 21 | var arr []string 22 | for scanner.Scan() { 23 | arr = append(arr, scanner.Text()) 24 | } 25 | 26 | lookupMPH = mph.Build(arr) 27 | 28 | } 29 | 30 | func MatchMPH(s string) bool { 31 | _, ok := lookupMPH.Lookup(s) 32 | return ok 33 | } 34 | -------------------------------------------------------------------------------- /matcher/radix.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "log" 6 | "os" 7 | 8 | "github.com/armon/go-radix" 9 | ) 10 | 11 | var lookupTree = radix.New() 12 | 13 | func initRadix() { 14 | 15 | f, err := os.Open("domains.txt") 16 | if err != nil { 17 | log.Fatalf("err=%v", err) 18 | } 19 | 20 | scanner := bufio.NewScanner(f) 21 | for scanner.Scan() { 22 | lookupTree.Insert(scanner.Text(), struct{}{}) 23 | } 24 | } 25 | 26 | func MatchRadix(s string) bool { 27 | _, ok := lookupTree.Get(s) 28 | return ok 29 | } 30 | -------------------------------------------------------------------------------- /morse/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "log" 7 | "os" 8 | "regexp" 9 | ) 10 | 11 | var nato = map[rune]string{ 12 | 'a': "alpha", 13 | 'b': "bravo", 14 | 'c': "charlie", 15 | 'd': "delta", 16 | 'e': "echo", 17 | 'f': "foxtrot", 18 | 'g': "golf", 19 | 'h': "hotel", 20 | 'i': "india", 21 | 'j': "juliett", 22 | 'k': "kilo", 23 | 'l': "lima", 24 | 'm': "mike", 25 | 'n': "november", 26 | 'o': "oscar", 27 | 'p': "papa", 28 | 'q': "quebec", 29 | 'r': "romeo", 30 | 's': "sierra", 31 | 't': "tango", 32 | 'u': "uniform", 33 | 'v': "victor", 34 | 'w': "whiskey", 35 | 'x': "xray", 36 | 'y': "yankee", 37 | 'z': "zulu", 38 | } 39 | 40 | var morse = map[rune]string{ 41 | 'a': ".-", 42 | 'b': "-...", 43 | 'c': "-.-.", 44 | 'd': "-..", 45 | 'e': ".", 46 | 'f': "..-.", 47 | 'g': "--.", 48 | 'h': "....", 49 | 'i': "..", 50 | 'j': ".---", 51 | 'k': "-.-", 52 | 'l': ".-..", 53 | 'm': "--", 54 | 'n': "-.", 55 | 'o': "---", 56 | 'p': ".--.", 57 | 'q': "--.-", 58 | 'r': ".-.", 59 | 's': "...", 60 | 't': "-", 61 | 'u': "..-", 62 | 'v': "...-", 63 | 'w': ".--", 64 | 'x': "-..-", 65 | 'y': "-.--", 66 | 'z': "--..", 67 | } 68 | 69 | var regexps = map[rune]string{ 70 | '.': "[a-m]", 71 | '-': "[n-z]", 72 | } 73 | 74 | type target struct { 75 | r rune 76 | re string 77 | } 78 | 79 | func slurp(path string) ([]string, error) { 80 | file, err := os.Open(path) 81 | if err != nil { 82 | return nil, err 83 | } 84 | defer file.Close() 85 | 86 | var lines []string 87 | scanner := bufio.NewScanner(file) 88 | for scanner.Scan() { 89 | lines = append(lines, scanner.Text()) 90 | } 91 | return lines, scanner.Err() 92 | } 93 | 94 | func main() { 95 | var targets []target 96 | for r := rune('a'); r <= 'z'; r++ { 97 | var re string 98 | for _, c := range morse[r] { 99 | re += regexps[c] 100 | } 101 | targets = append(targets, target{r: r, re: re}) 102 | } 103 | 104 | var words []string 105 | 106 | words, err := slurp("/usr/share/dict/words") 107 | if err != nil { 108 | log.Fatalln(err) 109 | } 110 | 111 | for _, t := range targets { 112 | re := regexp.MustCompile("^" + t.re + "$") 113 | for _, v := range words { 114 | if re.MatchString(v) { 115 | fmt.Println(nato[t.r], v) 116 | } 117 | } 118 | 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /msgrpc/cli.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | codec "github.com/ugorji/go/codec" 6 | "log" 7 | "net" 8 | "net/rpc" 9 | ) 10 | 11 | // dup'd in srv.go 12 | type EchoArgument struct { 13 | A string 14 | B string 15 | C string 16 | } 17 | 18 | func main() { 19 | 20 | conn, err := net.Dial("tcp", "localhost:9000") 21 | if err != nil { 22 | log.Fatal("dialing:", err) 23 | } 24 | 25 | var h codec.MsgpackHandle 26 | rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, &h) 27 | client := rpc.NewClientWithCodec(rpcCodec) 28 | 29 | args := []string{"foo1", "foo2", "foo3"} 30 | var reply string 31 | err = client.Call("Echo123", args, &reply) 32 | if err != nil { 33 | log.Fatal("Echo123 error:", err) 34 | } 35 | fmt.Println("reply=", reply) 36 | 37 | arg := &EchoArgument{A: "a string", B: "b string", C: "c string"} 38 | err = client.Call("EchoStruct", arg, &reply) 39 | if err != nil { 40 | log.Fatal("EchoStruct error:", err) 41 | } 42 | fmt.Println("reply=", reply) 43 | 44 | } 45 | -------------------------------------------------------------------------------- /msgrpc/cli.py: -------------------------------------------------------------------------------- 1 | import msgpackrpc 2 | 3 | address = msgpackrpc.Address('localhost', 9000) 4 | client = msgpackrpc.Client(address, unpack_encoding='utf-8') 5 | 6 | # go server 7 | #print client.call("Echo.Echo123", ["foo1", "foo2", "foo3"]) 8 | #print client.call("Echo.EchoStruct", {"A" :"a string", "B":"b string", "C":"c string"}) 9 | # python server 10 | print client.call("Echo123", ["foo1", "foo2", "foo3"]) 11 | print client.call("EchoStruct", {"A" :"a string", "B":"b string", "C":"c string"}) 12 | -------------------------------------------------------------------------------- /msgrpc/srv.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | codec "github.com/ugorji/go/codec" 6 | "log" 7 | "net" 8 | "net/rpc" 9 | ) 10 | 11 | type Echo struct{} 12 | 13 | func (e *Echo) Echo123(args []string, response *string) error { 14 | log.Printf("args=%#v\n", args) 15 | *response = fmt.Sprintf("%#v", args) 16 | return nil 17 | } 18 | 19 | // dup'd in cli.go 20 | type EchoArgument struct { 21 | A string 22 | B string 23 | C string 24 | } 25 | 26 | func (e *Echo) EchoStruct(arg EchoArgument, response *string) error { 27 | log.Printf("args=%#v\n", arg) 28 | *response = fmt.Sprintf("%#v", arg) 29 | return nil 30 | } 31 | 32 | func main() { 33 | 34 | echo := &Echo{} 35 | rpc.Register(echo) 36 | 37 | ln, e := net.Listen("tcp", ":9000") 38 | if e != nil { 39 | log.Fatal("listen error:", e) 40 | } 41 | 42 | log.Println("rpc server starting") 43 | 44 | for { 45 | conn, err := ln.Accept() 46 | if err != nil { 47 | log.Println(err) 48 | continue 49 | } 50 | var h codec.MsgpackHandle 51 | c := codec.MsgpackSpecRpc.ServerCodec(conn, &h) 52 | 53 | go rpc.ServeCodec(c) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /msgrpc/srv.py: -------------------------------------------------------------------------------- 1 | import msgpackrpc 2 | 3 | class EchoHandler(object): 4 | 5 | def Echo123(self, msg): 6 | (msg1, msg2, msg3) = msg 7 | return ("1:%s 2:%s 3:%s" % (msg1, msg2, msg3)) 8 | 9 | def EchoStruct(self, msg): 10 | return ("msg:%s" % msg) 11 | 12 | if __name__ == '__main__': 13 | addr = msgpackrpc.Address('localhost', 9000) 14 | server = msgpackrpc.Server(EchoHandler()) 15 | server.listen(addr) 16 | server.start() 17 | -------------------------------------------------------------------------------- /mtest/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "math/big" 7 | "os" 8 | ) 9 | 10 | func main() { 11 | 12 | scanner := bufio.NewScanner(os.Stdin) 13 | 14 | var z big.Int 15 | var a, b, c big.Int 16 | var r big.Int 17 | 18 | two := big.NewInt(2) 19 | 20 | var n_add, n_sub, n_mul2, n_div2, n_gcd, n_sqr, n_invmod, n_exptmod int 21 | 22 | for scanner.Scan() { 23 | cmd := scanner.Text() 24 | 25 | fmt.Println(n_add, n_sub, n_mul2, n_div2, n_gcd, n_sqr, n_invmod) 26 | 27 | switch cmd { 28 | 29 | case "add_d", "add": 30 | n_add++ 31 | next(scanner, &a) 32 | next(scanner, &b) 33 | next(scanner, &r) 34 | 35 | z.Add(&a, &b) 36 | if z.Cmp(&r) != 0 { 37 | fmt.Println(a, b, r, z) 38 | } 39 | 40 | case "sub_d", "sub": 41 | n_sub++ 42 | next(scanner, &a) 43 | next(scanner, &b) 44 | next(scanner, &r) 45 | 46 | z.Sub(&a, &b) 47 | if z.Cmp(&r) != 0 { 48 | fmt.Println(a, b, r, z) 49 | } 50 | 51 | case "mul2": 52 | n_mul2++ 53 | next(scanner, &a) 54 | next(scanner, &r) 55 | 56 | z.Mul(&a, two) 57 | if z.Cmp(&r) != 0 { 58 | fmt.Println(a, b, r, z) 59 | } 60 | 61 | case "div2": 62 | n_div2++ 63 | next(scanner, &a) 64 | next(scanner, &r) 65 | 66 | z.Div(&a, two) 67 | if z.Cmp(&r) != 0 { 68 | fmt.Println(a, b, r, z) 69 | } 70 | case "sqr": 71 | n_sqr++ 72 | next(scanner, &a) 73 | next(scanner, &r) 74 | 75 | z.Mul(&a, &a) 76 | if false && z.Cmp(&r) != 0 { 77 | fmt.Printf("sqr: %s %s %s\n", a.String(), r.String(), z.String()) 78 | } 79 | 80 | case "gcd": 81 | n_gcd++ 82 | next(scanner, &a) 83 | next(scanner, &b) 84 | next(scanner, &r) 85 | 86 | z.GCD(nil, nil, &a, &b) 87 | if z.Cmp(&r) != 0 { 88 | fmt.Println(a, b, r, z) 89 | } 90 | 91 | case "invmod": 92 | n_invmod++ 93 | next(scanner, &a) 94 | next(scanner, &b) 95 | next(scanner, &r) 96 | 97 | z.ModInverse(&a, &b) 98 | if z.Cmp(&r) != 0 { 99 | fmt.Println(a, b, r, z) 100 | } 101 | 102 | case "exptmod": 103 | n_exptmod++ 104 | next(scanner, &a) 105 | next(scanner, &b) 106 | next(scanner, &c) 107 | next(scanner, &r) 108 | 109 | z.Exp(&a, &b, &c) 110 | if z.Cmp(&r) != 0 { 111 | fmt.Println(a, b, r, z) 112 | } 113 | 114 | default: 115 | // nothing 116 | } 117 | } 118 | } 119 | 120 | func next(scanner *bufio.Scanner, n *big.Int) { 121 | 122 | ok := scanner.Scan() 123 | if !ok { 124 | panic("end of input") 125 | } 126 | 127 | txt := scanner.Text() 128 | 129 | n.SetString(txt, 10) 130 | } 131 | -------------------------------------------------------------------------------- /nlz/bench_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func BenchmarkBSR(b *testing.B) { 8 | 9 | // var x uint64 = 0xFFFFFFFF 10 | 11 | N := uint64(b.N) 12 | for i := uint64(0); i < N; i++ { 13 | /* x += x 14 | if int(x) < 0 { 15 | x ^= 0x88888EEF 16 | } */ 17 | nlz(uint64(i)) 18 | } 19 | 20 | } 21 | 22 | func BenchmarkGo(b *testing.B) { 23 | 24 | // var x uint64 = 0xFFFFFFFF 25 | 26 | N := uint64(b.N) 27 | for i := uint64(0); i < N; i++ { 28 | /* x += x 29 | if int(x) < 0 { 30 | x ^= 0x88888EEF 31 | } */ 32 | nlz2(uint64(i)) 33 | } 34 | } 35 | 36 | func BenchmarkPopcount(b *testing.B) { 37 | 38 | N := uint64(b.N) 39 | for i := uint64(0); i < N; i++ { 40 | popcount(uint64(i)) 41 | } 42 | 43 | } 44 | 45 | func BenchmarkPopcountGo(b *testing.B) { 46 | 47 | N := uint64(b.N) 48 | for i := uint64(0); i < N; i++ { 49 | popcountGo(uint64(i)) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /nlz/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | func nlz2(x uint64) uint64 { 6 | var n uint64 7 | 8 | n = 1 9 | 10 | if (x >> 32) == 0 { 11 | n = n + 32 12 | x = x << 32 13 | } 14 | if (x >> (32 + 16)) == 0 { 15 | n = n + 16 16 | x = x << 16 17 | } 18 | 19 | if (x >> (32 + 16 + 8)) == 0 { 20 | n = n + 8 21 | x = x << 8 22 | } 23 | 24 | if (x >> (32 + 16 + 8 + 4)) == 0 { 25 | n = n + 4 26 | x = x << 4 27 | } 28 | 29 | if (x >> (32 + 16 + 8 + 4 + 2)) == 0 { 30 | n = n + 2 31 | x = x << 2 32 | } 33 | 34 | n = n - (x >> 63) 35 | return uint64(n) 36 | } 37 | 38 | func main() { 39 | 40 | // From http://www.hackersdelight.org/hdcodetxt/nlz.c.txt 41 | tests := []uint64{0, 32, 1, 31, 2, 30, 3, 30, 4, 29, 5, 29, 6, 29, 42 | 7, 29, 8, 28, 9, 28, 16, 27, 32, 26, 64, 25, 128, 24, 255, 24, 256, 23, 43 | 512, 22, 1024, 21, 2048, 20, 4096, 19, 8192, 18, 16384, 17, 32768, 16, 44 | 65536, 15, 0x20000, 14, 0x40000, 13, 0x80000, 12, 0x100000, 11, 45 | 0x200000, 10, 0x400000, 9, 0x800000, 8, 0x1000000, 7, 0x2000000, 6, 46 | 0x4000000, 5, 0x8000000, 4, 0x0FFFFFFF, 4, 0x10000000, 3, 47 | 0x3000FFFF, 2, 0x50003333, 1, 0x7FFFFFFF, 1, 0x80000000, 0, 48 | 0xFFFFFFFF, 0} 49 | 50 | for i := 0; i < len(tests); i += 2 { 51 | if lz := nlz(tests[i]); lz != tests[i+1]+32 { 52 | fmt.Printf("failed for test %d -> nlz(%b)=%d, got %d\n", i/2, tests[i], tests[i+1]+32, lz) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /nlz/nlz.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func nlz(x uint64) uint64 4 | -------------------------------------------------------------------------------- /nlz/nlz_amd64.s: -------------------------------------------------------------------------------- 1 | // func nlz(x uint64) uint64 2 | TEXT ·nlz(SB),4,$0-16 3 | BSRQ x+0(FP), AX 4 | JZ zero 5 | SUBQ $63, AX 6 | NEGQ AX 7 | MOVQ AX, ret+8(FP) 8 | RET 9 | zero: 10 | MOVQ $64, ret+8(FP) 11 | RET 12 | -------------------------------------------------------------------------------- /nlz/popcount.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func popcount(x uint64) uint64 4 | 5 | func popcountGo(x uint64) uint64 { 6 | 7 | // bit population count, see 8 | // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel 9 | x -= (x >> 1) & 0x5555555555555555 10 | x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 11 | x += x >> 4 12 | x &= 0x0f0f0f0f0f0f0f0f 13 | x *= 0x0101010101010101 14 | return x >> 56 15 | } 16 | -------------------------------------------------------------------------------- /nlz/popcount_amd64.s: -------------------------------------------------------------------------------- 1 | 2 | #define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 3 | 4 | // func popcount(x uint64) uint64 5 | 6 | TEXT ·popcount(SB),4,$0-16 7 | MOVQ x+0(FP), DX 8 | POPCNTQ_DX_DX 9 | MOVQ DX, ret+8(FP) 10 | RET 11 | -------------------------------------------------------------------------------- /numerical/integrate_test.go: -------------------------------------------------------------------------------- 1 | package numerical 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "testing" 7 | ) 8 | 9 | func f(x float64) float64 { 10 | return 6 + 2*x*x*math.Cos(x*x) 11 | } 12 | 13 | func TestIntegrators(t *testing.T) { 14 | 15 | const samples = 200 16 | 17 | algorithms := []struct { 18 | name string 19 | method func(float64, float64, int, func(float64) float64) float64 20 | }{ 21 | {"simpson", Simpsons}, 22 | {"simpson38", Simpsons38}, 23 | {"simpsoncomposite", SimpsonsComposite}, 24 | {"simpson38composite", Simpsons38Composite}, 25 | {"simpson_adaptive", SimpsonsAdaptive}, 26 | {"trapezoid", Trapezoid}, 27 | {"trapezoidfast", TrapezoidFast}, 28 | {"trapezoiditerative", TrapezoidIterative}, 29 | {"romberg", Romberg}, 30 | } 31 | 32 | for _, algorithm := range algorithms { 33 | 34 | totalTicks := uint64(0) 35 | 36 | var area float64 37 | 38 | for i := 0; i < samples; i++ { 39 | ticks := rdtsc() 40 | area = algorithm.method(0, math.SqrtPi, 100, f) 41 | totalTicks += rdtsc() - ticks 42 | } 43 | 44 | fmt.Printf("integrate: %-20s\t=> %.10f (avg. ticks=%d)\n", algorithm.name, area, totalTicks/samples) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /numerical/rdtsc.go: -------------------------------------------------------------------------------- 1 | package numerical 2 | 3 | func rdtsc() uint64 4 | -------------------------------------------------------------------------------- /numerical/rdtsc_amd64.s: -------------------------------------------------------------------------------- 1 | TEXT ·rdtsc(SB),0,$0-8 2 | RDTSC 3 | SHLQ $32, DX 4 | ADDQ DX, AX 5 | MOVQ AX, ret+0(FP) 6 | RET 7 | 8 | -------------------------------------------------------------------------------- /numerical/root.go: -------------------------------------------------------------------------------- 1 | package numerical 2 | 3 | import ( 4 | "math" 5 | ) 6 | 7 | func Bisect(f func(x float64) float64, x0, x1 float64) (root float64) { 8 | 9 | eps := 10e-8 10 | 11 | fx0 := f(x0) 12 | 13 | for x1-x0 > eps { 14 | root = (x0 + x1) / 2 15 | froot := f(root) 16 | if froot*fx0 < 0 { 17 | x1 = root 18 | } else { 19 | x0 = root 20 | fx0 = froot 21 | } 22 | } 23 | 24 | return root 25 | } 26 | 27 | func Secant(f func(x float64) float64, x0, x1 float64) (root float64) { 28 | 29 | eps := 10e-8 30 | 31 | fx0 := f(x0) 32 | for math.Abs(x1-x0) > eps { 33 | fx1 := f(x1) 34 | x2 := x1 - fx1*((x1-x0)/(fx1-fx0)) 35 | x1, x0 = x2, x1 36 | fx0 = fx1 37 | } 38 | 39 | return x1 40 | } 41 | 42 | // wikipedia 43 | 44 | func Falsi(f func(x float64) float64, s, t float64) (r float64) { 45 | 46 | side := 0 47 | 48 | fs := f(s) 49 | ft := f(t) 50 | 51 | m := 100 // max iterations 52 | 53 | e := 5e-15 // eps 54 | 55 | for n := 1; n <= m; n++ { 56 | r = (fs*t - ft*s) / (fs - ft) 57 | if math.Abs(t-s) < e*math.Abs(t+s) { 58 | break 59 | } 60 | fr := f(r) 61 | 62 | if fr*ft > 0 { 63 | t = r 64 | ft = fr 65 | if side == -1 { 66 | fs /= 2 67 | } 68 | side = -1 69 | } else if fs*fr > 0 { 70 | s = r 71 | fs = fr 72 | if side == +1 { 73 | ft /= 2 74 | } 75 | side = +1 76 | } else { 77 | break 78 | } 79 | } 80 | return r 81 | } 82 | 83 | func max(a, b float64) float64 { 84 | if a > b { 85 | return a 86 | } 87 | return b 88 | } 89 | 90 | // Numerical Methods in Engineering with Python -> Go 91 | 92 | func Ridder(f func(x float64) float64, a, b float64) (r float64) { 93 | 94 | tol := 1.0e-9 95 | 96 | fa := f(a) 97 | if fa == 0.0 { 98 | return a 99 | } 100 | fb := f(b) 101 | if fb == 0.0 { 102 | return b 103 | } 104 | 105 | // if fa*fb > 0.0: errror.err('Root is not bracketed') 106 | 107 | var xOld float64 108 | 109 | for i := 0; i < 30; i++ { 110 | // Compute the improved root x from Ridder's formula 111 | c := 0.5 * (a + b) 112 | fc := f(c) 113 | s := math.Sqrt(fc*fc - fa*fb) 114 | if s == 0.0 { 115 | return 0.0 116 | } 117 | dx := (c - a) * fc / s 118 | if (fa - fb) < 0.0 { 119 | dx = -dx 120 | } 121 | x := c + dx 122 | fx := f(x) 123 | // Test for convergence 124 | if i > 0 && math.Abs(x-xOld) < tol*max(math.Abs(x), 1.0) { 125 | return x 126 | } 127 | xOld = x 128 | // Re-bracket the root as tightly as possible 129 | if fc*fx > 0.0 { 130 | if fa*fx < 0.0 { 131 | b = x 132 | fb = fx 133 | } else { 134 | a = x 135 | fa = fx 136 | } 137 | } else { 138 | a = c 139 | b = x 140 | fa = fc 141 | fb = fx 142 | } 143 | 144 | } 145 | 146 | // too many iterations 147 | return 0 148 | 149 | } 150 | -------------------------------------------------------------------------------- /numerical/root_test.go: -------------------------------------------------------------------------------- 1 | package numerical 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "testing" 7 | ) 8 | 9 | func f1(x float64) float64 { 10 | return x - math.Tan(x) 11 | } 12 | 13 | func TestRootSolve(t *testing.T) { 14 | 15 | const samples = 200 16 | 17 | algorithms := []struct { 18 | name string 19 | method func(func(float64) float64, float64, float64) float64 20 | }{ 21 | {"bisect", Bisect}, 22 | {"secant", Secant}, 23 | {"falsi", Falsi}, 24 | {"ridder", Ridder}, 25 | } 26 | 27 | for _, algorithm := range algorithms { 28 | 29 | totalTicks := uint64(0) 30 | 31 | var r float64 32 | 33 | for i := 0; i < samples; i++ { 34 | ticks := rdtsc() 35 | r = algorithm.method(f1, 4.4, 4.6) 36 | totalTicks += rdtsc() - ticks 37 | 38 | } 39 | 40 | fmt.Printf("solve: %-20s\t=> %.10f (ticks=%d)\n", algorithm.name, r, totalTicks/samples) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /oma/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "math/rand" 7 | "time" 8 | ) 9 | 10 | type roll int 11 | 12 | const ( 13 | sideHome roll = iota 14 | sideFirst 15 | sideSecond 16 | sideEndOfLine 17 | ) 18 | 19 | func rollDie() roll { 20 | 21 | switch rand.Intn(6) { 22 | case 0, 1: 23 | return sideHome 24 | case 2: 25 | return sideFirst 26 | case 3: 27 | return sideSecond 28 | case 4, 5: 29 | return sideEndOfLine 30 | } 31 | 32 | panic("unreached") 33 | } 34 | 35 | type queue []int 36 | 37 | func (q *queue) moveToEnd(pos int) { 38 | e := (*q)[pos] 39 | copy((*q)[pos:], (*q)[pos+1:]) 40 | (*q)[len(*q)-1] = e 41 | } 42 | 43 | func (q *queue) moveHome(player int) { 44 | for i, v := range *q { 45 | if v == player { 46 | q.moveToEnd(i) 47 | *q = (*q)[:len(*q)-1] 48 | return 49 | } 50 | } 51 | } 52 | 53 | func playGame(players int) int { 54 | var q queue 55 | 56 | for i := 0; i < 5; i++ { 57 | q = append(q, 0) 58 | for p := players - 1; p > 0; p-- { 59 | q = append(q, p) 60 | } 61 | } 62 | 63 | scores := make([]int, players) 64 | 65 | var player int 66 | 67 | for { 68 | switch rollDie() { 69 | 70 | case sideHome: 71 | q.moveHome(player) 72 | scores[player]++ 73 | if scores[player] == 5 { 74 | return player 75 | } 76 | 77 | case sideFirst: 78 | p := q[0] 79 | q.moveHome(p) 80 | scores[p]++ 81 | if scores[p] == 5 { 82 | return p 83 | } 84 | 85 | case sideSecond: 86 | p := q[1] 87 | q.moveHome(p) 88 | scores[p]++ 89 | if scores[p] == 5 { 90 | return p 91 | } 92 | 93 | case sideEndOfLine: 94 | q.moveToEnd(0) 95 | } 96 | 97 | player = (player + 1) % players 98 | } 99 | } 100 | 101 | func main() { 102 | 103 | players := flag.Int("p", 4, "players") 104 | runs := flag.Int("r", 1000, "games to simulate") 105 | flag.Parse() 106 | 107 | rand.Seed(time.Now().UnixNano()) 108 | 109 | scores := make([]int, *players) 110 | 111 | for i := 0; i < *runs; i++ { 112 | scores[playGame(*players)]++ 113 | } 114 | 115 | fmt.Println(scores) 116 | } 117 | -------------------------------------------------------------------------------- /photocmp/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "html/template" 7 | "log" 8 | "math/rand" 9 | "net/http" 10 | "path/filepath" 11 | "sort" 12 | "strconv" 13 | "time" 14 | ) 15 | 16 | func main() { 17 | port := flag.Int("p", 8080, "port to listen on") 18 | setNoCache := flag.Bool("no-cache", false, "set no-cache header on requests") 19 | shufflePhotos := flag.Bool("shuffle", false, "shuffle photo order") 20 | photosPerPage := flag.Int("n", 30, "photos per page") 21 | 22 | flag.Parse() 23 | 24 | dir1 := flag.Arg(0) 25 | dir2 := flag.Arg(1) 26 | 27 | log.Println("Comparing images from", dir1, "and", dir2, "on port", *port) 28 | 29 | d1files, _ := filepath.Glob(dir1 + "/*") 30 | d2files, _ := filepath.Glob(dir2 + "/*") 31 | 32 | if len(d1files) != len(d2files) { 33 | log.Println("file count mismatch: dir1=", len(d1files), "dir2=", len(d2files)) 34 | } 35 | 36 | m := make(map[string]struct{}) 37 | addFileNames(m, d1files) 38 | addFileNames(m, d2files) 39 | 40 | var photos []string 41 | for k := range m { 42 | photos = append(photos, k) 43 | } 44 | 45 | sort.Strings(photos) 46 | 47 | if *shufflePhotos { 48 | rand.Seed(time.Now().UnixNano()) 49 | perm := rand.Perm(len(photos)) 50 | dst := make([]string, len(photos)) 51 | for i, v := range perm { 52 | dst[i] = photos[v] 53 | } 54 | 55 | photos = dst 56 | } 57 | 58 | // set up handler to serve our files 59 | http.Handle("/dir1/", http.StripPrefix("/dir1/", http.FileServer(http.Dir(dir1)))) 60 | http.Handle("/dir2/", http.StripPrefix("/dir2/", http.FileServer(http.Dir(dir2)))) 61 | 62 | http.HandleFunc("/compare", func(w http.ResponseWriter, r *http.Request) { 63 | r.ParseForm() 64 | if *setNoCache { 65 | w.Header().Set("Cache-Control", "no-cache") 66 | } 67 | 68 | pageStr := r.FormValue("page") 69 | 70 | page, err := strconv.Atoi(pageStr) 71 | if err != nil { 72 | log.Println("invalid param for page, zeroing") 73 | page = 0 74 | } 75 | 76 | offset := page * *photosPerPage 77 | limit := offset + *photosPerPage 78 | 79 | if offset > len(photos) { 80 | offset = len(photos) 81 | } 82 | 83 | if limit > len(photos) { 84 | limit = len(photos) 85 | } 86 | 87 | prev := page - 1 88 | if prev < 0 { 89 | prev = 0 90 | } 91 | 92 | tmplParams := struct { 93 | Photos []string 94 | Next int 95 | Prev int 96 | }{ 97 | Photos: photos[offset:limit], 98 | Next: page + 1, 99 | Prev: prev, 100 | } 101 | 102 | compareTMPL.Execute(w, tmplParams) 103 | }) 104 | 105 | portStr := fmt.Sprintf(":%d", *port) 106 | log.Fatal("ListenAndServe:", http.ListenAndServe(portStr, nil)) 107 | } 108 | 109 | func addFileNames(m map[string]struct{}, paths []string) { 110 | for _, fname := range paths { 111 | _, file := filepath.Split(fname) 112 | m[file] = struct{}{} 113 | } 114 | } 115 | 116 | var compareTMPL = template.Must(template.New("compare").Parse( 117 | ` 118 | 119 | 120 | {{ range .Photos }} 121 | 122 | 123 | 124 | 125 | {{ end }} 126 |
127 | 128 | Next 129 | Prev 130 | `)) 131 | -------------------------------------------------------------------------------- /pphrase/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "crypto/rand" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | ) 11 | 12 | func main() { 13 | 14 | wordlist := flag.String("w", "words.txt", "word list to use") 15 | phraseLen := flag.Int("n", 5, "length of passphrase") 16 | bracketed := flag.Bool("bracketed", false, "return passphrase inside brackets") 17 | 18 | flag.Parse() 19 | 20 | f, err := os.Open(*wordlist) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | 25 | scanner := bufio.NewScanner(f) 26 | 27 | var words []string 28 | 29 | for scanner.Scan() { 30 | words = append(words, scanner.Text()) 31 | } 32 | 33 | if err := scanner.Err(); err != nil { 34 | log.Fatal(err) 35 | } 36 | 37 | if len(words) != 2048 { 38 | log.Fatal("wordlist must contain 2048 words") 39 | } 40 | 41 | var phrase []string 42 | for i := 0; i < *phraseLen; i++ { 43 | var buf [2]byte 44 | _, err := rand.Read(buf[:]) 45 | if err != nil { 46 | log.Fatal("error reading crypto/rand:", err) 47 | } 48 | word := int(buf[0]&0x07)<<8 + int(buf[1]) 49 | phrase = append(phrase, words[word]) 50 | } 51 | 52 | if *bracketed { 53 | fmt.Println(phrase) 54 | } else { 55 | for _, word := range phrase { 56 | fmt.Print(word + " ") 57 | } 58 | fmt.Println() 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /prior/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | "sort" 11 | "strings" 12 | "text/tabwriter" 13 | 14 | "github.com/google/go-github/github" 15 | "golang.org/x/oauth2" 16 | ) 17 | 18 | func main() { 19 | 20 | pretty := flag.Bool("pretty", false, "pretty output") 21 | flag.Parse() 22 | 23 | ctx := context.Background() 24 | ts := oauth2.StaticTokenSource( 25 | &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, 26 | ) 27 | tc := oauth2.NewClient(ctx, ts) 28 | 29 | client := github.NewClient(tc) 30 | 31 | var w io.Writer = os.Stdout 32 | 33 | if *pretty { 34 | w = tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0) 35 | } 36 | 37 | var all []*github.Repository 38 | 39 | for page := 1; ; page++ { 40 | repos, _, err := client.Repositories.List(ctx, "", &github.RepositoryListOptions{ListOptions: github.ListOptions{Page: page}}) 41 | if err != nil { 42 | log.Printf("error fetching repositories page %d = %+v\n", page, err) 43 | break 44 | } 45 | 46 | log.Printf("fetched %d results", len(repos)) 47 | 48 | if len(repos) == 0 { 49 | log.Printf("done") 50 | break 51 | } 52 | 53 | all = append(all, repos...) 54 | } 55 | 56 | log.Printf("total %d repos", len(all)) 57 | 58 | sort.Sort(ByCreatedAt(all)) 59 | 60 | fmt.Fprintln(w, strings.Join([]string{ 61 | "Name", 62 | "Description", 63 | "Created", 64 | "Last Modification"}, "\t")) 65 | 66 | for _, r := range all { 67 | // ignore forks 68 | if r.GetFork() { 69 | log.Printf("ignoring fork %q", r.GetFullName()) 70 | continue 71 | } 72 | 73 | if r.GetDescription() == "" { 74 | log.Printf("missing description for %q", r.GetFullName()) 75 | } 76 | 77 | fmt.Fprintln(w, strings.Join([]string{ 78 | r.GetFullName(), 79 | r.GetDescription(), 80 | r.GetCreatedAt().Format("2006-01-02"), 81 | r.GetPushedAt().Format("2006-01-02")}, "\t")) 82 | } 83 | 84 | if f, ok := w.(interface { 85 | Flush() error 86 | }); ok { 87 | f.Flush() 88 | } 89 | } 90 | 91 | type ByCreatedAt []*github.Repository 92 | 93 | func (b ByCreatedAt) Len() int { return len(b) } 94 | func (b ByCreatedAt) Less(i, j int) bool { return b[i].GetCreatedAt().Before(b[j].GetCreatedAt().Time) } 95 | func (b ByCreatedAt) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 96 | -------------------------------------------------------------------------------- /qrshow/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "code.google.com/p/rsc/qr" 5 | "flag" 6 | "fmt" 7 | "log" 8 | ) 9 | 10 | func main() { 11 | 12 | level := flag.String("level", "m", "QR level: L/M/Q/H") 13 | 14 | flag.Parse() 15 | 16 | var qrLevel qr.Level 17 | 18 | switch *level { 19 | case "L", "l": 20 | qrLevel = qr.L 21 | case "M", "m": 22 | qrLevel = qr.M 23 | case "H", "h": 24 | qrLevel = qr.H 25 | case "Q", "q": 26 | qrLevel = qr.Q 27 | default: 28 | log.Fatal("unknown QR level:", *level) 29 | } 30 | 31 | data := flag.Arg(0) 32 | 33 | code, _ := qr.Encode(data, qrLevel) 34 | 35 | black := "\033[30;40m" 36 | white := "\033[30;47m" 37 | reset := "\033[0m" 38 | 39 | fmt.Print(reset) 40 | 41 | // two pixel border 42 | for i := 0; i < 2; i++ { 43 | fmt.Print(white + " ") 44 | for x := 0; x < code.Size; x++ { 45 | fmt.Print(" ") 46 | } 47 | fmt.Println(" " + reset) 48 | } 49 | 50 | for x := 0; x < code.Size; x++ { 51 | fmt.Print(white + " ") // two pixel border 52 | for y := 0; y < code.Size; y++ { 53 | if code.Black(x, y) { 54 | fmt.Print(black + " ") 55 | } else { 56 | fmt.Print(white + " ") 57 | } 58 | } 59 | fmt.Println(white + " " + reset) // two pixel border 60 | } 61 | 62 | // two pixel border 63 | for i := 0; i < 2; i++ { 64 | fmt.Print(white + " ") 65 | for x := 0; x < code.Size; x++ { 66 | fmt.Print(" ") 67 | } 68 | fmt.Println(white + " " + reset) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /quantiles/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "math/rand" 10 | "os" 11 | "runtime/pprof" 12 | "sort" 13 | "strconv" 14 | "time" 15 | 16 | "github.com/VividCortex/gohistogram" 17 | bquantile "github.com/bmizerany/perks/quantile" 18 | "github.com/caio/go-tdigest" 19 | gogk "github.com/dgryski/go-gk" 20 | "github.com/dgryski/go-kll" 21 | squantile "github.com/streadway/quantile" 22 | ) 23 | 24 | func main() { 25 | 26 | cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file") 27 | 28 | f := flag.String("f", "", "file to read") 29 | 30 | rand.Seed(time.Now().UnixNano()) 31 | 32 | flag.Parse() 33 | 34 | if *cpuprofile != "" { 35 | f, err := os.Create(*cpuprofile) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | pprof.StartCPUProfile(f) 40 | defer pprof.StopCPUProfile() 41 | } 42 | 43 | ch := make(chan int) 44 | 45 | var r io.Reader 46 | 47 | if *f == "" { 48 | r = os.Stdin 49 | } else { 50 | var err error 51 | r, err = os.Open(*f) 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | } 57 | 58 | go sendInts(r, ch) 59 | 60 | var stream []int 61 | 62 | bmq := bquantile.NewBiased() 63 | gk := gogk.New(0.0001) 64 | sq := squantile.New(squantile.Unknown(0.0001)) 65 | vvh := gohistogram.NewHistogram(80) 66 | td := tdigest.New(100) 67 | kl := kll.New(1024) 68 | 69 | for v := range ch { 70 | stream = append(stream, v) 71 | bmq.Insert(float64(v)) 72 | gk.Insert(float64(v)) 73 | sq.Add(float64(v)) 74 | vvh.Add(float64(v)) 75 | td.Add(float64(v), 1) 76 | kl.Update(float64(v)) 77 | } 78 | 79 | qq := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 0.9999, 0.99999} 80 | 81 | fmt.Println("bq:") 82 | for _, q := range qq { 83 | fmt.Printf(" % 4g", bmq.Query(q)) 84 | } 85 | fmt.Println() 86 | 87 | fmt.Println("sq:") 88 | for _, q := range qq { 89 | fmt.Printf(" % 4g", sq.Get(q)) 90 | } 91 | fmt.Println() 92 | 93 | fmt.Println("gk:") 94 | for _, q := range qq { 95 | fmt.Printf(" % 4g", gk.Query(q)) 96 | } 97 | fmt.Println() 98 | 99 | fmt.Println("vv:") 100 | for _, q := range qq { 101 | fmt.Printf(" % 4d", int(vvh.Quantile(q))) 102 | } 103 | fmt.Println() 104 | 105 | fmt.Println("td: ") 106 | for _, q := range qq { 107 | fmt.Printf(" % 4d", int(td.Quantile(q))) 108 | } 109 | fmt.Println() 110 | 111 | klcdf := kl.CDF() 112 | fmt.Println("kl: ") 113 | for _, q := range qq { 114 | fmt.Printf(" % 4d", int(klcdf.Query(q))) 115 | } 116 | fmt.Println() 117 | 118 | sort.Ints(stream) 119 | fmt.Println("ex: ") 120 | for _, q := range qq { 121 | fmt.Printf(" % 4d", stream[int(float64(len(stream))*q)]) 122 | } 123 | fmt.Println() 124 | } 125 | 126 | func sendInts(r io.Reader, ch chan<- int) { 127 | sc := bufio.NewScanner(r) 128 | for sc.Scan() { 129 | b := sc.Bytes() 130 | v, err := strconv.ParseInt(string(b), 10, 64) 131 | if err != nil { 132 | log.Fatal(err) 133 | } 134 | ch <- int(v) 135 | } 136 | if sc.Err() != nil { 137 | log.Fatal(sc.Err()) 138 | } 139 | close(ch) 140 | } 141 | -------------------------------------------------------------------------------- /range2cidr/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "net" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | type network struct { 12 | ip uint32 13 | bits uint32 14 | } 15 | 16 | func (n network) String() string { 17 | return fmt.Sprintf("%d.%d.%d.%d/%d", byte(n.ip>>24), byte(n.ip>>16), byte(n.ip>>8), byte(n.ip), n.bits) 18 | } 19 | 20 | func range2cidr(ipStart uint32, ipEnd uint32) []network { 21 | var nets []network 22 | newip := ipStart 23 | for newip <= ipEnd { 24 | bits, mask := uint32(1), uint32(1) 25 | for bits < 32 { 26 | newip = ipStart | mask 27 | if (newip > ipEnd) || ((ipStart>>bits)<>= 1 30 | break 31 | } 32 | bits++ 33 | mask = (mask << 1) + 1 34 | } 35 | nets = append(nets, network{ipStart, 32 - bits}) 36 | ipStart = (ipStart | mask) + 1 37 | newip = ipStart 38 | } 39 | return nets 40 | } 41 | 42 | func main() { 43 | scanner := bufio.NewScanner(os.Stdin) 44 | for scanner.Scan() { 45 | fields := strings.Fields(scanner.Text()) 46 | if len(fields) != 2 { 47 | fmt.Println("bad line:", scanner.Text()) 48 | continue 49 | } 50 | ipStart, ipEnd := str2ip32(fields[0]), str2ip32(fields[1]) 51 | if ipStart == 0 || ipEnd == 0 { 52 | fmt.Println("error parsing:", scanner.Text()) 53 | continue 54 | } 55 | fmt.Printf("%s - %s: %v", fields[0], fields[1], range2cidr(ipStart, ipEnd)) 56 | } 57 | } 58 | 59 | func str2ip32(s string) uint32 { 60 | netip := net.ParseIP(s) 61 | if netip == nil { 62 | return 0 63 | } 64 | ip4 := netip.To4() 65 | return uint32(ip4[0])<<24 | uint32(ip4[1])<<16 | uint32(ip4[2])<<8 | uint32(ip4[3]) 66 | } 67 | -------------------------------------------------------------------------------- /rangebench/bench_test.go: -------------------------------------------------------------------------------- 1 | package rangebench 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | ) 7 | 8 | var buf = make([]int, 1024*1024) 9 | 10 | var sink int 11 | 12 | func benchmarkRangeIndex(b *testing.B, n int) { 13 | 14 | var sum = 0 15 | 16 | for i := 0; i < b.N; i++ { 17 | for idx := range buf[:n] { 18 | sum += buf[idx] 19 | } 20 | } 21 | 22 | sink += sum 23 | 24 | } 25 | 26 | func benchmarkRangeIndexValue(b *testing.B, n int) { 27 | 28 | var sum = 0 29 | 30 | for i := 0; i < b.N; i++ { 31 | for _, val := range buf[:n] { 32 | sum += val 33 | } 34 | } 35 | 36 | sink += sum 37 | } 38 | 39 | var benchSizes = []int{5, 10, 50, 100, 1024, 8192, 64 * 1024, 512 * 1024, 1024 * 1024} 40 | 41 | func BenchmarkRangeIndexValue(b *testing.B) { 42 | for _, n := range benchSizes { 43 | b.Run(strconv.Itoa(n), func(b *testing.B) { benchmarkRangeIndexValue(b, n) }) 44 | } 45 | } 46 | 47 | func BenchmarkRangeIndex(b *testing.B) { 48 | for _, n := range benchSizes { 49 | b.Run(strconv.Itoa(n), func(b *testing.B) { benchmarkRangeIndex(b, n) }) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /repl/repl.go: -------------------------------------------------------------------------------- 1 | package repl 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "sort" 7 | "strings" 8 | 9 | "github.com/flynn/go-shlex" 10 | "github.com/peterh/liner" 11 | ) 12 | 13 | type Cmd func(args []string) error 14 | 15 | func Run(prompt string, commands map[string]Cmd) { 16 | 17 | line := liner.NewLiner() 18 | defer line.Close() 19 | 20 | var keys []string 21 | 22 | for k := range commands { 23 | keys = append(keys, k) 24 | } 25 | 26 | sort.Strings(keys) 27 | 28 | line.SetCompleter(func(line string) []string { 29 | var completions []string 30 | for _, k := range keys { 31 | if strings.HasPrefix(k, line) { 32 | completions = append(completions, k+" ") 33 | } 34 | } 35 | return completions 36 | }) 37 | 38 | REPL: 39 | for { 40 | var err error 41 | var command string 42 | command, err = line.Prompt(prompt) 43 | if err == io.EOF { 44 | break 45 | } 46 | if err != nil { 47 | fmt.Println("error reading line:", err) 48 | continue 49 | } 50 | 51 | args, _ := shlex.Split(command) 52 | 53 | if len(args) == 0 { 54 | continue 55 | } 56 | 57 | line.AppendHistory(command) 58 | 59 | switch args[0] { 60 | 61 | case "h", "help": 62 | fmt.Println("h[elp] -- this help") 63 | fmt.Println("q[uit] -- quit") 64 | fmt.Print("known commands:\n", strings.Join(keys, "\n"), "\n") 65 | 66 | case "q", "quit": 67 | break REPL 68 | 69 | default: 70 | 71 | if f, ok := commands[args[0]]; ok { 72 | err := f(args[1:]) 73 | if err != nil { 74 | fmt.Println("error:", err) 75 | } 76 | } else { 77 | fmt.Println("unknown command, try `help`") 78 | } 79 | } 80 | } 81 | 82 | fmt.Println("bye") 83 | } 84 | -------------------------------------------------------------------------------- /rndsample/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "math/rand" 9 | "os" 10 | "time" 11 | ) 12 | 13 | func main() { 14 | 15 | sampleSize := flag.Int("n", 1000, "sample size") 16 | 17 | flag.Parse() 18 | 19 | if *sampleSize < 0 { 20 | log.Fatal("sample size must be >0: got ", *sampleSize) 21 | } 22 | 23 | rand.Seed(time.Now().UnixNano()) 24 | 25 | r := newReservoir(*sampleSize) 26 | 27 | scanner := bufio.NewScanner(os.Stdin) 28 | 29 | for scanner.Scan() { 30 | r.push(scanner.Text()) 31 | } 32 | 33 | for _, s := range r.data { 34 | fmt.Println(s) 35 | } 36 | } 37 | 38 | type reservoir struct { 39 | data []string 40 | n int 41 | } 42 | 43 | func newReservoir(capacity int) *reservoir { 44 | return &reservoir{ 45 | data: make([]string, 0, capacity), 46 | } 47 | } 48 | 49 | func (r *reservoir) push(s string) { 50 | 51 | index := r.n 52 | 53 | r.n++ 54 | 55 | // not enough samples yet -- add it 56 | if index < cap(r.data) { 57 | r.data = append(r.data, s) 58 | return 59 | } 60 | 61 | ridx := rand.Intn(r.n) // == index+1, so we're 0..index inclusive 62 | 63 | if ridx >= len(r.data) { 64 | // ignore this one 65 | return 66 | } 67 | 68 | // add to our sample 69 | r.data[ridx] = s 70 | } 71 | -------------------------------------------------------------------------------- /rndtxt/randomtext.go: -------------------------------------------------------------------------------- 1 | // Package rndtxt generates random text strings 2 | package rndtxt 3 | 4 | // Generate generates ln bytes of random text 5 | func Generate(ln int) []byte { 6 | // yoinked from a go-nuts post by rsc 7 | var x = 0xFFFFFFFF 8 | text := make([]byte, ln) 9 | for i := range text { 10 | x += x 11 | if int(x) < 0 { 12 | x ^= 0x88888EEF 13 | } 14 | v := byte(uint(x) % 95) 15 | if v == 0 { 16 | text[i] = '\n' 17 | } else { 18 | text[i] = v + ' ' 19 | } 20 | } 21 | return text 22 | } 23 | -------------------------------------------------------------------------------- /servedir/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | ) 9 | 10 | func main() { 11 | dir := flag.String("d", ".", "directory to serve") 12 | port := flag.Int("p", 8080, "port to listen on") 13 | setNoCache := flag.Bool("no-cache", false, "set no-cache header on requests") 14 | 15 | flag.Parse() 16 | 17 | log.Println("Serving files from", *dir, "on port", *port) 18 | 19 | fserver := http.FileServer(http.Dir(*dir)) 20 | 21 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 22 | log.Println(r.RemoteAddr, r.URL) 23 | if *setNoCache { 24 | w.Header().Set("Cache-Control", "no-cache") 25 | } 26 | fserver.ServeHTTP(w, r) 27 | }) 28 | 29 | portStr := fmt.Sprintf(":%d", *port) 30 | err := http.ListenAndServe(portStr, nil) 31 | if err != nil { 32 | log.Fatal("ListenAndServe: ", err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /shellsort/shell.go: -------------------------------------------------------------------------------- 1 | package shellsort 2 | 3 | import "github.com/dgryski/go-pcgr" 4 | import "time" 5 | 6 | var gaps = []int{701, 301, 132, 57, 23, 10, 4, 1} 7 | 8 | func Shell(a []int) { 9 | // Sort an array a[0...n-1]. 10 | 11 | n := len(a) 12 | 13 | // Start with the largest gap and work down to a gap of 1 14 | for _, gap := range gaps { 15 | // Do a gapped insertion sort for this gap size. 16 | // The first gap elements a[0..gap-1] are already in gapped order 17 | // keep adding one more element until the entire array is gap sorted 18 | for i := gap; i < n; i++ { 19 | // add a[i] to the elements that have been gap sorted 20 | // save a[i] in temp and make a hole at position i 21 | temp := a[i] 22 | // shift earlier gap-sorted elements up until the correct location for a[i] is found 23 | var j int 24 | for j = i; j >= gap && a[j-gap] > temp; j -= gap { 25 | a[j] = a[j-gap] 26 | } 27 | // put temp (the original a[i]) in its correct location 28 | a[j] = temp 29 | } 30 | } 31 | } 32 | 33 | func permuteRandom(array []int, r *pcgr.Rand) { 34 | for i := len(array) - 1; i >= 1; i-- { 35 | j := r.Bound(uint32(i + 1)) 36 | array[i], array[j] = array[j], array[i] 37 | } 38 | } 39 | 40 | const _C = 16 41 | 42 | // compare-exchange two regions of length offset each 43 | func compareRegions(a []int, s, t, offset, round int, r *pcgr.Rand) { 44 | mate := make([]int, offset) 45 | for count := 0; count < _C; count++ { 46 | for i := 0; i < offset; i++ { 47 | mate[i] = i 48 | } 49 | permuteRandom(mate, r) 50 | for i := 0; i < offset; i++ { 51 | 52 | i2 := s + i 53 | j2 := t + mate[i] 54 | 55 | if i2 >= len(a) || j2 >= len(a) || r.Bound(uint32(round)) == 0 { 56 | continue 57 | } 58 | 59 | if ((i2 < j2) && (a[i2] > a[j2])) || ((i2 > j2) && (a[i2] < a[j2])) { 60 | a[i2], a[j2] = a[j2], a[i2] 61 | } 62 | } 63 | } 64 | } 65 | 66 | func RandShell(a []int) { 67 | 68 | r := &pcgr.Rand{} 69 | r.Seed(time.Now().UnixNano()) 70 | 71 | n := len(a) 72 | round := 1 73 | 74 | for _, offset := range gaps { 75 | 76 | for i := 0; i < n-offset; i += offset { // compare-exchange up 77 | compareRegions(a, i, i+offset, offset, round, r) 78 | } 79 | for i := n - offset; i >= offset; i -= offset { // compare-exchange down 80 | compareRegions(a, i-offset, i, offset, round, r) 81 | } 82 | for i := 0; i < n-3*offset; i += offset { // compare 3 hops up 83 | compareRegions(a, i, i+3*offset, offset, round, r) 84 | } 85 | for i := 0; i < n-2*offset; i += offset { // compare 2 hops up 86 | compareRegions(a, i, i+2*offset, offset, round, r) 87 | } 88 | for i := 0; i < n; i += 2 * offset { // compare odd-even regions 89 | compareRegions(a, i, i+offset, offset, round, r) 90 | } 91 | for i := offset; i < n-offset; i += 2 * offset { // compare even-odd regions 92 | compareRegions(a, i, i+offset, offset, round, r) 93 | } 94 | 95 | round++ 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /shellsort/shell_test.go: -------------------------------------------------------------------------------- 1 | package shellsort 2 | 3 | import ( 4 | "math/rand" 5 | "reflect" 6 | "testing" 7 | "testing/quick" 8 | ) 9 | 10 | type bigarray []int 11 | 12 | func (b bigarray) Generate(rand *rand.Rand, size int) reflect.Value { 13 | nv := make([]int, size*1000) 14 | 15 | for i := range nv { 16 | nv[i] = rand.Intn(1000 * size) 17 | } 18 | 19 | return reflect.ValueOf(nv) 20 | } 21 | 22 | func testSort(t *testing.T, sort func(a []int), which string) { 23 | 24 | rand.Seed(0) 25 | 26 | f := func(a bigarray) bool { 27 | sort(a) 28 | for i := 1; i < len(a); i++ { 29 | if a[i-1] > a[i] { 30 | return false 31 | } 32 | } 33 | return true 34 | } 35 | 36 | if err := quick.Check(f, nil); err != nil { 37 | t.Errorf("%s: %v", which, err) 38 | } 39 | } 40 | 41 | func TestShell(t *testing.T) { testSort(t, Shell, "shell") } 42 | func TestRand(t *testing.T) { testSort(t, RandShell, "rand") } 43 | -------------------------------------------------------------------------------- /shlines/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/dchest/siphash" 9 | ) 10 | 11 | func main() { 12 | 13 | scan := bufio.NewScanner(os.Stdin) 14 | 15 | for scan.Scan() { 16 | u := siphash.Hash(0, 0, scan.Bytes()) 17 | fmt.Printf("%016x\n", u) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /shufsecs/shufsecs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | func outputShuf(currentSecond string, requests []string) { 13 | perm := rand.Perm(len(requests)) 14 | for _, v := range perm { 15 | fmt.Print(currentSecond, "\t", requests[v], "\n") 16 | } 17 | } 18 | 19 | func main() { 20 | scanner := bufio.NewScanner(os.Stdin) 21 | 22 | var requests []string 23 | var currentSecond string 24 | 25 | for scanner.Scan() { 26 | line := scanner.Text() 27 | fields := strings.SplitN(line, "\t", 3) 28 | if currentSecond != fields[0] { 29 | outputShuf(currentSecond, requests) 30 | currentSecond = fields[0] 31 | requests = requests[:0] 32 | } 33 | 34 | count := 1 35 | if len(fields) == 3 { 36 | count, _ = strconv.Atoi(fields[2]) 37 | } 38 | for i := 0; i < count; i++ { 39 | requests = append(requests, fields[1]) 40 | } 41 | } 42 | 43 | outputShuf(currentSecond, requests) 44 | } 45 | -------------------------------------------------------------------------------- /simhash/scanners.go: -------------------------------------------------------------------------------- 1 | package simhash 2 | 3 | // TODO: channel scanner 4 | 5 | // Return features one-at-a-time to be considered by SimHash. 6 | // This matches (partially) the scanner interface for bufio.Scanner, so those scanner can be reused here. 7 | type FeatureScanner interface { 8 | Scan() bool 9 | Bytes() []byte 10 | Err() error 11 | } 12 | 13 | type SliceScanner struct { 14 | tokens [][]byte 15 | i int 16 | } 17 | 18 | // NewSliceScanner creates a scanner that returns the byte slices in tokens 19 | func NewSliceScanner(tokens [][]byte) FeatureScanner { 20 | return &SliceScanner{tokens: tokens} 21 | } 22 | 23 | func (s *SliceScanner) Err() error { 24 | return nil 25 | } 26 | 27 | func (s *SliceScanner) Scan() bool { 28 | 29 | if s.i >= len(s.tokens) { 30 | return false 31 | } 32 | s.i++ 33 | return true 34 | } 35 | 36 | func (s *SliceScanner) Bytes() []byte { 37 | return s.tokens[s.i-1] 38 | } 39 | -------------------------------------------------------------------------------- /simhash/simhash.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package simhash implements the simhash document similarity hashing function. 3 | 4 | By itself, this isn't that useful. If I ever decide to create more fully 5 | featured SimHash implementation, it will get moved to its own repository. 6 | 7 | http://www.cs.princeton.edu/courses/archive/spr04/cos598B/bib/CharikarEstim.pdf 8 | http://infolab.stanford.edu/~manku/papers/07www-duplicates.pdf 9 | http://irl.cse.tamu.edu/people/sadhan/papers/cikm2011.pdf 10 | 11 | Left here for legacy reasons, but you probably want 12 | http://github.com/dgryski/go-simstore now. 13 | */ 14 | package simhash 15 | 16 | import ( 17 | "github.com/dchest/siphash" 18 | ) 19 | 20 | // SimHash returns a simhash value for the document returned by the scanner 21 | func SimHash(scanner FeatureScanner) uint64 { 22 | var signs [64]int64 23 | 24 | for scanner.Scan() { 25 | h := siphash.Hash(0, 0, scanner.Bytes()) 26 | 27 | for i := 0; i < 64; i++ { 28 | negate := int(h) & 1 29 | // if negate is 1, we will negate '-1', below 30 | r := (-1 ^ -negate) + negate 31 | signs[i] += int64(r) 32 | h >>= 1 33 | } 34 | } 35 | 36 | var shash uint64 37 | 38 | // TODO: can probably be done with SSE? 39 | for i := 63; i >= 0; i-- { 40 | shash <<= 1 41 | shash |= uint64(signs[i]>>63) & 1 42 | } 43 | 44 | return shash 45 | } 46 | 47 | func hammingDistance(v1 uint64, v2 uint64) int { 48 | // TODO: replace with magic multiplication or lookup? 49 | 50 | v := v1 ^ v2 51 | var c int 52 | for c = 0; v != 0; c++ { 53 | v &= v - 1 54 | } 55 | 56 | return c 57 | } 58 | -------------------------------------------------------------------------------- /simhash/simhash_test.go: -------------------------------------------------------------------------------- 1 | package simhash 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func simhashString(s string) uint64 { 11 | scanner := bufio.NewScanner(strings.NewReader(s)) 12 | scanner.Split(bufio.ScanWords) 13 | 14 | return SimHash(scanner) 15 | } 16 | 17 | func TestSimHash(t *testing.T) { 18 | 19 | h1 := simhashString("Now is the winter of our discontent and also the time for all good people to come to the aid of the party") 20 | fmt.Printf("h=%016x\n", h1) 21 | 22 | h2 := simhashString("Now is the winter of our discontent and also the time for all good people to come and party") 23 | fmt.Printf("h=%016x\n", h2) 24 | 25 | h3 := simhashString("The more we get together together together the more we get together the happier we'll be") 26 | fmt.Printf("h=%016x\n", h3) 27 | 28 | fmt.Printf("d(h1,h2)=%d\n", hammingDistance(h1, h2)) 29 | fmt.Printf("d(h1,h3)=%d\n", hammingDistance(h1, h3)) 30 | fmt.Printf("d(h2,h3)=%d\n", hammingDistance(h2, h3)) 31 | 32 | } 33 | -------------------------------------------------------------------------------- /sipsum/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "strconv" 11 | "strings" 12 | 13 | "github.com/dchest/siphash" 14 | ) 15 | 16 | func main() { 17 | 18 | keys := flag.String("k", "", "key0,key1") 19 | 20 | flag.Parse() 21 | 22 | var k0, k1 uint64 23 | 24 | if *keys != "" { 25 | var err error 26 | k0, k1, err = parseKeys(*keys) 27 | 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | } 32 | 33 | if flag.NArg() == 0 { 34 | hashOne(k0, k1, os.Stdin, "stdin") 35 | return 36 | } 37 | 38 | for _, arg := range flag.Args() { 39 | f, err := os.Open(arg) 40 | if err != nil { 41 | log.Println(f, err) 42 | continue 43 | } 44 | hashOne(k0, k1, f, arg) 45 | } 46 | } 47 | 48 | func hashOne(k0, k1 uint64, r io.Reader, name string) { 49 | buf, err := ioutil.ReadAll(r) 50 | if err != nil { 51 | log.Println(name, err) 52 | return 53 | } 54 | h := siphash.Hash(k0, k1, buf) 55 | fmt.Printf("%016x %s\n", h, name) 56 | } 57 | 58 | func parseKeys(keys string) (uint64, uint64, error) { 59 | 60 | kstr := strings.SplitN(keys, ",", 2) 61 | 62 | var k64 []uint64 63 | 64 | for _, k := range kstr { 65 | k0, err := strconv.ParseUint(k, 16, 64) 66 | if err != nil { 67 | return 0, 0, err 68 | } 69 | k64 = append(k64, k0) 70 | } 71 | 72 | return k64[0], k64[1], nil 73 | } 74 | -------------------------------------------------------------------------------- /skvchk/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "compress/gzip" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "log" 10 | "os" 11 | "strings" 12 | 13 | "github.com/dchest/siphash" 14 | "github.com/dgryski/go-onlinestats" 15 | "github.com/dgryski/go-shardedkv" 16 | "github.com/dgryski/go-shardedkv/choosers/chash" 17 | "github.com/dgryski/go-shardedkv/choosers/jump" 18 | "github.com/dgryski/go-shardedkv/choosers/ketama" 19 | ) 20 | 21 | func main() { 22 | 23 | chooserType := flag.String("chooser", "jump", "shardedkv chooser") 24 | expectedShard := flag.String("shard", "", "expected shard name") 25 | numShards := flag.Int("n", 16, "number of shards") 26 | fname := flag.String("f", "", "input file name") 27 | 28 | flag.Parse() 29 | 30 | var buckets []string 31 | for i := 1; i <= *numShards; i++ { 32 | buckets = append(buckets, fmt.Sprintf("shard%d", i)) 33 | } 34 | 35 | var chooser shardedkv.Chooser 36 | 37 | switch *chooserType { 38 | case "jump": 39 | chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) }) 40 | case "ketama": 41 | chooser = ketama.New() 42 | case "chash": 43 | chooser = chash.New() 44 | default: 45 | log.Fatalf("unknown chooser: %s; known jump,ketama,chash", *chooserType) 46 | } 47 | 48 | chooser.SetBuckets(buckets) 49 | 50 | results := make(map[string]int) 51 | 52 | var f io.Reader 53 | 54 | f, err := os.Open(*fname) 55 | if err != nil { 56 | log.Fatal("error loading input file", fname, ":", err) 57 | } 58 | 59 | if strings.HasSuffix(*fname, ".gz") { 60 | f, _ = gzip.NewReader(f) 61 | } 62 | 63 | scan := bufio.NewScanner(f) 64 | for scan.Scan() { 65 | shard := chooser.Choose(scan.Text()) 66 | if *expectedShard != "" && shard != *expectedShard { 67 | fmt.Printf("wrong shard %s %+v\n", shard, scan.Text()) 68 | } 69 | results[shard]++ 70 | } 71 | 72 | stats := onlinestats.NewRunning() 73 | 74 | fmt.Println("chooser", *chooserType) 75 | for _, b := range buckets { 76 | v := results[b] 77 | fmt.Printf("%-10s %d\n", b, v) 78 | stats.Push(float64(v)) 79 | } 80 | 81 | if *expectedShard == "" { 82 | fmt.Println("mean", stats.Mean()) 83 | fmt.Println("stdev", stats.Stddev(), stats.Stddev()/stats.Mean()) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /skvdist/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | 10 | "github.com/dchest/siphash" 11 | "github.com/dgryski/go-onlinestats" 12 | "github.com/dgryski/go-shardedkv" 13 | "github.com/dgryski/go-shardedkv/choosers/chash" 14 | "github.com/dgryski/go-shardedkv/choosers/jump" 15 | "github.com/dgryski/go-shardedkv/choosers/ketama" 16 | "github.com/dgryski/go-shardedkv/choosers/mpc" 17 | ) 18 | 19 | func main() { 20 | 21 | numBuckets := flag.Int("b", 8, "buckets") 22 | chooserType := flag.String("chooser", "jump", "shardedkv chooser") 23 | verbose := flag.Bool("v", false, "verbose") 24 | replicas := flag.Int("r", 0, "replicas") 25 | 26 | flag.Parse() 27 | 28 | var buckets []string 29 | for i := 0; i < *numBuckets; i++ { 30 | // buckets = append(buckets, fmt.Sprintf("%016x", siphash.Hash(0x0ddc0ffeebadf00d, 0xcafebabedeadbeef, []byte(fmt.Sprintf("192.168.0.%d", 10+i))))) 31 | // buckets = append(buckets, fmt.Sprintf("%016x", siphash.Hash(0, 0, []byte(fmt.Sprintf("192.168.0.%d", 10+i))))) 32 | buckets = append(buckets, fmt.Sprintf("192.168.0.%d", 10+i)) 33 | } 34 | 35 | var chooser shardedkv.Chooser 36 | 37 | switch *chooserType { 38 | case "jump": 39 | chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) }) 40 | case "ketama": 41 | chooser = ketama.New() 42 | case "mpc": 43 | chooser = mpc.New(func(b []byte, seed uint64) uint64 { return siphash.Hash(seed, 0, b) }, [2]uint64{0x0ddc0ffeebadf00d, 0xcafebabedeadbeef}, 21) 44 | case "chash": 45 | // 5120*8 + 5120*(8+16)*2 bytes 46 | chooser = chash.New() 47 | default: 48 | log.Fatalf("unknown chooser: %s; known jump,ketama,chash,mpc", *chooserType) 49 | } 50 | 51 | chooser.SetBuckets(buckets) 52 | 53 | results := make(map[string]int) 54 | 55 | scan := bufio.NewScanner(os.Stdin) 56 | for scan.Scan() { 57 | key := scan.Text() 58 | shard := chooser.Choose(key) 59 | if *verbose { 60 | fmt.Printf("%s %s\n", shard, key) 61 | } 62 | results[shard]++ 63 | } 64 | 65 | stats := onlinestats.NewRunning() 66 | 67 | min := int(^uint(0) / 2) 68 | max := 0 69 | fmt.Println("chooser", *chooserType) 70 | for _, b := range buckets { 71 | v := results[b] 72 | if v < min { 73 | min = v 74 | } 75 | if v > max { 76 | max = v 77 | } 78 | fmt.Printf("%-10s %d\n", b, v) 79 | stats.Push(float64(v)) 80 | } 81 | 82 | fmt.Printf("min=%d max=%d\n", min, max) 83 | fmt.Printf("mean %.2f\n", stats.Mean()) 84 | fmt.Printf("stdev %.2f %.2f%%\n", stats.Stddev(), 100*stats.Stddev()/stats.Mean()) 85 | fmt.Printf("peak/mean %.2f\n", float64(max)/stats.Mean()) 86 | } 87 | -------------------------------------------------------------------------------- /skvdist/skvdist.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | 4 | use strict; 5 | 6 | use ShardedKV::Continuum::Jump; 7 | 8 | my $continuum = ShardedKV::Continuum::Jump->new( 9 | from => { 10 | ids => [map "shard$_" , 1..32], 11 | } 12 | ); 13 | 14 | my $processed = 0; 15 | while(<>) { 16 | chomp; 17 | my ($shard, $key) = split / /, $_, 2; 18 | my $dst = $continuum->choose($key); 19 | die "mismatch for $key: $shard != $dst" if $shard ne $dst; 20 | $processed++; 21 | if (($processed % (1024*1024)) == 0) { 22 | print scalar localtime, " ", $processed, "\n"; 23 | } 24 | } 25 | 26 | print "processed $processed files\n"; 27 | -------------------------------------------------------------------------------- /sshdregex/asm/fsmasm_stub.go: -------------------------------------------------------------------------------- 1 | //go:build amd64 && !tinygo 2 | 3 | package asm 4 | 5 | //go:noescape 6 | func Match(data []byte) int 7 | -------------------------------------------------------------------------------- /sshdregex/asm/fsmasm_tinygo.go: -------------------------------------------------------------------------------- 1 | //go:build wasi || wasip1 || tinygo 2 | 3 | package asm 4 | 5 | func Match(data []byte) int { 6 | return 0 7 | } 8 | -------------------------------------------------------------------------------- /sshdregex/bench_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "regexp" 5 | "testing" 6 | 7 | "github.com/dgryski/trifles/sshdregex/asm" 8 | "github.com/dgryski/trifles/sshdregex/c" 9 | ) 10 | 11 | var data = []byte(`Jan 18 06:41:30 corecompute sshd[42327]: Failed keyboard-interactive/pam for root from 112.100.68.182 port 48803 ssh2`) 12 | 13 | var sink int 14 | 15 | func BenchmarkRagel(b *testing.B) { 16 | var hits int 17 | for i := 0; i < b.N; i++ { 18 | if MatchRagel(data) { 19 | hits++ 20 | } 21 | } 22 | 23 | sink += hits 24 | } 25 | 26 | func BenchmarkRegex(b *testing.B) { 27 | 28 | // Compile regular expression first 29 | r, _ := regexp.Compile(`sshd\[\d{5}\]:\s*Failed`) 30 | 31 | b.ResetTimer() 32 | 33 | var hits int 34 | 35 | for i := 0; i < b.N; i++ { 36 | if r.Match(data) { 37 | hits++ 38 | } 39 | } 40 | 41 | sink += hits 42 | } 43 | 44 | func BenchmarkFSM(b *testing.B) { 45 | var hits int 46 | 47 | for i := 0; i < b.N; i++ { 48 | var d []byte 49 | if i >= len(data) { 50 | d = data[i%len(data):] 51 | } else { 52 | d = data[:i] 53 | } 54 | 55 | if Match(d) != -1 { 56 | hits++ 57 | } 58 | } 59 | 60 | sink += hits 61 | } 62 | 63 | func BenchmarkFSMUnsafe(b *testing.B) { 64 | var hits int 65 | 66 | for i := 0; i < b.N; i++ { 67 | var d []byte 68 | if i >= len(data) { 69 | d = data[i%len(data):] 70 | } else { 71 | d = data[:i] 72 | } 73 | 74 | if UnsafeMatch(d) != -1 { 75 | hits++ 76 | } 77 | } 78 | 79 | sink += hits 80 | } 81 | 82 | func BenchmarkFSMC(b *testing.B) { 83 | var hits int 84 | 85 | for i := 0; i < b.N; i++ { 86 | var d []byte 87 | if i >= len(data) { 88 | d = data[i%len(data):] 89 | } else { 90 | d = data[:i] 91 | } 92 | 93 | if c.Match(d) != -1 { 94 | hits++ 95 | } 96 | } 97 | 98 | sink += hits 99 | } 100 | 101 | func BenchmarkMatchASM(b *testing.B) { 102 | var hits int 103 | 104 | for i := 0; i < b.N; i++ { 105 | if asm.Match(data) != -1 { 106 | hits++ 107 | } 108 | } 109 | 110 | sink += hits 111 | } 112 | -------------------------------------------------------------------------------- /sshdregex/c/c.go: -------------------------------------------------------------------------------- 1 | package c 2 | 3 | /* 4 | int sshdmain(const char *b, const char *e); 5 | */ 6 | import "C" 7 | 8 | import "unsafe" 9 | 10 | func Match(data []byte) int { 11 | var b, e unsafe.Pointer 12 | if len(data) > 0 { 13 | b = unsafe.Pointer(&data[0]) 14 | e = unsafe.Add(b, len(data)) 15 | } 16 | return int(C.sshdmain((*C.char)(b), (*C.char)(e))) 17 | } 18 | -------------------------------------------------------------------------------- /sshdregex/c/fsmc.c: -------------------------------------------------------------------------------- 1 | 2 | int 3 | sshdmain(const char *b, const char *e) 4 | { 5 | const char *p; 6 | int c; 7 | 8 | p = b; 9 | 10 | 11 | l0: 12 | if (p == e) return -1; 13 | 14 | c = (unsigned char) *p++; 15 | if (c != 's') goto l0; 16 | 17 | l1: /* e.g. "s" */ 18 | if (p == e) return -1; 19 | 20 | c = (unsigned char) *p++; 21 | if (c != 's') goto l0; 22 | 23 | l2: /* e.g. "ss" */ 24 | if (p == e) return -1; 25 | 26 | c = (unsigned char) *p++; 27 | if (c == 'h') goto l3; 28 | if (c == 's') goto l2; 29 | goto l0; 30 | 31 | l3: /* e.g. "ssh" */ 32 | if (p == e) return -1; 33 | 34 | c = (unsigned char) *p++; 35 | if (c == 'd') goto l4; 36 | if (c == 's') goto l1; 37 | goto l0; 38 | 39 | l4: /* e.g. "sshd" */ 40 | if (p == e) return -1; 41 | 42 | c = (unsigned char) *p++; 43 | if (c == '[') goto l5; 44 | if (c == 's') goto l1; 45 | goto l0; 46 | 47 | l5: /* e.g. "sshd[" */ 48 | if (p == e) return -1; 49 | 50 | c = (unsigned char) *p++; 51 | if (c <= '/') goto l0; 52 | if (c <= '9') goto l6; 53 | if (c <= 'r') goto l0; 54 | if (c == 's') goto l1; 55 | goto l0; 56 | 57 | l6: /* e.g. "sshd[0" */ 58 | if (p == e) return -1; 59 | 60 | c = (unsigned char) *p++; 61 | if (c <= '/') goto l0; 62 | if (c <= '9') goto l7; 63 | if (c <= 'r') goto l0; 64 | if (c == 's') goto l1; 65 | goto l0; 66 | 67 | l7: /* e.g. "sshd[00" */ 68 | if (p == e) return -1; 69 | 70 | c = (unsigned char) *p++; 71 | if (c <= '/') goto l0; 72 | if (c <= '9') goto l8; 73 | if (c <= 'r') goto l0; 74 | if (c == 's') goto l1; 75 | goto l0; 76 | 77 | l8: /* e.g. "sshd[000" */ 78 | if (p == e) return -1; 79 | 80 | c = (unsigned char) *p++; 81 | if (c <= '/') goto l0; 82 | if (c <= '9') goto l9; 83 | if (c <= 'r') goto l0; 84 | if (c == 's') goto l1; 85 | goto l0; 86 | 87 | l9: /* e.g. "sshd[0000" */ 88 | if (p == e) return -1; 89 | 90 | c = (unsigned char) *p++; 91 | if (c <= '/') goto l0; 92 | if (c <= '9') goto l10; 93 | if (c <= 'r') goto l0; 94 | if (c == 's') goto l1; 95 | goto l0; 96 | 97 | l10: /* e.g. "sshd[00000" */ 98 | if (p == e) return -1; 99 | 100 | c = (unsigned char) *p++; 101 | if (c == ']') goto l11; 102 | if (c == 's') goto l1; 103 | goto l0; 104 | 105 | l11: /* e.g. "sshd[00000]" */ 106 | if (p == e) return -1; 107 | 108 | c = (unsigned char) *p++; 109 | if (c == ':') goto l12; 110 | if (c == 's') goto l1; 111 | goto l0; 112 | 113 | l12: /* e.g. "sshd[00000]:" */ 114 | if (p == e) return -1; 115 | 116 | c = (unsigned char) *p++; 117 | if (c <= '\b') goto l0; 118 | if (c <= '\r') goto l12; 119 | if (c <= '\x1f') goto l0; 120 | if (c == ' ') goto l12; 121 | if (c <= 'E') goto l0; 122 | if (c == 'F') goto l13; 123 | if (c <= 'r') goto l0; 124 | if (c == 's') goto l1; 125 | goto l0; 126 | 127 | l13: /* e.g. "sshd[00000]:F" */ 128 | if (p == e) return -1; 129 | 130 | c = (unsigned char) *p++; 131 | if (c == 'a') goto l14; 132 | if (c == 's') goto l1; 133 | goto l0; 134 | 135 | l14: /* e.g. "sshd[00000]:Fa" */ 136 | if (p == e) return -1; 137 | 138 | c = (unsigned char) *p++; 139 | if (c == 'i') goto l15; 140 | if (c == 's') goto l1; 141 | goto l0; 142 | 143 | l15: /* e.g. "sshd[00000]:Fai" */ 144 | if (p == e) return -1; 145 | 146 | c = (unsigned char) *p++; 147 | if (c == 'l') goto l16; 148 | if (c == 's') goto l1; 149 | goto l0; 150 | 151 | l16: /* e.g. "sshd[00000]:Fail" */ 152 | if (p == e) return -1; 153 | 154 | c = (unsigned char) *p++; 155 | if (c == 'e') goto l17; 156 | if (c == 's') goto l1; 157 | goto l0; 158 | 159 | l17: /* e.g. "sshd[00000]:Faile" */ 160 | if (p == e) return -1; 161 | 162 | c = (unsigned char) *p++; 163 | if (c == 'd') goto l18; 164 | if (c == 's') goto l1; 165 | goto l0; 166 | 167 | l18: /* e.g. "sshd[00000]:Failed" */ 168 | return 0x1; /* "sshd\[\d{5}\]:\s*Failed" */ 169 | } 170 | 171 | -------------------------------------------------------------------------------- /sshdregex/gen.sh: -------------------------------------------------------------------------------- 1 | re -r pcre -k pair -e '' -E main -pl go 'sshd\[\d{5}\]:\s*Failed' |sed -e 's/package mainfsm/package main/' >fsm.go 2 | re -r pcre -k pair -e 'Unsafe' -E main -pl go_unsafe 'sshd\[\d{5}\]:\s*Failed' |sed -e 's/package mainfsm/package main/' >fsmunsafe.go 3 | re -r pcre -k pair -e sshd -pl vmc 'sshd\[\d{5}\]:\s*Failed' > c/fsmc.c 4 | re -r pcre -k pair -e '' -E asm -pl amd64_go 'sshd\[\d{5}\]:\s*Failed' > asm/fsmasm_amd64.s 5 | ragel-go -G2 ragel.rl && gofmt -w ragel.go 6 | -------------------------------------------------------------------------------- /sshdregex/ragel.rl: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //go:nobounds 4 | func MatchRagel(data []byte) bool { 5 | 6 | %% machine scanner; 7 | %% write data; 8 | 9 | cs, p, pe, eof := 0, 0, len(data), len(data) 10 | 11 | _ = eof 12 | 13 | %%{ 14 | main := any* 'sshd[' digit{5} ']:' space* 'Failed' @{ return true } ; 15 | 16 | write init; 17 | write exec; 18 | }%% 19 | 20 | return false 21 | } 22 | -------------------------------------------------------------------------------- /stablepart/bench/bench_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | ) 7 | 8 | func benchmarkInts(b *testing.B, n int) { 9 | 10 | rand.Seed(0) 11 | 12 | ints := make([]int, n) 13 | for i := range ints { 14 | ints[i] = rand.Intn(1000) 15 | } 16 | 17 | orig := make([]int, n) 18 | copy(orig, ints) 19 | 20 | b.ResetTimer() 21 | 22 | for i := 0; i < b.N; i++ { 23 | stablePartitionInts(ints, 0, len(ints), func(i int) bool { return i&1 == 0 }) 24 | copy(ints, orig) 25 | } 26 | } 27 | 28 | func benchmarkMove(b *testing.B, n int) { 29 | 30 | rand.Seed(0) 31 | 32 | ints := make([]int, n) 33 | for i := range ints { 34 | ints[i] = rand.Intn(1000) 35 | } 36 | 37 | orig := make([]int, n) 38 | copy(orig, ints) 39 | 40 | b.ResetTimer() 41 | 42 | for i := 0; i < b.N; i++ { 43 | stablePartitionMove(ints, len(ints), func(i int) bool { return i&1 == 0 }) 44 | copy(ints, orig) 45 | } 46 | } 47 | 48 | func BenchmarkInts10(b *testing.B) { benchmarkInts(b, 10) } 49 | func BenchmarkInts20(b *testing.B) { benchmarkInts(b, 20) } 50 | func BenchmarkInts50(b *testing.B) { benchmarkInts(b, 50) } 51 | func BenchmarkInts100(b *testing.B) { benchmarkInts(b, 100) } 52 | func BenchmarkInts200(b *testing.B) { benchmarkInts(b, 200) } 53 | func BenchmarkInts500(b *testing.B) { benchmarkInts(b, 500) } 54 | func BenchmarkInts1000(b *testing.B) { benchmarkInts(b, 1000) } 55 | func BenchmarkInts2000(b *testing.B) { benchmarkInts(b, 2000) } 56 | func BenchmarkInts5000(b *testing.B) { benchmarkInts(b, 5000) } 57 | func BenchmarkInts10000(b *testing.B) { benchmarkInts(b, 10000) } 58 | 59 | func BenchmarkMove10(b *testing.B) { benchmarkMove(b, 10) } 60 | func BenchmarkMove20(b *testing.B) { benchmarkMove(b, 20) } 61 | func BenchmarkMove50(b *testing.B) { benchmarkMove(b, 50) } 62 | func BenchmarkMove100(b *testing.B) { benchmarkMove(b, 100) } 63 | func BenchmarkMove200(b *testing.B) { benchmarkMove(b, 200) } 64 | func BenchmarkMove500(b *testing.B) { benchmarkMove(b, 500) } 65 | func BenchmarkMove1000(b *testing.B) { benchmarkMove(b, 1000) } 66 | func BenchmarkMove2000(b *testing.B) { benchmarkMove(b, 2000) } 67 | func BenchmarkMove5000(b *testing.B) { benchmarkMove(b, 5000) } 68 | func BenchmarkMove10000(b *testing.B) { benchmarkMove(b, 10000) } 69 | -------------------------------------------------------------------------------- /stablepart/bench/part.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "log" 4 | 5 | func stablePartitionInts(d []int, f, l int, p func(i int) bool) int { 6 | 7 | n := l - f 8 | if n == 0 { 9 | return f 10 | } 11 | 12 | if n == 1 { 13 | r := f 14 | if p(d[f]) { 15 | r++ 16 | } 17 | return r 18 | } 19 | 20 | m := f + n/2 21 | 22 | return rotateInts(d, stablePartitionInts(d, f, m, p), m, stablePartitionInts(d, m, l, p)) 23 | } 24 | 25 | func rotateInts(d []int, f, k, l int) int { 26 | reverseInts(d, f, k) 27 | reverseInts(d, k, l) 28 | reverseInts(d, f, l) 29 | return f + l - k 30 | } 31 | 32 | func reverseInts(d []int, f, l int) { 33 | lend := (l - f) 34 | for i, j := 0, lend-1; i < lend/2; i, j = i+1, j-1 { 35 | d[f+i], d[f+j] = d[f+j], d[f+i] 36 | } 37 | } 38 | 39 | func stablePartitionMove(array []int, sz int, p func(i int) bool) { 40 | 41 | var shift_index int 42 | for shift_index < sz && p(array[shift_index]) { 43 | shift_index++ 44 | } 45 | 46 | index := shift_index + 1 47 | 48 | for index < sz { 49 | // find the next number that needs to be shifted left 50 | if !p(array[index]) { 51 | index++ 52 | } else { 53 | value := array[index] 54 | num_elements_to_move := index - shift_index 55 | copy(array[shift_index+1:shift_index+1+num_elements_to_move], array[shift_index:shift_index+num_elements_to_move]) 56 | array[shift_index] = value // move the element to the hole left at the end 57 | shift_index++ 58 | index++ 59 | } 60 | } 61 | } 62 | 63 | func main() { 64 | array := []int{3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9} 65 | 66 | ints := append([]int(nil), array...) 67 | stablePartitionMove(ints, len(ints), func(i int) bool { return i%2 == 0 }) 68 | log.Printf("ints=%+v\n", ints) 69 | 70 | copy(ints, array) 71 | stablePartitionMove(ints, len(ints), func(i int) bool { return i%2 == 0 }) 72 | log.Printf("ints=%+v\n", ints) 73 | } 74 | -------------------------------------------------------------------------------- /stablepart/stablepart.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "math/rand" 7 | "sort" 8 | ) 9 | 10 | func StablePartition(d sort.Interface, f, l int, p func(i int) bool) int { 11 | 12 | n := l - f 13 | if n == 0 { 14 | return f 15 | } 16 | 17 | if n == 1 { 18 | r := f 19 | if p(f) { 20 | r++ 21 | } 22 | return r 23 | } 24 | 25 | m := f + n/2 26 | 27 | return Rotate(d, StablePartition(d, f, m, p), m, StablePartition(d, m, l, p)) 28 | } 29 | 30 | func Rotate(d sort.Interface, f, k, l int) int { 31 | Reverse(d, f, k) 32 | Reverse(d, k, l) 33 | Reverse(d, f, l) 34 | return f + l - k 35 | } 36 | 37 | func Reverse(d sort.Interface, f, l int) { 38 | lend := (l - f) 39 | for i, j := 0, lend-1; i < lend/2; i, j = i+1, j-1 { 40 | d.Swap(f+i, f+j) 41 | } 42 | } 43 | 44 | type element struct { 45 | val int 46 | idx int 47 | } 48 | 49 | type elements []element 50 | 51 | func (e elements) Len() int { return len(e) } 52 | func (e elements) Swap(i, j int) { e[i], e[j] = e[j], e[i] } 53 | func (e elements) Less(i, j int) bool { return e[i].val < e[j].val } 54 | 55 | func verify(pre, elts elements, f, l int, which string, p func(i int) bool) { 56 | 57 | // check the predicate 58 | for i := f; i < l; i++ { 59 | if !p(f) { 60 | fmt.Printf("%s predicate failed %v\n", which, elts[0]) 61 | fmt.Printf("pre =%+v\n", pre) 62 | fmt.Printf("elts=%+v\n", elts) 63 | log.Fatalln("verify failed") 64 | } 65 | 66 | } 67 | 68 | // check the order 69 | for i := f + 1; i < l; i++ { 70 | if elts[i-1].idx > elts[i].idx { 71 | fmt.Printf("%s partition mismatch: %v %v\n", which, elts[i-1], elts[i]) 72 | fmt.Printf("pre =%+v\n", pre) 73 | fmt.Printf("elts=%+v\n", elts) 74 | log.Fatalln("verify failed") 75 | } 76 | } 77 | } 78 | 79 | func main() { 80 | 81 | const numTests = 100000 82 | 83 | for j := 0; j < numTests; j++ { 84 | 85 | // generate random slice 86 | sz := rand.Intn(100) 87 | elts := make(elements, sz) 88 | for i := range elts { 89 | elts[i].val = rand.Intn(1000) 90 | elts[i].idx = i 91 | } 92 | 93 | // save a copy 94 | pre := make(elements, sz) 95 | copy(pre, elts) 96 | 97 | predicate := func(i int) bool { return elts[i].val&1 == 0 } 98 | 99 | // partition 100 | p := StablePartition(elts, 0, elts.Len(), predicate) 101 | 102 | // verify it's correct 103 | verify(pre, elts, 0, p, "pre", predicate) 104 | verify(pre, elts, p, sz, "post", func(i int) bool { return !predicate(i) }) 105 | } 106 | 107 | fmt.Printf("%d tests succeeded\n", numTests) 108 | } 109 | -------------------------------------------------------------------------------- /staticset/staticset.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "log" 7 | "math" 8 | ) 9 | 10 | func search(s []uint32, nbits uint) uint32 { 11 | 12 | slots := 1 << nbits 13 | 14 | log.Printf("searching for multiplier to fit %d keys in %d slots\n", len(s), slots) 15 | 16 | if len(s) > slots { 17 | return 0 18 | } 19 | 20 | dst := make([]uint32, slots) 21 | 22 | for i := 1; i < math.MaxUint32; i++ { 23 | 24 | if i&((1<<20)-1) == 0 { 25 | log.Println(i) 26 | } 27 | 28 | // try with multiplier `i` 29 | for d := range dst { 30 | dst[d] = 0 31 | } 32 | 33 | var placed int 34 | for _, v := range s { 35 | h := (v * uint32(i) >> (32 - nbits)) 36 | if dst[h] != 0 { 37 | break 38 | } 39 | dst[h] = v 40 | placed++ 41 | } 42 | if placed == len(s) { 43 | return uint32(i) 44 | } 45 | } 46 | return 0 47 | } 48 | 49 | func u32(s string) uint32 { 50 | return binary.BigEndian.Uint32([]byte(s)) 51 | } 52 | 53 | func main() { 54 | 55 | keys := []string{ 56 | "SIP/", 57 | "INVI", 58 | "ACK ", 59 | "CANC", 60 | "BYE ", 61 | "PRAC", 62 | "REGI", 63 | "OPTI", 64 | "INFO", 65 | "UPDA", 66 | "SUBS", 67 | "NOTI", 68 | "MESS", 69 | "REFE", 70 | "PUBL", 71 | } 72 | 73 | var k32s []uint32 74 | 75 | for _, s := range keys { 76 | k32s = append(k32s, u32(s)) 77 | } 78 | 79 | m := search(k32s, 4) 80 | 81 | if m == 0 { 82 | log.Fatalf("unable to find multiplier") 83 | } 84 | 85 | fmt.Println("m=", m) 86 | for _, v := range k32s { 87 | fmt.Println((v * m) >> 28) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /superbat/README: -------------------------------------------------------------------------------- 1 | Playing around with batmanjs and go-restful. 2 | 3 | The batmanjs example is from http://www.speg.com/batman/ 4 | 5 | The go code is modified sample from https://github.com/emicklei/go-restful/blob/master/examples/restful-user-resource.go 6 | -------------------------------------------------------------------------------- /superbat/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Batman! 6 | 7 | 8 | 9 | 10 | 76 | 77 | 78 | 79 |
80 |
81 | 82 |
83 | 84 | 89 | 90 | Stats 91 |
92 | 93 |
94 | Number of heroes:
95 | Average Power:
96 | back 97 |
98 | 99 |
100 |
101 | Hero: 102 | Power: (reroll) 103 |
104 | back 105 |
106 | 107 |
108 |
109 | Hero:
110 | Power:
111 |
112 | back 113 |
114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /superbat/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "net/http" 7 | "strconv" 8 | 9 | "github.com/go-martini/martini" 10 | "github.com/martini-contrib/render" 11 | ) 12 | 13 | type HeroDB interface{} 14 | 15 | type heroDB struct { 16 | heros map[int]Hero 17 | numHeros int 18 | } 19 | 20 | type Hero struct { 21 | Id int 22 | Name string 23 | Power int 24 | } 25 | 26 | type BatmanRequest struct { 27 | Hero Hero 28 | } 29 | 30 | func allHeros(db HeroDB, parms martini.Params, r render.Render) { 31 | u := (db).(*heroDB) 32 | var heros []Hero 33 | 34 | for _, v := range u.heros { 35 | heros = append(heros, v) 36 | } 37 | r.JSON(http.StatusOK, heros) 38 | } 39 | 40 | func findHero(db HeroDB, parms martini.Params, r render.Render) { 41 | u := (db).(*heroDB) 42 | 43 | id, _ := strconv.Atoi(parms["id"]) 44 | 45 | hero, ok := u.heros[id] 46 | if !ok { 47 | r.Error(http.StatusNotFound) 48 | return 49 | } 50 | 51 | r.JSON(http.StatusOK, hero) 52 | } 53 | 54 | func updateHero(db HeroDB, req *http.Request, r render.Render) { 55 | u := (db).(*heroDB) 56 | 57 | breq := new(BatmanRequest) 58 | body, _ := ioutil.ReadAll(req.Body) 59 | req.Body.Close() 60 | 61 | err := json.Unmarshal(body, &breq) 62 | hero := breq.Hero 63 | 64 | if err != nil { 65 | r.Error(http.StatusInternalServerError) 66 | return 67 | } 68 | 69 | u.heros[hero.Id] = hero 70 | r.JSON(http.StatusOK, hero) 71 | } 72 | 73 | func createHero(db HeroDB, req *http.Request, r render.Render) { 74 | u := (db).(*heroDB) 75 | 76 | breq := new(BatmanRequest) 77 | body, _ := ioutil.ReadAll(req.Body) 78 | req.Body.Close() 79 | err := json.Unmarshal(body, &breq) 80 | hero := breq.Hero 81 | 82 | u.numHeros++ 83 | 84 | hero.Id = u.numHeros 85 | 86 | if err != nil { 87 | r.Error(http.StatusInternalServerError) 88 | } 89 | 90 | u.heros[hero.Id] = hero 91 | r.JSON(http.StatusCreated, hero) 92 | } 93 | 94 | func removeHero(db HeroDB, parms martini.Params, r render.Render) { 95 | u := (db).(*heroDB) 96 | id, _ := strconv.Atoi(parms["id"]) 97 | 98 | delete(u.heros, id) 99 | 100 | r.JSON(http.StatusOK, nil) 101 | } 102 | 103 | func main() { 104 | 105 | m := martini.Classic() 106 | 107 | m.Use(render.Renderer()) 108 | 109 | // Setup routes 110 | r := martini.NewRouter() 111 | r.Get(`/api/heros`, allHeros) 112 | r.Get(`/api/heros/:id`, findHero) 113 | r.Post(`/api/heros`, createHero) 114 | r.Put(`/api/heros/:id`, updateHero) 115 | r.Delete(`/api/heros/:id`, removeHero) 116 | 117 | // Inject database 118 | u := &heroDB{make(map[int]Hero), 0} 119 | m.MapTo(u, (*HeroDB)(nil)) 120 | 121 | // Add the router action 122 | m.Action(r.Handle) 123 | 124 | m.Run() 125 | } 126 | -------------------------------------------------------------------------------- /superbat/public: -------------------------------------------------------------------------------- 1 | . -------------------------------------------------------------------------------- /testlibfsm/fishasm/fishasm_stub.go: -------------------------------------------------------------------------------- 1 | package fishasm 2 | 3 | //go:noescape 4 | func Match(data []byte) int 5 | -------------------------------------------------------------------------------- /testlibfsm/fuzz.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x -e 4 | 5 | iter=30 6 | items=10 7 | 8 | input=../matcher/domains.txt 9 | regex_opt=regex-opt 10 | 11 | for i in $(seq $iter); do 12 | $regex_opt $(shuf $input |head -n $items | tr '\n' '|' | sed 's/|$//; s/\./\\./g; ') >regexp.in 13 | re -r pcre -l go -k pair -e fish "$(cat regexp.in)" >fishfsm/fishfsm.go 14 | re -r pcre -l amd64_go -k pair -e fish "$(cat regexp.in)" >fishasm/fishasm.s 15 | re -r pcre -l vmc -k pair -e fish "$(cat regexp.in)" >fishrx.c 16 | printf 'package main; import "regexp"; var fishrx = regexp.MustCompile(`%s`);' "$(cat regexp.in)" > fishrx.go 17 | go build 18 | ./testlibfsm 19 | done 20 | -------------------------------------------------------------------------------- /testlibfsm/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "log" 7 | "unsafe" 8 | 9 | "github.com/dgryski/trifles/testlibfsm/fishasm" 10 | "github.com/dgryski/trifles/testlibfsm/fishfsm" 11 | ) 12 | 13 | /* 14 | 15 | int fishmain(const char *b, const char *e); 16 | 17 | */ 18 | import "C" 19 | 20 | func main() { 21 | input := flag.String("-i", "../matcher/domains.txt", "input file") 22 | flag.Parse() 23 | 24 | data, err := ioutil.ReadFile(*input) 25 | if err != nil { 26 | log.Fatalf("Error reading %q: %v", *input, err) 27 | } 28 | 29 | var b []byte 30 | var caughtfsm int 31 | var caughtasm int 32 | var caughtvmc int 33 | var caughtrx int 34 | 35 | b = data 36 | for len(b) > 20 { 37 | if fishfsm.Match(b) != -1 { 38 | caughtfsm++ 39 | } 40 | b = b[20:] 41 | } 42 | 43 | b = data 44 | for len(b) > 20 { 45 | if fishasm.Match(b) != -1 { 46 | caughtasm++ 47 | } 48 | b = b[20:] 49 | } 50 | 51 | b = data 52 | for len(b) > 20 { 53 | if vmcMatch(b) != -1 { 54 | caughtvmc++ 55 | } 56 | b = b[20:] 57 | } 58 | 59 | b = data 60 | for len(b) > 20 { 61 | if fishrx.Match(b) { 62 | caughtrx++ 63 | } 64 | b = b[20:] 65 | } 66 | 67 | if caughtfsm != caughtasm || caughtasm != caughtrx || caughtfsm != caughtvmc { 68 | log.Fatalf("matcher count mismatch: fsm=%d asm=%d vmc=%d rx=%d", caughtfsm, caughtasm, caughtvmc, caughtrx) 69 | } 70 | } 71 | 72 | func vmcMatch(s []byte) int { 73 | return int(C.fishmain((*C.char)(unsafe.Pointer(&s[0])), (*C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(&s[0]))+uintptr(len(s))*unsafe.Sizeof(s[0]))))) 74 | } 75 | -------------------------------------------------------------------------------- /tinyjsonbench/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "embed" 5 | "encoding/json" 6 | "flag" 7 | ) 8 | 9 | //go:embed repos.json 10 | var response []byte 11 | 12 | func main() { 13 | n := flag.Int("n", 1000, "number of iterations") 14 | flag.Parse() 15 | 16 | var total int 17 | 18 | var repos []*Repository 19 | for i := 0; i < *n; i++ { 20 | if err := json.Unmarshal(response, &repos); err != nil { 21 | println("error unmarshaling", err) 22 | return 23 | } 24 | total += len(repos) 25 | } 26 | println(total) 27 | } 28 | -------------------------------------------------------------------------------- /tinyjsonbench/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | n=20 4 | 5 | for t in new; do 6 | echo "build $t compiler, then press enter" 7 | read 8 | tinygo build -o tinyjsonbench.$t 9 | echo -n "warmup $t: " 10 | for i in $(seq 3); do ./tinyjsonbench.$t >/dev/null; echo -n "."; done; echo 11 | echo -n "benchmark $t: " 12 | for i in $(seq $n); do time ./tinyjsonbench.$t >/dev/null ; echo -n "."; done 2> $t.out; echo 13 | awk '/real/{sub(/0m/, "", $2); sub(/s/, "", $2); print $2}' $t.out > $t.times 14 | dist < $t.times 15 | done 16 | 17 | tinystat old.times new.times 18 | -------------------------------------------------------------------------------- /toepoch/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | func main() { 14 | 15 | field := flag.Int("f", 0, "field to transform") 16 | format := flag.String("d", "", "date format") 17 | 18 | flag.Parse() 19 | 20 | scanner := bufio.NewScanner(os.Stdin) 21 | 22 | var d time.Time 23 | var prev string 24 | 25 | for scanner.Scan() { 26 | fields := strings.Fields(scanner.Text()) 27 | t := fields[*field] 28 | if t != prev { 29 | var err error 30 | d, err = time.Parse(*format, t) 31 | if err != nil { 32 | log.Println("time.Parse():", err) 33 | continue 34 | } 35 | } 36 | 37 | fields[*field] = fmt.Sprint(d.Unix()) 38 | fmt.Printf("%s\n", strings.Join(fields, " ")) 39 | prev = t 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /toms/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | func main() { 14 | 15 | field := flag.Int("f", 0, "field to transform") 16 | dur := flag.Duration("d", 1*time.Millisecond, "output duration") 17 | 18 | flag.Parse() 19 | 20 | scanner := bufio.NewScanner(os.Stdin) 21 | 22 | if *field == 0 { 23 | for scanner.Scan() { 24 | 25 | d, err := time.ParseDuration(scanner.Text()) 26 | if err != nil { 27 | log.Println("time.Parse():", err) 28 | continue 29 | } 30 | 31 | fmt.Println(int(float64(d.Nanoseconds())/float64(*dur) + 0.5)) 32 | } 33 | 34 | return 35 | } 36 | 37 | for scanner.Scan() { 38 | 39 | var d time.Duration 40 | 41 | var duration string 42 | 43 | fields := strings.Fields(scanner.Text()) 44 | duration = fields[*field] 45 | d, err := time.ParseDuration(duration) 46 | if err != nil { 47 | log.Println("time.Parse():", err) 48 | continue 49 | } 50 | 51 | fields[*field] = fmt.Sprint(int(float64(d.Nanoseconds())/float64(*dur) + 0.5)) 52 | fmt.Printf("%s\n", strings.Join(fields, " ")) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /topkerr/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "log" 7 | "os" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | func main() { 13 | 14 | k := flag.Int("n", 500, "k") 15 | goldenName := flag.String("golden", "", "golden file to read") 16 | gotName := flag.String("f", "", "file name to test") 17 | 18 | flag.Parse() 19 | 20 | golden := loadCounts(*goldenName, *k*2) 21 | got := loadCounts(*gotName, *k) 22 | 23 | var total int 24 | 25 | for k, v := range got { 26 | e := golden[k] - v 27 | total += e * e 28 | delete(got, k) 29 | delete(golden, k) 30 | } 31 | 32 | // add the missing 33 | if false { 34 | for _, v := range golden { 35 | total += v * v 36 | } 37 | } 38 | 39 | log.Printf("float64(total)/float64(*k) = %+v\n", float64(total)/float64(*k)) 40 | } 41 | 42 | func loadCounts(fname string, k int) map[string]int { 43 | 44 | f, err := os.Open(fname) 45 | if err != nil { 46 | log.Fatal(err) 47 | } 48 | 49 | counts := make(map[string]int) 50 | 51 | sc := bufio.NewScanner(f) 52 | 53 | var lines int 54 | 55 | for sc.Scan() { 56 | line := sc.Text() 57 | tab := strings.IndexByte(line, ' ') 58 | count, _ := strconv.Atoi(line[tab+1:]) 59 | item := line[:tab] 60 | counts[item] = count 61 | lines++ 62 | if lines > k { 63 | break 64 | } 65 | } 66 | 67 | if err := sc.Err(); err != nil { 68 | log.Fatal(err) 69 | } 70 | 71 | return counts 72 | } 73 | -------------------------------------------------------------------------------- /tsip/Makefile: -------------------------------------------------------------------------------- 1 | 2 | CFLAGS=-O2 3 | 4 | .c.o: 5 | g++ -c $(CFLAGS) $< -o $@ 6 | 7 | bench: main.cc tsip.o oaath.o oaathu.o 8 | g++ $(CFLAGS) $^ -o $@ -lbenchmark -lpthread 9 | 10 | tsiptest: tsip.o tsip_test.o 11 | g++ $(CFLAGS) $^ -o $@ 12 | -------------------------------------------------------------------------------- /tsip/bench.txt: -------------------------------------------------------------------------------- 1 | Benchmark Time CPU Iterations 2 | ----------------------------------------------------- 3 | Hash_tsip/1 8 ns 8 ns 85827082 4 | Hash_tsip/2 8 ns 8 ns 83002087 5 | Hash_tsip/3 9 ns 9 ns 78468853 6 | Hash_tsip/4 9 ns 9 ns 80605654 7 | Hash_tsip/5 9 ns 9 ns 78349997 8 | Hash_tsip/6 11 ns 11 ns 74711545 9 | Hash_tsip/7 11 ns 11 ns 62197898 10 | Hash_tsip/8 10 ns 10 ns 66830457 11 | Hash_tsip/9 10 ns 10 ns 70486611 12 | Hash_tsip/10 10 ns 10 ns 69089089 13 | Hash_tsip/15 13 ns 13 ns 58135275 14 | Hash_tsip/20 13 ns 13 ns 55830703 15 | Hash_tsip/30 15 ns 15 ns 42539494 16 | Hash_tsip/40 20 ns 20 ns 34829886 17 | Hash_tsip/50 23 ns 23 ns 31133988 18 | Hash_tsip/100 38 ns 38 ns 17774835 19 | Hash_oaath/1 8 ns 8 ns 78008590 20 | Hash_oaath/2 10 ns 10 ns 69254103 21 | Hash_oaath/3 11 ns 11 ns 63823175 22 | Hash_oaath/4 13 ns 13 ns 55653548 23 | Hash_oaath/5 14 ns 14 ns 48008833 24 | Hash_oaath/6 16 ns 16 ns 44874644 25 | Hash_oaath/7 17 ns 17 ns 39378575 26 | Hash_oaath/8 19 ns 19 ns 35293509 27 | Hash_oaath/9 21 ns 21 ns 34424358 28 | Hash_oaath/10 23 ns 23 ns 30490933 29 | Hash_oaath/15 32 ns 32 ns 22460797 30 | Hash_oaath/20 43 ns 43 ns 15742641 31 | Hash_oaath/30 62 ns 62 ns 11443452 32 | Hash_oaath/40 75 ns 75 ns 9202069 33 | Hash_oaath/50 90 ns 90 ns 7423678 34 | Hash_oaath/100 165 ns 165 ns 3701639 35 | Hash_oaathu/0 7 ns 7 ns 95071806 36 | Hash_oaathu/1 8 ns 8 ns 84811197 37 | Hash_oaathu/2 10 ns 10 ns 72640609 38 | Hash_oaathu/3 11 ns 11 ns 64820437 39 | Hash_oaathu/4 13 ns 13 ns 55556969 40 | Hash_oaathu/5 14 ns 14 ns 47807668 41 | Hash_oaathu/6 16 ns 16 ns 43074336 42 | Hash_oaathu/7 17 ns 17 ns 39150325 43 | Hash_oaathu/8 19 ns 19 ns 37578063 44 | Hash_oaathu/9 21 ns 21 ns 33491817 45 | Hash_oaathu/10 22 ns 22 ns 30764955 46 | Hash_oaathu/11 23 ns 23 ns 28735454 47 | Hash_oaathu/12 26 ns 26 ns 27742854 48 | Hash_oaathu/13 27 ns 27 ns 25150772 49 | Hash_oaathu/14 29 ns 29 ns 24143989 50 | Hash_oaathu/15 32 ns 32 ns 21441371 51 | Hash_oaathu/16 34 ns 34 ns 19536187 52 | -------------------------------------------------------------------------------- /tsip/go/asm.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "strconv" 7 | 8 | . "github.com/mmcloughlin/avo/build" 9 | . "github.com/mmcloughlin/avo/operand" 10 | . "github.com/mmcloughlin/avo/reg" 11 | ) 12 | 13 | func sipround(v0, v1 Register) { 14 | ADDQ(v1, v0) 15 | 16 | ROLQ(Imm(13), v1) 17 | XORQ(v0, v1) 18 | 19 | ROLQ(Imm(35), v0) 20 | ADDQ(v1, v0) 21 | 22 | ROLQ(Imm(17), v1) 23 | XORQ(v0, v1) 24 | 25 | ROLQ(Imm(21), v0) 26 | } 27 | 28 | func main() { 29 | Package("github.com/dgryski/trifles/tsip/go/") 30 | 31 | TEXT("HashASM", NOSPLIT, "func(k0, k1 uint64, p []byte) uint64") 32 | 33 | reg_v0 := GP64() 34 | reg_v1 := GP64() 35 | 36 | Load(Param("k0"), reg_v0) 37 | Load(Param("k1"), reg_v1) 38 | 39 | reg_magic := GP64() 40 | MOVQ(Imm(0x736f6d6570736575), reg_magic) 41 | XORQ(reg_magic, reg_v0) 42 | MOVQ(Imm(0x646f72616e646f6d), reg_magic) 43 | XORQ(reg_magic, reg_v1) 44 | 45 | reg_p := GP64() 46 | reg_p_len := GP64() 47 | 48 | Load(Param("p").Base(), reg_p) 49 | Load(Param("p").Len(), reg_p_len) 50 | 51 | reg_b := GP64() 52 | MOVQ(reg_p_len, reg_b) 53 | SHLQ(Imm(56), reg_b) 54 | 55 | reg_m := GP64() 56 | 57 | loop_end := "loop_end" 58 | loop_begin := "loop_begin" 59 | CMPQ(reg_p_len, Imm(8)) 60 | JL(LabelRef(loop_end)) 61 | 62 | Label(loop_begin) 63 | MOVQ(Mem{Base: reg_p}, reg_m) 64 | XORQ(reg_m, reg_v1) 65 | sipround(reg_v0, reg_v1) 66 | XORQ(reg_m, reg_v0) 67 | 68 | ADDQ(Imm(8), reg_p) 69 | SUBQ(Imm(8), reg_p_len) 70 | CMPQ(reg_p_len, Imm(8)) 71 | JGE(LabelRef(loop_begin)) 72 | Label(loop_end) 73 | 74 | var labels []string 75 | for i := 0; i < 8; i++ { 76 | labels = append(labels, "sw"+strconv.Itoa(i)) 77 | } 78 | 79 | for i := 0; i < 7; i++ { 80 | CMPQ(reg_p_len, Imm(uint64(i))) 81 | JE(LabelRef(labels[i])) 82 | } 83 | 84 | char := GP64() 85 | for i := 7; i > 0; i-- { 86 | Label(labels[i]) 87 | MOVBQZX(Mem{Base: reg_p, Disp: i - 1}, char) 88 | SHLQ(Imm(uint64(i-1)*8), char) 89 | ORQ(char, reg_b) 90 | 91 | } 92 | 93 | Label(labels[0]) 94 | 95 | XORQ(reg_b, reg_v1) 96 | sipround(reg_v0, reg_v1) 97 | XORQ(reg_b, reg_v0) 98 | 99 | XORQ(Imm(0xff), reg_v1) 100 | sipround(reg_v0, reg_v1) 101 | ROLQ(Imm(32), reg_v1) 102 | sipround(reg_v0, reg_v1) 103 | ROLQ(Imm(32), reg_v1) 104 | 105 | XORQ(reg_v1, reg_v0) 106 | 107 | Store(reg_v0, ReturnIndex(0)) 108 | RET() 109 | 110 | Generate() 111 | } 112 | -------------------------------------------------------------------------------- /tsip/go/cmd/mapdist/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/dchest/siphash" 10 | "github.com/dgryski/go-marvin32" 11 | "github.com/dgryski/go-sip13" 12 | tsip "github.com/dgryski/trifles/tsip/go" 13 | ) 14 | 15 | func main() { 16 | 17 | bits := flag.Uint("s", 8, "table bits") 18 | f := flag.String("f", "tsip", "function to test") 19 | 20 | flag.Parse() 21 | 22 | var h func(uint64, uint64, []byte) uint64 23 | 24 | switch *f { 25 | case "tsip": 26 | h = tsip.Hash 27 | case "siphash": 28 | h = siphash.Hash 29 | case "sip13": 30 | h = sip13.Sum64 31 | case "marvin32": 32 | h = func(k0, k1 uint64, m []byte) uint64 { return uint64(marvin32.Sum32(0, m)) } 33 | } 34 | 35 | size := uint64(1<<*bits) - 1 36 | 37 | t := make([]uint32, size+1) 38 | 39 | scanner := bufio.NewScanner(os.Stdin) 40 | 41 | for scanner.Scan() { 42 | b := scanner.Bytes() 43 | a := h(0, 0, b) 44 | t[a&size]++ 45 | } 46 | 47 | for i, v := range t { 48 | fmt.Println(i, v) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tsip/go/testdata/tsip.txt: -------------------------------------------------------------------------------- 1 | 855990dc87035b18 2 | d5f355eb52737021 3 | 87ae5ec174a61b3e 4 | 6fdba8e069d8b491 5 | 9a875f44c535f356 6 | ab45c9ec3922556d 7 | c87b5eea313a8cc2 8 | ff1a0f574cee48cb 9 | 0b8e8caac1e824c9 10 | 6eaa0e1b564ad875 11 | ff7b01b454c8d6ad 12 | cbd0e43cfeb0de18 13 | 778559a82e70164d 14 | 20bbf6f1d0f15003 15 | 99eda79b58fa6067 16 | f4ade85db31dc456 17 | 68c44a2442cc998e 18 | 3b579b758b0a22e2 19 | 3e668bd55a6c90a0 20 | 3ba053135e746e0e 21 | 118430e80b721395 22 | 32dd1273612934d9 23 | 17b3b243c6473fe3 24 | afb440d4df2ac3f6 25 | 431e30355edbf160 26 | 4b79d419de325167 27 | 08c0413a6caa6d97 28 | 8832b282a233439c 29 | b680c35d986e6ebb 30 | e25a41bb5c53bf94 31 | 125f068793fb27d5 32 | f2c706205d6adb3e 33 | 31d4d851ef96a6e0 34 | 403a5b2e2519b213 35 | 13da7c68c22a8359 36 | a8d6cf5da023d62d 37 | 7c1c717944bffd6d 38 | d00e4f8e37ebb2b6 39 | 861dc5e276098f7f 40 | c7f020e5edaba21f 41 | a2975f463a4b2c30 42 | 1674283fff391555 43 | ee9c044eecc4d9fc 44 | c14c872b01746822 45 | 2f57aac5624bad86 46 | bef8a187e66299bd 47 | e16739b75e5441c8 48 | 976987a330acbaa7 49 | dc8649ab927a880c 50 | 741820b53b8f1754 51 | 0acda223aa3545ec 52 | 5a8c9bf54e525876 53 | 3dbd9fcd5337fb66 54 | 8f11c2754d82bed0 55 | fc2305d8800547c1 56 | 3dc77b5ec54e4bcb 57 | 820358898965bafc 58 | 2cbca5e142cc8ac4 59 | ae19ad2e8d886fb4 60 | a9e6ca050e67d9ff 61 | 14cd8986ef83e359 62 | 394f46208535c695 63 | 83b3efb94edd8929 64 | a046ab3234f4e3c7 65 | -------------------------------------------------------------------------------- /tsip/go/tsip.go: -------------------------------------------------------------------------------- 1 | package tsip 2 | 3 | import ( 4 | "encoding/binary" 5 | "math/bits" 6 | ) 7 | 8 | type sip struct { 9 | v0, v1 uint64 10 | } 11 | 12 | func (s *sip) round() { 13 | s.v0 += s.v1 14 | s.v1 = bits.RotateLeft64(s.v1, 13) ^ s.v0 15 | s.v0 = bits.RotateLeft64(s.v0, 35) + s.v1 16 | s.v1 = bits.RotateLeft64(s.v1, 17) ^ s.v0 17 | s.v0 = bits.RotateLeft64(s.v0, 21) 18 | } 19 | 20 | func Hash(k0, k1 uint64, p []byte) uint64 { 21 | 22 | s := sip{ 23 | v0: k0 ^ 0x736f6d6570736575, 24 | v1: k1 ^ 0x646f72616e646f6d, 25 | } 26 | b := uint64(len(p)) << 56 27 | 28 | for len(p) >= 8 { 29 | m := binary.LittleEndian.Uint64(p[:8]) 30 | s.v1 ^= m 31 | s.round() 32 | s.v0 ^= m 33 | p = p[8:] 34 | } 35 | 36 | switch len(p) { 37 | case 7: 38 | b |= uint64(p[6]) << 48 39 | fallthrough 40 | case 6: 41 | b |= uint64(p[5]) << 40 42 | fallthrough 43 | case 5: 44 | b |= uint64(p[4]) << 32 45 | fallthrough 46 | case 4: 47 | b |= uint64(p[3]) << 24 48 | fallthrough 49 | case 3: 50 | b |= uint64(p[2]) << 16 51 | fallthrough 52 | case 2: 53 | b |= uint64(p[1]) << 8 54 | fallthrough 55 | case 1: 56 | b |= uint64(p[0]) 57 | } 58 | 59 | // last block 60 | s.v1 ^= b 61 | s.round() 62 | s.v0 ^= b 63 | 64 | // finalization 65 | s.v1 ^= 0xff 66 | s.round() 67 | s.v1 = bits.RotateLeft64(s.v1, 32) 68 | s.round() 69 | s.v1 = bits.RotateLeft64(s.v1, 32) 70 | 71 | return s.v0 ^ s.v1 72 | } 73 | -------------------------------------------------------------------------------- /tsip/go/tsip_amd64.s: -------------------------------------------------------------------------------- 1 | // Code generated by command: go run asm.go -out tsip_amd64.s. DO NOT EDIT. 2 | 3 | #include "textflag.h" 4 | 5 | // func HashASM(k0 uint64, k1 uint64, p []byte) uint64 6 | TEXT ·HashASM(SB), NOSPLIT, $0-48 7 | MOVQ k0+0(FP), AX 8 | MOVQ k1+8(FP), CX 9 | MOVQ $0x736f6d6570736575, DX 10 | XORQ DX, AX 11 | MOVQ $0x646f72616e646f6d, DX 12 | XORQ DX, CX 13 | MOVQ p_base+16(FP), DX 14 | MOVQ p_len+24(FP), BX 15 | MOVQ BX, BP 16 | SHLQ $0x38, BP 17 | CMPQ BX, $0x08 18 | JL loop_end 19 | 20 | loop_begin: 21 | MOVQ (DX), SI 22 | XORQ SI, CX 23 | ADDQ CX, AX 24 | ROLQ $0x0d, CX 25 | XORQ AX, CX 26 | ROLQ $0x23, AX 27 | ADDQ CX, AX 28 | ROLQ $0x11, CX 29 | XORQ AX, CX 30 | ROLQ $0x15, AX 31 | XORQ SI, AX 32 | ADDQ $0x08, DX 33 | SUBQ $0x08, BX 34 | CMPQ BX, $0x08 35 | JGE loop_begin 36 | 37 | loop_end: 38 | CMPQ BX, $0x00 39 | JE sw0 40 | CMPQ BX, $0x01 41 | JE sw1 42 | CMPQ BX, $0x02 43 | JE sw2 44 | CMPQ BX, $0x03 45 | JE sw3 46 | CMPQ BX, $0x04 47 | JE sw4 48 | CMPQ BX, $0x05 49 | JE sw5 50 | CMPQ BX, $0x06 51 | JE sw6 52 | 53 | sw7: 54 | MOVBQZX 6(DX), BX 55 | SHLQ $0x30, BX 56 | ORQ BX, BP 57 | 58 | sw6: 59 | MOVBQZX 5(DX), BX 60 | SHLQ $0x28, BX 61 | ORQ BX, BP 62 | 63 | sw5: 64 | MOVBQZX 4(DX), BX 65 | SHLQ $0x20, BX 66 | ORQ BX, BP 67 | 68 | sw4: 69 | MOVBQZX 3(DX), BX 70 | SHLQ $0x18, BX 71 | ORQ BX, BP 72 | 73 | sw3: 74 | MOVBQZX 2(DX), BX 75 | SHLQ $0x10, BX 76 | ORQ BX, BP 77 | 78 | sw2: 79 | MOVBQZX 1(DX), BX 80 | SHLQ $0x08, BX 81 | ORQ BX, BP 82 | 83 | sw1: 84 | MOVBQZX (DX), BX 85 | SHLQ $0x00, BX 86 | ORQ BX, BP 87 | 88 | sw0: 89 | XORQ BP, CX 90 | ADDQ CX, AX 91 | ROLQ $0x0d, CX 92 | XORQ AX, CX 93 | ROLQ $0x23, AX 94 | ADDQ CX, AX 95 | ROLQ $0x11, CX 96 | XORQ AX, CX 97 | ROLQ $0x15, AX 98 | XORQ BP, AX 99 | XORQ $0xff, CX 100 | ADDQ CX, AX 101 | ROLQ $0x0d, CX 102 | XORQ AX, CX 103 | ROLQ $0x23, AX 104 | ADDQ CX, AX 105 | ROLQ $0x11, CX 106 | XORQ AX, CX 107 | ROLQ $0x15, AX 108 | ROLQ $0x20, CX 109 | ADDQ CX, AX 110 | ROLQ $0x0d, CX 111 | XORQ AX, CX 112 | ROLQ $0x23, AX 113 | ADDQ CX, AX 114 | ROLQ $0x11, CX 115 | XORQ AX, CX 116 | ROLQ $0x15, AX 117 | ROLQ $0x20, CX 118 | XORQ CX, AX 119 | MOVQ AX, ret+40(FP) 120 | RET 121 | -------------------------------------------------------------------------------- /tsip/go/tsip_stub.go: -------------------------------------------------------------------------------- 1 | // +build amd64 2 | 3 | package tsip 4 | 5 | //go:generate go run asm.go -out tsip_amd64.s 6 | //go:noescape 7 | 8 | func HashASM(k0, k1 uint64, p []byte) uint64 9 | -------------------------------------------------------------------------------- /tsip/go/tsip_test.go: -------------------------------------------------------------------------------- 1 | package tsip 2 | 3 | import ( 4 | "bufio" 5 | "os" 6 | "strconv" 7 | "testing" 8 | ) 9 | 10 | func TestTSIP(t *testing.T) { 11 | 12 | var k0 uint64 = 0x0706050403020100 13 | var k1 uint64 = 0x0f0e0d0c0b0a0908 14 | 15 | f, err := os.Open("testdata/tsip.txt") 16 | if err != nil { 17 | t.Fatalf("unable to open expected results: %v", err) 18 | } 19 | 20 | scanner := bufio.NewScanner(f) 21 | 22 | var buf []byte 23 | 24 | var i int 25 | 26 | for scanner.Scan() { 27 | 28 | wantStr := scanner.Text() 29 | 30 | want, err := strconv.ParseUint(wantStr, 16, 64) 31 | if err != nil { 32 | t.Errorf("error parsing want: %v: %v", wantStr, err) 33 | } 34 | 35 | got := Hash(k0, k1, buf) 36 | 37 | if got != want { 38 | t.Errorf("failed: %d: got=%v, want=%v\n", i, got, want) 39 | } 40 | 41 | got = HashASM(k0, k1, buf) 42 | 43 | if got != want { 44 | t.Errorf("failed asm: %d: got=%v, want=%v\n", i, got, want) 45 | } 46 | 47 | buf = append(buf, byte(i)) 48 | i++ 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tsip/main.cc: -------------------------------------------------------------------------------- 1 | 2 | #include "benchmark/benchmark_api.h" 3 | 4 | unsigned char buffer[8192]; 5 | 6 | #include "tsip.h" 7 | 8 | uint32_t one_at_a_time_hard(const unsigned char *seed, const unsigned char *str, size_t len); 9 | uint64_t oaathu(const unsigned char *seed, const unsigned char *message, size_t len); 10 | 11 | 12 | void Hash_oaath(benchmark::State& state) { 13 | unsigned char seed[16]; 14 | while (state.KeepRunning()) { 15 | benchmark::DoNotOptimize( 16 | one_at_a_time_hard(seed, buffer, state.range(0)) 17 | ); 18 | } 19 | } 20 | 21 | void Hash_oaathu(benchmark::State& state) { 22 | unsigned char seed[16]; 23 | while (state.KeepRunning()) { 24 | benchmark::DoNotOptimize( 25 | oaathu(seed, buffer, state.range(0)) 26 | ); 27 | } 28 | } 29 | 30 | void Hash_tsip(benchmark::State& state) { 31 | unsigned char seed[16]; 32 | while (state.KeepRunning()) { 33 | benchmark::DoNotOptimize( 34 | tsip(seed, buffer, state.range(0)) 35 | ); 36 | } 37 | } 38 | 39 | 40 | static void CustomArguments(benchmark::internal::Benchmark *b) { 41 | uint64_t sizes[] = {1,2,3,4,5,6,7,8,9,10,15,20,30,40,50,100, 0}; 42 | for (int i=0; sizes[i] != 0; i++) { 43 | b->Arg(sizes[i]); 44 | } 45 | } 46 | 47 | static void Custom0To16(benchmark::internal::Benchmark *b) { 48 | for (int i=0; i <= 16; i++) { 49 | b->Arg(i); 50 | } 51 | } 52 | 53 | BENCHMARK(Hash_tsip)->Apply(CustomArguments); 54 | BENCHMARK(Hash_oaath)->Apply(CustomArguments); 55 | BENCHMARK(Hash_oaathu)->Apply(Custom0To16); 56 | 57 | BENCHMARK_MAIN() 58 | -------------------------------------------------------------------------------- /tsip/oaath.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | 5 | uint32_t one_at_a_time_hard(const unsigned char *seed, const unsigned char *str, 6 | size_t len) { 7 | const unsigned char *const end = (const unsigned char *)str + len; 8 | uint32_t hash = *((uint32_t *)seed) + len; 9 | 10 | while (str < end) { 11 | hash += (hash << 10); 12 | hash ^= (hash >> 6); 13 | hash += *str++; 14 | } 15 | 16 | hash += (hash << 10); 17 | hash ^= (hash >> 6); 18 | hash += seed[4]; 19 | 20 | hash += (hash << 10); 21 | hash ^= (hash >> 6); 22 | hash += seed[5]; 23 | 24 | hash += (hash << 10); 25 | hash ^= (hash >> 6); 26 | hash += seed[6]; 27 | 28 | hash += (hash << 10); 29 | hash ^= (hash >> 6); 30 | hash += seed[7]; 31 | 32 | hash += (hash << 10); 33 | hash ^= (hash >> 6); 34 | 35 | hash += (hash << 3); 36 | hash ^= (hash >> 11); 37 | return (hash + (hash << 15)); 38 | } 39 | -------------------------------------------------------------------------------- /tsip/oaathu.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | 5 | uint32_t oaathu(const unsigned char *seed, const unsigned char *str, size_t len) { 6 | uint32_t hash = *((uint32_t *)seed) + len; 7 | 8 | switch (len) { 9 | case 16: 10 | hash += (hash << 10); 11 | hash ^= (hash >> 6); 12 | hash += str[15]; 13 | case 15: 14 | hash += (hash << 10); 15 | hash ^= (hash >> 6); 16 | hash += str[14]; 17 | case 14: 18 | hash += (hash << 10); 19 | hash ^= (hash >> 6); 20 | hash += str[13]; 21 | case 13: 22 | hash += (hash << 10); 23 | hash ^= (hash >> 6); 24 | hash += str[12]; 25 | case 12: 26 | hash += (hash << 10); 27 | hash ^= (hash >> 6); 28 | hash += str[11]; 29 | case 11: 30 | hash += (hash << 10); 31 | hash ^= (hash >> 6); 32 | hash += str[10]; 33 | case 10: 34 | hash += (hash << 10); 35 | hash ^= (hash >> 6); 36 | hash += str[9]; 37 | case 9: 38 | hash += (hash << 10); 39 | hash ^= (hash >> 6); 40 | hash += str[8]; 41 | case 8: 42 | hash += (hash << 10); 43 | hash ^= (hash >> 6); 44 | hash += str[7]; 45 | case 7: 46 | hash += (hash << 10); 47 | hash ^= (hash >> 6); 48 | hash += str[6]; 49 | case 6: 50 | hash += (hash << 10); 51 | hash ^= (hash >> 6); 52 | hash += str[5]; 53 | case 5: 54 | hash += (hash << 10); 55 | hash ^= (hash >> 6); 56 | hash += str[4]; 57 | case 4: 58 | hash += (hash << 10); 59 | hash ^= (hash >> 6); 60 | hash += str[3]; 61 | case 3: 62 | hash += (hash << 10); 63 | hash ^= (hash >> 6); 64 | hash += str[2]; 65 | case 2: 66 | hash += (hash << 10); 67 | hash ^= (hash >> 6); 68 | hash += str[1]; 69 | case 1: 70 | hash += (hash << 10); 71 | hash ^= (hash >> 6); 72 | hash += str[0]; 73 | case 0: 74 | hash += (hash << 10); 75 | hash ^= (hash >> 6); 76 | hash += seed[4]; 77 | hash += (hash << 10); 78 | hash ^= (hash >> 6); 79 | hash += seed[5]; 80 | hash += (hash << 10); 81 | hash ^= (hash >> 6); 82 | hash += seed[6]; 83 | hash += (hash << 10); 84 | hash ^= (hash >> 6); 85 | hash += seed[7]; 86 | hash += (hash << 10); 87 | hash ^= (hash >> 6); 88 | 89 | hash += (hash << 3); 90 | hash ^= (hash >> 11); 91 | } 92 | return (hash + (hash << 15)); 93 | } 94 | -------------------------------------------------------------------------------- /tsip/tsip.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | 5 | static uint64_t inline U8TO64_LE(const unsigned char *p) { 6 | return *(const uint64_t *)p; 7 | } 8 | 9 | static uint64_t rotl64(uint64_t x, int k) { 10 | return (((x) << (k)) | ((x) >> (64 - k))); 11 | } 12 | 13 | uint64_t tsip(const unsigned char *seed, const unsigned char *m, size_t len) { 14 | 15 | uint64_t v0, v1; 16 | uint64_t mi, k0, k1; 17 | uint64_t last7; 18 | 19 | k0 = U8TO64_LE(seed); 20 | k1 = U8TO64_LE(seed + 8); 21 | 22 | v0 = k0 ^ 0x736f6d6570736575ull; 23 | v1 = k1 ^ 0x646f72616e646f6dull; 24 | 25 | last7 = (uint64_t)(len & 0xff) << 56; 26 | 27 | #define sipcompress() \ 28 | do { \ 29 | v0 += v1; \ 30 | v1 = rotl64(v1, 13) ^ v0; \ 31 | v0 = rotl64(v0, 35) + v1; \ 32 | v1 = rotl64(v1, 17) ^ v0; \ 33 | v0 = rotl64(v0, 21); \ 34 | } while (0) 35 | 36 | const unsigned char *end = m + (len & ~7); 37 | 38 | while (m < end) { 39 | mi = U8TO64_LE(m); 40 | v1 ^= mi; 41 | sipcompress(); 42 | v0 ^= mi; 43 | m += 8; 44 | } 45 | 46 | switch (len & 7) { 47 | case 7: 48 | last7 |= (uint64_t)m[6] << 48; 49 | case 6: 50 | last7 |= (uint64_t)m[5] << 40; 51 | case 5: 52 | last7 |= (uint64_t)m[4] << 32; 53 | case 4: 54 | last7 |= (uint64_t)m[3] << 24; 55 | case 3: 56 | last7 |= (uint64_t)m[2] << 16; 57 | case 2: 58 | last7 |= (uint64_t)m[1] << 8; 59 | case 1: 60 | last7 |= (uint64_t)m[0]; 61 | case 0: 62 | default:; 63 | }; 64 | 65 | v1 ^= last7; 66 | sipcompress(); 67 | v0 ^= last7; 68 | 69 | // finalization 70 | v1 ^= 0xff; 71 | sipcompress(); 72 | v1 = rotl64(v1, 32); 73 | sipcompress(); 74 | v1 = rotl64(v1, 32); 75 | 76 | return v0 ^ v1; 77 | } 78 | -------------------------------------------------------------------------------- /tsip/tsip.h: -------------------------------------------------------------------------------- 1 | uint64_t tsip(const unsigned char *seed, const unsigned char *message, size_t len); 2 | 3 | -------------------------------------------------------------------------------- /tsip/tsip_test.c: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include 4 | #include 5 | 6 | #include "tsip.h" 7 | 8 | int main() { 9 | 10 | const unsigned char seed[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}; 11 | 12 | size_t i; 13 | unsigned char buf[64]; 14 | uint64_t h; 15 | 16 | for (i=0;i< 64;i++) { 17 | buf[i]=i; 18 | h = tsip(seed, (const unsigned char *)buf, i); 19 | printf("%016lx\n", h); 20 | } 21 | 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /u64cmp/bench_test.go: -------------------------------------------------------------------------------- 1 | package u64cmp 2 | 3 | import "testing" 4 | 5 | var dataA = make([]uint64, 500*1024) 6 | var dataB = make([]uint64, 500*1024) 7 | 8 | var Sink bool 9 | 10 | func BenchmarkNaive(b *testing.B) { 11 | for i := 0; i < b.N; i++ { 12 | Sink = naive(dataA, dataB) 13 | } 14 | } 15 | 16 | func BenchmarkUnrollXor(b *testing.B) { 17 | for i := 0; i < b.N; i++ { 18 | Sink = unrollxor(dataA, dataB) 19 | } 20 | } 21 | 22 | func BenchmarkBytesEqual(b *testing.B) { 23 | for i := 0; i < b.N; i++ { 24 | Sink = bytesEq(dataA, dataB) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /u64cmp/cmp.go: -------------------------------------------------------------------------------- 1 | package u64cmp 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "unsafe" 7 | ) 8 | 9 | func naive(a, b []uint64) bool { 10 | 11 | if len(a) != len(b) { 12 | return false 13 | } 14 | 15 | for i, v := range a { 16 | if v != b[i] { 17 | return false 18 | } 19 | } 20 | 21 | return true 22 | } 23 | 24 | func unrollxor(a, b []uint64) bool { 25 | 26 | if len(a) != len(b) { 27 | return false 28 | } 29 | 30 | var xor uint64 31 | 32 | for len(a) >= 32 && len(b) >= 32 && xor == 0 { 33 | xor |= a[0+0] ^ b[0+0] 34 | xor |= a[0+1] ^ b[0+1] 35 | xor |= a[0+2] ^ b[0+2] 36 | xor |= a[0+3] ^ b[0+3] 37 | xor |= a[0+4] ^ b[0+4] 38 | xor |= a[0+5] ^ b[0+5] 39 | xor |= a[0+6] ^ b[0+6] 40 | xor |= a[0+7] ^ b[0+7] 41 | 42 | xor |= a[8+0] ^ b[8+0] 43 | xor |= a[8+1] ^ b[8+1] 44 | xor |= a[8+2] ^ b[8+2] 45 | xor |= a[8+3] ^ b[8+3] 46 | xor |= a[8+4] ^ b[8+4] 47 | xor |= a[8+5] ^ b[8+5] 48 | xor |= a[8+6] ^ b[8+6] 49 | xor |= a[8+7] ^ b[8+7] 50 | 51 | xor |= a[16+0] ^ b[16+0] 52 | xor |= a[16+1] ^ b[16+1] 53 | xor |= a[16+2] ^ b[16+2] 54 | xor |= a[16+3] ^ b[16+3] 55 | xor |= a[16+4] ^ b[16+4] 56 | xor |= a[16+5] ^ b[16+5] 57 | xor |= a[16+6] ^ b[16+6] 58 | xor |= a[16+7] ^ b[16+7] 59 | 60 | xor |= a[24+0] ^ b[24+0] 61 | xor |= a[24+1] ^ b[24+1] 62 | xor |= a[24+2] ^ b[24+2] 63 | xor |= a[24+3] ^ b[24+3] 64 | xor |= a[24+4] ^ b[24+4] 65 | xor |= a[24+5] ^ b[24+5] 66 | xor |= a[24+6] ^ b[24+6] 67 | xor |= a[24+7] ^ b[24+7] 68 | 69 | a, b = a[32:], b[32:] 70 | } 71 | 72 | if xor != 0 { 73 | return false 74 | } 75 | 76 | for i, v := range a { 77 | if v != b[i] { 78 | return false 79 | } 80 | } 81 | 82 | return true 83 | } 84 | 85 | func bytesEq(a, b []uint64) bool { 86 | 87 | p := unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(&a)).Data) 88 | 89 | var aBytes []byte 90 | hdr := (*reflect.SliceHeader)(unsafe.Pointer(&aBytes)) 91 | hdr.Data = uintptr(p) 92 | hdr.Len = len(a) * 8 93 | hdr.Cap = len(a) * 8 94 | 95 | p = unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(&b)).Data) 96 | 97 | var bBytes []byte 98 | hdr = (*reflect.SliceHeader)(unsafe.Pointer(&bBytes)) 99 | hdr.Data = uintptr(p) 100 | hdr.Len = len(b) * 8 101 | hdr.Cap = len(b) * 8 102 | 103 | return bytes.Equal(aBytes, bBytes) 104 | } 105 | -------------------------------------------------------------------------------- /udprelay/udpclient.py: -------------------------------------------------------------------------------- 1 | # udp tests 2 | import socket 3 | import time 4 | 5 | UDP_IP = "127.0.0.1" 6 | UDP_PORT = 12233 7 | 8 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 9 | 10 | message = "hello, world" 11 | 12 | for i in xrange(100): 13 | for j in xrange(5000): sock.sendto(message, (UDP_IP, UDP_PORT)) 14 | time.sleep(0.1) 15 | 16 | -------------------------------------------------------------------------------- /urlq/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "net/url" 9 | "os" 10 | ) 11 | 12 | func main() { 13 | 14 | qparam := flag.String("q", "q", "query parameter to extract") 15 | 16 | flag.Parse() 17 | 18 | scanner := bufio.NewScanner(os.Stdin) 19 | 20 | for scanner.Scan() { 21 | s := scanner.Text() 22 | u, err := url.Parse(s) 23 | if err != nil { 24 | log.Printf("%v: err %+v\n", s, err) 25 | continue 26 | } 27 | query := u.Query() 28 | if query == nil { 29 | continue 30 | } 31 | for _, qq := range query[*qparam] { 32 | fmt.Println(qq) 33 | } 34 | } 35 | 36 | if err := scanner.Err(); err != nil { 37 | log.Printf("error during scan: %+v\n", err) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /uuid/uuid.go: -------------------------------------------------------------------------------- 1 | package uuid 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "io" 7 | "log" 8 | ) 9 | 10 | // UUIDv4 returns a type 4 (`random') UUID 11 | func UUIDv4() string { 12 | b := make([]byte, 16) 13 | _, err := io.ReadFull(rand.Reader, b) 14 | if err != nil { 15 | // probably "shouldn't happen" 16 | log.Fatal(err) 17 | } 18 | b[6] = (b[6] & 0x0F) | 0x40 19 | b[8] = (b[8] &^ 0x40) | 0x80 20 | return fmt.Sprintf("%x-%x-%x-%x-%x", b[:4], b[4:6], b[6:8], b[8:10], b[10:]) 21 | } 22 | -------------------------------------------------------------------------------- /wasmlink/Makefile: -------------------------------------------------------------------------------- 1 | export PATH := /usr/local/Cellar/llvm/16.0.1/bin:$(PATH) 2 | TINYGOROOT=$(shell tinygo env TINYGOROOT) 3 | SHELL := /bin/sh 4 | MAIN=wasmlink 5 | OBJS=main.o task.o runtime.o add.o 6 | 7 | build: 8 | @ PATH=$(PATH) make build_all 9 | 10 | build_all: main.o task.o runtime.o add.o 11 | 12 | link_dog: 13 | EXTRA=rematch-dog/matcher.o $(MAKE) link 14 | 15 | link_cat: 16 | EXTRA=rematch-cat/matcher.o $(MAKE) link 17 | 18 | link: $(OBJS) $(EXTRA) 19 | wasm-ld --stack-first --no-demangle -o linked.wasm $(OBJS) $(EXTRA) /Users/dgryski/go/src/github.com/tinygo-org/tinygo/lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a 20 | wasm-opt --asyncify linked.wasm -o linked-asyncify.wasm && rm -f linked.wasm 21 | mv linked-asyncify.wasm $(MAIN).wasm 22 | 23 | task.o: $(TINYGOROOT)/src/internal/task/task_asyncify_wasm.S 24 | clang --target=wasm32-unknown-wasi -c -o $@ $< 25 | 26 | runtime.o: $(TINYGOROOT)/src/runtime/asm_tinygowasm.S 27 | clang --target=wasm32-unknown-wasi -c -o $@ $< 28 | 29 | add.o: add.wat 30 | wat2wasm --relocatable add.wat -o add.o 31 | 32 | main.o: main.go 33 | tinygo build -target=wasi -o main.o 34 | 35 | clean: 36 | rm -f main.o add.o runtime.o task.o linked.wasm linked-asyncify.wasm wasmlink.wasm 37 | 38 | rematch-cat/matcher.o: rematch-cat/matcher.c 39 | clang --target=wasm32-unknown-wasi -c -o $@ $< 40 | rematch-dog/matcher.o: rematch-dog/matcher.c 41 | clang --target=wasm32-unknown-wasi -c -o $@ $< 42 | 43 | matchers: 44 | re -r pcre -k pair -e matcher -pl vmc meow >rematch-cat/matcher.c 45 | re -r pcre -k pair -e matcher -pl vmc woof >rematch-dog/matcher.c 46 | -------------------------------------------------------------------------------- /wasmlink/add.wat: -------------------------------------------------------------------------------- 1 | (module 2 | (func $add (param $lhs i32) (param $rhs i32) (result i32) 3 | local.get $lhs 4 | local.get $rhs 5 | i32.add) 6 | (export "add" (func $add)) 7 | ) 8 | -------------------------------------------------------------------------------- /wasmlink/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | //export add 6 | func add(a, b int32) int32 7 | 8 | //export matchermain 9 | func matchermain(a, b *byte) int32 10 | 11 | func main() { 12 | fmt.Println(add(2, 3)) 13 | 14 | runMatcher("meow ") 15 | runMatcher("woof ") 16 | } 17 | 18 | func runMatcher(s string) { 19 | buf := []byte(s) 20 | 21 | b := &buf[0] 22 | e := &buf[len(buf)-1] 23 | 24 | if matchermain(b, e) != -1 { 25 | fmt.Println("matcher hit for string", s) 26 | } else { 27 | fmt.Println("matcher miss for string", s) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /worker/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "io" 7 | "log" 8 | "os" 9 | ) 10 | 11 | func worker(urlch chan string, donech chan struct{}) { 12 | for u := range urlch { 13 | log.Println("processing u=", u) 14 | } 15 | 16 | donech <- struct{}{} 17 | } 18 | 19 | func main() { 20 | 21 | infname := flag.String("f", "", "input file name") 22 | workers := flag.Int("w", 4, "workers") 23 | 24 | urlch := make(chan string) 25 | donech := make(chan struct{}) 26 | 27 | log.Println("using", *workers, "workers") 28 | 29 | for i := 0; i < *workers; i++ { 30 | go worker(urlch, donech) 31 | } 32 | 33 | var f io.Reader 34 | if *infname == "" { 35 | f = os.Stdin 36 | } else { 37 | var err error 38 | f, err = os.Open(*infname) 39 | 40 | if err != nil { 41 | log.Fatal("error loading input file", *infname, ":", err) 42 | } 43 | } 44 | 45 | scan := bufio.NewScanner(f) 46 | var items []string 47 | for scan.Scan() { 48 | items = append(items, scan.Text()) 49 | } 50 | 51 | log.Println("Total items:", len(items)) 52 | 53 | processedItems := 0 54 | 55 | // send off the work 56 | for _, item := range items { 57 | urlch <- item 58 | processedItems++ 59 | } 60 | 61 | close(urlch) 62 | 63 | log.Println("waiting for workers to complete") 64 | for i := 0; i < *workers; i++ { 65 | <-donech 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /wscat/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "net/url" 9 | "os" 10 | 11 | "golang.org/x/net/websocket" 12 | ) 13 | 14 | func main() { 15 | flag.Parse() 16 | rawurl := flag.Arg(0) 17 | 18 | // ws://echo.websocket.org:80 19 | u, err := url.Parse(rawurl) 20 | if err != nil { 21 | log.Fatal("error parsing: ", err) 22 | } 23 | 24 | conn, err := websocket.Dial(rawurl, "tcp", "http://"+u.Host) 25 | 26 | if err != nil { 27 | log.Fatal("error dialing: ", err) 28 | } 29 | 30 | go func(conn *websocket.Conn) { 31 | scanner := bufio.NewScanner(os.Stdin) 32 | 33 | for scanner.Scan() { 34 | s := scanner.Text() 35 | fmt.Println("send=", s) 36 | websocket.Message.Send(conn, scanner.Text()) 37 | } 38 | if err := scanner.Err(); err != nil { 39 | log.Fatalf("error scanning stdin: %s", err) 40 | } 41 | }(conn) 42 | 43 | for { 44 | var msg string 45 | err := websocket.Message.Receive(conn, &msg) 46 | if err != nil { 47 | log.Fatal("error recv: ", err) 48 | } 49 | fmt.Println("recv=", msg) 50 | } 51 | } 52 | --------------------------------------------------------------------------------