├── DesignNotes.txt ├── LICENSE ├── README.md ├── app.yaml ├── code_of_conduct.md ├── entropyheader.go ├── index.yaml ├── notify.go ├── randomsanity.go ├── randomsanitystat.go ├── randomsanitystat_test.go ├── ratelimit.go ├── unique.go └── usage.go /DesignNotes.txt: -------------------------------------------------------------------------------- 1 | An infinite number of heuristic/statistical tests could be added; I 2 | will be much more likely to consider pull requests that add more if 3 | you can point to somebody else's code on github that screws up 4 | pseudorandom number generation in a way that your code catches. 5 | 6 | The first rough implementation of the duplication detection used a 7 | really big bloom filter with a very small false positive rate. That 8 | was fun to code, but it added a lot of complication and was only three 9 | or four times smaller on disk than the more straightforward solution 10 | using AppEngine's key/value store. 11 | 12 | The performance bottleneck is datastore reads for the duplication 13 | detection. Testing a 64-byte byte array is 48 reads (and 2 writes). 14 | If this service becomes very popular and AppEngine costs become an 15 | issue that is the first place to optimize. 16 | 17 | I've done some preliminary testing and benchmarking of an algorithm 18 | that uses 6 reads and 3 writes but sacrifices detection if the byte 19 | arrays overlap in fewer than 32 bytes. 20 | 21 | 22 | Possible future work, if there is demand: 23 | 24 | Public keys should be globally unique and look random. A tool that 25 | finds all the public keys on a system and submits them would be 26 | useful. It would be even more useful if you could safely run the 27 | tool twice and not get false-positive reports of non-uniqueness 28 | (if the query was tagged with the name or IP of the machine the 29 | server could ignore duplicates with the same tag). 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Gavin Andresen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | AppEngine-based service to sanity test what should be 2 | cryptographically secure random bytestreams. 3 | 4 | See http://www.randomsanity.org/ for documentation and information. 5 | -------------------------------------------------------------------------------- /app.yaml: -------------------------------------------------------------------------------- 1 | runtime: go 2 | api_version: go1 3 | 4 | handlers: 5 | - url: /.* 6 | script: _go_app 7 | -------------------------------------------------------------------------------- /code_of_conduct.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of 4 | fostering an open and welcoming community, we pledge to respect all people who 5 | contribute through reporting issues, posting feature requests, updating 6 | documentation, submitting pull requests or patches, and other activities. 7 | 8 | We are committed to making participation in this project a harassment-free 9 | experience for everyone, regardless of level of experience, gender, gender 10 | identity and expression, sexual orientation, disability, personal appearance, 11 | body size, race, ethnicity, age, religion, or nationality. 12 | 13 | Examples of unacceptable behavior by participants include: 14 | 15 | * The use of sexualized language or imagery 16 | * Personal attacks 17 | * Trolling or insulting/derogatory comments 18 | * Public or private harassment 19 | * Publishing other's private information, such as physical or electronic 20 | addresses, without explicit permission 21 | * Other unethical or unprofessional conduct 22 | 23 | Project maintainers have the right and responsibility to remove, edit, or 24 | reject comments, commits, code, wiki edits, issues, and other contributions 25 | that are not aligned to this Code of Conduct, or to ban temporarily or 26 | permanently any contributor for other behaviors that they deem inappropriate, 27 | threatening, offensive, or harmful. 28 | 29 | By adopting this Code of Conduct, project maintainers commit themselves to 30 | fairly and consistently applying these principles to every aspect of managing 31 | this project. Project maintainers who do not follow or enforce the Code of 32 | Conduct may be permanently removed from the project team. 33 | 34 | This Code of Conduct applies both within project spaces and in public spaces 35 | when an individual is representing the project or its community. 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 38 | reported by contacting a project maintainer (see below). All 39 | complaints will be reviewed and investigated and will result in a response that 40 | is deemed necessary and appropriate to the circumstances. Maintainers are 41 | obligated to maintain confidentiality with regard to the reporter of an 42 | incident. 43 | 44 | You may send reports to [our Conduct email](mailto:gavinandresen@gmail.com). 45 | 46 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 47 | version 1.3.0, available at 48 | [http://contributor-covenant.org/version/1/3/0/][version] 49 | 50 | [homepage]: http://contributor-covenant.org 51 | [version]: http://contributor-covenant.org/version/1/3/0/ 52 | -------------------------------------------------------------------------------- /entropyheader.go: -------------------------------------------------------------------------------- 1 | package randomsanity 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/hex" 6 | "net/http" 7 | ) 8 | 9 | func addEntropyHeader(w http.ResponseWriter) { 10 | // This assumes server has a good crypto/rand 11 | // implementation. We could memcache an array 12 | // that is initialized to crypto/rand but updated 13 | // with every request that comes in with random data. 14 | var b [32]byte 15 | n, err := rand.Read(b[:]) 16 | if err == nil && n == len(b) { 17 | w.Header().Add("X-Entropy", hex.EncodeToString(b[:])) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /index.yaml: -------------------------------------------------------------------------------- 1 | indexes: 2 | 3 | - kind: NotifyViaEmail 4 | properties: 5 | - name: UserId 6 | - name: Address 7 | -------------------------------------------------------------------------------- /notify.go: -------------------------------------------------------------------------------- 1 | package randomsanity 2 | 3 | import ( 4 | "appengine" 5 | "appengine/datastore" 6 | "appengine/mail" 7 | "crypto/rand" 8 | "encoding/hex" 9 | "fmt" 10 | "log" 11 | "net/http" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | import netmail "net/mail" 17 | 18 | // Code to notify customer when a rng failure is detected 19 | 20 | type NotifyViaEmail struct { 21 | UserID string 22 | Address string 23 | } 24 | 25 | // Return userID associated with request (or empty string) 26 | func userID(ctx appengine.Context, id string) (*datastore.Key, error) { 27 | // Only pay attention to ?id=123456 if they've done an authentication loop 28 | // and are already in the database 29 | if len(id) == 0 { 30 | return nil, nil 31 | } 32 | q := datastore.NewQuery("NotifyViaEmail").Filter("UserID =", id).Limit(1).KeysOnly() 33 | keys, err := q.GetAll(ctx, nil) 34 | if err != nil || len(keys) == 0 { 35 | return nil, err 36 | } 37 | return keys[0], nil 38 | } 39 | 40 | // Register an email address. To authenticate ownership of the 41 | // address, the server assigns a random user id and emails it. 42 | // To mitigate abuse, this method is heavily rate-limited per 43 | // IP and email address 44 | func registerEmailHandler(w http.ResponseWriter, r *http.Request) { 45 | // Requests generated by web browsers are not allowed: 46 | if r.Header.Get("Origin") != "" { 47 | http.Error(w, "CORS requests are not allowed", http.StatusForbidden) 48 | return 49 | } 50 | ua := r.Header.Get("User-Agent") 51 | if len(ua) < 4 || (!strings.EqualFold(ua[0:4], "curl") && !strings.EqualFold(ua[0:4], "wget")) { 52 | http.Error(w, "Email registration must be done via curl or wget", http.StatusForbidden) 53 | return 54 | } 55 | 56 | w.Header().Add("Content-Type", "text/plain") 57 | parts := strings.Split(r.URL.Path, "/") 58 | if len(parts) < 4 { 59 | http.Error(w, "Missing email", http.StatusBadRequest) 60 | return 61 | } 62 | if len(parts) > 4 { 63 | http.Error(w, "URL path too long", http.StatusBadRequest) 64 | return 65 | } 66 | 67 | addresses, err := netmail.ParseAddressList(parts[len(parts)-1]) 68 | if err != nil || len(addresses) != 1 { 69 | http.Error(w, "Invalid email address", http.StatusBadRequest) 70 | return 71 | } 72 | address := addresses[0] 73 | 74 | ctx := appengine.NewContext(r) 75 | 76 | // 2 registrations per IP per day 77 | limited, err := RateLimitResponse(ctx, w, IPKey("emailreg", r.RemoteAddr), 2, time.Hour*24) 78 | if err != nil || limited { 79 | return 80 | } 81 | // ... and 1 per email per week 82 | limited, err = RateLimitResponse(ctx, w, "emailreg"+address.Address, 1, time.Hour*24*7) 83 | if err != nil || limited { 84 | return 85 | } 86 | // ... and global 10 signups per hour (so a botnet with lots of IPs cannot 87 | // generate a huge surge of bogus registrations) 88 | limited, err = RateLimitResponse(ctx, w, "emailreg", 10, time.Hour) 89 | if err != nil || limited { 90 | return 91 | } 92 | // Note: the AppEngine dashboard can also be used to set quotas. 93 | // If somebody with a bunch of IP addresses is persistently annoying, 94 | // we'll switch to a web page with a CAPTCHA or require sign-in with 95 | // a Google account to register or require payment to register. 96 | 97 | var notify []NotifyViaEmail 98 | q := datastore.NewQuery("NotifyViaEmail").Filter("Address =", address.Address) 99 | if _, err := q.GetAll(ctx, ¬ify); err != nil { 100 | http.Error(w, "Datastore error", http.StatusInternalServerError) 101 | return 102 | } 103 | if len(notify) > 0 { 104 | sendNewID(ctx, address.Address, notify[0].UserID) 105 | fmt.Fprintf(w, "Check your email, ID sent to %s\n", address.Address) 106 | return 107 | } 108 | bytes := make([]byte, 8) 109 | if _, err := rand.Read(bytes); err != nil { 110 | http.Error(w, "rand.Read error", http.StatusInternalServerError) 111 | return 112 | } 113 | id := hex.EncodeToString(bytes) 114 | n := NotifyViaEmail{id, address.Address} 115 | k := datastore.NewIncompleteKey(ctx, "NotifyViaEmail", nil) 116 | if _, err := datastore.Put(ctx, k, &n); err != nil { 117 | http.Error(w, "Datastore error", http.StatusInternalServerError) 118 | return 119 | } 120 | sendNewID(ctx, address.Address, id) 121 | // HTTP response MUST NOT contain the id 122 | fmt.Fprintf(w, "Check your email, ID sent to %s", address.Address) 123 | } 124 | 125 | // Unregister, given userID 126 | func unRegisterIDHandler(w http.ResponseWriter, r *http.Request) { 127 | if r.Method != "DELETE" { 128 | http.Error(w, "unregister method must be DELETE", http.StatusBadRequest) 129 | return 130 | } 131 | parts := strings.Split(r.URL.Path, "/") 132 | if len(parts) < 4 { 133 | http.Error(w, "Missing userID", http.StatusBadRequest) 134 | return 135 | } 136 | if len(parts) > 4 { 137 | http.Error(w, "URL path too long", http.StatusBadRequest) 138 | return 139 | } 140 | ctx := appengine.NewContext(r) 141 | 142 | uID := parts[len(parts)-1] 143 | dbKey, err := userID(ctx, uID) 144 | if err != nil { 145 | http.Error(w, "datastore error", http.StatusInternalServerError) 146 | return 147 | } 148 | if dbKey == nil { 149 | http.Error(w, "User ID not found", http.StatusNotFound) 150 | return 151 | } 152 | err = datastore.Delete(ctx, dbKey) 153 | if err != nil { 154 | http.Error(w, "Error deleting key", http.StatusInternalServerError) 155 | return 156 | } 157 | fmt.Fprintf(w, "id %s unregistered\n", uID) 158 | } 159 | 160 | func sendNewID(ctx appengine.Context, address string, id string) { 161 | msg := &mail.Message{ 162 | Sender: "randomsanityalerts@gmail.com", 163 | To: []string{address}, 164 | Subject: "Random Sanity id request", 165 | } 166 | msg.Body = fmt.Sprintf("Somebody requested an id for this email address (%s)\n"+ 167 | "for the randomsanity.org service.\n"+ 168 | "\n"+ 169 | "id: %s\n"+ 170 | "\n"+ 171 | "Append ?id=%s to API calls to be notified of failures via email.\n"+ 172 | "\n"+ 173 | "If somebody is pretending to be you and you don't use the randomsanity.org\n"+ 174 | "service, please ignore this message.\n", 175 | address, id, id) 176 | if err := mail.Send(ctx, msg); err != nil { 177 | log.Printf("mail.Send failed: %s", err) 178 | } 179 | } 180 | 181 | func sendEmail(ctx appengine.Context, address string, tag string, b []byte, reason string) { 182 | // Don't spam if there are hundreds of failures, limit to 183 | // a handful per day: 184 | limit, err := RateLimit(ctx, address, 5, time.Hour*24) 185 | if err != nil || limit { 186 | return 187 | } 188 | 189 | msg := &mail.Message{ 190 | Sender: "randomsanityalerts@gmail.com", 191 | To: []string{address}, 192 | Subject: "Random Number Generator Failure Detected", 193 | } 194 | msg.Body = fmt.Sprintf("The randomsanity.org service has detected a failure.\n"+ 195 | "\n"+ 196 | "Failure reason: %s\n"+ 197 | "Data: 0x%s\n"+ 198 | "Tag: %s\n", reason, hex.EncodeToString(b), tag) 199 | if err := mail.Send(ctx, msg); err != nil { 200 | log.Printf("mail.Send failed: %s", err) 201 | } 202 | } 203 | 204 | func notify(ctx appengine.Context, uid string, tag string, b []byte, reason string) { 205 | if len(uid) == 0 { 206 | return 207 | } 208 | q := datastore.NewQuery("NotifyViaEmail").Filter("UserID =", uid) 209 | for t := q.Run(ctx); ; { 210 | var d NotifyViaEmail 211 | _, err := t.Next(&d) 212 | if err == datastore.Done { 213 | break 214 | } 215 | if err != nil { 216 | log.Printf("Datastore error: %s", err.Error()) 217 | return 218 | } 219 | sendEmail(ctx, d.Address, tag, b, reason) 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /randomsanity.go: -------------------------------------------------------------------------------- 1 | // AppEngine-based server to sanity check byte arrays 2 | // that are supposed to be random. 3 | package randomsanity 4 | 5 | import ( 6 | "appengine" 7 | "encoding/hex" 8 | "fmt" 9 | "net/http" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | func init() { 15 | // Main API point, sanity check hex bytes 16 | http.HandleFunc("/v1/q/", submitBytesHandler) 17 | 18 | // Start an email loop to get an id token, to be 19 | // notified via email of failures: 20 | http.HandleFunc("/v1/registeremail/", registerEmailHandler) 21 | 22 | // Remove an id token 23 | http.HandleFunc("/v1/unregister/", unRegisterIDHandler) 24 | 25 | // Get usage stats 26 | http.HandleFunc("/v1/usage", usageHandler) 27 | 28 | // Development/testing... 29 | http.HandleFunc("/v1/debug", debugHandler) 30 | 31 | // Redirect to www. home page 32 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 33 | if r.URL.Path != "/" { 34 | http.NotFound(w, r) 35 | return 36 | } 37 | http.Redirect(w, r, "https://www.randomsanity.org/", 301) 38 | }) 39 | } 40 | 41 | func debugHandler(w http.ResponseWriter, r *http.Request) { 42 | w.Header().Add("Content-Type", "text/plain") 43 | 44 | // Code useful for development/testing: 45 | // fmt.Fprint(w, "***r.Header headers***\n") 46 | // r.Header.Write(w) 47 | 48 | // ctx := appengine.NewContext(r) 49 | // fmt.Fprint(w, "Usage data:\n") 50 | // for _, u := range GetUsage(ctx) { 51 | // fmt.Fprintf(w, "%s,%d\n", u.Key, u.N) 52 | // } 53 | } 54 | 55 | func submitBytesHandler(w http.ResponseWriter, r *http.Request) { 56 | parts := strings.Split(r.URL.Path, "/") 57 | if len(parts) != 4 { 58 | http.Error(w, "Invalid GET", http.StatusBadRequest) 59 | return 60 | } 61 | b, err := hex.DecodeString(parts[len(parts)-1]) 62 | if err != nil { 63 | http.Error(w, "Invalid hex", http.StatusBadRequest) 64 | return 65 | } 66 | // Need at least 16 bytes to hit the 1-in-2^60 false positive rate 67 | if len(b) < 16 { 68 | http.Error(w, "Must provide 16 or more bytes", http.StatusBadRequest) 69 | return 70 | } 71 | 72 | ctx := appengine.NewContext(r) 73 | 74 | // Users that register can append id=....&tag=.... so 75 | // they're notified if somebody else submits 76 | // the same random bytes 77 | uID := r.FormValue("id") 78 | dbKey, _ := userID(ctx, uID) 79 | tag := "" 80 | if dbKey == nil { 81 | uID = "" 82 | } else { 83 | tag = r.FormValue("tag") 84 | if len(tag) > 64 { 85 | tag = "" // Tags must be short 86 | } 87 | } 88 | 89 | // Rate-limit by IP address, with a much higher limit for registered users 90 | // If more complicated logic is needed because of abuse a per-user limit 91 | // could be stored in the datastore, but running into the 600-per-hour-per-ip 92 | // limit should be rare (maybe a sysadmin has 200 virtual machines 93 | // behind the same IP address and restarts them more than three times in a hour....) 94 | var ratelimit uint64 = 60 95 | if len(uID) > 0 { 96 | ratelimit = 600 97 | } 98 | limited, err := RateLimitResponse(ctx, w, IPKey("q", r.RemoteAddr), ratelimit, time.Hour) 99 | if err != nil || limited { 100 | return 101 | } 102 | 103 | w.Header().Add("Content-Type", "application/json") 104 | 105 | // Returns some randomness caller can use to mix in to 106 | // their PRNG: 107 | addEntropyHeader(w) 108 | 109 | // First, some simple tests for non-random input: 110 | result, reason := LooksRandom(b) 111 | if !result { 112 | RecordUsage(ctx, "Fail_"+reason, 1) 113 | fmt.Fprint(w, "false") 114 | notify(ctx, uID, tag, b, reason) 115 | return 116 | } 117 | 118 | // Try to catch two machines with insufficient starting 119 | // entropy generating identical streams of random bytes. 120 | if len(b) > 64 { 121 | b = b[0:64] // Prevent DoS from excessive datastore lookups 122 | } 123 | unique, err := looksUnique(ctx, w, b, uID, tag) 124 | if err != nil { 125 | return 126 | } 127 | if unique { 128 | RecordUsage(ctx, "Success", 1) 129 | fmt.Fprint(w, "true") 130 | } else { 131 | RecordUsage(ctx, "Fail_Nonunique", 1) 132 | fmt.Fprint(w, "false") 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /randomsanitystat.go: -------------------------------------------------------------------------------- 1 | // Fast, simple statistical tests for short (e.g. 256-bit) bitstreams 2 | // 3 | // These are written for a 1-in-2^60 (one in a quintillion) false 4 | // positive rate, approximately, overall. Since multiple tests are 5 | // run, the false positive rate for each should be evern lower; 6 | // individual tests work on 8 byte chunks so have a 1-in-2^64 7 | // false positive rate. 8 | // 9 | // They are meant to catch catastrophic failures of software or hardware, 10 | // NOT to detect subtle biases. 11 | // 12 | // If you want to detect subtle biases, use one of these extensive 13 | // test suites: 14 | // NIST SP 800-22 15 | // DieHarder 16 | // TestU01 17 | // 18 | // If you are a certain type of programmer, you will be tempted to optimize 19 | // the snot out of these; there are lots of clever optimizations that could 20 | // make some of these tests an order or three of magnitude faster. 21 | // Don't. Find something more productive to do. CPU time is really cheap; 22 | // finding somebody willing to spend a half a day reviewing your awesomely 23 | // clever algorithm for detecting stuck bits is expensive. 24 | package randomsanity 25 | 26 | import ( 27 | "encoding/binary" 28 | ) 29 | 30 | type decodeF func([]byte) uint64 31 | 32 | func incrementing(b []byte, bytesPerNum int, fp decodeF) bool { 33 | // Need at least one number plus 64-bits-worth of items 34 | // to be under the 2^60 false positive rate 35 | if len(b) < bytesPerNum+8 { 36 | return false 37 | } 38 | first := fp(b[0:bytesPerNum]) 39 | nNums := len(b) / bytesPerNum 40 | allmatch := true 41 | for i := 1; i < nNums && allmatch; i++ { 42 | n := fp(b[bytesPerNum*i : bytesPerNum*(i+1)]) 43 | if first+uint64(i) != n { 44 | allmatch = false 45 | } 46 | } 47 | return allmatch 48 | } 49 | 50 | // Counting returns true if b contains bytes that can be interpreted 51 | // as incrementing numbers: 8/16/32/64 bytes, big or little endian. 52 | // It is meant to catch programming errors where an array index is used 53 | // instead of some source of random bytes. 54 | func Counting(b []byte) bool { 55 | if incrementing(b, 1, func(b []byte) uint64 { return uint64(b[0]) }) { 56 | return true 57 | } 58 | if incrementing(b, 2, func(b []byte) uint64 { return uint64(binary.LittleEndian.Uint16(b[0:2])) }) { 59 | return true 60 | } 61 | if incrementing(b, 2, func(b []byte) uint64 { return uint64(binary.BigEndian.Uint16(b[0:2])) }) { 62 | return true 63 | } 64 | if incrementing(b, 4, func(b []byte) uint64 { return uint64(binary.LittleEndian.Uint32(b[0:4])) }) { 65 | return true 66 | } 67 | if incrementing(b, 4, func(b []byte) uint64 { return uint64(binary.BigEndian.Uint32(b[0:4])) }) { 68 | return true 69 | } 70 | if incrementing(b, 8, func(b []byte) uint64 { return uint64(binary.LittleEndian.Uint64(b[0:8])) }) { 71 | return true 72 | } 73 | if incrementing(b, 8, func(b []byte) uint64 { return uint64(binary.BigEndian.Uint64(b[0:8])) }) { 74 | return true 75 | } 76 | return false 77 | } 78 | 79 | // Repeated returns true if b contains long runs of repeated bytes 80 | func Repeated(b []byte) bool { 81 | nBytes := len(b) 82 | nRepeated := 0 83 | 84 | for i := 1; i <= nBytes; i++ { 85 | if b[i-1] == b[i%nBytes] { 86 | nRepeated += 1 87 | if nRepeated >= 8 { 88 | return true 89 | } 90 | } else { 91 | nRepeated = 0 92 | } 93 | } 94 | return false 95 | } 96 | 97 | // BitStuck returns true if a bit in b is always set or unset 98 | // (and b is 64 or more bytes long) 99 | func BitStuck(b []byte) bool { 100 | if len(b) < 64 { 101 | return false 102 | } 103 | 104 | allOr := byte(0) 105 | allAnd := byte(0xff) 106 | 107 | for _, v := range b { 108 | allOr |= v 109 | allAnd &= v 110 | } 111 | if allOr != 0xff || allAnd != 0x0 { 112 | return true 113 | } 114 | return false 115 | } 116 | 117 | // DecimalHex detects confusing decimal and hex (no A-F hex digits) 118 | func DecimalHex(b []byte) bool { 119 | // ... need 45 or more bytes (89 or more digits) to be over the 2^60 fp rate... 120 | if len(b) < 45 { 121 | return false 122 | } 123 | for i := 0; i < len(b); i++ { 124 | if ((b[i] >> 4) >= 10) || ((b[i] & 0x0f) >= 10) { 125 | return false 126 | } 127 | } 128 | return true 129 | } 130 | 131 | // LooksRandom returns true and an empty string if b passes all 132 | // the tests; otherwise it returns false and a short string describing 133 | // which test failed. 134 | func LooksRandom(b []byte) (bool, string) { 135 | if Repeated(b) { 136 | return false, "Repeated bytes" 137 | } 138 | if Counting(b) { 139 | return false, "Counting" 140 | } 141 | if DecimalHex(b) { 142 | return false, "Decimal digits as hex" 143 | } 144 | if BitStuck(b) { 145 | return false, "Bit stuck" 146 | } 147 | 148 | return true, "" 149 | } 150 | -------------------------------------------------------------------------------- /randomsanitystat_test.go: -------------------------------------------------------------------------------- 1 | package randomsanity 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/hex" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func TestLooksRandom(t *testing.T) { 11 | var tests = []struct { 12 | hexbytes string 13 | want bool 14 | }{ 15 | // Software failure: use counter instead of random source 16 | // (rngstat.Counting tests) 17 | 18 | // 8-bit: start with a random 8-bit value, 19 | // chances that the next 8 bytes (64 bits) happen to look like 20 | // counting up are 1 in 2^64, less than our false-positive rate 21 | {"01 02 03 04 05 06 07 08 09", false}, 22 | {"18 19 1a 1b 1c 1d 1e 1f 20", false}, 23 | 24 | // 16-bit: 25 | {"0000 0001 0002 0003 0004", false}, // big-endian 26 | {"9991 9992 9993 9994 9995", false}, 27 | {"0000 0100 0200 0300 0400", false}, // little-endian 28 | {"9199 9299 9399 9499 9599", false}, 29 | 30 | // 32-bit: 31 | {"00000001 00000002 00000003", false}, // big-endian 32 | {"1111111f 11111120 11111121", false}, 33 | {"01000000 02000000 03000000", false}, // little-endian 34 | {"1f111111 20111111 21111111", false}, 35 | 36 | // 64-bit. Just one 64-bit sequence is enough to be under the 37 | // 2^60 false positive rate. 38 | {"0000000000000001 0000000000000002", false}, // big-endian 39 | {"ac80d400f8cd5946 ac80d400f8cd5947", false}, 40 | {"4edc2837e54241ff 4edc2837e5424200", false}, 41 | {"0100000000000000 0200000000000000", false}, // little-endian 42 | {"ff4132e53728dc4e 004232e53728dc4e", false}, 43 | 44 | // repeated bytes tests 45 | // (rngstat.Repeated tests) 46 | {"00", true}, 47 | {"ff", true}, 48 | {"00000000000000", true}, 49 | {"0000000000000000", false}, 50 | {"ffffffffffffffff", false}, 51 | {"fffffffeffffffff", true}, 52 | {"0100000000000000", true}, 53 | {"ff000000000000000000ff", false}, 54 | {"00ffffffffffffffffff00", false}, 55 | {"aaaaaaaaaaaaaaab", true}, 56 | {"aaaaaaaaaaaaaaaa", false}, 57 | {"ffaaaaaaaaaaaaaaaaaabb", false}, 58 | {"39393939393939ab", true}, 59 | {"3939393939393939", false}, 60 | {"ff393939393939393939bb", false}, 61 | 62 | // stuck bits tests (need 64 bytes for one bit set) 63 | {"136d3d153516244b2a366d7b401131523d453b701f4b7c6d39480710561b5e0a136d3d153516244b2a366d7b401131523d453b701f4b7c6d39480710561b5e0a", false}, // 0x80 bit unset 64 | {"13adbd95b516248baa36ad3b8011b1123d053bb09f0b3c2db9080790961b1e0a13adbd95b516248baa36ad3b8011b1123d053bb09f0b3c2db9080790961b1e0a", false}, // 0x40 bit unset 65 | {"13cd9d95951604cb8a16cd5bc01191521d451bd09f4b5c4d99480790d61b5e0a13cd9d95951604cb8a16cd5bc01191521d451bd09f4b5c4d99480790d61b5e0a", false}, // 0x20 bit unset 66 | {"11edbd95b51424c9a834ed79c011b1503d4539f09d497c6db9480590d4195c0811edbd95b51424c9a834ed79c011b1503d4539f09d497c6db9480590d4195c08", false}, // 0x02 bit unset 67 | {"12ecbc94b41624caaa36ec7ac010b0523c443af09e4a7c6cb8480690d61a5e0a12ecbc94b41624caaa36ec7ac010b0523c443af09e4a7c6cb8480690d61a5e0a", false}, // 0x01 bit unset 68 | {"13efbf97b71626cbaa36ef7bc213b3523f473bf29f4b7e6fbb4a0792d61b5e0a13efbf97b71626cbaa36ef7bc213b3523f473bf29f4b7e6fbb4a0792d61b5e0a", false}, // 0x02 bit set 69 | 70 | // Confusing decimal and hex (no A-F hex digits) 71 | // ... need 45 or more bytes (89 or more digits) to be over the 2^60 fp rate... 72 | {"5687699284852334922144442080814130522504026209026917517342398099734044916617241681431665", true}, 73 | {"568769928485233492214444208081413052250402620902691751734239809973404491661724168143166566", false}, 74 | {"a68769928485233492214444208081413052250402620902691751734239809973404491661724168143166566", true}, 75 | {"5f8769928485233492214444208081413052250402620902691751734239809973404491661724168143166566", true}, 76 | {"56876992848523349221444420808141305225040262090269175173423980997340449166172416814316656a", true}, 77 | {"56876992848523349221444420808141305225040262090F691751734239809973404491661724168143166566", true}, 78 | 79 | // Actual random bitstreams, 1 to 32 bytes 80 | {"8b", true}, 81 | {"6c72", true}, 82 | {"307dd9", true}, 83 | {"69f3171e", true}, 84 | {"64980ad616", true}, 85 | {"bb039395f8de", true}, 86 | {"0eee58c404c82b", true}, 87 | {"b45b237eeca0c59d", true}, 88 | {"1d69df683069246282", true}, 89 | {"81a6cefa3675ed6f04b9", true}, 90 | {"143d92cc0ac0c594169967", true}, 91 | {"a3d5be02d5b77a44793dccb4", true}, 92 | {"98aa8d91d6d732d88c39c8ceec", true}, 93 | {"3b1d9551df40c9330541c17a7ed2", true}, 94 | {"356982f3f3a0a48a13df95245a7330", true}, 95 | {"e47d253e45ccfa65f44493677aaf56ae", true}, 96 | {"92f4752dbfcc23da433c9a8759cc67b330", true}, 97 | {"17c7a1fae0f4a2d9efab4e4081f61afc4970", true}, 98 | {"da8445a72b1c80affd49346f36cb63429eae10", true}, 99 | {"be5d96f4a70273c960b3ce27997d6e388aac5e6b", true}, 100 | {"17872e3aadb230cdeec35335fc6d3e4bf4ccc45b29", true}, 101 | {"e9c5f8819c861b6e58af10e77233eac07328a1b51466", true}, 102 | {"48fd3700fea9515416527f5834519ab25ce418e152e7c2", true}, 103 | {"db80540a4bca01e1f218fb3162afe3ed6d4552fea89228bb", true}, 104 | {"c96c862bc74fa6d6d2f026868b7a611e1650ab28500eb161db", true}, 105 | {"44fce84f7a38be9532caf56ad5b8911f5756629e8402778a61f1", true}, 106 | {"8d637674c809bd2ab7b20a6dae939176a4ed7fb54e95e1a4a31db6", true}, 107 | {"4e811093195e9e7236a071c6c386650c374661d50cd802b86cfbe4a3", true}, 108 | {"194d61bdd628f380916746f6804eaa83f7919fa87dffd3bee80c1b4be8", true}, 109 | {"d1d648be784a79b0fde0a2f79562c1576643f0d322ff73163dd960c9a7a0", true}, 110 | {"4724b307af612288395831874016ede4f3ba2d41df40c3884f1ff1b9c05ac3", true}, 111 | {"13edbd95b51624cbaa36ed7bc011b1523d453bf09f4b7c6db9480790d61b5e0a", true}, 112 | } 113 | for _, test := range tests { 114 | b, err := hex.DecodeString(strings.Replace(test.hexbytes, " ", "", -1)) 115 | if err != nil { 116 | panic(err) 117 | } 118 | if got, which := LooksRandom(b); got != test.want { 119 | if which != "" { 120 | t.Errorf("LooksRandom(%q) = %v (%s)", test.hexbytes, got, which) 121 | } else { 122 | t.Errorf("LooksRandom(%q) = %v", test.hexbytes, got) 123 | } 124 | } 125 | } 126 | } 127 | 128 | func BenchmarkLooksRandom(b *testing.B) { 129 | var rhash [128]byte 130 | for i := 0; i < b.N; i++ { 131 | _, err := rand.Read(rhash[:]) 132 | if err != nil { 133 | panic(err) 134 | } 135 | r, t := LooksRandom(rhash[:]) 136 | if r == false { 137 | b.Errorf("%s failed LooksRandom (%s)", hex.EncodeToString(rhash[:]), t) 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /ratelimit.go: -------------------------------------------------------------------------------- 1 | package randomsanity 2 | 3 | import ( 4 | "appengine" 5 | "appengine/memcache" 6 | "fmt" 7 | "net/http" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | // Limit something (identified by key) to at most max per timespan 13 | // State stored in the memcache, so this is "best-effort" 14 | // Returns true if rate limit is hit. 15 | func RateLimit(ctx appengine.Context, key string, max uint64, timespan time.Duration) (bool, error) { 16 | value, err := memcache.Increment(ctx, key, -1, max+1) 17 | if err != nil { 18 | return false, err 19 | } 20 | // value 0 : ran into request limit 21 | if value == 0 { 22 | return true, nil 23 | } 24 | // value max means it wasn't set before, so 25 | // rewrite to set correct expiration time: 26 | if value == max { 27 | item, err := memcache.Get(ctx, key) 28 | if err != nil { 29 | return false, err 30 | } 31 | item.Expiration = timespan 32 | // There is a race condition here, but it is mostly harmless 33 | // (extra requests above the rate limit could slip through) 34 | memcache.Set(ctx, item) 35 | } 36 | return false, nil 37 | } 38 | 39 | // Rate limit, and write stuff to w: 40 | func RateLimitResponse(ctx appengine.Context, w http.ResponseWriter, key string, max uint64, timespan time.Duration) (bool, error) { 41 | limit, err := RateLimit(ctx, key, max, timespan) 42 | if err != nil { 43 | http.Error(w, "RateLimit error", http.StatusInternalServerError) 44 | return false, err 45 | } 46 | if limit { 47 | w.Header().Add("Content-Type", "text/plain") 48 | w.WriteHeader(http.StatusTooManyRequests) 49 | fmt.Fprint(w, "Request limit exceeded") 50 | return true, nil 51 | } 52 | return false, nil 53 | } 54 | 55 | // Get a reasonable memcache key from IPv4 or IPv6 address 56 | func IPKey(prefix string, ipaddr string) string { 57 | // If it is a super-long IPv6: use first four parts 58 | ipv6parts := strings.Split(ipaddr, ":") 59 | if len(ipv6parts) > 4 { 60 | return prefix + strings.Join(ipv6parts[0:4], ":") 61 | } 62 | return prefix + ipaddr 63 | } 64 | -------------------------------------------------------------------------------- /unique.go: -------------------------------------------------------------------------------- 1 | package randomsanity 2 | 3 | // Best-effort "have we ever seen this array of bytes before?" 4 | 5 | import ( 6 | "appengine" 7 | "appengine/datastore" 8 | "bytes" 9 | "crypto/rand" 10 | "crypto/sha256" 11 | "net/http" 12 | "time" 13 | ) 14 | 15 | func looksUnique(ctx appengine.Context, w http.ResponseWriter, b []byte, uID string, tag string) (bool, error) { 16 | // Test every 16-byte (128-bit) sequence in the input against our database 17 | 18 | // if we get a match, complain! 19 | match, i, err := unique(ctx, b[:], uID, tag) 20 | 21 | if err != nil { 22 | http.Error(w, err.Error(), http.StatusInternalServerError) 23 | return true, err 24 | } 25 | if match != nil { 26 | notify(ctx, uID, tag, b[i:i+16], "Non Unique") 27 | if len(match.UserID) > 0 && match.UserID != uID { 28 | notify(ctx, match.UserID, match.Tag, b[i:i+16], "Non Unique") 29 | } 30 | return false, nil 31 | } 32 | return true, nil 33 | } 34 | 35 | // 36 | // Entities in the 'RBH' datastore; 37 | // storing 16 "random we hope" bytes. 38 | // 39 | // First prefixBytes bytes are used as they key, 40 | // the rest are stored as the value, collisions just 41 | // result in multiple values under one key, oldest 42 | // entries first. 43 | // 44 | // The simplest possible storage scheme would be 45 | // 16-byte keys, but that is HUGELY inefficient. 46 | // 47 | // Why 128 bits? We want a false positive rate under 48 | // 1-in-2^60. We're basically running a 'birthday attack' 49 | // so comparing random 128-bit chunks we get 50 | // a chance of collision of any pair of about 1-in-2^64 51 | // 52 | 53 | const prefixBytes = 4 // Use 4 for production, 1 for development/testing collisions 54 | 55 | type RngUniqueBytesEntry struct { 56 | Trailing []byte `datastore:",noindex"` 57 | Time int64 `datastore:",noindex"` 58 | UserID string `datastore:",noindex"` 59 | Tag string `datastore:",noindex"` 60 | } 61 | type RngUniqueBytes struct { 62 | Hits []RngUniqueBytesEntry `datastore:",noindex"` 63 | } 64 | 65 | type SecretBytes struct { 66 | Secret []byte `datastore:",noindex"` 67 | CreationTime int64 68 | } 69 | 70 | func secretKey(ctx appengine.Context) ([]byte, error) { 71 | var result []byte 72 | 73 | // Create random secret if it doesn't already exist: 74 | var secrets []SecretBytes 75 | 76 | q := datastore.NewQuery("SecretBytes") 77 | if _, err := q.GetAll(ctx, &secrets); err != nil { 78 | return result, err 79 | } 80 | if len(secrets) == 0 { 81 | var b [16]byte 82 | if _, err := rand.Read(b[:]); err != nil { 83 | return result, err 84 | } 85 | result = b[:] 86 | secret := SecretBytes{result, time.Now().Unix()} 87 | k := datastore.NewIncompleteKey(ctx, "SecretBytes", nil) 88 | if _, err := datastore.Put(ctx, k, &secret); err != nil { 89 | return result, err 90 | } 91 | } else { 92 | result = secrets[0].Secret 93 | } 94 | return result, nil 95 | } 96 | 97 | func i64(b []byte) int64 { 98 | var result int64 99 | for i := uint(0); i < uint(len(b)) && i < 8; i++ { 100 | result = result | (int64(b[i]) << (i * 8)) 101 | } 102 | return result 103 | } 104 | 105 | func dealWithMultiError(err error) error { 106 | // GetMulti returns either plain errors OR 107 | // an appengine.MultiError that is an array 108 | // of errors. We're OK if all the 'errors' 109 | // are ErrNoSuchEntity; otherwise, 110 | // we'll report the first error 111 | switch err.(type) { 112 | case nil: 113 | return nil 114 | case appengine.MultiError: 115 | m := err.(appengine.MultiError) 116 | for _, e := range m { 117 | if e == nil || e == datastore.ErrNoSuchEntity { 118 | continue 119 | } 120 | return e 121 | } 122 | return nil 123 | default: 124 | return err 125 | } 126 | return err 127 | } 128 | 129 | // Given secret and data, return 16-byte hash 130 | func hash16(secret []byte, data []byte) []byte { 131 | b := bytes.Join([][]byte{secret, data}, []byte{}) 132 | h := sha256.Sum224(b) 133 | return h[0:16] 134 | } 135 | 136 | func unique(ctx appengine.Context, b []byte, uID string, tag string) (*RngUniqueBytesEntry, int, error) { 137 | n := len(b) - 15 // Number of queries 138 | keys := make([]*datastore.Key, n) 139 | vals := make([]*RngUniqueBytes, n) 140 | 141 | // Input is first hashed with a secret, to prevent an attacker 142 | // from intentionally causing database entry collisions. 143 | secret, err := secretKey(ctx) 144 | if err != nil { 145 | return nil, 0, err 146 | } 147 | 148 | chunks := make([][]byte, n) 149 | for i := 0; i < n; i++ { 150 | chunks[i] = hash16(secret, b[i:i+16]) 151 | 152 | keys[i] = datastore.NewKey(ctx, "RBH", "", 1+i64(chunks[i][0:prefixBytes]), nil) 153 | vals[i] = new(RngUniqueBytes) 154 | } 155 | err = datastore.GetMulti(ctx, keys, vals) 156 | err = dealWithMultiError(err) 157 | 158 | if err != nil { 159 | return nil, 0, err 160 | } 161 | for i, hit := range vals { 162 | for _, h := range hit.Hits { 163 | if bytes.Equal(h.Trailing, chunks[i][prefixBytes:]) { 164 | // Rewriting keeps this entry from getting evicted 165 | // and overwriting the userid prevents the 166 | // user from getting too many notifications 167 | write(ctx, chunks[i][:], time.Now().Unix(), "", h.Tag) 168 | return &h, i, nil // ... full match! 169 | } 170 | } 171 | } 172 | // If no matches, store the first and last 16 bytes. Any future 173 | // overlapping sequences will trigger a match. 174 | err = write(ctx, chunks[0][:], time.Now().Unix(), uID, tag) 175 | if err == nil && n > 1 { 176 | err = write(ctx, chunks[n-1][:], time.Now().Unix(), uID, tag) 177 | } 178 | if err != nil { 179 | return nil, 0, err 180 | } 181 | return nil, 0, nil 182 | } 183 | 184 | func write(ctx appengine.Context, b []byte, t int64, uID string, tag string) error { 185 | const maxEntriesPerKey = 100 186 | 187 | key := datastore.NewKey(ctx, "RBH", "", 1+i64(b[0:prefixBytes]), nil) 188 | 189 | err := datastore.RunInTransaction(ctx, func(ctx appengine.Context) error { 190 | hit := new(RngUniqueBytes) 191 | err := datastore.Get(ctx, key, hit) 192 | if err != nil && err != datastore.ErrNoSuchEntity { 193 | return err 194 | } 195 | // Find and remove old entry (if any): 196 | hits := hit.Hits[:0] 197 | for _, h := range hit.Hits { 198 | if !bytes.Equal(h.Trailing, b[prefixBytes:]) { 199 | hits = append(hits, h) 200 | } 201 | } 202 | // Append new: 203 | e := RngUniqueBytesEntry{Trailing: b[prefixBytes:], Time: t, UserID: uID, Tag: tag} 204 | hit.Hits = append(hits, e) 205 | // Throw out half the old if bucket overflows: 206 | if len(hit.Hits) > maxEntriesPerKey { 207 | hit.Hits = hit.Hits[len(hit.Hits)/2:] 208 | } 209 | _, err = datastore.Put(ctx, key, hit) 210 | return err 211 | }, nil) 212 | return err 213 | } 214 | -------------------------------------------------------------------------------- /usage.go: -------------------------------------------------------------------------------- 1 | package randomsanity 2 | 3 | import ( 4 | "appengine" 5 | "appengine/datastore" 6 | "encoding/json" 7 | "log" 8 | "math/rand" // don't need cryptographically secure randomness here 9 | "net/http" 10 | ) 11 | 12 | // Keep track of usage stats 13 | 14 | // If frequency of database writes becomes a problem, increase SAMPLING_FACTOR 15 | // to only write about every SAMPLING_FACTOR usages. 16 | const SAMPLING_FACTOR = 1 17 | 18 | type UsageRecord struct { 19 | K string 20 | N int64 `datastore:",noindex"` 21 | } 22 | 23 | func RecordUsage(ctx appengine.Context, k string, n int64) { 24 | if rand.Intn(SAMPLING_FACTOR) != 0 { 25 | return 26 | } 27 | key := datastore.NewKey(ctx, "UsageRecord", k, 0, nil) 28 | 29 | err := datastore.RunInTransaction(ctx, func(ctx appengine.Context) error { 30 | r := UsageRecord{K: k, N: 0} 31 | err := datastore.Get(ctx, key, &r) 32 | if err != nil && err != datastore.ErrNoSuchEntity { 33 | return err 34 | } 35 | r.N += n * SAMPLING_FACTOR 36 | _, err = datastore.Put(ctx, key, &r) 37 | return err 38 | }, nil) 39 | if err != nil { 40 | log.Printf("Datastore error: %s", err.Error()) 41 | } 42 | } 43 | 44 | func GetUsage(ctx appengine.Context) []UsageRecord { 45 | var results []UsageRecord 46 | 47 | q := datastore.NewQuery("UsageRecord") 48 | _, err := q.GetAll(ctx, &results) 49 | if err != nil { 50 | log.Printf("Datastore error: %s", err.Error()) 51 | } 52 | return results 53 | } 54 | 55 | func usageHandler(w http.ResponseWriter, r *http.Request) { 56 | w.Header().Add("Content-Type", "application/json") 57 | ctx := appengine.NewContext(r) 58 | usage := GetUsage(ctx) 59 | m := make(map[string]int64) 60 | for _, rr := range usage { 61 | m[rr.K] = rr.N 62 | } 63 | enc := json.NewEncoder(w) 64 | enc.Encode(m) 65 | } 66 | --------------------------------------------------------------------------------