├── misc ├── distance-2500-10000000.png └── rend-spec.txt ├── .gitignore ├── src ├── go_reduced_rsa_primalty.diff ├── hstools │ ├── stats_test.go │ ├── consensus_test.go │ ├── descid_test.go │ ├── hashring_test.go │ ├── format_test.go │ ├── brute_test.go │ ├── format.go │ ├── brute.go │ ├── hashring.go │ ├── descid.go │ ├── keysdb.go │ ├── consensus.go │ └── stats.go ├── cmd │ ├── grind.go │ ├── montecarlo.go │ ├── preprocess.go │ ├── curiosity.go │ ├── lookmeup.go │ ├── announce.html │ ├── announce.go │ ├── brute.go │ └── scrolls.go └── retrieve_hs_descriptor.py ├── .gitmodules ├── Makefile ├── LICENSE └── README.md /misc/distance-2500-10000000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FiloSottile/hstools/HEAD/misc/distance-2500-10000000.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | 24 | /bin 25 | /pkg 26 | /data 27 | -------------------------------------------------------------------------------- /src/go_reduced_rsa_primalty.diff: -------------------------------------------------------------------------------- 1 | diff --git a/src/crypto/rand/util.go b/src/crypto/rand/util.go 2 | index 5f74407..f6d7bbc 100644 3 | --- a/src/crypto/rand/util.go 4 | +++ b/src/crypto/rand/util.go 5 | @@ -96,7 +96,7 @@ func Prime(rand io.Reader, bits int) (p *big.Int, err error) { 6 | // There is a tiny possibility that, by adding delta, we caused 7 | // the number to be one bit too long. Thus we check BitLen 8 | // here. 9 | - if p.ProbablyPrime(20) && p.BitLen() == bits { 10 | + if p.ProbablyPrime(1) && p.BitLen() == bits { 11 | return 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/hstools/stats_test.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import "testing" 4 | 5 | func TestAnalyzePartitionData(t *testing.T) { 6 | h := RandomHashring(3000) 7 | res := AnalyzePartitionData(h.Distance4Data()) 8 | t.Log(res) 9 | 10 | var meanHi, meanLo, devHi, devLo bool 11 | for i := 0; i < 10; i++ { 12 | resSample := SampleDistance4(h) 13 | t.Log(resSample) 14 | meanHi = meanHi || resSample.Mean.Cmp(res.Mean) > 0 15 | meanLo = meanLo || resSample.Mean.Cmp(res.Mean) < 0 16 | devHi = devHi || resSample.AbsDev.Cmp(res.AbsDev) > 0 17 | devLo = devLo || resSample.AbsDev.Cmp(res.AbsDev) < 0 18 | } 19 | 20 | if !(meanHi && meanLo && devHi && devLo) { 21 | t.Failed() 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tor"] 2 | path = tor 3 | url = https://git.torproject.org/tor.git 4 | [submodule "src/github.com/boltdb/bolt"] 5 | path = src/github.com/boltdb/bolt 6 | url = https://github.com/boltdb/bolt 7 | [submodule "src/github.com/ActiveState/tail"] 8 | path = src/github.com/ActiveState/tail 9 | url = https://github.com/ActiveState/tail 10 | [submodule "src/github.com/howeyc/fsnotify"] 11 | path = src/github.com/howeyc/fsnotify 12 | url = https://github.com/howeyc/fsnotify 13 | [submodule "src/gopkg.in/tomb.v1"] 14 | path = src/gopkg.in/tomb.v1 15 | url = https://gopkg.in/tomb.v1 16 | [submodule "src/golang.org/x/net"] 17 | path = src/golang.org/x/net 18 | url = https://go.googlesource.com/net 19 | [submodule "src/github.com/mgutz/ansi"] 20 | path = src/github.com/mgutz/ansi 21 | url = https://github.com/mgutz/ansi 22 | -------------------------------------------------------------------------------- /src/hstools/consensus_test.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "log" 5 | "math/big" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestParseConsensus(t *testing.T) { 11 | c, err := ParseConsensus("../../misc/2015-04-11-19-00-00-consensus") 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | h := NewHashring(HashesToIntSlice(c.K)) 16 | if h.Len() != 2983 { 17 | t.Fatalf("wrong number of points: %d", h.Len()) 18 | } 19 | 20 | if len(c.IP) != len(c.K) { 21 | log.Fatal("mismatch keys to IPs") 22 | } 23 | 24 | tt, _ := time.Parse(time.RFC3339, "2015-04-11T19:30:00Z") 25 | desc, err := OnionToDescID("facebookcorewwwi.onion", tt) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | 30 | hsdir := h.Next(new(big.Int).SetBytes(desc[0])).Bytes() 31 | if ToHex(hsdir) != "274D66DC037FE344C58371B17C606988CBC37DFB" { 32 | t.Fatalf("wrong hsdir: %s", hsdir) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/hstools/descid_test.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestFacebookOnion(t *testing.T) { 9 | tt, _ := time.Parse(time.RFC3339, "2015-04-11T19:30:00Z") 10 | desc, err := OnionToDescID("facebookcorewwwi.onion", tt) 11 | if err != nil { 12 | t.Fatal(err) 13 | } 14 | if ToBase32(desc[0]) != "e4jiuabozanwqxdobx44w47mx2hi2auz" { 15 | t.Errorf("Wrong desc[0]: %v (!= e4jiuabozanwqxdobx44w47mx2hi2auz)", ToBase32(desc[0])) 16 | } 17 | if ToBase32(desc[1]) != "tyvtyaqd4trmgoopqktv4aawelu6skes" { 18 | t.Errorf("Wrong desc[0]: %v (!= tyvtyaqd4trmgoopqktv4aawelu6skes)", ToBase32(desc[1])) 19 | } 20 | } 21 | 22 | func TestCurrentOnion(t *testing.T) { 23 | if testing.Short() { 24 | t.SkipNow() 25 | } 26 | desc, err := OnionToDescID("facebookcorewwwi.onion", time.Now()) 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | t.Log(ToBase32(desc[0]), ToBase32(desc[1])) 31 | } 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test clean all 2 | .PHONY: preprocess brute lookmeup announce grind curiosity scrolls montecarlo 3 | all: preprocess brute lookmeup announce grind curiosity scrolls 4 | 5 | GO ?= go 6 | 7 | montecarlo: 8 | GOPATH="$(CURDIR)" $(GO) build -o bin/montecarlo src/cmd/montecarlo.go 9 | 10 | preprocess: 11 | GOPATH="$(CURDIR)" $(GO) build -o bin/preprocess src/cmd/preprocess.go 12 | 13 | brute: 14 | GOPATH="$(CURDIR)" $(GO) build -o bin/brute src/cmd/brute.go 15 | 16 | lookmeup: 17 | GOPATH="$(CURDIR)" $(GO) build -o bin/lookmeup src/cmd/lookmeup.go 18 | 19 | announce: 20 | GOPATH="$(CURDIR)" $(GO) build -o bin/announce src/cmd/announce.go 21 | 22 | grind: 23 | GOPATH="$(CURDIR)" $(GO) build -o bin/grind src/cmd/grind.go 24 | 25 | curiosity: 26 | GOPATH="$(CURDIR)" $(GO) build -o bin/curiosity src/cmd/curiosity.go 27 | 28 | scrolls: 29 | GOPATH="$(CURDIR)" $(GO) build -o bin/scrolls src/cmd/scrolls.go 30 | 31 | test: 32 | GOPATH="$(CURDIR)" $(GO) test hstools -race -v -short 33 | 34 | clean: 35 | - rm -r bin pkg 36 | -------------------------------------------------------------------------------- /src/hstools/hashring_test.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | ) 7 | 8 | func TestHashringNext(t *testing.T) { 9 | h := NewHashring([]*big.Int{ 10 | big.NewInt(500), 11 | big.NewInt(200), 12 | big.NewInt(300), 13 | big.NewInt(400), 14 | big.NewInt(100), 15 | }) 16 | 17 | for i := int64(1); i <= 5; i++ { 18 | expected := i * 100 19 | n := h.Next(big.NewInt(expected - 50)) 20 | if n.Int64() != expected { 21 | t.Fatal(n.Int64(), expected) 22 | } 23 | } 24 | 25 | n := h.Next(big.NewInt(550)) 26 | if n.Int64() != 100 { 27 | t.Fatal(n.Int64(), 100) 28 | } 29 | } 30 | 31 | func TestHashringDistance(t *testing.T) { 32 | h := NewHashring([]*big.Int{ 33 | big.NewInt(500), 34 | big.NewInt(200), 35 | big.NewInt(300), 36 | big.NewInt(400), 37 | big.NewInt(100), 38 | }) 39 | 40 | n := h.Distance(big.NewInt(450)) 41 | if n.Int64() != 50 { 42 | t.Fatal(n.Int64()) 43 | } 44 | 45 | n = h.Distance(big.NewInt(550)) 46 | exp := new(big.Int).Sub(HashringLimit, big.NewInt(550-100)) 47 | if n.Cmp(exp) != 0 { 48 | t.Fatal(n, exp) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Filippo Valsorda 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/hstools/format_test.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "math/big" 7 | "testing" 8 | ) 9 | 10 | func exitIfErr(t *testing.T, err error) { 11 | if err != nil { 12 | t.Fatal(err) 13 | } 14 | } 15 | 16 | func TestHashDistance(t *testing.T) { 17 | var a, b, A, B, d Hash 18 | var aInt, bInt = new(big.Int), new(big.Int) 19 | var dInt, DInt = new(big.Int), new(big.Int) 20 | for i := 0; i < 10000; i++ { 21 | _, err := rand.Read(a[:]) 22 | exitIfErr(t, err) 23 | copy(A[:], a[:]) 24 | _, err = rand.Read(b[:]) 25 | exitIfErr(t, err) 26 | copy(B[:], b[:]) 27 | 28 | a.Distance(&b, &d) 29 | DInt = DInt.SetBytes(d[:]) 30 | 31 | if !bytes.Equal(a[:], A[:]) || !bytes.Equal(b[:], B[:]) { 32 | t.Fatal("input changed") 33 | } 34 | 35 | // Compute the distance by big.Int to check 36 | aInt = aInt.SetBytes(a[:]) 37 | bInt = bInt.SetBytes(b[:]) 38 | if aInt.Cmp(bInt) < 0 { 39 | dInt = dInt.Sub(bInt, aInt) 40 | } else { 41 | dInt = dInt.Sub(HashringLimit, aInt) 42 | dInt = dInt.Add(dInt, bInt) 43 | } 44 | 45 | if DInt.Cmp(dInt) != 0 { 46 | t.Log(DInt.Bytes()) 47 | t.Log(dInt.Bytes()) 48 | t.Fatal("different result") 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/cmd/grind.go: -------------------------------------------------------------------------------- 1 | // +build manually 2 | 3 | // grind: compute the average distance to 1st and 4th node and its average 4 | // deviation and output it as JSON lines 5 | package main 6 | 7 | import ( 8 | "bufio" 9 | "encoding/json" 10 | "hstools" 11 | "log" 12 | "net/http" 13 | _ "net/http/pprof" 14 | "os" 15 | "runtime" 16 | ) 17 | 18 | func fatalIfErr(err error) { 19 | if err != nil { 20 | log.Fatal(err) 21 | } 22 | } 23 | 24 | func main() { 25 | runtime.GOMAXPROCS(runtime.NumCPU()) 26 | 27 | go func() { 28 | log.Println(http.ListenAndServe("localhost:6060", nil)) 29 | }() 30 | 31 | if len(os.Args) != 2 { 32 | log.Fatal("usage: grind pckcns.dat > stats.jsonl") 33 | } 34 | 35 | w := bufio.NewWriterSize(os.Stdout, 1024*1024) 36 | jsonEncoder := json.NewEncoder(w) 37 | r := hstools.NewPackReader(os.Args[1]) 38 | for i := 0; r.Load(); i++ { 39 | c := r.Consensus() 40 | h := hstools.NewHashring(hstools.HashesToIntSlice(c.K)) 41 | 42 | fatalIfErr(jsonEncoder.Encode(hstools.AnalyzedConsensus{ 43 | T: c.Time, 44 | Distance: hstools.AnalyzePartitionData(h.DistanceData()), 45 | Distance4: hstools.AnalyzePartitionData(h.Distance4Data()), 46 | })) 47 | 48 | if i%1000 == 0 { 49 | log.Println(hstools.HourToTime(c.Time)) 50 | } 51 | } 52 | fatalIfErr(r.Err()) 53 | fatalIfErr(w.Flush()) 54 | } 55 | -------------------------------------------------------------------------------- /src/cmd/montecarlo.go: -------------------------------------------------------------------------------- 1 | // +build manually 2 | 3 | package main 4 | 5 | import ( 6 | "hstools" 7 | "log" 8 | "math/big" 9 | "math/rand" 10 | "os" 11 | "strconv" 12 | "time" 13 | 14 | "code.google.com/p/plotinum/plot" 15 | "code.google.com/p/plotinum/plotter" 16 | ) 17 | 18 | var random = rand.New(rand.NewSource(time.Now().UnixNano())) 19 | 20 | func RandomDistance(ring *hstools.Hashring) *big.Int { 21 | origin := new(big.Int).Rand(random, hstools.HashringLimit) 22 | return ring.Distance(origin) 23 | } 24 | 25 | func plotDistance() { 26 | if len(os.Args) < 3 { 27 | log.Fatal("usage: montecarlo HSDirs RUNS") 28 | } 29 | 30 | hsdirs, err := strconv.Atoi(os.Args[1]) 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | runs, err := strconv.Atoi(os.Args[2]) 35 | if err != nil { 36 | log.Fatal(err) 37 | } 38 | 39 | ring := hstools.RandomHashring(hsdirs) 40 | 41 | v := make(plotter.Values, runs) 42 | for i := 0; i < runs; i++ { 43 | d := RandomDistance(ring) 44 | // keep 32 bits of precision 45 | v[i] = float64(d.Rsh(d, 160-32).Int64()) 46 | } 47 | 48 | p, err := plot.New() 49 | if err != nil { 50 | log.Fatal(err) 51 | } 52 | p.Title.Text = "Distance" 53 | 54 | h, err := plotter.NewHist(v, 100) 55 | if err != nil { 56 | log.Fatal(err) 57 | } 58 | p.Add(h) 59 | 60 | if err := p.Save(20, 5, "distance.png"); err != nil { 61 | log.Fatal(err) 62 | } 63 | } 64 | 65 | func main() { 66 | plotDistance() 67 | } 68 | -------------------------------------------------------------------------------- /src/hstools/brute_test.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "bytes" 5 | "crypto/x509" 6 | "encoding/pem" 7 | "testing" 8 | ) 9 | 10 | func TestHashIdentity(t *testing.T) { 11 | block, _ := pem.Decode([]byte(`-----BEGIN RSA PRIVATE KEY----- 12 | MIICXAIBAAKBgQDX11Z88VBf+4ZJiczyTjTHMS9x1ZbC5qBLQj4LhOWkKJZe9ObK 13 | lcbGd+oyVNip4FTaY5RFenMYOt1ESlYn8jaU/vAi0IMA/E70x9c0p6eLwSr+zCEU 14 | CL/S6ISxwnaYiP92fLfL9keGErKoMbN3t01tAmaDN5jdaaiREVGsHgFVoQIDAQAB 15 | AoGALUw6EHqsfZhR9HkBFBEprmw6Is/KlhjEp0a9srkvYKZL+J25GecZEmn0Mp/v 16 | 4Kb9599iLLqoEPu5mC1pq3R/055F97x/IGxxhP/80LmXLCIeeNG+m3s/ezwUNgny 17 | jT+rsCQAxs/r6sjIcCIAfM8rKtXuqcgUew+d8G3hoSwYv+kCQQD9Fc79mdV8sL/c 18 | ChCY9ryxFwofSn8Ljpm4SJ1RssBsXF3+RnG/G6P80k3/wcae/1w1m/KpoqvZT0Qw 19 | fUfMe87XAkEA2lO4P+2oNkjlaqHVlBJYShBm4QoBPls0boX4aB4hjb+AlQ8P024+ 20 | Pis7qXa4glxlumlDL6CXQx/cRjsdXyXIRwJBAPEeI/SM6U5Afqm+lQ2GlUMKtkQV 21 | j3CNTXq7A9bgPF+AqLQmnRv704J9Qn6WOQsmMs2IY+ql5p/E2yxvT0ZL9kUCQDkC 22 | bXU8AJWUOVu7wIJ2u9kzKToQG70Foc5Oa0v8ujRCUjgaA77o5ZXkQiMBHjLkH6gq 23 | fmG8ZGMhuaoZG5VRz1cCQGox7SskO48AyaynCKNXM3+vWDNtiwrsxBeX3T2nIWWO 24 | x8IyevfhgPIzX0bajUEqm+phNXWBMUTobyTJbkJQ4NQ= 25 | -----END RSA PRIVATE KEY-----`)) 26 | if block.Type != "RSA PRIVATE KEY" { 27 | t.Fatal("wrong type") 28 | } 29 | key, err := x509.ParsePKCS1PrivateKey(block.Bytes) 30 | exitIfErr(t, err) 31 | fing, err := FromHex("EC816FBE76CD94C9064C8F22AF5A468CC46953EA") 32 | exitIfErr(t, err) 33 | res := HashIdentity(key.PublicKey) 34 | if !bytes.Equal(res[:], fing) { 35 | t.Fail() 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/cmd/preprocess.go: -------------------------------------------------------------------------------- 1 | // +build manually 2 | 3 | package main 4 | 5 | import ( 6 | "hstools" 7 | "log" 8 | "net/http" 9 | _ "net/http/pprof" 10 | "os" 11 | "runtime" 12 | "sync" 13 | "time" 14 | ) 15 | 16 | func main() { 17 | runtime.GOMAXPROCS(runtime.NumCPU()) 18 | 19 | go func() { 20 | log.Println(http.ListenAndServe("localhost:6060", nil)) 21 | }() 22 | 23 | if len(os.Args) != 4 { 24 | log.Fatal("usage: preprocess /data/dir/ 2014-01-01-00 2014-01-31-23") 25 | } 26 | 27 | pckFile, err := os.Create("pckcns.dat") 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | keysDB, err := hstools.OpenKeysDb("keys.db") 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | 36 | since, err := time.Parse("2006-01-02-15", os.Args[2]) 37 | if err != nil { 38 | log.Fatal(err) 39 | } 40 | until, err := time.Parse("2006-01-02-15", os.Args[3]) 41 | if err != nil { 42 | log.Fatal(err) 43 | } 44 | 45 | wg := &sync.WaitGroup{} 46 | 47 | ch := hstools.ReadConsensuses(os.Args[1], 48 | hstools.Hour(since.Unix()/3600), hstools.Hour(until.Unix()/3600)) 49 | for c := range ch { 50 | if c.Error != nil { 51 | log.Println(c.Error) 52 | continue 53 | } 54 | 55 | if err := hstools.WritePackedConsensus(pckFile, c); err != nil { 56 | log.Fatal(err) 57 | } 58 | 59 | keysDB.Seen(c.K, c.IP, c.Time, wg) 60 | 61 | log.Println(c.Filename, len(c.K)) 62 | } 63 | 64 | wg.Wait() 65 | if err := keysDB.Close(); err != nil { 66 | log.Fatal(err) 67 | } 68 | if err := pckFile.Close(); err != nil { 69 | log.Fatal(err) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/cmd/curiosity.go: -------------------------------------------------------------------------------- 1 | // +build manually 2 | 3 | package main 4 | 5 | import ( 6 | "encoding/json" 7 | "fmt" 8 | "hstools" 9 | "log" 10 | "net/http" 11 | _ "net/http/pprof" 12 | "os" 13 | "runtime" 14 | 15 | "github.com/boltdb/bolt" 16 | ) 17 | 18 | func main() { 19 | runtime.GOMAXPROCS(runtime.NumCPU()) 20 | 21 | go func() { 22 | log.Println(http.ListenAndServe("localhost:6060", nil)) 23 | }() 24 | 25 | if len(os.Args) != 3 { 26 | log.Fatal("usage: curiosity keys.db {ip,key,colocated}") 27 | } 28 | 29 | keysDB, err := hstools.OpenKeysDb(os.Args[1]) 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | 34 | keysDB.View(func(tx *bolt.Tx) error { 35 | switch os.Args[2] { 36 | case "ip": 37 | c := tx.Bucket([]byte("IPs")).Cursor() 38 | for k, v := c.First(); k != nil; k, v = c.Next() { 39 | var res hstools.IPMeta 40 | if err := json.Unmarshal(v, &res); err != nil { 41 | log.Fatal(err) 42 | } 43 | fmt.Printf("%d %s\n", len(res.Keys), k) 44 | } 45 | case "keys": 46 | c := tx.Bucket([]byte("Keys")).Cursor() 47 | for k, v := c.First(); k != nil; k, v = c.Next() { 48 | var res hstools.KeyMeta 49 | if err := json.Unmarshal(v, &res); err != nil { 50 | log.Fatal(err) 51 | } 52 | fmt.Printf("%d %s\n", len(res.IPs), hstools.ToHex(k)) 53 | } 54 | case "colocated": 55 | c := tx.Bucket([]byte("Keys")).Cursor() 56 | for k, _ := c.First(); k != nil; k, _ = c.Next() { 57 | coloNum, ips := hstools.ColocatedKeys(k, keysDB) 58 | fmt.Printf("%d keys on %d IPs - %s %v\n", 59 | coloNum, len(ips), hstools.ToHex(k), ips) 60 | } 61 | default: 62 | log.Fatal("usage: curiosity keys.db {ip,key,colocated}") 63 | } 64 | 65 | return nil 66 | }) 67 | 68 | if err := keysDB.Close(); err != nil { 69 | log.Fatal(err) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/cmd/lookmeup.go: -------------------------------------------------------------------------------- 1 | // +build manually 2 | 3 | // lookmeup: bruteforce a onion address that when looked up will have the given 4 | // HSDir, so that you can look it up and see it in the logs of the HSDir 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "crypto/rand" 10 | "hstools" 11 | "log" 12 | "math/big" 13 | "net/http" 14 | _ "net/http/pprof" 15 | "os" 16 | "runtime" 17 | "time" 18 | ) 19 | 20 | func fatalIfErr(err error) { 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | } 25 | 26 | func main() { 27 | runtime.GOMAXPROCS(runtime.NumCPU()) 28 | 29 | go func() { 30 | log.Println(http.ListenAndServe("localhost:6060", nil)) 31 | }() 32 | 33 | if len(os.Args) < 3 { 34 | log.Fatal("usage: lookmeup consensus fingerprint") 35 | } 36 | 37 | var hash hstools.Hash 38 | h, err := hstools.FromHex(os.Args[2]) 39 | fatalIfErr(err) 40 | copy(hash[:], h[:]) 41 | log.Println("[*] Your node is", hstools.ToHex(hash[:])) 42 | 43 | log.Println("[*] Loading consensus...") 44 | c, err := hstools.ParseConsensus(os.Args[1]) 45 | fatalIfErr(err) 46 | hashring := hstools.NewHashring(hstools.HashesToIntSlice(c.K)) 47 | hsDirInt := hashring.Prev(new(big.Int).SetBytes(hash[:])) // first HSDir 48 | hsDirInt = hashring.Prev(hsDirInt) // second HSDir 49 | hsDirInt = hashring.Prev(hsDirInt) // third HSDir 50 | hsDir := hstools.IntToHash(hsDirInt) 51 | log.Println(" Lowest HSDir:", hstools.ToHex(hsDir[:])) 52 | 53 | log.Println("[*] Starting bruteforce...") 54 | b := make([]byte, 10) 55 | for { 56 | _, err := rand.Read(b) 57 | fatalIfErr(err) 58 | onion := hstools.ToBase32(b) 59 | res, err := hstools.OnionToDescID(onion, time.Now()) 60 | fatalIfErr(err) 61 | if bytes.Compare(hsDir[:], res[0]) < 0 && bytes.Compare(res[0], hash[:]) < 0 { 62 | log.Println(" Here, use this:", onion+".onion") 63 | break 64 | } 65 | } 66 | 67 | log.Println("[*] Done!") 68 | } 69 | -------------------------------------------------------------------------------- /src/hstools/format.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "crypto/sha1" 5 | "encoding/base32" 6 | "encoding/base64" 7 | "encoding/hex" 8 | "math" 9 | "math/big" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | type Hash [20]byte 15 | 16 | // Distance efficiently calculates the difference (b - a) mod 2^160, or 17 | // distance a -> b on a 20-byte ring and stores it in d. a and b unchanged. 18 | func (a *Hash) Distance(b, d *Hash) { 19 | var carry bool 20 | for i := len(a) - 1; i >= 0; i-- { 21 | B := b[i] 22 | if carry { 23 | B-- 24 | } 25 | d[i] = B - a[i] 26 | carry = B < a[i] || (carry && B == math.MaxUint8) 27 | } 28 | } 29 | 30 | func SHA1(data []byte) *Hash { 31 | h := Hash(sha1.Sum(data)) 32 | return &h 33 | } 34 | 35 | func ToBase32(b []byte) string { 36 | return strings.ToLower(base32.StdEncoding.EncodeToString(b)) 37 | } 38 | 39 | func FromBase32(s string) ([]byte, error) { 40 | return base32.StdEncoding.DecodeString(strings.ToUpper(s)) 41 | } 42 | 43 | func ToHex(b []byte) string { 44 | return strings.ToUpper(hex.EncodeToString(b)) 45 | } 46 | 47 | func FromHex(s string) ([]byte, error) { 48 | return hex.DecodeString(s) 49 | } 50 | 51 | func FromBase64(s string) ([]byte, error) { 52 | if r := len(s) % 4; r != 0 { 53 | s += strings.Repeat("=", 4-r) 54 | } 55 | return base64.StdEncoding.DecodeString(s) 56 | } 57 | 58 | func HourToTime(h Hour) time.Time { 59 | return time.Unix(int64(h*3600), 0) 60 | } 61 | 62 | func HashesToIntSlice(keys []Hash) []*big.Int { 63 | ints := make([]*big.Int, len(keys)) 64 | for i, k := range keys { 65 | ints[i] = new(big.Int).SetBytes(k[:]) 66 | } 67 | return ints 68 | } 69 | 70 | func IntsToHashSlice(ints []*big.Int) []*Hash { 71 | hashes := make([]*Hash, len(ints)) 72 | for n, i := range ints { 73 | var k Hash 74 | b := i.Bytes() 75 | copy(k[len(k)-len(b):], b) 76 | hashes[n] = &k 77 | } 78 | return hashes 79 | } 80 | 81 | func IntToHash(i *big.Int) Hash { 82 | var k Hash 83 | b := i.Bytes() 84 | copy(k[len(k)-len(b):], b) 85 | return k 86 | } 87 | 88 | func HashToInt(k Hash) *big.Int { 89 | return new(big.Int).SetBytes(k[:]) 90 | } 91 | -------------------------------------------------------------------------------- /src/cmd/announce.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | WebSocket Test 7 | 52 | 63 | 64 | 65 | 66 |
67 | 68 | 69 | -------------------------------------------------------------------------------- /src/cmd/announce.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "hstools" 5 | "io" 6 | "log" 7 | "net/http" 8 | "os" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | "github.com/ActiveState/tail" 14 | "golang.org/x/net/websocket" 15 | ) 16 | 17 | var channelsLock sync.RWMutex 18 | var channels = make(map[*websocket.Conn]chan string) 19 | 20 | func WSServer(ws *websocket.Conn) { 21 | onion := make([]byte, 16) 22 | if _, err := io.ReadFull(ws, onion); err != nil { 23 | log.Println(err) 24 | return 25 | } 26 | log.Printf("%s", string(onion)) 27 | res, err := hstools.OnionToDescID(string(onion), time.Now()) 28 | if err != nil { 29 | log.Println(err) 30 | return 31 | } 32 | log.Printf("%s %s", hstools.ToBase32(res[0]), hstools.ToBase32(res[1])) 33 | announce := make(chan string) 34 | channelsLock.Lock() 35 | channels[ws] = announce 36 | channelsLock.Unlock() 37 | defer func() { 38 | channelsLock.Lock() 39 | delete(channels, ws) 40 | channelsLock.Unlock() 41 | }() 42 | for desc := range announce { 43 | if hstools.ToBase32(res[0]) != desc && hstools.ToBase32(res[1]) != desc { 44 | continue 45 | } 46 | if _, err := ws.Write([]byte("1")); err != nil { 47 | log.Println(err) 48 | return 49 | } 50 | } 51 | } 52 | 53 | func main() { 54 | defer tail.Cleanup() 55 | if len(os.Args) < 4 { 56 | log.Fatal("usage: announce cert.pem key.pem logfile") 57 | } 58 | 59 | http.Handle("/announce", websocket.Handler(WSServer)) 60 | go func() { 61 | log.Fatal(http.ListenAndServeTLS("0.0.0.0:14242", os.Args[1], os.Args[2], nil)) 62 | }() 63 | 64 | t, err := tail.TailFile(os.Args[3], tail.Config{ 65 | Location: &tail.SeekInfo{ 66 | Offset: 0, 67 | Whence: os.SEEK_END, 68 | }, 69 | MustExist: true, 70 | Follow: true, 71 | ReOpen: true, 72 | }) 73 | if err != nil { 74 | log.Fatal(err) 75 | } 76 | for line := range t.Lines { 77 | if strings.Contains(line.Text, "Got a v2 rendezvous descriptor request for ID") { 78 | i := strings.Index(line.Text, `'"`) + 2 79 | log.Println(line.Text[i : i+32]) 80 | channelsLock.RLock() 81 | for _, ch := range channels { 82 | ch <- line.Text[i : i+32] 83 | } 84 | channelsLock.RUnlock() 85 | } 86 | } 87 | log.Fatal(t.Err()) 88 | } 89 | -------------------------------------------------------------------------------- /src/cmd/brute.go: -------------------------------------------------------------------------------- 1 | // +build manually 2 | 3 | // brute: bruteforce Identity Keys that will be the 6 HSDir for the given onion 4 | // at the given time, considering the given consensus state 5 | package main 6 | 7 | import ( 8 | "crypto/rsa" 9 | "crypto/x509" 10 | "encoding/pem" 11 | "hstools" 12 | "log" 13 | "math/big" 14 | "net/http" 15 | _ "net/http/pprof" 16 | "os" 17 | "runtime" 18 | "time" 19 | ) 20 | 21 | func fatalIfErr(err error) { 22 | if err != nil { 23 | log.Fatal(err) 24 | } 25 | } 26 | 27 | func main() { 28 | runtime.GOMAXPROCS(runtime.NumCPU()) 29 | 30 | go func() { 31 | log.Println(http.ListenAndServe("localhost:6060", nil)) 32 | }() 33 | 34 | if len(os.Args) < 4 { 35 | log.Fatal("usage: brute consensus onion RFC3339time") 36 | } 37 | 38 | log.Println("[*] Computing HS descriptors...") 39 | t, err := time.Parse(time.RFC3339, os.Args[3]) 40 | fatalIfErr(err) 41 | desc, err := hstools.OnionToDescID(os.Args[2], t) 42 | fatalIfErr(err) 43 | var keyA, keyB hstools.Hash 44 | copy(keyA[:], desc[0]) 45 | copy(keyB[:], desc[1]) 46 | log.Printf(" Onion '%s' at time '%s'\n", os.Args[2], t) 47 | log.Println(" Descriptor A:", hstools.ToHex(keyA[:])) 48 | log.Println(" Descriptor B:", hstools.ToHex(keyB[:])) 49 | 50 | log.Println("[*] Loading consensus...") 51 | // Note: this should maybe also consider potential future HSDir 52 | c, err := hstools.ParseConsensus(os.Args[1]) 53 | fatalIfErr(err) 54 | hashring := hstools.NewHashring(hstools.HashesToIntSlice(c.K)) 55 | nextA := hstools.IntToHash(hashring.Next(new(big.Int).SetBytes(keyA[:]))) 56 | nextB := hstools.IntToHash(hashring.Next(new(big.Int).SetBytes(keyB[:]))) 57 | log.Println(" First HSDir A:", hstools.ToHex(nextA[:])) 58 | log.Println(" First HSDir B:", hstools.ToHex(nextB[:])) 59 | 60 | log.Println("[*] Starting bruteforce...") 61 | keysA, keysB := hstools.Brute(keyA, keyB, nextA, nextB, 3, 62 | runtime.NumCPU(), log.Println) 63 | 64 | log.Println("[*] Done!") 65 | 66 | for _, keys := range [][]*rsa.PrivateKey{keysA, keysB} { 67 | for i := 0; i < 3; i++ { 68 | if err := pem.Encode(os.Stderr, &pem.Block{ 69 | Type: "RSA PRIVATE KEY", 70 | Bytes: x509.MarshalPKCS1PrivateKey(keys[i]), 71 | }); err != nil { 72 | log.Fatal(err) 73 | } 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/hstools/brute.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "crypto/rsa" 7 | "crypto/sha1" 8 | "encoding/asn1" 9 | ) 10 | 11 | func HashIdentity(pk rsa.PublicKey) Hash { 12 | // tor-spec.txt#n108 13 | // When we refer to "the hash of a public key", we mean the SHA-1 hash of the 14 | // DER encoding of an ASN.1 RSA public key (as specified in PKCS.1). 15 | // rfc3447#appendix-A.1.1 16 | // RSAPublicKey ::= SEQUENCE { 17 | // modulus INTEGER, -- n 18 | // publicExponent INTEGER -- e 19 | // } 20 | der, err := asn1.Marshal(pk) 21 | if err != nil { 22 | panic(err) 23 | } 24 | return Hash(sha1.Sum(der)) 25 | } 26 | 27 | func checkKey(key *rsa.PrivateKey) bool { 28 | for _, p := range key.Primes { 29 | if !p.ProbablyPrime(20) { 30 | return false 31 | } 32 | } 33 | return true 34 | } 35 | 36 | func Brute(targetA, targetB, maxA, maxB Hash, numKeys, numP int, 37 | log func(v ...interface{})) (a []*rsa.PrivateKey, b []*rsa.PrivateKey) { 38 | finished := false 39 | keys := make(chan *rsa.PrivateKey) 40 | for p := 0; p < numP; p++ { 41 | go func(p int) { 42 | for i := 0; i%100 != 0 || !finished; i++ { 43 | // We generate real keys because e++ ones are detectable 44 | // Set the ProbablyPrime rounds to 1 in rand.Prime, we check later 45 | key, err := rsa.GenerateKey(rand.Reader, 1024) 46 | if err != nil { 47 | panic(err) 48 | } 49 | id := HashIdentity(key.PublicKey) 50 | 51 | if (bytes.Compare(targetA[:], id[:]) < 0 && bytes.Compare(id[:], maxA[:]) < 0) || 52 | (bytes.Compare(targetB[:], id[:]) < 0 && bytes.Compare(id[:], maxB[:]) < 0) { 53 | keys <- key 54 | } 55 | 56 | if i%1000 == 0 && i != 0 { 57 | log("Process #", p, "- iteration #", i) 58 | } 59 | } 60 | }(p) 61 | } 62 | for { 63 | key := <-keys 64 | if !checkKey(key) { 65 | log("scrapped bad key") 66 | continue 67 | } 68 | id := HashIdentity(key.PublicKey) 69 | switch { 70 | case bytes.Compare(targetA[:], id[:]) < 0 && bytes.Compare(id[:], maxA[:]) < 0: 71 | a = append(a, key) 72 | case bytes.Compare(targetB[:], id[:]) < 0 && bytes.Compare(id[:], maxB[:]) < 0: 73 | b = append(b, key) 74 | default: 75 | log("weird, this key is not valid anymore?") 76 | continue 77 | } 78 | log("FOUND ONE!", "A", len(a), "B", len(b)) 79 | if len(a) >= numKeys && len(b) >= numKeys { 80 | finished = true 81 | return 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/hstools/hashring.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "math/big" 5 | "math/rand" 6 | "sort" 7 | "time" 8 | ) 9 | 10 | var ( 11 | bigOne = big.NewInt(1) 12 | bigTwo = big.NewInt(2) 13 | bigThree = big.NewInt(3) 14 | 15 | HashringLimit = new(big.Int).Lsh(bigOne, 160) 16 | 17 | random = rand.New(rand.NewSource(time.Now().UnixNano())) 18 | ) 19 | 20 | type Hashring struct { 21 | // points is the sorted list of values present on the ring 22 | points []*big.Int 23 | } 24 | 25 | type bigIntSlice []*big.Int 26 | 27 | func (b bigIntSlice) Len() int { return len(b) } 28 | func (b bigIntSlice) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 29 | func (b bigIntSlice) Less(i, j int) bool { return b[i].Cmp(b[j]) < 0 } 30 | 31 | // NewHashring returns a Hashring with the given unsorted points. 32 | func NewHashring(points []*big.Int) *Hashring { 33 | h := &Hashring{ 34 | points: make([]*big.Int, len(points)), 35 | } 36 | copy(h.points, points) 37 | sort.Sort(bigIntSlice(h.points)) 38 | return h 39 | } 40 | 41 | func RandomHashring(entries int) *Hashring { 42 | points := make([]*big.Int, entries) 43 | for i := 0; i < entries; i++ { 44 | points[i] = new(big.Int).Rand(random, HashringLimit) 45 | } 46 | 47 | return NewHashring(points) 48 | } 49 | 50 | func (h *Hashring) Len() int { 51 | return len(h.points) 52 | } 53 | 54 | func (h *Hashring) Next(p *big.Int) *big.Int { 55 | i := sort.Search(len(h.points), func(i int) bool { 56 | return h.points[i].Cmp(p) > 0 57 | }) 58 | return h.points[i%len(h.points)] 59 | } 60 | 61 | func (h *Hashring) Next3(p *big.Int) []*big.Int { 62 | i := sort.Search(len(h.points), func(i int) bool { 63 | return h.points[i].Cmp(p) > 0 64 | }) 65 | return []*big.Int{ 66 | h.points[i%len(h.points)], 67 | h.points[(i+1)%len(h.points)], 68 | h.points[(i+2)%len(h.points)], 69 | } 70 | } 71 | 72 | func (h *Hashring) Fourth(p *big.Int) *big.Int { 73 | i := sort.Search(len(h.points), func(i int) bool { 74 | return h.points[i].Cmp(p) > 0 75 | }) 76 | return h.points[(i+3)%len(h.points)] 77 | } 78 | 79 | func (h *Hashring) Prev(p *big.Int) *big.Int { 80 | i := sort.Search(len(h.points), func(i int) bool { 81 | return h.points[i].Cmp(p) >= 0 82 | }) 83 | return h.points[(i-1)%len(h.points)] 84 | } 85 | 86 | func (*Hashring) Diff(from, to *big.Int) *big.Int { 87 | if from.Cmp(to) < 0 { 88 | return new(big.Int).Sub(to, from) 89 | } else { 90 | res := new(big.Int).Sub(HashringLimit, from) 91 | return res.Add(res, to) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/hstools/descid.go: -------------------------------------------------------------------------------- 1 | // Tool for predicting the HSDirs responsible for a particular hidden service at a 2 | // given time using the rendevous v2 scheme as specified in rend-spec.txt. Heavily 3 | // influenced by Tor source and Donncha O'Cearbhaill's retrieve_hs_descriptor.py 4 | // 5 | // author: George Tankersley 6 | // author: Filippo Valsorda 7 | 8 | package hstools 9 | 10 | import ( 11 | "crypto/sha1" 12 | "encoding/binary" 13 | "errors" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | const ( 19 | REPLICAS = 2 20 | REND_TIME_PERIOD_V2_DESC_VALIDITY = 24 * 60 * 60 // 86400 21 | ) 22 | 23 | func OnionToDescID(onion string, t time.Time) ([][]byte, error) { 24 | onion = strings.ToLower(onion) 25 | 26 | switch len(onion) { 27 | case 16 + len(".onion"): 28 | if onion[16:] == ".onion" { 29 | onion = onion[:16] 30 | } else { 31 | return nil, errors.New("wrong suffix") 32 | } 33 | case 16: 34 | // good like this 35 | default: 36 | return nil, errors.New("wrong length") 37 | } 38 | 39 | first, err := ComputeRendV2DescID(onion, 0, t.Unix(), "") 40 | if err != nil { 41 | return nil, err 42 | } 43 | second, err := ComputeRendV2DescID(onion, 1, t.Unix(), "") 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | return [][]byte{first, second}, nil 49 | } 50 | 51 | func ComputeRendV2DescID(serviceID string, replica byte, time int64, descCookie string) ([]byte, error) { 52 | // Convert service ID to binary. 53 | serviceIDBinary, err := FromBase32(serviceID) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | // Calculate current time-period. 59 | timePeriod := getTimePeriod(time, 0, serviceIDBinary) 60 | 61 | // Calculate secret-id-part = h(time-period | cookie | replica). 62 | secretIDPart := getSecretIDPartBytes(timePeriod, descCookie, replica) 63 | 64 | // Calculate descriptor ID. 65 | descID := rendGetDescriptorIDBytes(serviceIDBinary, secretIDPart) 66 | 67 | return descID, nil 68 | } 69 | 70 | func getTimePeriod(time int64, deviation int64, serviceIDBinary []byte) int64 { 71 | return (time+int64(serviceIDBinary[0])*REND_TIME_PERIOD_V2_DESC_VALIDITY/256)/REND_TIME_PERIOD_V2_DESC_VALIDITY + int64(deviation) 72 | } 73 | 74 | func getSecretIDPartBytes(timePeriod int64, descCookie string, replica byte) []byte { 75 | h := sha1.New() 76 | htonlTime := make([]byte, 4) 77 | binary.BigEndian.PutUint32(htonlTime, uint32(timePeriod)) 78 | h.Write(htonlTime) 79 | if descCookie != "" { 80 | h.Write([]byte(descCookie)) 81 | } 82 | h.Write([]byte{replica}) 83 | return h.Sum(nil) 84 | } 85 | 86 | func rendGetDescriptorIDBytes(serviceIDBinary, secretIDPart []byte) []byte { 87 | h := sha1.New() 88 | h.Write(serviceIDBinary) 89 | h.Write(secretIDPart) 90 | return h.Sum(nil) 91 | } 92 | -------------------------------------------------------------------------------- /src/hstools/keysdb.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "log" 7 | "sync" 8 | "time" 9 | 10 | "github.com/boltdb/bolt" 11 | ) 12 | 13 | type KeyMeta struct { 14 | FirstSeen Hour 15 | LastSeen Hour 16 | IPs []string 17 | } 18 | 19 | type IPMeta struct { 20 | Keys [][]byte 21 | } 22 | 23 | type KeysDB struct { 24 | db *bolt.DB 25 | } 26 | 27 | func OpenKeysDb(filename string) (*KeysDB, error) { 28 | d := &KeysDB{} 29 | var err error 30 | 31 | if d.db, err = bolt.Open(filename, 0664, nil); err != nil { 32 | return nil, err 33 | } 34 | 35 | d.db.MaxBatchDelay = 5 * time.Second 36 | 37 | if err = d.db.Update(func(tx *bolt.Tx) error { 38 | if _, err := tx.CreateBucketIfNotExists([]byte("Keys")); err != nil { 39 | return err 40 | } 41 | if _, err := tx.CreateBucketIfNotExists([]byte("IPs")); err != nil { 42 | return err 43 | } 44 | return nil 45 | }); err != nil { 46 | return nil, err 47 | } 48 | 49 | return d, nil 50 | } 51 | 52 | func (d *KeysDB) Seen(keys []Hash, ips []string, h Hour, wg *sync.WaitGroup) { 53 | fn := func(tx *bolt.Tx) error { 54 | for i, k := range keys { 55 | ip := ips[i] 56 | b := tx.Bucket([]byte("Keys")) 57 | oldJSON := b.Get(k[:]) 58 | meta := KeyMeta{ 59 | FirstSeen: h, 60 | LastSeen: h, 61 | IPs: []string{ip}, 62 | } 63 | if oldJSON != nil { 64 | var oldMeta KeyMeta 65 | if err := json.Unmarshal(oldJSON, &oldMeta); err != nil { 66 | return err 67 | } 68 | if oldMeta.FirstSeen < meta.FirstSeen { 69 | meta.FirstSeen = oldMeta.FirstSeen 70 | } 71 | if oldMeta.LastSeen > meta.LastSeen { 72 | meta.LastSeen = oldMeta.LastSeen 73 | } 74 | for _, oldIP := range oldMeta.IPs { 75 | if oldIP != ip { 76 | meta.IPs = append(meta.IPs, oldIP) 77 | } 78 | } 79 | } 80 | encoded, err := json.Marshal(meta) 81 | if err != nil { 82 | return err 83 | } 84 | if err := b.Put(k[:], encoded); err != nil { 85 | return err 86 | } 87 | 88 | b = tx.Bucket([]byte("IPs")) 89 | oldJSON = b.Get([]byte(ip)) 90 | ipMeta := IPMeta{ 91 | Keys: [][]byte{k[:]}, 92 | } 93 | if oldJSON != nil { 94 | var oldMeta IPMeta 95 | if err := json.Unmarshal(oldJSON, &oldMeta); err != nil { 96 | return err 97 | } 98 | for _, oldKey := range oldMeta.Keys { 99 | if !bytes.Equal(oldKey, k[:]) { 100 | ipMeta.Keys = append(ipMeta.Keys, oldKey) 101 | } 102 | } 103 | } 104 | encoded, err = json.Marshal(ipMeta) 105 | if err != nil { 106 | return err 107 | } 108 | if err := b.Put([]byte(ip), encoded); err != nil { 109 | return err 110 | } 111 | } 112 | return nil 113 | } 114 | go func() { 115 | wg.Add(1) 116 | if err := d.db.Batch(fn); err != nil { 117 | log.Fatal(err) 118 | } else { 119 | log.Println("recorded", HourToTime(h)) 120 | wg.Done() 121 | } 122 | }() 123 | } 124 | 125 | func (d *KeysDB) Lookup(key Hash) (res KeyMeta, err error) { 126 | err = d.db.View(func(tx *bolt.Tx) error { 127 | b := tx.Bucket([]byte("Keys")) 128 | v := b.Get([]byte(key[:])) 129 | if err := json.Unmarshal(v, &res); err != nil { 130 | return err 131 | } 132 | return nil 133 | }) 134 | return 135 | } 136 | 137 | func (d *KeysDB) View(fn func(tx *bolt.Tx) error) error { 138 | return d.db.View(fn) 139 | } 140 | 141 | func (d *KeysDB) Close() error { 142 | return d.db.Close() 143 | } 144 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is a Go library and a collection of tools to interact with and analyze Tor HSDirs. 2 | 3 | ## Bruteforcing tool 4 | 5 | `brute` will bruteforce Identity Keys that will be the 6 HSDir for the given onion 6 | at the given time, considering the given consensus state (which should probably just be the most 7 | [recent](https://collector.torproject.org/recent/relay-descriptors/consensuses/). 8 | 9 | ./bin/brute 2015-05-23-23-00-00-consensus facebookcorewwwi.onion 2015-05-29T12:00:00Z 10 | 11 | You should really patch your Go standard library with the provided reduced-primalty patch, it will make the bruteforce 12 | 20 times as fast and complete checks are performed anyway by the tool itself (ProTip: have a separate Go tree for this 13 | security-shattering patch!) 14 | 15 | ## Preprocessing 16 | 17 | First, download consensus files from the following two addresses: 18 | 19 | https://collector.torproject.org/archive/relay-descriptors/consensuses/ 20 | https://collector.torproject.org/recent/relay-descriptors/consensuses/ 21 | 22 | and make sure they are named like this (this is what you will get by just unpacking the tarballs): 23 | 24 | $DIR/consensuses-2015-01/01/2015-01-01-00-00-00-consensus 25 | 26 | Then there are two steps to preprocessing: 27 | 28 | preprocess $DIR 2015-01-01-00 2015-05-28-10 29 | grind pckcns.dat > stats.jsonl 30 | 31 | This should leave you with the following three files: `pckcns.dat`, `keys.db` and `stats.jsonl` 32 | 33 | ## Analysis tools 34 | 35 | ### scrolls 36 | 37 | `scrolls` will go through consensuses in a time frame and print the HSDir set every time it changed, 38 | with color-coded suspiciousness scores. 39 | 40 | ./bin/scrolls pckcns.dat keys.db stats.jsonl facebookcorewwwi.onion 2015-05-01-01 2015-05-28-10 41 | 42 | ![image](https://cloud.githubusercontent.com/assets/1225294/7873542/e206bdf6-05a3-11e5-99db-d9d89a2bf4ec.png) 43 | 44 | * "Dist score" is a normalized score of how near the first HSDir is to the Descriptor ID, compared to the 45 | expected mean and average deviation on the ring. 46 | * "Dist4 score" is the same, but for the distance of the 4th node. 47 | * "Age" is how many hours before the node got the HSDir flag (or appeared). ∞ means since the beginning of the time frame. 48 | * "Long" is how many hours after the node lost the HSDir flag (or disappeared). ∞ means until the end of the time frame. 49 | * "Colo keys" is how many other Identity Keys have been observed on IPs that hosted this Identity Key. 50 | 51 | ### curiosity 52 | 53 | `curiosity` will generate statistics about correlations between IPs and multiple Identity Keys. 54 | 55 | ./bin/curiosity data/keys.db colocated | sort -g 56 | 57 | [...] 58 | 57 keys on 1 IPs - EFA4BF05D097357832B8C7EE3391C8A534CB06F1 [178.32.143.175] 59 | 57 keys on 1 IPs - FADD1E620505CD8B1E22DC9CC503D06592C24EF9 [178.32.143.175] 60 | 57 keys on 1 IPs - FFCCB39F28DB9926F9D34D97F1D98EEA834EED38 [178.32.143.175] 61 | 57 keys on 3 IPs - 0045D088EB8E8E3634F3C82A8010A50BC18938D4 [54.242.144.249 23.20.33.81 23.22.192.88] 62 | 57 keys on 3 IPs - 01120D3FA96BA5B5F5FBB5E1581E8C73E0EFE24D [54.242.144.249 23.20.33.81 23.22.192.88] 63 | 57 keys on 3 IPs - 01C5B224668F77D3662956DDEC15DA46A252CC23 [54.242.144.249 23.20.33.81 23.22.192.88] 64 | 57 keys on 3 IPs - 027B74EEB1A43CFA490D3AE7C4CCC11F5F26A3A7 [54.242.144.249 23.20.33.81 23.22.192.88] 65 | 57 keys on 3 IPs - 03D401D7B671098945BD5B14E3D54278CBCF5067 [54.242.144.249 23.20.33.81 23.22.192.88] 66 | 67 | ## Coming soon 68 | 69 | * automated orchestration tool to easily deploy defensive arrays of HSDirs 70 | * real-time targeted and generic statistical monitoring tools 71 | -------------------------------------------------------------------------------- /src/hstools/consensus.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | "os" 10 | "path/filepath" 11 | ) 12 | 13 | const consensusFilename = "consensuses-2006-01/02/2006-01-02-15-00-00-consensus" 14 | 15 | // Hour is just a Unix timestamp divided by 3600, a unique index for an hour 16 | type Hour int32 17 | 18 | type Consensus struct { 19 | Time Hour 20 | Filename string 21 | Error error 22 | K []Hash 23 | IP []string 24 | } 25 | 26 | // ReadConsensuses reads consensus files from a folder structure like 27 | // DIR/consensuses-2011-02/04/2011-02-04-02-00-00-consensus and sends them 28 | // on the returned channel. From since to until included. 29 | func ReadConsensuses(dir string, since, until Hour) chan *Consensus { 30 | ch := make(chan *Consensus) 31 | go func() { 32 | for h := since; h <= until; h++ { 33 | filename := HourToTime(h).Format(consensusFilename) 34 | filename = filepath.Join(dir, filename) 35 | 36 | c, err := ParseConsensus(filename) 37 | if err != nil { 38 | c = &Consensus{Error: err} 39 | } else { 40 | c.Time = h 41 | } 42 | 43 | ch <- c 44 | } 45 | close(ch) 46 | }() 47 | return ch 48 | } 49 | 50 | // ParseConsensus parses a consensus file and extracts the HSDir Hashring 51 | func ParseConsensus(filename string) (*Consensus, error) { 52 | f, err := os.Open(filename) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | var fingerprint, ip string 58 | c := &Consensus{Filename: filename} 59 | 60 | scanner := bufio.NewScanner(bufio.NewReaderSize(f, 2000000)) 61 | for scanner.Scan() { 62 | b := scanner.Bytes() 63 | 64 | if bytes.Equal(b[:2], []byte("r ")) { 65 | parts := bytes.Split(b, []byte(" ")) 66 | fingerprint = string(parts[2]) 67 | ip = string(parts[6]) 68 | continue 69 | } 70 | 71 | if bytes.Equal(b[:2], []byte("s ")) && bytes.Contains(b, []byte("HSDir")) { 72 | f, err := FromBase64(fingerprint) 73 | if err != nil { 74 | return nil, fmt.Errorf("%v (%s)", err, fingerprint) 75 | } 76 | var k Hash 77 | copy(k[len(k)-len(f):], f) 78 | c.K = append(c.K, k) 79 | c.IP = append(c.IP, ip) 80 | } 81 | } 82 | if err := scanner.Err(); err != nil { 83 | return nil, err 84 | } 85 | 86 | return c, nil 87 | } 88 | 89 | type PackedConsensusHdr struct { 90 | Time Hour 91 | Len int32 92 | } 93 | 94 | func WritePackedConsensus(w io.Writer, c *Consensus) error { 95 | hdr := PackedConsensusHdr{Time: c.Time, Len: int32(len(c.K))} 96 | if err := binary.Write(w, binary.BigEndian, hdr); err != nil { 97 | return err 98 | } 99 | for _, k := range c.K { 100 | if _, err := w.Write(k[:]); err != nil { 101 | return err 102 | } 103 | } 104 | return nil 105 | } 106 | 107 | type PackReader struct { 108 | rd io.Reader 109 | cl io.Closer 110 | c *Consensus 111 | err error 112 | } 113 | 114 | func NewPackReader(filename string) (p *PackReader) { 115 | f, err := os.Open(filename) 116 | if err != nil { 117 | return &PackReader{ 118 | err: err, 119 | } 120 | } 121 | return &PackReader{ 122 | rd: bufio.NewReaderSize(f, 1024*1024*100), 123 | cl: f, 124 | } 125 | } 126 | 127 | func (p *PackReader) Load() bool { 128 | var hdr PackedConsensusHdr 129 | if err := binary.Read(p.rd, binary.BigEndian, &hdr); err == io.EOF { 130 | p.cl.Close() 131 | return false 132 | } else if err != nil { 133 | p.err = err 134 | p.cl.Close() 135 | return false 136 | } 137 | p.c = &Consensus{ 138 | Time: hdr.Time, 139 | K: make([]Hash, hdr.Len), 140 | } 141 | for i := int32(0); i < hdr.Len; i++ { 142 | _, err := io.ReadFull(p.rd, p.c.K[i][:]) 143 | if err != nil { 144 | p.err = err 145 | p.cl.Close() 146 | return false 147 | } 148 | } 149 | return true 150 | } 151 | 152 | func (p *PackReader) Consensus() *Consensus { 153 | return p.c 154 | } 155 | 156 | func (p *PackReader) Err() error { 157 | return p.err 158 | } 159 | -------------------------------------------------------------------------------- /src/retrieve_hs_descriptor.py: -------------------------------------------------------------------------------- 1 | """ 2 | List a Hidden Service HSDir for a given consensus or at the current instant. 3 | 4 | - Donncha O'Cearbhaill - donncha@donncha.is 5 | - Filippo Valsorda 6 | """ 7 | 8 | from time import mktime, time 9 | from base64 import b32encode, b32decode 10 | from hashlib import sha1 11 | from struct import pack, unpack 12 | from stem.descriptor import parse_file, DocumentHandler 13 | from stem.descriptor.remote import DescriptorDownloader 14 | import argparse 15 | from bisect import bisect_left 16 | 17 | # Returns base_32 encode desc_id - descriptor-id = H(permanent-id | H(time-period | descriptor-cookie | replica)) 18 | def rend_compute_v2_desc_id(service_id_base32, replica, time, descriptor_cookie = ""):# 19 | service_id = b32decode(service_id_base32, 1) 20 | time_period = get_time_period(time, 0, service_id) 21 | secret_id_part = get_secret_id_part_bytes(time_period, descriptor_cookie, replica) 22 | desc_id = rend_get_descriptor_id_bytes(service_id, secret_id_part) 23 | return b32encode(desc_id).lower() 24 | 25 | # Calculates time period - time-period = (current-time + permanent-id-byte * 86400 / 256) / 86400 26 | def get_time_period(time, deviation, service_id): 27 | REND_TIME_PERIOD_V2_DESC_VALIDITY = 24 * 60 * 60 28 | return int(((time + ((unpack('B', service_id[0])[0] * REND_TIME_PERIOD_V2_DESC_VALIDITY) ) / 256) ) / REND_TIME_PERIOD_V2_DESC_VALIDITY + deviation) 29 | 30 | # Calculate secret_id_part - secret-id-part = H(time-period | descriptor-cookie | replica) 31 | def get_secret_id_part_bytes(time_period, descriptor_cookie, replica): 32 | secret_id_part = sha1() 33 | secret_id_part.update(pack('>I', time_period)[:4]); 34 | if descriptor_cookie: 35 | secret_id_part.update(descriptor_cookie) 36 | secret_id_part.update('{0:02X}'.format(replica).decode('hex')) 37 | return secret_id_part.digest() 38 | 39 | def rend_get_descriptor_id_bytes(service_id, secret_id_part): 40 | descriptor_id = sha1() 41 | descriptor_id.update(service_id) 42 | descriptor_id.update(secret_id_part) 43 | return descriptor_id.digest() 44 | 45 | def find_responsible_HSDir(descriptor_id, consensus): 46 | fingerprint_list = [] 47 | for _, router in consensus.routers.items(): 48 | if "HSDir" in router.flags: 49 | fingerprint_list.append(router.fingerprint.decode("hex")) 50 | fingerprint_list.sort() 51 | 52 | descriptor_position = bisect_left(fingerprint_list, b32decode(descriptor_id, 1)) 53 | 54 | responsible_HSDirs = [] 55 | for i in range(0, 3): 56 | fingerprint = fingerprint_list[descriptor_position + i] 57 | router = consensus.routers[fingerprint.encode("hex").upper()] 58 | responsible_HSDirs.append({ 59 | 'nickname': router.nickname, 60 | 'fingerprint': router.fingerprint, 61 | 'address': router.address, 62 | 'dir_port': router.dir_port, 63 | 'descriptor_id': descriptor_id 64 | }) 65 | 66 | return responsible_HSDirs 67 | 68 | def main(): 69 | REPLICAS = 2 70 | 71 | parser = argparse.ArgumentParser() 72 | parser.add_argument('onion_address', help='The hidden service address - e.g. (idnxcnkne4qt76tg.onion)') 73 | parser.add_argument('--consensus', help='The optional consensus file', required=False) 74 | args = parser.parse_args() 75 | 76 | if args.consensus is None: 77 | downloader = DescriptorDownloader() 78 | consensus = downloader.get_consensus(document_handler = DocumentHandler.DOCUMENT).run()[0] 79 | t = time() 80 | else: 81 | with open(args.consensus) as f: 82 | consensus = next(parse_file(f, 'network-status-consensus-3 1.0', document_handler = DocumentHandler.DOCUMENT)) 83 | t = mktime(consensus.valid_after.timetuple()) 84 | 85 | service_id, tld = args.onion_address.split(".") 86 | if tld == 'onion' and len(service_id) == 16 and service_id.isalnum(): 87 | for replica in range(0, REPLICAS): 88 | descriptor_id = rend_compute_v2_desc_id(service_id, replica, t) 89 | print descriptor_id + '\t' + b32decode(descriptor_id, True).encode('hex') 90 | for router in find_responsible_HSDir(descriptor_id, consensus): 91 | print router['fingerprint'] + '\t' + router['nickname'] 92 | 93 | else: 94 | print "[!] The onion address you provided is not valid" 95 | 96 | if __name__ == '__main__': 97 | main() 98 | -------------------------------------------------------------------------------- /src/cmd/scrolls.go: -------------------------------------------------------------------------------- 1 | // +build manually 2 | 3 | package main 4 | 5 | import ( 6 | "bufio" 7 | "bytes" 8 | "encoding/json" 9 | "fmt" 10 | "hstools" 11 | "log" 12 | "math/big" 13 | "net/http" 14 | _ "net/http/pprof" 15 | "os" 16 | "runtime" 17 | "runtime/debug" 18 | "time" 19 | 20 | "github.com/mgutz/ansi" 21 | ) 22 | 23 | var ( 24 | yellow = ansi.ColorFunc("yellow+bh") 25 | red = ansi.ColorFunc("red+bh") 26 | black = ansi.ColorFunc("red+b:white+h") 27 | ) 28 | 29 | func fatalIfErr(err error) { 30 | if err != nil { 31 | debug.PrintStack() 32 | log.Fatal(err) 33 | } 34 | } 35 | 36 | func getAge(h *hstools.Hash, now hstools.Hour, since time.Time, keysDB *hstools.KeysDB) string { 37 | v, err := keysDB.Lookup(*h) 38 | fatalIfErr(err) 39 | if hstools.HourToTime(v.FirstSeen).Before(since) || hstools.HourToTime(v.FirstSeen).Equal(since) { 40 | return "∞" 41 | } 42 | age := now - v.FirstSeen 43 | switch { 44 | case age < 24: 45 | return black(fmt.Sprintf("%d", age)) 46 | case age < 7*24: 47 | return red(fmt.Sprintf("%d", age)) 48 | case age < 15*24: 49 | return yellow(fmt.Sprintf("%d", age)) 50 | default: 51 | return fmt.Sprintf("%d", age) 52 | } 53 | } 54 | 55 | func getLongevity(h *hstools.Hash, now hstools.Hour, until time.Time, keysDB *hstools.KeysDB) string { 56 | v, err := keysDB.Lookup(*h) 57 | fatalIfErr(err) 58 | if hstools.HourToTime(v.LastSeen).After(until) || hstools.HourToTime(v.LastSeen).Equal(until) { 59 | return "∞" 60 | } 61 | long := v.LastSeen - now 62 | switch { 63 | case long < 24: 64 | return black(fmt.Sprintf("%d", long)) 65 | case long < 7*24: 66 | return red(fmt.Sprintf("%d", long)) 67 | case long < 15*24: 68 | return yellow(fmt.Sprintf("%d", long)) 69 | default: 70 | return fmt.Sprintf("%d", long) 71 | } 72 | } 73 | 74 | func getColo(h *hstools.Hash, keysDB *hstools.KeysDB) string { 75 | res, _ := hstools.ColocatedKeys(h[:], keysDB) 76 | switch { 77 | case res > 20: 78 | return black(fmt.Sprintf("%d", res)) 79 | case res > 10: 80 | return red(fmt.Sprintf("%d", res)) 81 | case res > 5: 82 | return yellow(fmt.Sprintf("%d", res)) 83 | default: 84 | return fmt.Sprintf("%d", res) 85 | } 86 | } 87 | 88 | func colorScore(v int64) string { 89 | switch { 90 | case v > 200: 91 | return black(fmt.Sprintf("%d", v)) 92 | case v > 150: 93 | return red(fmt.Sprintf("%d", v)) 94 | case v > 100: 95 | return yellow(fmt.Sprintf("%d", v)) 96 | default: 97 | return fmt.Sprintf("%d", v) 98 | } 99 | } 100 | 101 | func main() { 102 | runtime.GOMAXPROCS(runtime.NumCPU()) 103 | 104 | log.SetFlags(0) 105 | 106 | go func() { 107 | log.Println(http.ListenAndServe("localhost:6060", nil)) 108 | }() 109 | 110 | if len(os.Args) != 7 { 111 | log.Fatal("usage: scavenge pckcns.dat keys.db stats.jsonl xxx.onion 2015-05-01-01 2015-05-31-23") 112 | } 113 | 114 | keysDB, err := hstools.OpenKeysDb(os.Args[2]) 115 | fatalIfErr(err) 116 | 117 | since, err := time.Parse("2006-01-02-15", os.Args[5]) 118 | if err != nil { 119 | log.Fatal(err) 120 | } 121 | until, err := time.Parse("2006-01-02-15", os.Args[6]) 122 | if err != nil { 123 | log.Fatal(err) 124 | } 125 | 126 | statsFile, err := os.Open(os.Args[3]) 127 | fatalIfErr(err) 128 | jsonStats := json.NewDecoder(bufio.NewReaderSize(statsFile, 1024*1024)) 129 | var metrics hstools.AnalyzedConsensus 130 | var last0, last1 []byte 131 | 132 | r := hstools.NewPackReader(os.Args[1]) 133 | for r.Load() { 134 | c := r.Consensus() 135 | jsonStats.Decode(&metrics) 136 | if metrics.T != c.Time { 137 | log.Fatal("unaligned data") 138 | } 139 | 140 | if since.After(hstools.HourToTime(c.Time)) { 141 | continue 142 | } 143 | if until.Before(hstools.HourToTime(c.Time)) { 144 | break 145 | } 146 | 147 | h := hstools.NewHashring(hstools.HashesToIntSlice(c.K)) 148 | 149 | desc, err := hstools.OnionToDescID(os.Args[4], hstools.HourToTime(c.Time)) 150 | fatalIfErr(err) 151 | desc0 := new(big.Int).SetBytes(desc[0]) 152 | desc1 := new(big.Int).SetBytes(desc[1]) 153 | 154 | hsDir0 := hstools.IntsToHashSlice(h.Next3(desc0)) 155 | allHSDir0 := append(append(append([]byte{}, hsDir0[0][:]...), hsDir0[1][:]...), hsDir0[2][:]...) 156 | hsDir1 := hstools.IntsToHashSlice(h.Next3(desc1)) 157 | allHSDir1 := append(append(append([]byte{}, hsDir1[0][:]...), hsDir1[1][:]...), hsDir1[2][:]...) 158 | if bytes.Equal(last0, allHSDir0) && bytes.Equal(last1, allHSDir1) { 159 | continue 160 | } 161 | last0, last1 = allHSDir0, allHSDir1 162 | 163 | // Metric 1. Age 164 | // age0, err := h.Age(desc0, c.Time, keysDB) 165 | // fatalIfErr(err) 166 | // age1, err := h.Age(desc1, c.Time, keysDB) 167 | // fatalIfErr(err) 168 | 169 | // Metric 2. Longevity 170 | // long0, err := h.Longevity(desc0, c.Time, keysDB) 171 | // fatalIfErr(err) 172 | // long1, err := h.Longevity(desc1, c.Time, keysDB) 173 | // fatalIfErr(err) 174 | 175 | // Metric 3. Distance 176 | dist0 := hstools.Score(h.Distance(desc0), metrics.Distance) 177 | dist1 := hstools.Score(h.Distance(desc1), metrics.Distance) 178 | 179 | // Metric 4. Distance4 180 | dist40 := hstools.Score(h.Distance4(desc0), metrics.Distance4) 181 | dist41 := hstools.Score(h.Distance4(desc1), metrics.Distance4) 182 | 183 | // Metric 5. Colocated keys 184 | // colo0 := h.Colocated(desc0, keysDB) 185 | // colo1 := h.Colocated(desc1, keysDB) 186 | 187 | fmt.Printf(` 188 | ###### %s 189 | ###### Replica 0 - Dist score %s - Dist4 score %s 190 | %s - Age %s - Long %s - Colo keys %s 191 | %s - Age %s - Long %s - Colo keys %s 192 | %s - Age %s - Long %s - Colo keys %s 193 | ###### Replica 1 - Dist score %s - Dist4 score %s 194 | %s - Age %s - Long %s - Colo keys %s 195 | %s - Age %s - Long %s - Colo keys %s 196 | %s - Age %s - Long %s - Colo keys %s 197 | `, 198 | hstools.HourToTime(c.Time), colorScore(dist0), colorScore(dist40), 199 | 200 | hstools.ToHex(hsDir0[0][:]), getAge(hsDir0[0], c.Time, since, keysDB), 201 | getLongevity(hsDir0[0], c.Time, until, keysDB), getColo(hsDir0[0], keysDB), 202 | 203 | hstools.ToHex(hsDir0[1][:]), getAge(hsDir0[1], c.Time, since, keysDB), 204 | getLongevity(hsDir0[1], c.Time, until, keysDB), getColo(hsDir0[1], keysDB), 205 | 206 | hstools.ToHex(hsDir0[2][:]), getAge(hsDir0[2], c.Time, since, keysDB), 207 | getLongevity(hsDir0[2], c.Time, until, keysDB), getColo(hsDir0[2], keysDB), 208 | 209 | colorScore(dist1), colorScore(dist41), 210 | 211 | hstools.ToHex(hsDir1[0][:]), getAge(hsDir1[0], c.Time, since, keysDB), 212 | getLongevity(hsDir1[0], c.Time, until, keysDB), getColo(hsDir1[0], keysDB), 213 | 214 | hstools.ToHex(hsDir1[1][:]), getAge(hsDir1[1], c.Time, since, keysDB), 215 | getLongevity(hsDir1[1], c.Time, until, keysDB), getColo(hsDir1[1], keysDB), 216 | 217 | hstools.ToHex(hsDir1[2][:]), getAge(hsDir1[2], c.Time, since, keysDB), 218 | getLongevity(hsDir1[2], c.Time, until, keysDB), getColo(hsDir1[2], keysDB), 219 | ) 220 | 221 | // if colo0 > 20 || colo1 > 20 { 222 | // log.Println(hstools.HourToTime(c.Time), colo0, colo1) 223 | // } 224 | 225 | // if dist0 > 200 || dist1 > 200 || dist40 > 250 || dist41 > 250 || 226 | // age0 < 24*7 || age1 < 24*7 || long0 < 24*3 || long1 < 24*3 || 227 | // colo0 > 20 || colo1 > 20 { 228 | // log.Println(hstools.HourToTime(c.Time), 229 | // "# AGE", age0, age1, 230 | // "# LONG", long0, long1, 231 | // "# DIST", dist0, dist1, 232 | // "# DIST4", dist40, dist41, 233 | // "# COLO", colo0, colo1) 234 | // } 235 | } 236 | fatalIfErr(r.Err()) 237 | } 238 | -------------------------------------------------------------------------------- /src/hstools/stats.go: -------------------------------------------------------------------------------- 1 | package hstools 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "math/big" 7 | 8 | "github.com/boltdb/bolt" 9 | ) 10 | 11 | type MetricData struct { 12 | Mean *big.Int 13 | AbsDev *big.Int 14 | } 15 | 16 | type AnalyzedConsensus struct { 17 | T Hour 18 | Distance *MetricData 19 | Distance4 *MetricData 20 | } 21 | 22 | type PartitionData struct { 23 | x0 *big.Int 24 | x1 *big.Int 25 | l *big.Int 26 | } 27 | 28 | func bigMean(nums []*big.Int) *big.Int { 29 | avg := big.NewInt(0) 30 | size := big.NewInt(int64(len(nums))) 31 | for _, n := range nums { 32 | avg.Add(avg, new(big.Int).Div(n, size)) 33 | } 34 | return avg 35 | } 36 | 37 | func bigSqrt(n *big.Int) *big.Int { 38 | // adapted from mini-gmp 39 | u, t := new(big.Int), new(big.Int) 40 | t.SetBit(t, n.BitLen()/2+1, 1) 41 | for { 42 | u.Set(t) 43 | t.Quo(n, u) 44 | t.Add(t, u) 45 | t.Rsh(t, 1) 46 | if t.Cmp(u) >= 0 { 47 | return u 48 | } 49 | } 50 | } 51 | 52 | func bigCubeRoot(n *big.Int) *big.Int { 53 | // http://math.stackexchange.com/a/263113 54 | cube, x := new(big.Int), new(big.Int) 55 | 56 | a := new(big.Int).Set(n) 57 | for cube.Exp(a, bigThree, nil).Cmp(n) > 0 { 58 | // a = (2*a + n/a^2) / 3 59 | x.Quo(n, x.Mul(a, a)) 60 | x.Add(x.Add(x, a), a) 61 | a.Quo(x, bigThree) 62 | } 63 | 64 | return a 65 | } 66 | 67 | func bigStdDev(nums []*big.Int, mean *big.Int) *big.Int { 68 | avg := big.NewInt(0) 69 | size := big.NewInt(int64(len(nums)) - 1) 70 | for _, n := range nums { 71 | d := new(big.Int) 72 | d.Exp(d.Sub(n, mean), bigTwo, nil) 73 | avg.Add(avg, d.Div(d, size)) 74 | } 75 | return bigSqrt(avg) 76 | } 77 | 78 | // func bigAbsDev(nums []*big.Int, mean *big.Int) *big.Int { 79 | // avg := big.NewInt(0) 80 | // size := big.NewInt(int64(len(nums))) 81 | // for _, n := range nums { 82 | // d := new(big.Int) 83 | // d.Abs(d.Sub(mean, n)) 84 | // avg.Add(avg, d.Div(d, size)) 85 | // } 86 | // return avg 87 | // } 88 | 89 | func bigAbsDev(nums []*big.Int, mean *big.Int) *big.Int { 90 | devs := make([]*big.Int, len(nums)) 91 | for i, n := range nums { 92 | devs[i] = new(big.Int).Sub(n, mean) 93 | devs[i].Abs(devs[i]) 94 | } 95 | return bigMean(devs) 96 | } 97 | 98 | // The SampleX functions have been replaced by the analytical (not random) 99 | // XData + AnalyzePartitionData functions 100 | const ROUNDS = 100000 101 | 102 | func (h *Hashring) Distance(p *big.Int) *big.Int { 103 | return h.Diff(p, h.Next(p)) 104 | } 105 | 106 | func SampleDistance(h *Hashring) (res *MetricData) { 107 | samples := make([]*big.Int, ROUNDS) 108 | origin := new(big.Int) 109 | for i := 0; i < ROUNDS; i++ { 110 | origin.Rand(random, HashringLimit) 111 | samples[i] = h.Distance(origin) 112 | } 113 | mean := bigMean(samples) 114 | return &MetricData{ 115 | Mean: mean, 116 | // StdDev: bigStdDev(samples, mean), 117 | AbsDev: bigAbsDev(samples, mean), 118 | } 119 | } 120 | 121 | func (h *Hashring) Distance4(p *big.Int) *big.Int { 122 | return h.Diff(p, h.Fourth(p)) 123 | } 124 | 125 | func SampleDistance4(h *Hashring) (res *MetricData) { 126 | samples := make([]*big.Int, ROUNDS) 127 | origin := new(big.Int) 128 | for i := 0; i < ROUNDS; i++ { 129 | origin.Rand(random, HashringLimit) 130 | samples[i] = h.Distance4(origin) 131 | } 132 | mean := bigMean(samples) 133 | return &MetricData{ 134 | Mean: mean, 135 | // StdDev: bigStdDev(samples, mean), 136 | AbsDev: bigAbsDev(samples, mean), 137 | } 138 | } 139 | 140 | func (h *Hashring) Distance4Data() (res []*PartitionData) { 141 | for i, p := range h.points { 142 | x0 := h.Diff(p, h.points[(i+4)%len(h.points)]) 143 | l := h.Diff(p, h.points[(i+1)%len(h.points)]) 144 | x1 := new(big.Int).Sub(x0, l) 145 | res = append(res, &PartitionData{ 146 | x0: x0, x1: x1, l: l, 147 | }) 148 | } 149 | return 150 | } 151 | 152 | func (h *Hashring) DistanceData() (res []*PartitionData) { 153 | for i, p := range h.points { 154 | x0 := h.Diff(p, h.points[(i+1)%len(h.points)]) 155 | res = append(res, &PartitionData{ 156 | x0: x0, x1: big.NewInt(0), l: x0, 157 | }) 158 | } 159 | return 160 | } 161 | 162 | func AnalyzePartitionData(data []*PartitionData) *MetricData { 163 | m := &MetricData{ 164 | Mean: new(big.Int), 165 | AbsDev: new(big.Int), 166 | } 167 | for _, part := range data { 168 | u := new(big.Int).Add(part.x0, part.x1) 169 | u.Abs(u.Div(u, bigTwo)) 170 | u.Div(u.Mul(u, part.l), HashringLimit) 171 | m.Mean.Add(m.Mean, u) 172 | } 173 | for _, part := range data { 174 | d0 := new(big.Int).Sub(part.x0, m.Mean) 175 | d1 := new(big.Int).Sub(part.x1, m.Mean) 176 | if d0.Sign() == d1.Sign() { 177 | w := new(big.Int).Add(d0, d1) 178 | w.Abs(w.Div(w, bigTwo)) 179 | w.Div(w.Mul(w, part.l), HashringLimit) 180 | m.AbsDev.Add(m.AbsDev, w) 181 | } else { 182 | // Assumes l = x0 - x1 and x0 > x1 183 | if part.l.Cmp(new(big.Int).Sub(part.x0, part.x1)) != 0 || 184 | !(part.x0.Cmp(part.x1) > 0) { 185 | log.Fatal(part.l, part.x0, part.x1) 186 | } 187 | d1.Abs(d1) 188 | 189 | w0 := new(big.Int).Div(d0, bigTwo) 190 | w0.Div(w0.Mul(w0.Abs(w0), d0), HashringLimit) 191 | m.AbsDev.Add(m.AbsDev, w0) 192 | 193 | w1 := new(big.Int).Div(d1, bigTwo) 194 | w1.Div(w1.Mul(w1.Abs(w1), d1), HashringLimit) 195 | m.AbsDev.Add(m.AbsDev, w1) 196 | } 197 | } 198 | return m 199 | } 200 | 201 | var mask = new(big.Int).Exp(big.NewInt(2), big.NewInt(160-30), nil) 202 | 203 | func Score(v *big.Int, res *MetricData) int64 { 204 | dev := new(big.Int).Sub(res.Mean, v) 205 | return dev.Div(dev.Mul(dev, big.NewInt(100)), res.AbsDev).Int64() 206 | } 207 | 208 | func (h *Hashring) Age(p *big.Int, now Hour, keysDB *KeysDB) (Hour, error) { 209 | var res Hour 210 | for _, p := range h.Next3(p) { 211 | v, err := keysDB.Lookup(IntToHash(p)) 212 | if err != nil { 213 | return 0, err 214 | } 215 | res += now - v.FirstSeen 216 | } 217 | return res / 3, nil 218 | } 219 | 220 | func (h *Hashring) Longevity(p *big.Int, now Hour, keysDB *KeysDB) (Hour, error) { 221 | var res Hour 222 | for _, p := range h.Next3(p) { 223 | v, err := keysDB.Lookup(IntToHash(p)) 224 | if err != nil { 225 | return 0, err 226 | } 227 | res += v.LastSeen - now 228 | } 229 | return res / 3, nil 230 | } 231 | 232 | func (h *Hashring) Colocated(p *big.Int, keysDB *KeysDB) int { 233 | var tot int 234 | for _, p := range h.Next3(p) { 235 | h := IntToHash(p) 236 | n, _ := ColocatedKeys(h[:], keysDB) 237 | tot += n 238 | } 239 | return tot - 3 240 | } 241 | 242 | // AgeData is not really in use, Age and Longevity are assessed in absolute 243 | func (h *Hashring) AgeData(now Hour, keysDB *KeysDB) (res []*PartitionData, err error) { 244 | for i, p := range h.points { 245 | age, err := h.Age(p, now, keysDB) 246 | if err != nil { 247 | return nil, err 248 | } 249 | l := h.Diff(p, h.points[(i+1)%len(h.points)]) 250 | res = append(res, &PartitionData{ 251 | x0: big.NewInt(int64(age)), x1: big.NewInt(int64(age)), l: l, 252 | }) 253 | } 254 | return 255 | } 256 | 257 | func ColocatedKeys(k []byte, keysDB *KeysDB) (coloNum int, ips []string) { 258 | if err := keysDB.View(func(tx *bolt.Tx) error { 259 | b := tx.Bucket([]byte("Keys")) 260 | var res KeyMeta 261 | if err := json.Unmarshal(b.Get(k), &res); err != nil { 262 | return err 263 | } 264 | colocated := make(map[string]struct{}) 265 | for _, ip := range res.IPs { 266 | ipMetaJSON := tx.Bucket([]byte("IPs")).Get([]byte(ip)) 267 | var ipMeta IPMeta 268 | if err := json.Unmarshal(ipMetaJSON, &ipMeta); err != nil { 269 | return err 270 | } 271 | for _, key := range ipMeta.Keys { 272 | colocated[ToHex(key)] = struct{}{} 273 | } 274 | } 275 | ips = res.IPs 276 | coloNum = len(colocated) 277 | return nil 278 | }); err != nil { 279 | log.Fatal(err) 280 | } 281 | return 282 | } 283 | -------------------------------------------------------------------------------- /misc/rend-spec.txt: -------------------------------------------------------------------------------- 1 | 2 | Tor Rendezvous Specification 3 | 4 | 0. Overview and preliminaries 5 | 6 | The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL 7 | NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and 8 | "OPTIONAL" in this document are to be interpreted as described in 9 | RFC 2119. 10 | 11 | Read 12 | https://svn.torproject.org/svn/projects/design-paper/tor-design.html#sec:rendezvous 13 | before you read this specification. It will make more sense. 14 | 15 | Rendezvous points provide location-hidden services (server 16 | anonymity) for the onion routing network. With rendezvous points, 17 | Bob can offer a TCP service (say, a webserver) via the onion 18 | routing network, without revealing the IP of that service. 19 | 20 | Bob does this by anonymously advertising a public key for his 21 | service, along with a list of onion routers to act as "Introduction 22 | Points" for his service. He creates forward circuits to those 23 | introduction points, and tells them about his service. To 24 | connect to Bob, Alice first builds a circuit to an OR to act as 25 | her "Rendezvous Point." She then connects to one of Bob's chosen 26 | introduction points, and asks it to tell him about her Rendezvous 27 | Point (RP). If Bob chooses to answer, he builds a circuit to her 28 | RP, and tells it to connect him to Alice. The RP joins their 29 | circuits together, and begins relaying cells. Alice's 'BEGIN' 30 | cells are received directly by Bob's OP, which passes data to 31 | and from the local server implementing Bob's service. 32 | 33 | Below we describe a network-level specification of this service, 34 | along with interfaces to make this process transparent to Alice 35 | (so long as she is using an OP). 36 | 37 | 0.1. Notation, conventions and prerequisites 38 | 39 | In the specifications below, we use the same notation and terminology 40 | as in "tor-spec.txt". The service specified here also requires the 41 | existence of an onion routing network as specified in that file. 42 | 43 | H(x) is a SHA1 digest of x. 44 | PKSign(SK,x) is a PKCS.1-padded RSA signature of x with SK. 45 | PKEncrypt(SK,x) is a PKCS.1-padded RSA encryption of x with SK. 46 | Public keys are all RSA, and encoded in ASN.1. 47 | All integers are stored in network (big-endian) order. 48 | All symmetric encryption uses AES in counter mode, except where 49 | otherwise noted. 50 | 51 | In all discussions, "Alice" will refer to a user connecting to a 52 | location-hidden service, and "Bob" will refer to a user running a 53 | location-hidden service. 54 | 55 | An OP is (as defined elsewhere) an "Onion Proxy" or Tor client. 56 | 57 | An OR is (as defined elsewhere) an "Onion Router" or Tor server. 58 | 59 | An "Introduction point" is a Tor server chosen to be Bob's medium-term 60 | 'meeting place'. A "Rendezvous point" is a Tor server chosen by Alice to 61 | be a short-term communication relay between her and Bob. All Tor servers 62 | potentially act as introduction and rendezvous points. 63 | 64 | 0.2. Protocol outline 65 | 66 | 1. Bob->Bob's OP: "Offer IP:Port as public-key-name:Port". [configuration] 67 | (We do not specify this step; it is left to the implementor of 68 | Bob's OP.) 69 | 70 | 2. Bob's OP generates a long-term keypair. 71 | 72 | 3. Bob's OP->Introduction point via Tor: [introduction setup] 73 | "This public key is (currently) associated to me." 74 | 75 | 4. Bob's OP->directory service via Tor: publishes Bob's service descriptor 76 | [advertisement] 77 | "Meet public-key X at introduction point A, B, or C." (signed) 78 | 79 | 5. Out of band, Alice receives a z.onion:port address. 80 | She opens a SOCKS connection to her OP, and requests z.onion:port. 81 | 82 | 6. Alice's OP retrieves Bob's descriptor via Tor. [descriptor lookup.] 83 | 84 | 7. Alice's OP chooses a rendezvous point, opens a circuit to that 85 | rendezvous point, and establishes a rendezvous circuit. [rendezvous 86 | setup.] 87 | 88 | 8. Alice connects to the Introduction point via Tor, and tells it about 89 | her rendezvous point. (Encrypted to Bob.) [Introduction 1] 90 | 91 | 9. The Introduction point passes this on to Bob's OP via Tor, along the 92 | introduction circuit. [Introduction 2] 93 | 94 | 10. Bob's OP decides whether to connect to Alice, and if so, creates a 95 | circuit to Alice's RP via Tor. Establishes a shared circuit. 96 | [Rendezvous 1] 97 | 98 | 11. The Rendezvous point forwards Bob's confirmation to Alice's OP. 99 | [Rendezvous 2] 100 | 101 | 12. Alice's OP sends begin cells to Bob's OP. [Connection] 102 | 103 | 0.3. Constants and new cell types 104 | 105 | Relay cell types 106 | 32 -- RELAY_COMMAND_ESTABLISH_INTRO 107 | 33 -- RELAY_COMMAND_ESTABLISH_RENDEZVOUS 108 | 34 -- RELAY_COMMAND_INTRODUCE1 109 | 35 -- RELAY_COMMAND_INTRODUCE2 110 | 36 -- RELAY_COMMAND_RENDEZVOUS1 111 | 37 -- RELAY_COMMAND_RENDEZVOUS2 112 | 38 -- RELAY_COMMAND_INTRO_ESTABLISHED 113 | 39 -- RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 114 | 40 -- RELAY_COMMAND_INTRODUCE_ACK 115 | 116 | 0.4. Version overview 117 | 118 | There are several parts in the hidden service protocol that have 119 | changed over time, each of them having its own version number, whereas 120 | other parts remained the same. The following list of potentially 121 | versioned protocol parts should help reduce some confusion: 122 | 123 | - Hidden service descriptor: the binary-based v0 was the default for a 124 | long time, and an ASCII-based v2 has been added by proposal 114. The 125 | v0 descriptor format has been deprecated in 0.2.2.1-alpha. See 1.3. 126 | 127 | - Hidden service descriptor propagation mechanism: currently related to 128 | the hidden service descriptor version -- v0 publishes to the original 129 | hs directory authorities, whereas v2 publishes to a rotating subset 130 | of relays with the "HSDir" flag; see 1.4 and 1.6. 131 | 132 | - Introduction protocol for how to generate an introduction cell: 133 | v0 specified a nickname for the rendezvous point and assumed the 134 | relay would know about it, whereas v2 now specifies IP address, 135 | port, and onion key so the relay doesn't need to already recognize 136 | it. See 1.8. 137 | 138 | 1. The Protocol 139 | 140 | 1.1. Bob configures his local OP. 141 | 142 | We do not specify a format for the OP configuration file. However, 143 | OPs SHOULD allow Bob to provide more than one advertised service 144 | per OP, and MUST allow Bob to specify one or more virtual ports per 145 | service. Bob provides a mapping from each of these virtual ports 146 | to a local IP:Port pair. 147 | 148 | 1.2. Bob's OP establishes his introduction points. 149 | 150 | The first time the OP provides an advertised service, it generates 151 | a public/private keypair (stored locally). 152 | 153 | The OP chooses a small number of Tor servers as introduction points. 154 | The OP establishes a new introduction circuit to each introduction 155 | point. These circuits MUST NOT be used for anything but hidden service 156 | introduction. To establish the introduction, Bob sends a 157 | RELAY_COMMAND_ESTABLISH_INTRO cell, containing: 158 | 159 | KL Key length [2 octets] 160 | PK Bob's public key or service key [KL octets] 161 | HS Hash of session info [20 octets] 162 | SIG Signature of above information [variable] 163 | 164 | KL is the length of PK, in octets. 165 | 166 | To prevent replay attacks, the HS field contains a SHA-1 hash based on the 167 | shared secret KH between Bob's OP and the introduction point, as 168 | follows: 169 | HS = H(KH | "INTRODUCE") 170 | That is: 171 | HS = H(KH | [49 4E 54 52 4F 44 55 43 45]) 172 | (KH, as specified in tor-spec.txt, is H(g^xy | [00]) .) 173 | 174 | Upon receiving such a cell, the OR first checks that the signature is 175 | correct with the included public key. If so, it checks whether HS is 176 | correct given the shared state between Bob's OP and the OR. If either 177 | check fails, the OP discards the cell; otherwise, it associates the 178 | circuit with Bob's public key, and dissociates any other circuits 179 | currently associated with PK. On success, the OR sends Bob a 180 | RELAY_COMMAND_INTRO_ESTABLISHED cell with an empty payload. 181 | 182 | Bob's OP uses either Bob's public key or a freshly generated, single-use 183 | service key in the RELAY_COMMAND_ESTABLISH_INTRO cell, depending on the 184 | configured hidden service descriptor version. The public key is used for 185 | v0 descriptors, the service key for v2 descriptors. In the latter case, the 186 | service keys of all introduction points are included in the v2 hidden 187 | service descriptor together with the other introduction point information. 188 | The reason is that the introduction point does not need to and therefore 189 | should not know for which hidden service it works, so as to prevent it from 190 | tracking the hidden service's activity. If the hidden service is configured 191 | to publish both v0 and v2 descriptors, two separate sets of introduction 192 | points are established. 193 | 194 | 1.3. Bob's OP generates service descriptors. 195 | 196 | For versions before 0.2.2.1-alpha, Bob's OP periodically generates and 197 | publishes a descriptor of type "V0". 198 | 199 | The "V0" descriptor contains: 200 | 201 | KL Key length [2 octets] 202 | PK Bob's public key [KL octets] 203 | TS A timestamp [4 octets] 204 | NI Number of introduction points [2 octets] 205 | Ipt A list of NUL-terminated ORs [variable] 206 | SIG Signature of above fields [variable] 207 | 208 | TS is the number of seconds elapsed since Jan 1, 1970. 209 | 210 | The members of Ipt may be either (a) nicknames, or (b) identity key 211 | digests, encoded in hex, and prefixed with a '$'. Clients must 212 | accept both forms. Services must only generate the second form. 213 | Once 0.0.9.x is obsoleted, we can drop the first form. 214 | 215 | [It's ok for Bob to advertise 0 introduction points. He might want 216 | to do that if he previously advertised some introduction points, 217 | and now he doesn't have any. -RD] 218 | 219 | Beginning with 0.2.0.10-alpha, Bob's OP encodes "V2" descriptors in 220 | addition to (or instead of) "V0" descriptors. The format of a "V2" 221 | descriptor is as follows: 222 | 223 | "rendezvous-service-descriptor" SP descriptor-id NL 224 | 225 | [At start, exactly once] 226 | 227 | Indicates the beginning of the descriptor. "descriptor-id" is a 228 | periodically changing identifier of 160 bits formatted as 32 base32 229 | chars that is calculated by the hidden service and its clients. The 230 | "descriptor-id" is calculated by performing the following operation: 231 | 232 | descriptor-id = 233 | H(permanent-id | H(time-period | descriptor-cookie | replica)) 234 | 235 | "permanent-id" is the permanent identifier of the hidden service, 236 | consisting of 80 bits. It can be calculated by computing the hash value 237 | of the public hidden service key and truncating after the first 80 bits: 238 | 239 | permanent-id = H(public-key)[:10] 240 | 241 | Note: If Bob's OP has "stealth" authorization enabled (see Section 2.2), 242 | it uses the client key in place of the public hidden service key. 243 | 244 | "H(time-period | descriptor-cookie | replica)" is the (possibly 245 | secret) id part that is necessary to verify that the hidden service is 246 | the true originator of this descriptor and that is therefore contained 247 | in the descriptor, too. The descriptor ID can only be created by the 248 | hidden service and its clients, but the "signature" below can only be 249 | created by the service. 250 | 251 | "time-period" changes periodically as a function of time and 252 | "permanent-id". The current value for "time-period" can be calculated 253 | using the following formula: 254 | 255 | time-period = (current-time + permanent-id-byte * 86400 / 256) 256 | / 86400 257 | 258 | "current-time" contains the current system time in seconds since 259 | 1970-01-01 00:00, e.g. 1188241957. "permanent-id-byte" is the first 260 | (unsigned) byte of the permanent identifier (which is in network 261 | order), e.g. 143. Adding the product of "permanent-id-byte" and 262 | 86400 (seconds per day), divided by 256, prevents "time-period" from 263 | changing for all descriptors at the same time of the day. The result 264 | of the overall operation is a (network-ordered) 32-bit integer, e.g. 265 | 13753 or 0x000035B9 with the example values given above. 266 | 267 | "descriptor-cookie" is an optional secret password of 128 bits that 268 | is shared between the hidden service provider and its clients. If the 269 | descriptor-cookie is left out, the input to the hash function is 128 270 | bits shorter. 271 | 272 | "replica" denotes the number of the replica. A service publishes 273 | multiple descriptors with different descriptor IDs in order to 274 | distribute them to different places on the ring. 275 | 276 | "version" SP version-number NL 277 | 278 | [Exactly once] 279 | 280 | The version number of this descriptor's format. Version numbers are a 281 | positive integer. 282 | 283 | "permanent-key" NL a public key in PEM format 284 | 285 | [Exactly once] 286 | 287 | The public key of the hidden service which is required to verify the 288 | "descriptor-id" and the "signature". 289 | 290 | "secret-id-part" SP secret-id-part NL 291 | 292 | [Exactly once] 293 | 294 | The result of the following operation as explained above, formatted as 295 | 32 base32 chars. Using this secret id part, everyone can verify that 296 | the signed descriptor belongs to "descriptor-id". 297 | 298 | secret-id-part = H(time-period | descriptor-cookie | replica) 299 | 300 | "publication-time" SP YYYY-MM-DD HH:MM:SS NL 301 | 302 | [Exactly once] 303 | 304 | A timestamp when this descriptor has been created. It should be 305 | rounded down to the nearest hour. 306 | 307 | "protocol-versions" SP version-string NL 308 | 309 | [Exactly once] 310 | 311 | A comma-separated list of recognized and permitted version numbers 312 | for use in INTRODUCE cells; these versions are described in section 313 | 1.8 below. Version numbers are positive integers. 314 | 315 | "introduction-points" NL encrypted-string 316 | 317 | [At most once] 318 | 319 | A list of introduction points. If the optional "descriptor-cookie" is 320 | used, this list is encrypted with AES in CTR mode with a random 321 | initialization vector of 128 bits that is written to 322 | the beginning of the encrypted string, and the "descriptor-cookie" as 323 | secret key of 128 bits length. 324 | 325 | The string containing the introduction point data (either encrypted 326 | or not) is encoded in base64, and surrounded with 327 | "-----BEGIN MESSAGE-----" and "-----END MESSAGE-----". 328 | 329 | The unencrypted string may begin with: 330 | 331 | "service-authentication" auth-type auth-data NL 332 | 333 | [Any number] 334 | 335 | The service-specific authentication data can be used to perform 336 | client authentication. This data is independent of the selected 337 | introduction point as opposed to "intro-authentication" below. The 338 | format of auth-data (base64-encoded or PEM format) depends on 339 | auth-type. See section 2 of this document for details on auth 340 | mechanisms. 341 | 342 | Subsequently, an arbitrary number of introduction point entries may 343 | follow, each containing the following data: 344 | 345 | "introduction-point" SP identifier NL 346 | 347 | [At start, exactly once] 348 | 349 | The identifier of this introduction point: the base32 encoded 350 | hash of this introduction point's identity key. 351 | 352 | "ip-address" SP ip4 NL 353 | 354 | [Exactly once] 355 | 356 | The IP address of this introduction point. 357 | 358 | "onion-port" SP port NL 359 | 360 | [Exactly once] 361 | 362 | The TCP port on which the introduction point is listening for 363 | incoming onion requests. 364 | 365 | "onion-key" NL a public key in PEM format 366 | 367 | [Exactly once] 368 | 369 | The public key that can be used to encrypt messages to this 370 | introduction point. 371 | 372 | "service-key" NL a public key in PEM format 373 | 374 | [Exactly once] 375 | 376 | The public key that can be used to encrypt messages to the hidden 377 | service. 378 | 379 | "intro-authentication" auth-type auth-data NL 380 | 381 | [Any number] 382 | 383 | The introduction-point-specific authentication data can be used 384 | to perform client authentication. This data depends on the 385 | selected introduction point as opposed to "service-authentication" 386 | above. The format of auth-data (base64-encoded or PEM format) 387 | depends on auth-type. See section 2 of this document for details 388 | on auth mechanisms. 389 | 390 | (This ends the fields in the encrypted portion of the descriptor.) 391 | 392 | [It's ok for Bob to advertise 0 introduction points. He might want 393 | to do that if he previously advertised some introduction points, 394 | and now he doesn't have any. -RD] 395 | 396 | "signature" NL signature-string 397 | 398 | [At end, exactly once] 399 | 400 | A signature of all fields above with the private key of the hidden 401 | service. 402 | 403 | 1.3.1. Other descriptor formats we don't use. 404 | 405 | Support for the V0 descriptor format was dropped in 0.2.2.0-alpha-dev: 406 | 407 | KL Key length [2 octets] 408 | PK Bob's public key [KL octets] 409 | TS A timestamp [4 octets] 410 | NI Number of introduction points [2 octets] 411 | Ipt A list of NUL-terminated ORs [variable] 412 | SIG Signature of above fields [variable] 413 | 414 | KL is the length of PK, in octets. 415 | TS is the number of seconds elapsed since Jan 1, 1970. 416 | 417 | The members of Ipt may be either (a) nicknames, or (b) identity key 418 | digests, encoded in hex, and prefixed with a '$'. 419 | 420 | The V1 descriptor format was understood and accepted from 421 | 0.1.1.5-alpha-cvs to 0.2.0.6-alpha-dev, but no Tors generated it and 422 | it was removed: 423 | 424 | V Format byte: set to 255 [1 octet] 425 | V Version byte: set to 1 [1 octet] 426 | KL Key length [2 octets] 427 | PK Bob's public key [KL octets] 428 | TS A timestamp [4 octets] 429 | PROTO Protocol versions: bitmask [2 octets] 430 | NI Number of introduction points [2 octets] 431 | For each introduction point: (as in INTRODUCE2 cells) 432 | IP Introduction point's address [4 octets] 433 | PORT Introduction point's OR port [2 octets] 434 | ID Introduction point identity ID [20 octets] 435 | KLEN Length of onion key [2 octets] 436 | KEY Introduction point onion key [KLEN octets] 437 | SIG Signature of above fields [variable] 438 | 439 | A hypothetical "V1" descriptor, that has never been used but might 440 | be useful for historical reasons, contains: 441 | 442 | V Format byte: set to 255 [1 octet] 443 | V Version byte: set to 1 [1 octet] 444 | KL Key length [2 octets] 445 | PK Bob's public key [KL octets] 446 | TS A timestamp [4 octets] 447 | PROTO Rendezvous protocol versions: bitmask [2 octets] 448 | NA Number of auth mechanisms accepted [1 octet] 449 | For each auth mechanism: 450 | AUTHT The auth type that is supported [2 octets] 451 | AUTHL Length of auth data [1 octet] 452 | AUTHD Auth data [variable] 453 | NI Number of introduction points [2 octets] 454 | For each introduction point: (as in INTRODUCE2 cells) 455 | ATYPE An address type (typically 4) [1 octet] 456 | ADDR Introduction point's IP address [4 or 16 octets] 457 | PORT Introduction point's OR port [2 octets] 458 | AUTHT The auth type that is supported [2 octets] 459 | AUTHL Length of auth data [1 octet] 460 | AUTHD Auth data [variable] 461 | ID Introduction point identity ID [20 octets] 462 | KLEN Length of onion key [2 octets] 463 | KEY Introduction point onion key [KLEN octets] 464 | SIG Signature of above fields [variable] 465 | 466 | AUTHT specifies which authentication/authorization mechanism is 467 | required by the hidden service or the introduction point. AUTHD 468 | is arbitrary data that can be associated with an auth approach. 469 | Currently only AUTHT of [00 00] is supported, with an AUTHL of 0. 470 | See section 2 of this document for details on auth mechanisms. 471 | 472 | 1.4. Bob's OP advertises his service descriptor(s). 473 | 474 | Bob's OP advertises his service descriptor to a fixed set of v0 hidden 475 | service directory servers and/or a changing subset of all v2 hidden service 476 | directories. 477 | 478 | For versions before 0.2.2.1-alpha, Bob's OP opens a stream to each v0 479 | directory server's directory port via Tor. (He may re-use old circuits for 480 | this.) Over this stream, Bob's OP makes an HTTP 'POST' request, to a URL 481 | "/tor/rendezvous/publish" relative to the directory server's root, 482 | containing as its body Bob's service descriptor. 483 | 484 | Upon receiving a descriptor, the directory server checks the signature, 485 | and discards the descriptor if the signature does not match the enclosed 486 | public key. Next, the directory server checks the timestamp. If the 487 | timestamp is more than 24 hours in the past or more than 1 hour in the 488 | future, or the directory server already has a newer descriptor with the 489 | same public key, the server discards the descriptor. Otherwise, the 490 | server discards any older descriptors with the same public key and 491 | version format, and associates the new descriptor with the public key. 492 | The directory server remembers this descriptor for at least 24 hours 493 | after its timestamp. At least every 18 hours, Bob's OP uploads a 494 | fresh descriptor. 495 | 496 | If Bob's OP is configured to publish v2 descriptors, it does so to a 497 | changing subset of all v2 hidden service directories instead of the 498 | authoritative directory servers. Therefore, Bob's OP opens a stream via 499 | Tor to each responsible hidden service directory. (He may re-use old 500 | circuits for this.) Over this stream, Bob's OP makes an HTTP 'POST' 501 | request to a URL "/tor/rendezvous2/publish" relative to the hidden service 502 | directory's root, containing as its body Bob's service descriptor. 503 | 504 | [XXX022 Reusing old circuits for HS dir posts is very bad. Do we really 505 | do that? --RR] 506 | 507 | At any time, there are 6 hidden service directories responsible for 508 | keeping replicas of a descriptor; they consist of 2 sets of 3 hidden 509 | service directories with consecutive onion IDs. Bob's OP learns about 510 | the complete list of hidden service directories by filtering the 511 | consensus status document received from the directory authorities. A 512 | hidden service directory is deemed responsible for a descriptor ID if 513 | it has the HSDir flag and its identity digest is one of the first three 514 | identity digests of HSDir relays following the descriptor ID in a 515 | circular list. A hidden service directory will only accept a descriptor 516 | whose timestamp is no more than three days before or one day after the 517 | current time according to the directory's clock. 518 | 519 | Bob's OP publishes a new v2 descriptor once an hour or whenever its 520 | content changes. V2 descriptors can be found by clients within a given 521 | time period of 24 hours, after which they change their ID as described 522 | under 1.3. If a published descriptor would be valid for less than 60 523 | minutes (= 2 x 30 minutes to allow the server to be 30 minutes behind 524 | and the client 30 minutes ahead), Bob's OP publishes the descriptor 525 | under the ID of both, the current and the next publication period. 526 | 527 | 1.5. Alice receives a z.onion address. 528 | 529 | When Alice receives a pointer to a location-hidden service, it is as a 530 | hostname of the form "z.onion", where z is a base32 encoding of a 531 | 10-octet hash of Bob's service's public key, computed as follows: 532 | 533 | 1. Let H = H(PK). 534 | 2. Let H' = the first 80 bits of H, considering each octet from 535 | most significant bit to least significant bit. 536 | 3. Generate a 16-character encoding of H', using base32 as defined 537 | in RFC 4648. 538 | 539 | (We only use 80 bits instead of the 160 bits from SHA1 because we 540 | don't need to worry about arbitrary collisions, and because it will 541 | make handling the url's more convenient.) 542 | 543 | [Yes, numbers are allowed at the beginning. See RFC 1123. -NM] 544 | 545 | 1.6. Alice's OP retrieves a service descriptor. 546 | 547 | Alice's OP fetches the service descriptor from the fixed set of v0 hidden 548 | service directory servers and/or a changing subset of all v2 hidden service 549 | directories. 550 | 551 | For versions before 0.2.2.1-alpha, Alice's OP opens a stream to a directory 552 | server via Tor, and makes an HTTP GET request for the document 553 | '/tor/rendezvous/', where '' is replaced with the encoding of Bob's 554 | public key as described above. (She may re-use old circuits for this.) The 555 | directory replies with a 404 HTTP response if it does not recognize , 556 | and otherwise returns Bob's most recently uploaded service descriptor. 557 | 558 | If Alice's OP receives a 404 response, it tries the other directory 559 | servers, and only fails the lookup if none recognize the public key hash. 560 | 561 | Upon receiving a service descriptor, Alice verifies with the same process 562 | as the directory server uses, described above in section 1.4. 563 | 564 | The directory server gives a 400 response if it cannot understand Alice's 565 | request. 566 | 567 | Alice should cache the descriptor locally, but should not use 568 | descriptors that are more than 24 hours older than their timestamp. 569 | [Caching may make her partitionable, but she fetched it anonymously, 570 | and we can't very well *not* cache it. -RD] 571 | 572 | If Alice's OP is running 0.2.1.10-alpha or higher, it fetches v2 hidden 573 | service descriptors. Versions before 0.2.2.1-alpha are fetching both v0 and 574 | v2 descriptors in parallel. Similar to the description in section 1.4, 575 | Alice's OP fetches a v2 descriptor from a randomly chosen hidden service 576 | directory out of the changing subset of 6 nodes. If the request is 577 | unsuccessful, Alice retries the other remaining responsible hidden service 578 | directories in a random order. Alice relies on Bob to care about a potential 579 | clock skew between the two by possibly storing two sets of descriptors (see 580 | end of section 1.4). 581 | 582 | Alice's OP opens a stream via Tor to the chosen v2 hidden service 583 | directory. (She may re-use old circuits for this.) Over this stream, 584 | Alice's OP makes an HTTP 'GET' request for the document 585 | "/tor/rendezvous2/", where z is replaced with the encoding of the 586 | descriptor ID. The directory replies with a 404 HTTP response if it does 587 | not recognize , and otherwise returns Bob's most recently uploaded 588 | service descriptor. 589 | 590 | 1.7. Alice's OP establishes a rendezvous point. 591 | 592 | When Alice requests a connection to a given location-hidden service, 593 | and Alice's OP does not have an established circuit to that service, 594 | the OP builds a rendezvous circuit. It does this by establishing 595 | a circuit to a randomly chosen OR, and sending a 596 | RELAY_COMMAND_ESTABLISH_RENDEZVOUS cell to that OR. The body of that cell 597 | contains: 598 | 599 | RC Rendezvous cookie [20 octets] 600 | 601 | The rendezvous cookie is an arbitrary 20-byte value, chosen randomly by 602 | Alice's OP. Alice SHOULD choose a new rendezvous cookie for each new 603 | connection attempt. 604 | 605 | Upon receiving a RELAY_COMMAND_ESTABLISH_RENDEZVOUS cell, the OR associates 606 | the RC with the circuit that sent it. It replies to Alice with an empty 607 | RELAY_COMMAND_RENDEZVOUS_ESTABLISHED cell to indicate success. 608 | 609 | Alice's OP MUST NOT use the circuit which sent the cell for any purpose 610 | other than rendezvous with the given location-hidden service. 611 | 612 | 1.8. Introduction: from Alice's OP to Introduction Point 613 | 614 | Alice builds a separate circuit to one of Bob's chosen introduction 615 | points, and sends it a RELAY_COMMAND_INTRODUCE1 cell containing: 616 | 617 | Cleartext 618 | PK_ID Identifier for Bob's PK [20 octets] 619 | Encrypted to Bob's PK: (in the v0 intro protocol) 620 | RP Rendezvous point's nickname [20 octets] 621 | RC Rendezvous cookie [20 octets] 622 | g^x Diffie-Hellman data, part 1 [128 octets] 623 | OR (in the v1 intro protocol) 624 | VER Version byte: set to 1. [1 octet] 625 | RP Rendezvous point nick or ID [42 octets] 626 | RC Rendezvous cookie [20 octets] 627 | g^x Diffie-Hellman data, part 1 [128 octets] 628 | OR (in the v2 intro protocol) 629 | VER Version byte: set to 2. [1 octet] 630 | IP Rendezvous point's address [4 octets] 631 | PORT Rendezvous point's OR port [2 octets] 632 | ID Rendezvous point identity ID [20 octets] 633 | KLEN Length of onion key [2 octets] 634 | KEY Rendezvous point onion key [KLEN octets] 635 | RC Rendezvous cookie [20 octets] 636 | g^x Diffie-Hellman data, part 1 [128 octets] 637 | OR (in the v3 intro protocol) 638 | VER Version byte: set to 3. [1 octet] 639 | AUTHT The auth type that is used [1 octet] 640 | If AUTHT != [00]: 641 | AUTHL Length of auth data [2 octets] 642 | AUTHD Auth data [variable] 643 | TS A timestamp [4 octets] 644 | IP Rendezvous point's address [4 octets] 645 | PORT Rendezvous point's OR port [2 octets] 646 | ID Rendezvous point identity ID [20 octets] 647 | KLEN Length of onion key [2 octets] 648 | KEY Rendezvous point onion key [KLEN octets] 649 | RC Rendezvous cookie [20 octets] 650 | g^x Diffie-Hellman data, part 1 [128 octets] 651 | 652 | PK_ID is the hash of Bob's public key or the service key, depending on the 653 | hidden service descriptor version. In case of a v0 descriptor, Alice's OP 654 | uses Bob's public key. If Alice has downloaded a v2 descriptor, she uses 655 | the contained public key ("service-key"). 656 | 657 | RP is NUL-padded and terminated. In version 0 of the intro protocol, RP 658 | must contain a nickname. In version 1, it must contain EITHER a nickname or 659 | an identity key digest that is encoded in hex and prefixed with a '$'. 660 | 661 | The hybrid encryption to Bob's PK works just like the hybrid 662 | encryption in CREATE cells (see tor-spec). Thus the payload of the 663 | version 0 RELAY_COMMAND_INTRODUCE1 cell on the wire will contain 664 | 20+42+16+20+20+128=246 bytes, and the version 1 and version 2 665 | introduction formats have other sizes. 666 | 667 | Through Tor 0.2.0.6-alpha, clients only generated the v0 introduction 668 | format, whereas hidden services have understood and accepted v0, 669 | v1, and v2 since 0.1.1.x. As of Tor 0.2.0.7-alpha and 0.1.2.18, 670 | clients switched to using the v2 intro format. 671 | 672 | The Timestampe (TS) field is no longer used in Tor 0.2.3.9-alpha and 673 | later. Clients MAY refrain from sending it; see the 674 | "Support022HiddenServices" parameter in dir-spec.txt. Clients SHOULD 675 | NOT send a precise timestamp, and should instead round to the nearest 676 | 10 minutes. 677 | 678 | 1.9. Introduction: From the Introduction Point to Bob's OP 679 | 680 | If the Introduction Point recognizes PK_ID as a public key which has 681 | established a circuit for introductions as in 1.2 above, it sends the body 682 | of the cell in a new RELAY_COMMAND_INTRODUCE2 cell down the corresponding 683 | circuit. (If the PK_ID is unrecognized, the RELAY_COMMAND_INTRODUCE1 cell is 684 | discarded.) 685 | 686 | After sending the RELAY_COMMAND_INTRODUCE2 cell to Bob, the OR replies to 687 | Alice with an empty RELAY_COMMAND_INTRODUCE_ACK cell. If no 688 | RELAY_COMMAND_INTRODUCE2 cell can be sent, the OR replies to Alice with a 689 | non-empty cell to indicate an error. (The semantics of the cell body may be 690 | determined later; the current implementation sends a single '1' byte on 691 | failure.) 692 | 693 | When Bob's OP receives the RELAY_COMMAND_INTRODUCE2 cell, it first checks 694 | for a replay. Because of the (undesirable!) malleability of the hybrid 695 | encryption, Bob's OP should only check whether the RSA-encrypted part is 696 | replayed. It does this by keeping, for each introduction key, a list of 697 | cryptographic digests of all the RSA-encrypted parts of the INTRODUCE2 698 | cells that it's seen, and dropping any INTRODUCE2 cell whose RSA-encrypted 699 | part it has seen before. When Bob's OP stops using a given introduction 700 | key, it drops the replay cache corresponding to that key. 701 | 702 | (Versions of Tor before 0.2.3.9-alpha used the timestamp in the INTRODUCE2 703 | cell to limit the lifetime of entries in the replay cache. This proved to 704 | be fragile, due to insufficiently synchronized clients.) 705 | 706 | Assuming that the cell has not been replayed, Bob's server decrypts it 707 | with the private key for the corresponding hidden service, and extracts the 708 | rendezvous point's nickname, the rendezvous cookie, and the value of g^x 709 | chosen by Alice. 710 | 711 | 1.10. Rendezvous 712 | 713 | Bob's OP builds a new Tor circuit ending at Alice's chosen rendezvous 714 | point, and sends a RELAY_COMMAND_RENDEZVOUS1 cell along this circuit, 715 | containing: 716 | RC Rendezvous cookie [20 octets] 717 | g^y Diffie-Hellman [128 octets] 718 | KH Handshake digest [20 octets] 719 | 720 | (Bob's OP MUST NOT use this circuit for any other purpose.) 721 | 722 | (By default, Bob builds a circuit of at least three hops, *not including* 723 | Alice's chosen rendezvous point.) 724 | 725 | If the RP recognizes RC, it relays the rest of the cell down the 726 | corresponding circuit in a RELAY_COMMAND_RENDEZVOUS2 cell, containing: 727 | 728 | g^y Diffie-Hellman [128 octets] 729 | KH Handshake digest [20 octets] 730 | 731 | (If the RP does not recognize the RC, it discards the cell and 732 | tears down the circuit.) 733 | 734 | When Alice's OP receives a RELAY_COMMAND_RENDEZVOUS2 cell on a circuit which 735 | has sent a RELAY_COMMAND_ESTABLISH_RENDEZVOUS cell but which has not yet 736 | received a reply, it uses g^y and H(g^xy) to complete the handshake as in 737 | the Tor circuit extend process: they establish a 60-octet string as 738 | K = SHA1(g^xy | [00]) | SHA1(g^xy | [01]) | SHA1(g^xy | [02]) 739 | and generate KH, Df, Db, Kf, and Kb as in the KDF-TOR key derivation 740 | approach documented in tor-spec.txt. 741 | 742 | As in the TAP handshake, if the KH value derived from KDF-Tor does not 743 | match the value in the RENDEZVOUS2 cell, the client must close the 744 | circuit. 745 | 746 | Subsequently, the rendezvous point passes relay cells, unchanged, from 747 | each of the two circuits to the other. When Alice's OP sends RELAY cells 748 | along the circuit, it authenticates with Df, and encrypts them with the 749 | Kf, then with all of the keys for the ORs in Alice's side of the circuit; 750 | and when Alice's OP receives RELAY cells from the circuit, it decrypts 751 | them with the keys for the ORs in Alice's side of the circuit, then 752 | decrypts them with Kb, and checks integrity with Db. Bob's OP does the 753 | same, with Kf and Kb interchanged. 754 | 755 | 1.11. Creating streams 756 | 757 | To open TCP connections to Bob's location-hidden service, Alice's OP sends 758 | a RELAY_COMMAND_BEGIN cell along the established circuit, using the special 759 | address "", and a chosen port. Bob's OP chooses a destination IP and 760 | port, based on the configuration of the service connected to the circuit, 761 | and opens a TCP stream. From then on, Bob's OP treats the stream as an 762 | ordinary exit connection. 763 | [ Except he doesn't include addr in the connected cell or the end 764 | cell. -RD] 765 | 766 | Alice MAY send multiple RELAY_COMMAND_BEGIN cells along the circuit, to open 767 | multiple streams to Bob. Alice SHOULD NOT send RELAY_COMMAND_BEGIN cells 768 | for any other address along her circuit to Bob; if she does, Bob MUST reject 769 | them. 770 | 771 | 1.12. Closing streams 772 | 773 | The payload of a RELAY_END cell begins with a single 'reason' byte to 774 | describe why the stream is closing, plus optional data (depending on the 775 | reason.) These can be found in section 6.3 of tor-spec. The following 776 | describes some of the hidden service related reasons. 777 | 778 | 1 -- REASON_MISC 779 | 780 | Catch-all for unlisted reasons. Shouldn't happen much in practice. 781 | 782 | 2 -- REASON_RESOLVEFAILED 783 | 784 | Tor tried to fetch the hidden service descriptor from the hsdirs but 785 | none of them had it. This implies that the hidden service has not 786 | been running in the past 24 hours. 787 | 788 | 3 -- REASON_CONNECTREFUSED 789 | 790 | Every step of the rendezvous worked great, and that the hidden 791 | service is indeed up and running and configured to use the virtual 792 | port you asked for, but there was nothing listening on the other end 793 | of that virtual port. For example, the HS's Tor client is running 794 | fine but its apache service is down. 795 | 796 | 4 -- REASON_EXITPOLICY 797 | 798 | The destination port that you tried is not configured on the hidden 799 | service side. That is, the hidden service was up and reachable, but 800 | it isn't listening on this port. Since Tor 0.2.6.2-alpha and later 801 | hidden service don't send this error code; instead they send back an 802 | END cell with reason DONE reason then close the circuit on you. This 803 | behavior can be controlled by a config option. 804 | 805 | 5 -- REASON_DESTROY 806 | 807 | The circuit closed before you could get a response back -- transient 808 | failure, e.g. a relay went down unexpectedly. Trying again might 809 | work. 810 | 811 | 6 -- REASON_DONE 812 | 813 | Anonymized TCP connection was closed. If you get an END cell with 814 | reason DONE, *before* you've gotten your CONNECTED cell, that 815 | indicates a similar situation to EXITPOLICY, but the hidden service 816 | is running 0.2.6.2-alpha or later, and it has now closed the circuit 817 | on you. 818 | 819 | 7 -- REASON_TIMEOUT 820 | 821 | Either like CONNECTREFUSED above but connect() got the ETIMEDOUT 822 | errno, or the client-side timeout of 120 seconds kicked in and we 823 | gave up. 824 | 825 | 8 -- REASON_NOROUTE 826 | 827 | Like CONNECTREFUSED except the errno at the hidden service when 828 | trying to connect() to the service was ENETUNREACH, EHOSTUNREACH, 829 | EACCES, or EPERM. 830 | 831 | 10 -- REASON_INTERNAL 832 | 833 | Internal error inside the Tor client -- hopefully you will not see 834 | this much. Please report if you do! 835 | 836 | 12 -- REASON_CONNRESET 837 | 838 | Like CONNECTREFUSED except the errno at the hidden service when 839 | trying to connect() to the service was ECONNRESET. 840 | 841 | 2. Authentication and authorization. 842 | 843 | The rendezvous protocol as described in Section 1 provides a few options 844 | for implementing client-side authorization. There are two steps in the 845 | rendezvous protocol that can be used for performing client authorization: 846 | when downloading and decrypting parts of the hidden service descriptor and 847 | at Bob's Tor client before contacting the rendezvous point. A service 848 | provider can restrict access to his service at these two points to 849 | authorized clients only. 850 | 851 | There are currently two authorization protocols specified that are 852 | described in more detail below: 853 | 854 | 1. The first protocol allows a service provider to restrict access 855 | to clients with a previously received secret key only, but does not 856 | attempt to hide service activity from others. 857 | 858 | 2. The second protocol, albeit being feasible for a limited set of about 859 | 16 clients, performs client authorization and hides service activity 860 | from everyone but the authorized clients. 861 | 862 | 2.1. Service with large-scale client authorization 863 | 864 | The first client authorization protocol aims at performing access control 865 | while consuming as few additional resources as possible. This is the "basic" 866 | authorization protocol. A service provider should be able to permit access 867 | to a large number of clients while denying access for everyone else. 868 | However, the price for scalability is that the service won't be able to hide 869 | its activity from unauthorized or formerly authorized clients. 870 | 871 | The main idea of this protocol is to encrypt the introduction-point part 872 | in hidden service descriptors to authorized clients using symmetric keys. 873 | This ensures that nobody else but authorized clients can learn which 874 | introduction points a service currently uses, nor can someone send a 875 | valid INTRODUCE1 message without knowing the introduction key. Therefore, 876 | a subsequent authorization at the introduction point is not required. 877 | 878 | A service provider generates symmetric "descriptor cookies" for his 879 | clients and distributes them outside of Tor. The suggested key size is 880 | 128 bits, so that descriptor cookies can be encoded in 22 base64 chars 881 | (which can hold up to 22 * 5 = 132 bits, leaving 4 bits to encode the 882 | authorization type (here: "0") and allow a client to distinguish this 883 | authorization protocol from others like the one proposed below). 884 | Typically, the contact information for a hidden service using this 885 | authorization protocol looks like this: 886 | 887 | v2cbb2l4lsnpio4q.onion Ll3X7Xgz9eHGKCCnlFH0uz 888 | 889 | When generating a hidden service descriptor, the service encrypts the 890 | introduction-point part with a single randomly generated symmetric 891 | 128-bit session key using AES-CTR as described for v2 hidden service 892 | descriptors in rend-spec. Afterwards, the service encrypts the session 893 | key to all descriptor cookies using AES. Authorized client should be able 894 | to efficiently find the session key that is encrypted for him/her, so 895 | that 4 octet long client ID are generated consisting of descriptor cookie 896 | and initialization vector. Descriptors always contain a number of 897 | encrypted session keys that is a multiple of 16 by adding fake entries. 898 | Encrypted session keys are ordered by client IDs in order to conceal 899 | addition or removal of authorized clients by the service provider. 900 | 901 | ATYPE Authorization type: set to 1. [1 octet] 902 | ALEN Number of clients := 1 + ((clients - 1) div 16) [1 octet] 903 | for each symmetric descriptor cookie: 904 | ID Client ID: H(descriptor cookie | IV)[:4] [4 octets] 905 | SKEY Session key encrypted with descriptor cookie [16 octets] 906 | (end of client-specific part) 907 | RND Random data [(15 - ((clients - 1) mod 16)) * 20 octets] 908 | IV AES initialization vector [16 octets] 909 | IPOS Intro points, encrypted with session key [remaining octets] 910 | 911 | An authorized client needs to configure Tor to use the descriptor cookie 912 | when accessing the hidden service. Therefore, a user adds the contact 913 | information that she received from the service provider to her torrc 914 | file. Upon downloading a hidden service descriptor, Tor finds the 915 | encrypted introduction-point part and attempts to decrypt it using the 916 | configured descriptor cookie. (In the rare event of two or more client 917 | IDs being equal a client tries to decrypt all of them.) 918 | 919 | Upon sending the introduction, the client includes her descriptor cookie 920 | as auth type "1" in the INTRODUCE2 cell that she sends to the service. 921 | The hidden service checks whether the included descriptor cookie is 922 | authorized to access the service and either responds to the introduction 923 | request, or not. 924 | 925 | 2.2. Authorization for limited number of clients 926 | 927 | A second, more sophisticated client authorization protocol goes the extra 928 | mile of hiding service activity from unauthorized clients. This is the 929 | "stealth" authorization protocol. With all else being equal to the preceding 930 | authorization protocol, the second protocol publishes hidden service 931 | descriptors for each user separately and gets along with encrypting the 932 | introduction-point part of descriptors to a single client. This allows the 933 | service to stop publishing descriptors for removed clients. As long as a 934 | removed client cannot link descriptors issued for other clients to the 935 | service, it cannot derive service activity any more. The downside of this 936 | approach is limited scalability. Even though the distributed storage of 937 | descriptors (cf. proposal 114) tackles the problem of limited scalability to 938 | a certain extent, this protocol should not be used for services with more 939 | than 16 clients. (In fact, Tor should refuse to advertise services for more 940 | than this number of clients.) 941 | 942 | A hidden service generates an asymmetric "client key" and a symmetric 943 | "descriptor cookie" for each client. The client key is used as 944 | replacement for the service's permanent key, so that the service uses a 945 | different identity for each of his clients. The descriptor cookie is used 946 | to store descriptors at changing directory nodes that are unpredictable 947 | for anyone but service and client, to encrypt the introduction-point 948 | part, and to be included in INTRODUCE2 cells. Once the service has 949 | created client key and descriptor cookie, he tells them to the client 950 | outside of Tor. The contact information string looks similar to the one 951 | used by the preceding authorization protocol (with the only difference 952 | that it has "1" encoded as auth-type in the remaining 4 of 132 bits 953 | instead of "0" as before). 954 | 955 | When creating a hidden service descriptor for an authorized client, the 956 | hidden service uses the client key and descriptor cookie to compute 957 | secret ID part and descriptor ID: 958 | 959 | secret-id-part = H(time-period | descriptor-cookie | replica) 960 | 961 | descriptor-id = H(client-key[:10] | secret-id-part) 962 | 963 | The hidden service also replaces permanent-key in the descriptor with 964 | client-key and encrypts introduction-points with the descriptor cookie. 965 | 966 | ATYPE Authorization type: set to 2. [1 octet] 967 | IV AES initialization vector [16 octets] 968 | IPOS Intro points, encr. with descriptor cookie [remaining octets] 969 | 970 | When uploading descriptors, the hidden service needs to make sure that 971 | descriptors for different clients are not uploaded at the same time (cf. 972 | Section 1.1) which is also a limiting factor for the number of clients. 973 | 974 | When a client is requested to establish a connection to a hidden service 975 | it looks up whether it has any authorization data configured for that 976 | service. If the user has configured authorization data for authorization 977 | protocol "2", the descriptor ID is determined as described in the last 978 | paragraph. Upon receiving a descriptor, the client decrypts the 979 | introduction-point part using its descriptor cookie. Further, the client 980 | includes its descriptor cookie as auth-type "2" in INTRODUCE2 cells that 981 | it sends to the service. 982 | 983 | 2.3. Hidden service configuration 984 | 985 | A hidden service that is meant to perform client authorization adds a 986 | new option HiddenServiceAuthorizeClient to its hidden service 987 | configuration. This option contains the authorization type which is 988 | either "basic" for the protocol described in 2.1 or "stealth" for the 989 | protocol in 2.2 and a comma-separated list of human-readable client 990 | names, so that Tor can create authorization data for these clients: 991 | 992 | HiddenServiceAuthorizeClient auth-type client-name,client-name,... 993 | 994 | If this option is configured, HiddenServiceVersion is automatically 995 | reconfigured to contain only version numbers of 2 or higher. There is 996 | a maximum of 512 client names for basic auth and a maximum of 16 for 997 | stealth auth. 998 | 999 | Tor stores all generated authorization data for the authorization 1000 | protocols described in Sections 2.1 and 2.2 in a new file using the 1001 | following file format: 1002 | 1003 | "client-name" human-readable client identifier NL 1004 | "descriptor-cookie" 128-bit key ^= 22 base64 chars NL 1005 | 1006 | If the authorization protocol of Section 2.2 is used, Tor also generates 1007 | and stores the following data: 1008 | 1009 | "client-key" NL a public key in PEM format 1010 | 1011 | 2.4. Client configuration 1012 | 1013 | To specify the cookie to use to access a given hidden service, 1014 | clients use the following syntax: 1015 | 1016 | HidServAuth onion-address auth-cookie [service-name]: 1017 | 1018 | Valid onion addresses contain 16 characters in a-z2-7 plus 1019 | ".onion", and valid auth cookies contain 22 characters in 1020 | A-Za-z0-9+/. The service name is only used for internal purposes, 1021 | e.g., for Tor controllers; nothing in Tor itself requires or uses 1022 | it. 1023 | 1024 | 1025 | 3. Hidden service directory operation 1026 | 1027 | This section has been introduced with the v2 hidden service descriptor 1028 | format. It describes all operations of the v2 hidden service descriptor 1029 | fetching and propagation mechanism that are required for the protocol 1030 | described in section 1 to succeed with v2 hidden service descriptors. 1031 | 1032 | 3.1. Configuring as hidden service directory 1033 | 1034 | Every onion router that has its directory port open can decide whether it 1035 | wants to store and serve hidden service descriptors. An onion router which 1036 | is configured as such includes the "hidden-service-dir" flag in its router 1037 | descriptors that it sends to directory authorities. 1038 | 1039 | The directory authorities include a new flag "HSDir" for routers that 1040 | decided to provide storage for hidden service descriptors and that 1041 | have been running for at least 24 hours. 1042 | 1043 | 3.2. Accepting publish requests 1044 | 1045 | Hidden service directory nodes accept publish requests for v2 hidden service 1046 | descriptors and store them to their local memory. (It is not necessary to 1047 | make descriptors persistent, because after restarting, the onion router 1048 | would not be accepted as a storing node anyway, because it has not been 1049 | running for at least 24 hours.) All requests and replies are formatted as 1050 | HTTP messages. Requests are initiated via BEGIN_DIR cells directed to 1051 | the router's directory port, and formatted as HTTP POST requests to the URL 1052 | "/tor/rendezvous2/publish" relative to the hidden service directory's root, 1053 | containing as its body a v2 service descriptor. 1054 | 1055 | A hidden service directory node parses every received descriptor and only 1056 | stores it when it thinks that it is responsible for storing that descriptor 1057 | based on its own routing table. See section 1.4 for more information on how 1058 | to determine responsibility for a certain descriptor ID. 1059 | 1060 | 3.3. Processing fetch requests 1061 | 1062 | Hidden service directory nodes process fetch requests for hidden service 1063 | descriptors by looking them up in their local memory. (They do not need to 1064 | determine if they are responsible for the passed ID, because it does no harm 1065 | if they deliver a descriptor for which they are not (any more) responsible.) 1066 | All requests and replies are formatted as HTTP messages. Requests are 1067 | initiated via BEGIN_DIR cells directed to the router's directory port, 1068 | and formatted as HTTP GET requests for the document "/tor/rendezvous2/", 1069 | where z is replaced with the encoding of the descriptor ID. 1070 | 1071 | --------------------------------------------------------------------------------