└── main.go /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/rand" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "os" 10 | "sync" 11 | "time" 12 | 13 | "github.com/prometheus/client_golang/prometheus" 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | "github.com/syndtr/goleveldb/leveldb" 16 | 17 | ds "github.com/ipfs/go-datastore" 18 | "github.com/ipfs/go-ipfs-addr" 19 | logging "github.com/ipfs/go-log" 20 | libp2p "github.com/libp2p/go-libp2p" 21 | host "github.com/libp2p/go-libp2p-host" 22 | dht "github.com/libp2p/go-libp2p-kad-dht" 23 | peer "github.com/libp2p/go-libp2p-peer" 24 | pstore "github.com/libp2p/go-libp2p-peerstore" 25 | ma "github.com/multiformats/go-multiaddr" 26 | mh "github.com/multiformats/go-multihash" 27 | ) 28 | 29 | var ( 30 | node_counts_g = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 31 | Name: "nodes_total", 32 | Subsystem: "stats", 33 | Namespace: "libp2p", 34 | Help: "total number of nodes seen in a given time period", 35 | }, []string{"interval", "version"}) 36 | 37 | protocols_g = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 38 | Name: "protocols", 39 | Subsystem: "stats", 40 | Namespace: "libp2p", 41 | Help: "protocol counts by name", 42 | }, []string{"interval", "protocol"}) 43 | 44 | query_lat_h = prometheus.NewHistogram(prometheus.HistogramOpts{ 45 | Name: "query", 46 | Subsystem: "dht", 47 | Namespace: "libp2p", 48 | Help: "dht 'findclosestpeers' latencies", 49 | Buckets: []float64{0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 2, 5, 10, 15, 20, 25, 30, 60}, 50 | }) 51 | 52 | addr_counts_s = prometheus.NewSummary(prometheus.SummaryOpts{ 53 | Name: "addr_counts", 54 | Subsystem: "stats", 55 | Namespace: "libp2p", 56 | Help: "address counts discovered by the dht crawls", 57 | }) 58 | ) 59 | 60 | func init() { 61 | prometheus.MustRegister(node_counts_g) 62 | prometheus.MustRegister(protocols_g) 63 | prometheus.MustRegister(query_lat_h) 64 | } 65 | 66 | var log = logging.Logger("dht_scrape") 67 | 68 | var bspi []pstore.PeerInfo 69 | 70 | var DefaultBootstrapAddresses = []string{ 71 | "/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io 72 | "/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM", // pluto.i.ipfs.io 73 | "/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu", // saturn.i.ipfs.io 74 | "/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64", // venus.i.ipfs.io 75 | "/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd", // earth.i.ipfs.io 76 | "/ip4/104.236.151.122/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx", 77 | "/ip4/188.40.114.11/tcp/4001/ipfs/QmZY7MtK8ZbG1suwrxc7xEYZ2hQLf1dAWPRHhjxC8rjq8E", 78 | "/ip4/5.9.59.34/tcp/4001/ipfs/QmRv1GNseNP1krEwHDjaQMeQVJy41879QcDwpJVhY8SWve", 79 | } 80 | 81 | func init() { 82 | for _, a := range DefaultBootstrapAddresses { 83 | ia, err := ipfsaddr.ParseString(a) 84 | if err != nil { 85 | panic(err) 86 | } 87 | 88 | bspi = append(bspi, pstore.PeerInfo{ 89 | ID: ia.ID(), 90 | Addrs: []ma.Multiaddr{ia.Transport()}, 91 | }) 92 | } 93 | } 94 | 95 | func handlePromWithAuth(w http.ResponseWriter, r *http.Request) { 96 | u, p, ok := r.BasicAuth() 97 | if !ok { 98 | w.WriteHeader(403) 99 | return 100 | } 101 | 102 | if !(u == "protocol" && p == os.Getenv("IPFS_METRICS_PASSWORD")) { 103 | w.WriteHeader(403) 104 | return 105 | } 106 | 107 | promhttp.Handler().ServeHTTP(w, r) 108 | } 109 | 110 | func main() { 111 | http.HandleFunc("/metrics", handlePromWithAuth) 112 | go func() { 113 | if err := http.ListenAndServe(":1234", nil); err != nil { 114 | panic(err) 115 | } 116 | }() 117 | 118 | db, err := leveldb.OpenFile("netdata", nil) 119 | if err != nil { 120 | panic(err) 121 | } 122 | defer db.Close() 123 | 124 | if err := getStats(db); err != nil { 125 | log.Error("get stats failed: ", err) 126 | } 127 | 128 | for { 129 | if err := buildHostAndScrapePeers(db); err != nil { 130 | log.Error("scrape failed: ", err) 131 | } 132 | } 133 | } 134 | 135 | func buildHostAndScrapePeers(db *leveldb.DB) error { 136 | fmt.Println("building new node to collect metrics with") 137 | ctx, cancel := context.WithCancel(context.Background()) 138 | defer cancel() 139 | h, err := libp2p.New(ctx, libp2p.ListenAddrStrings("/ip4/0.0.0.0/tcp/4001")) 140 | if err != nil { 141 | return err 142 | } 143 | 144 | defer func() { 145 | fmt.Println("closing host...") 146 | h.Close() 147 | }() 148 | 149 | mds := ds.NewMapDatastore() 150 | mdht := dht.NewDHT(ctx, h, mds) 151 | 152 | bootstrap(ctx, h) 153 | 154 | fmt.Println("starting new scrape round...") 155 | for i := 0; i < 15; i++ { 156 | if err := scrapePeers(db, h, mdht); err != nil { 157 | return err 158 | } 159 | time.Sleep(time.Second * 10) 160 | } 161 | return nil 162 | } 163 | 164 | func getRandomString() string { 165 | buf := make([]byte, 32) 166 | rand.Read(buf) 167 | o, err := mh.Encode(buf, mh.SHA2_256) 168 | if err != nil { 169 | panic(err) 170 | } 171 | return string(o) 172 | } 173 | 174 | type trackingInfo struct { 175 | Addresses []string `json:"a"` 176 | LastConnected time.Time `json:"lc"` 177 | LastSeen time.Time `json:"ls"` 178 | FirstSeen time.Time `json:"fs"` 179 | Sightings int `json:"n"` 180 | AgentVersion string `json:"av"` 181 | Protocols []string `json:"ps"` 182 | } 183 | 184 | func getStats(db *leveldb.DB) error { 185 | now := time.Now() 186 | 187 | protosCountDay := make(map[string]int) 188 | protosCountHour := make(map[string]int) 189 | dayVersCount := make(map[string]int) 190 | hourVersCount := make(map[string]int) 191 | var dayPeerCount int 192 | var hourPeerCount int 193 | 194 | iter := db.NewIterator(nil, nil) 195 | defer iter.Release() 196 | 197 | for iter.Next() { 198 | pid := peer.ID(iter.Key()) 199 | 200 | var ti trackingInfo 201 | err := json.Unmarshal(iter.Value(), &ti) 202 | if err != nil { 203 | log.Error("invalid json in leveldb for peer: ", pid.Pretty()) 204 | continue 205 | } 206 | 207 | age := now.Sub(ti.LastSeen) 208 | if age <= time.Hour*24 { 209 | dayVersCount[ti.AgentVersion]++ 210 | dayPeerCount++ 211 | 212 | for _, p := range ti.Protocols { 213 | protosCountDay[p]++ 214 | } 215 | } 216 | if age <= time.Hour { 217 | hourVersCount[ti.AgentVersion]++ 218 | hourPeerCount++ 219 | for _, p := range ti.Protocols { 220 | protosCountHour[p]++ 221 | } 222 | } 223 | 224 | } 225 | 226 | for k, v := range dayVersCount { 227 | node_counts_g.WithLabelValues("day", k).Set(float64(v)) 228 | } 229 | for k, v := range hourVersCount { 230 | node_counts_g.WithLabelValues("hour", k).Set(float64(v)) 231 | } 232 | for k, v := range protosCountDay { 233 | protocols_g.WithLabelValues("day", k).Set(float64(v)) 234 | } 235 | for k, v := range protosCountHour { 236 | protocols_g.WithLabelValues("hour", k).Set(float64(v)) 237 | } 238 | 239 | return nil 240 | } 241 | 242 | // bootstrap (TODO: choose from larger group of peers) 243 | func bootstrap(ctx context.Context, h host.Host) { 244 | var wg sync.WaitGroup 245 | for i := 0; i < len(bspi); i++ { 246 | wg.Add(1) 247 | go func(i int) { 248 | defer wg.Done() 249 | v := len(bspi) - (1 + i) 250 | if err := h.Connect(ctx, bspi[v]); err != nil { 251 | log.Error(bspi[v], err) 252 | } 253 | }(i) 254 | } 255 | wg.Wait() 256 | } 257 | 258 | func scrapePeers(db *leveldb.DB, h host.Host, mdht *dht.IpfsDHT) error { 259 | ctx, cancel := context.WithCancel(context.Background()) 260 | defer cancel() 261 | 262 | var wg sync.WaitGroup 263 | rlim := make(chan struct{}, 10) 264 | fmt.Printf("scraping") 265 | scrapeRound := func(k string) { 266 | mctx, cancel := context.WithTimeout(ctx, time.Second*30) 267 | defer cancel() 268 | defer wg.Done() 269 | defer fmt.Print(".") 270 | rlim <- struct{}{} 271 | defer func() { 272 | <-rlim 273 | }() 274 | 275 | start := time.Now() 276 | peers, err := mdht.GetClosestPeers(mctx, k) 277 | if err != nil { 278 | log.Error(err) 279 | return 280 | } 281 | 282 | done := false 283 | for !done { 284 | select { 285 | case _, ok := <-peers: 286 | if !ok { 287 | done = true 288 | } 289 | case <-mctx.Done(): 290 | done = true 291 | } 292 | } 293 | took := time.Since(start).Seconds() 294 | query_lat_h.Observe(took) 295 | } 296 | 297 | for i := 0; i < 15; i++ { 298 | wg.Add(1) 299 | go scrapeRound(getRandomString()) 300 | } 301 | wg.Wait() 302 | fmt.Println("done!") 303 | 304 | peers := h.Peerstore().Peers() 305 | conns := h.Network().Conns() 306 | 307 | connected := make(map[peer.ID]bool) 308 | for _, c := range conns { 309 | connected[c.RemotePeer()] = true 310 | } 311 | 312 | tx, err := db.OpenTransaction() 313 | if err != nil { 314 | return err 315 | } 316 | 317 | now := time.Now() 318 | var pstat *trackingInfo 319 | for _, p := range peers { 320 | if p == h.ID() { 321 | continue 322 | } 323 | val, err := db.Get([]byte(p), nil) 324 | switch err { 325 | case leveldb.ErrNotFound: 326 | pstat = &trackingInfo{ 327 | FirstSeen: now, 328 | } 329 | default: 330 | log.Error("getting data from leveldb: ", err) 331 | continue 332 | case nil: 333 | pstat = new(trackingInfo) 334 | if err := json.Unmarshal(val, pstat); err != nil { 335 | log.Error("leveldb had bad json data: ", err) 336 | } 337 | } 338 | 339 | pstat.Sightings++ 340 | if connected[p] { 341 | pstat.LastConnected = now 342 | } 343 | pstat.LastSeen = now 344 | 345 | addrs := h.Peerstore().Addrs(p) 346 | pstat.Addresses = nil // reset 347 | for _, a := range addrs { 348 | pstat.Addresses = append(pstat.Addresses, a.String()) 349 | } 350 | av, err := h.Peerstore().Get(p, "AgentVersion") 351 | if err == nil { 352 | pstat.AgentVersion = fmt.Sprint(av) 353 | } 354 | protos, _ := h.Peerstore().GetProtocols(p) 355 | if len(protos) != 0 { 356 | pstat.Protocols = protos 357 | } 358 | 359 | data, err := json.Marshal(pstat) 360 | if err != nil { 361 | log.Error("failed to json marshal pstat: ", err) 362 | continue 363 | } 364 | if err := tx.Put([]byte(p), data, nil); err != nil { 365 | log.Error("failed to write to leveldb: ", err) 366 | continue 367 | } 368 | } 369 | if err := tx.Commit(); err != nil { 370 | log.Error("failed to commit update transaction: ", err) 371 | } 372 | 373 | fmt.Printf("updating stats took %s\n", time.Since(now)) 374 | 375 | if err := getStats(db); err != nil { 376 | return err 377 | } 378 | 379 | return nil 380 | } 381 | --------------------------------------------------------------------------------