├── .gitignore ├── CHANGELOG.md ├── README.md ├── db └── db.go ├── downloader └── downloader.go ├── events └── site_events.go ├── interfaces └── interfaces.go ├── main.go ├── peer └── peer.go ├── peer_manager └── peer_manager.go ├── server ├── server.go └── wrapper.go ├── site └── site.go ├── site_manager └── site_manager.go ├── socket └── socket.go ├── tasks └── file_tasks.go └── utils └── utils.go /.gitignore: -------------------------------------------------------------------------------- 1 | data 2 | media 3 | zeronet 4 | *.exe 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | ## 0.0.1 4 | - Initial release 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ZeroNet Golang client 2 | 3 | Very basic ZeroNet client implementation. 4 | 5 | ## Usage 6 | 7 | * Just run it and wait for new browser tab openning 8 | 9 | ## TODO for next version 10 | 11 | - [ ] Bugfixes 12 | - [ ] Optimization of system resources and bandwidth utilization 13 | - [ ] SQLite support 14 | - [ ] Site update process 15 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "regexp" 11 | "strings" 12 | "time" 13 | 14 | log "github.com/Sirupsen/logrus" 15 | 16 | "github.com/G1itchZero/ZeroGo/utils" 17 | "github.com/Jeffail/gabs" 18 | _ "github.com/mattn/go-sqlite3" 19 | ) 20 | 21 | type DB struct { 22 | db *sql.DB 23 | schema *gabs.Container 24 | path string 25 | site string 26 | } 27 | 28 | func NewDB(site string, schema *gabs.Container, p string) *DB { 29 | dbFile := path.Join(p, schema.S("db_file").Data().(string)) 30 | 31 | os.MkdirAll(path.Dir(dbFile), 0777) 32 | db, err := sql.Open("sqlite3", dbFile) 33 | if err != nil { 34 | log.Fatal(fmt.Errorf("DB error: %v", err)) 35 | } 36 | return &DB{ 37 | db: db, 38 | schema: schema, 39 | path: path.Dir(dbFile), 40 | site: site, 41 | } 42 | } 43 | 44 | func (db *DB) createJSONTable() { 45 | _, err := db.db.Exec(` 46 | CREATE TABLE IF NOT EXISTS json (json_id INTEGER PRIMARY KEY AUTOINCREMENT, directory VARCHAR(255), file_name VARCHAR(255))`, nil) 47 | if err != nil { 48 | log.Fatal(fmt.Errorf("DB error: %v", err)) 49 | } 50 | _, err = db.db.Exec(` 51 | CREATE TABLE IF NOT EXISTS keyvalue (keyvalue_id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT, value INTEGER, json_id INTEGER)`, nil) 52 | if err != nil { 53 | log.Fatal(fmt.Errorf("DB error: %v", err)) 54 | } 55 | } 56 | 57 | func (db *DB) addJSON(dir string, filename string) int64 { 58 | name := "json" 59 | keys := []string{"directory", "file_name"} 60 | ph := []string{"?", "?"} 61 | q := fmt.Sprintf("INSERT OR REPLACE INTO %s (%s) values(%s)", name, strings.Join(keys, ", "), strings.Join(ph, ", ")) 62 | result, err := db.db.Exec(q, dir, filename) 63 | if err != nil { 64 | log.Fatal(fmt.Errorf("DB error: %v", err)) 65 | } 66 | index, _ := result.LastInsertId() 67 | // index, err := db.db.Exec("SELECT json_id from json where directory = ? and file_name = ?", dir, filename) 68 | // if err != nil { 69 | // log.Fatal(fmt.Errorf("DB error: %v", err)) 70 | // } 71 | return index 72 | } 73 | 74 | func (db *DB) createTables() { 75 | tables := db.schema.S("tables").Data().(map[string]interface{}) 76 | for name, t := range tables { 77 | table := t.(map[string]interface{}) 78 | var cols []string 79 | for _, col := range table["cols"].([]interface{}) { 80 | c := col.([]interface{}) 81 | cols = append(cols, fmt.Sprintf("%s %s", c[0], c[1])) 82 | } 83 | _, err := db.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", name, strings.Join(cols, ", ")), nil) 84 | if err != nil { 85 | log.Fatal(fmt.Errorf("DB error: %v", err)) 86 | } 87 | for _, index := range table["indexes"].([]interface{}) { 88 | db.db.Exec(index.(string), nil) 89 | } 90 | } 91 | } 92 | 93 | func (db *DB) mapToTable(dataTable string, data *gabs.Container, jid int64) { 94 | for _, p := range data.S(dataTable).Data().([]interface{}) { 95 | post := p.(map[string]interface{}) 96 | keys := []string{"json_id"} 97 | ph := []string{"?"} 98 | values := []interface{}{jid} 99 | for key, val := range post { 100 | keys = append(keys, key) 101 | ph = append(ph, "?") 102 | values = append(values, val) 103 | } 104 | q := fmt.Sprintf("INSERT OR REPLACE INTO %s (%s) values(%s)", dataTable, strings.Join(keys, ", "), strings.Join(ph, ", ")) 105 | _, err := db.db.Exec(q, values...) 106 | if err != nil { 107 | log.Fatal(fmt.Errorf("DB error: %v", err)) 108 | } 109 | } 110 | } 111 | 112 | func (db *DB) mapToField(dataField string, data *gabs.Container, jid int64) { 113 | keys := []string{"json_id", "key", "value"} 114 | ph := []string{"?", "?", "?"} 115 | q := fmt.Sprintf("INSERT OR REPLACE INTO %s (%s) values(%s)", "keyvalue", strings.Join(keys, ", "), strings.Join(ph, ", ")) 116 | _, err := db.db.Exec(q, jid, dataField, data.S(dataField).Data()) 117 | if err != nil { 118 | log.Fatal(fmt.Errorf("DB error: %v", err)) 119 | } 120 | } 121 | 122 | func (db *DB) Query(q string) (interface{}, error) { 123 | log.Println(q) 124 | if q == "" { 125 | return nil, errors.New("empty query") 126 | } 127 | for db == nil || db.db == nil { 128 | time.Sleep(time.Millisecond * 100) 129 | } 130 | rows, err := db.db.Query(q) 131 | if err != nil { 132 | return "", err 133 | } 134 | defer rows.Close() 135 | columns, err := rows.Columns() 136 | if err != nil { 137 | return "", err 138 | } 139 | count := len(columns) 140 | tableData := make([]map[string]interface{}, 0) 141 | values := make([]interface{}, count) 142 | valuePtrs := make([]interface{}, count) 143 | for rows.Next() { 144 | for i := 0; i < count; i++ { 145 | valuePtrs[i] = &values[i] 146 | } 147 | rows.Scan(valuePtrs...) 148 | entry := make(map[string]interface{}) 149 | for i, col := range columns { 150 | var v interface{} 151 | val := values[i] 152 | b, ok := val.([]byte) 153 | if ok { 154 | v = string(b) 155 | } else { 156 | v = val 157 | } 158 | entry[col] = v 159 | } 160 | tableData = append(tableData, entry) 161 | } 162 | log.Println(tableData) 163 | return tableData, nil 164 | } 165 | 166 | func (db *DB) Init() { 167 | db.createJSONTable() 168 | db.createTables() 169 | maps := db.schema.S("maps").Data().(map[string]interface{}) 170 | for mapRe, dm := range maps { 171 | filepath.Walk(path.Join(utils.GetDataPath(), db.site), func(filepath string, f os.FileInfo, err error) error { 172 | match, _ := regexp.MatchString(mapRe, filepath) 173 | if match { 174 | mapName := filepath 175 | dataMap := dm.(map[string]interface{}) 176 | data, err := utils.LoadJSON(mapName) 177 | if err != nil { 178 | log.Fatal(fmt.Errorf("DB match (%s) error: %v", mapName, err)) 179 | } 180 | jid := db.addJSON("", strings.Replace(mapName, db.path+"/", "", 1)) 181 | 182 | if dataMap["to_table"] != nil { 183 | dataTables := (dataMap["to_table"].([]interface{})) 184 | for _, dt := range dataTables { 185 | dataTable := dt.(string) 186 | db.mapToTable(dataTable, data, jid) 187 | } 188 | } 189 | if dataMap["to_keyvalue"] != nil { 190 | dataFields := (dataMap["to_keyvalue"].([]interface{})) 191 | for _, dt := range dataFields { 192 | dataField := dt.(string) 193 | db.mapToField(dataField, data, jid) 194 | } 195 | } 196 | } 197 | return nil 198 | }) 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /downloader/downloader.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path" 8 | "sort" 9 | "sync" 10 | "time" 11 | 12 | "github.com/G1itchZero/ZeroGo/events" 13 | "github.com/G1itchZero/ZeroGo/interfaces" 14 | "github.com/G1itchZero/ZeroGo/peer_manager" 15 | "github.com/G1itchZero/ZeroGo/tasks" 16 | "github.com/G1itchZero/ZeroGo/utils" 17 | "github.com/Jeffail/gabs" 18 | "github.com/fatih/color" 19 | "gopkg.in/cheggaaa/pb.v1" 20 | 21 | log "github.com/Sirupsen/logrus" 22 | ) 23 | 24 | type FilterFunc func(string) bool 25 | 26 | type Downloader struct { 27 | Address string 28 | Peers *peer_manager.PeerManager 29 | Tasks tasks.Tasks 30 | Files map[string]*tasks.FileTask 31 | Includes []string 32 | ContentRequested bool 33 | Content *gabs.Container 34 | TotalFiles int 35 | StartedTasks int 36 | OnChanges chan events.SiteEvent 37 | ProgressBar *pb.ProgressBar 38 | sync.Mutex 39 | } 40 | 41 | func NewDownloader(address string) *Downloader { 42 | green := color.New(color.FgGreen).SprintFunc() 43 | d := Downloader{ 44 | Peers: peer_manager.NewPeerManager(address), 45 | Address: address, 46 | OnChanges: make(chan events.SiteEvent, 400), 47 | Files: map[string]*tasks.FileTask{}, 48 | StartedTasks: 0, 49 | Includes: []string{}, 50 | } 51 | if !utils.GetDebug() { 52 | d.ProgressBar = pb.New(1).Prefix(green(address)) 53 | d.ProgressBar.SetRefreshRate(time.Millisecond * 50) 54 | d.ProgressBar.ShowTimeLeft = false 55 | } 56 | return &d 57 | } 58 | 59 | func (d *Downloader) Download(done chan int, filter FilterFunc, modified float64) bool { 60 | green := color.New(color.FgGreen).SprintFunc() 61 | // fmt.Println(fmt.Sprintf("Download site: %s", green(d.Address))) 62 | 63 | dir := path.Join(utils.GetDataPath(), d.Address) 64 | os.MkdirAll(dir, 0777) 65 | 66 | d.ContentRequested = false 67 | d.Tasks = tasks.Tasks{tasks.NewTask("content.json", "", 0, d.Address, d.OnChanges)} 68 | 69 | go d.Peers.Announce() 70 | d.processContent(filter) 71 | d.ContentRequested = true 72 | if d.Content.S("modified").Data().(float64) == modified { 73 | log.Println(fmt.Sprintf("Not modified: %v", modified)) 74 | for _, task := range d.Tasks { 75 | task.Done = true 76 | } 77 | done <- 0 78 | return true 79 | } 80 | log.Println(fmt.Sprintf("Files in queue: %s", green(len(d.Tasks)-1))) 81 | sort.Sort(d.Tasks) 82 | for _, task := range d.Tasks { 83 | go d.ScheduleFile(task) 84 | } 85 | for d.PendingTasksCount() > 0 { 86 | select { 87 | case p := <-d.Peers.OnPeers: 88 | sort.Sort(d.Tasks) 89 | n := 0 90 | t := d.Tasks[n] 91 | for t.Done { 92 | n++ 93 | if n >= len(d.Tasks) { 94 | n = -1 95 | break 96 | } 97 | t = d.Tasks[n] 98 | } 99 | if n >= 0 { 100 | log.WithFields(log.Fields{ 101 | "task": t, 102 | "peer": p, 103 | }).Debug("New new peer ->") 104 | go d.ScheduleFileForPeer(t, p) 105 | } 106 | } 107 | } 108 | done <- 0 109 | return true 110 | } 111 | 112 | func (d *Downloader) GetContent() (*gabs.Container, error) { 113 | filename := path.Join(utils.GetDataPath(), d.Address, "content.json") 114 | if _, err := os.Stat(filename); err != nil { 115 | log.Info("No downloaded content.json") 116 | return nil, errors.New("Not downloaded yet") 117 | } 118 | return utils.LoadJSON(filename) 119 | } 120 | 121 | func (d *Downloader) processContent(filter FilterFunc) *tasks.FileTask { 122 | d.ContentRequested = true 123 | task := d.ScheduleFile(d.Tasks[0]) 124 | content, _ := gabs.ParseJSON(task.GetContent()) 125 | d.Content = content 126 | files, _ := content.S("files").ChildrenMap() 127 | includes, _ := content.S("includes").ChildrenMap() 128 | if includes != nil { 129 | for name := range includes { 130 | info, _ := gabs.Consume(map[string]interface{}{ 131 | "sha512": "", "size": 2048000.0, 132 | }) 133 | files[name] = info 134 | d.Includes = append(d.Includes, name) 135 | // func(name string) { 136 | t := tasks.NewTask(name, "", 2048000.0, d.Address, d.OnChanges) 137 | d.Tasks = append(d.Tasks, t) 138 | t = d.ScheduleFile(t) 139 | // log.Fatal(t) 140 | content, err := gabs.ParseJSON(t.GetContent()) 141 | if err != nil { 142 | log.Fatalf("Include content error: %s", err) 143 | } 144 | users, err := content.S("user_contents").S("archived").ChildrenMap() 145 | if err != nil { 146 | // log.Fatalf("Include users list error: %s", err) 147 | continue 148 | } 149 | for u := range users { 150 | u = path.Join("data/users", u) 151 | info, _ := gabs.Consume(map[string]interface{}{ 152 | "sha512": "", "size": 2048000.0, 153 | }) 154 | files[path.Join(u, "content.json")] = info 155 | files[path.Join(u, "data.json")] = info 156 | } 157 | // }(k) 158 | } 159 | } 160 | d.TotalFiles = len(files) + 1 //content.json 161 | for filename, child := range files { 162 | if filter != nil && !filter(filename) { 163 | d.TotalFiles-- 164 | continue 165 | } 166 | file := child.Data().(map[string]interface{}) 167 | t := tasks.NewTask(filename, file["sha512"].(string), file["size"].(float64), d.Address, d.OnChanges) 168 | log.Println(filename) 169 | d.Tasks = append(d.Tasks, t) 170 | d.Files[t.Filename] = t 171 | log.WithFields(log.Fields{ 172 | "task": task, 173 | }).Debug("New task") 174 | } 175 | if d.ProgressBar != nil { 176 | d.ProgressBar.Total = int64(d.TotalFiles) 177 | } 178 | return task 179 | 180 | } 181 | 182 | func (d *Downloader) ScheduleFileForPeer(task *tasks.FileTask, peer interfaces.IPeer) *tasks.FileTask { 183 | // filename := path.Join(utils.GetDataPath(), d.Address, task.Filename) 184 | // if _, err := os.Stat(filename); err == nil && task.Filename != "content.json" { 185 | // log.WithFields(log.Fields{ 186 | // "task": task.Filename, 187 | // }).Info("File from disk") 188 | // task.Start() 189 | // task.Finish() 190 | // return task 191 | // } 192 | log.WithFields(log.Fields{ 193 | "task": task.Filename, 194 | "peer": peer.GetAddress(), 195 | }).Info("Requesting file") 196 | res := task.AddPeer(peer) 197 | if d.ProgressBar != nil && res == nil { 198 | d.ProgressBar.Increment() 199 | d.ProgressBar.Update() 200 | } 201 | return task 202 | } 203 | 204 | func (d *Downloader) ScheduleFile(task *tasks.FileTask) *tasks.FileTask { 205 | d.StartedTasks++ 206 | if d.PendingTasksCount() == 0 { 207 | return nil 208 | } 209 | peer := d.Peers.Get() 210 | 211 | for peer == nil { 212 | peer = d.Peers.Get() 213 | time.Sleep(100) 214 | } 215 | return d.ScheduleFileForPeer(task, peer) 216 | } 217 | 218 | func (d *Downloader) PendingTasksCount() int { 219 | n := 0 220 | for _, task := range d.Tasks { 221 | if !task.Done { 222 | n++ 223 | } 224 | } 225 | return n 226 | } 227 | 228 | func (d *Downloader) PendingTasks() tasks.Tasks { 229 | res := tasks.Tasks{} 230 | for _, task := range d.Tasks { 231 | if !task.Done { 232 | res = append(res, task) 233 | } 234 | } 235 | return res 236 | } 237 | 238 | func (d *Downloader) FinishedTasks() int { 239 | n := 0 240 | for _, task := range d.Tasks { 241 | if task.Done { 242 | n++ 243 | } 244 | } 245 | return n 246 | } 247 | -------------------------------------------------------------------------------- /events/site_events.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | type SiteEvent struct { 4 | Type string 5 | Payload interface{} 6 | } 7 | -------------------------------------------------------------------------------- /interfaces/interfaces.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | type ISite interface { 4 | } 5 | 6 | type ITask interface { 7 | GetSite() string 8 | GetFilename() string 9 | GetContent() []byte 10 | GetStarted() bool 11 | GetDone() bool 12 | AppendContent([]byte, int) 13 | GetSize() int64 14 | Start() 15 | Check() bool 16 | Finish() 17 | } 18 | type IPeer interface { 19 | AddTask(ITask) error 20 | GetAddress() string 21 | } 22 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "strings" 8 | "time" 9 | 10 | "net/http" 11 | _ "net/http/pprof" 12 | 13 | l "log" 14 | 15 | "github.com/G1itchZero/ZeroGo/server" 16 | "github.com/G1itchZero/ZeroGo/site_manager" 17 | "github.com/G1itchZero/ZeroGo/utils" 18 | log "github.com/Sirupsen/logrus" 19 | "github.com/pkg/browser" 20 | "github.com/urfave/cli" 21 | ) 22 | 23 | const VERSION string = "0.1.0" 24 | 25 | func main() { 26 | os.MkdirAll(utils.GetDataPath(), 0777) 27 | utils.CreateCerts() 28 | log.SetLevel(log.ErrorLevel) 29 | // l.SetOutput(ioutil.Discard) 30 | log.WithFields(log.Fields{ 31 | "id": utils.GetPeerID(), 32 | }).Info("Your Peer ID") 33 | 34 | app := cli.NewApp() 35 | app.Name = "ZeroGo" 36 | app.Usage = "ZeroNet gate" 37 | app.Version = VERSION 38 | 39 | app.Flags = []cli.Flag{ 40 | cli.BoolFlag{ 41 | Name: "debug", 42 | Usage: "enable debug mode", 43 | }, 44 | cli.BoolFlag{ 45 | Name: "no-tab", 46 | Usage: "dont open new tab", 47 | }, 48 | cli.IntFlag{ 49 | Name: "port", 50 | Value: 43210, 51 | Usage: "serving port", 52 | }, 53 | cli.StringFlag{ 54 | Name: "homepage", 55 | Value: utils.ZN_HOMEPAGE, 56 | Usage: "homepage", 57 | }, 58 | } 59 | 60 | sm := site_manager.NewSiteManager() 61 | 62 | app.Commands = []cli.Command{ 63 | { 64 | Name: "download", 65 | Aliases: []string{"d"}, 66 | Usage: "download site", 67 | Action: func(c *cli.Context) error { 68 | if c.Bool("debug") { 69 | log.SetLevel(log.DebugLevel) 70 | } 71 | address := c.Args().First() 72 | sm.Remove(address) 73 | site := sm.Get(address) 74 | site.Wait() 75 | return nil 76 | }, 77 | }, 78 | } 79 | 80 | app.Action = func(c *cli.Context) error { 81 | utils.SetDebug(c.Bool("debug")) 82 | utils.SetHomepage(c.String("homepage")) 83 | 84 | hasMedia, _ := utils.Exists(path.Join(utils.GetDataPath(), utils.ZN_UPDATE)) 85 | 86 | sync := make(chan int) 87 | if !hasMedia { 88 | go func() { 89 | site := sm.GetFiles(utils.ZN_UPDATE, func(filename string) bool { 90 | return strings.HasPrefix(filename, utils.ZN_MEDIA) 91 | }) 92 | site.Wait() 93 | sync <- 0 94 | }() 95 | } else { 96 | go func() { 97 | sync <- 0 98 | }() 99 | } 100 | names := sm.GetFiles(utils.ZN_NAMES, func(filename string) bool { 101 | return strings.HasPrefix(filename, "data/names.json") 102 | }) 103 | names.Wait() 104 | sm.Get(utils.ZN_ID) 105 | // ids.Wait() 106 | <-sync 107 | sm.LoadNames() 108 | s := server.NewServer(c.Int("port"), sm) 109 | if c.Bool("debug") { 110 | l.SetOutput(os.Stdout) 111 | log.SetLevel(log.DebugLevel) 112 | go func() { 113 | log.Println(http.ListenAndServe("localhost:6060", nil)) 114 | }() 115 | } 116 | if !c.Bool("no-tab") { 117 | go func() { 118 | time.Sleep(time.Second) 119 | addr := utils.ZN_HOMEPAGE 120 | if c.NArg() > 0 { 121 | addr = c.Args().First() 122 | } 123 | browser.OpenURL(fmt.Sprintf("http://127.0.0.1:%d/%s", c.Int("port"), addr)) 124 | }() 125 | } 126 | s.Serve() 127 | return nil 128 | } 129 | 130 | app.Run(os.Args) 131 | } 132 | -------------------------------------------------------------------------------- /peer/peer.go: -------------------------------------------------------------------------------- 1 | package peer 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/binary" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "math" 10 | r "math/rand" 11 | "net" 12 | "os" 13 | "path" 14 | "sync" 15 | "time" 16 | 17 | "github.com/G1itchZero/ZeroGo/interfaces" 18 | "github.com/G1itchZero/ZeroGo/utils" 19 | _ "github.com/Sirupsen/logrus" 20 | log "github.com/Sirupsen/logrus" 21 | msgpack "gopkg.in/vmihailenco/msgpack.v2" 22 | ) 23 | 24 | type State int 25 | 26 | const ( 27 | Disconnected State = iota 28 | Connecting 29 | Connected 30 | ) 31 | 32 | type Handshake struct { 33 | Version string `msgpack:"version"` 34 | Rev int `msgpack:"rev"` 35 | Protocol string `msgpack:"protocol"` 36 | PeerID string `msgpack:"peer_id"` 37 | FileserverPort int `msgpack:"fileserver_port"` 38 | PortOpened bool `msgpack:"port_opened"` 39 | TargetIP string `msgpack:"target_ip"` 40 | CryptSupported bool `msgpack:"crypt_supported"` 41 | Crypt string `msgpack:"crypt"` 42 | } 43 | 44 | type RequestFile struct { 45 | Site string `msgpack:"site"` 46 | InnerPath string `msgpack:"inner_path"` 47 | Location int `msgpack:"location"` 48 | } 49 | 50 | type Request struct { 51 | Cmd string `msgpack:"cmd"` 52 | ReqID int `msgpack:"req_id"` 53 | Params interface{} `msgpack:"params"` 54 | Size int64 55 | } 56 | 57 | type Response struct { 58 | Cmd string `msgpack:"cmd"` 59 | ReqID int `msgpack:"req_id"` 60 | Body string `msgpack:"body"` 61 | Size int `msgpack:"size"` 62 | StreamBytes int `msgpack:"stream_bytes"` 63 | To int `msgpack:"to"` 64 | Location int `msgpack:"location"` 65 | Buffer []byte 66 | } 67 | 68 | type Peer struct { 69 | State State 70 | Address string 71 | Port uint64 72 | Connection *tls.Conn 73 | ReqID int 74 | Tasks []interfaces.ITask 75 | ActiveTasks int 76 | Cancel chan struct{} 77 | buffers map[int][]byte 78 | chans map[int]chan Response 79 | Ticker *time.Ticker 80 | Listening bool 81 | Free chan *Peer 82 | wlock *sync.Mutex 83 | sizes map[int]int64 84 | sync.Mutex 85 | } 86 | 87 | func (peer *Peer) AddTask(task interfaces.ITask) error { 88 | peer.Tasks = append(peer.Tasks, task) 89 | _, res := peer.Download(task) 90 | peer.Free <- peer 91 | return res 92 | } 93 | 94 | func (peer *Peer) RemoveTask(task interfaces.ITask) { 95 | i := func() int { 96 | i := 0 97 | for _, b := range peer.Tasks { 98 | if b.GetSite() == task.GetSite() { 99 | return i 100 | } 101 | i++ 102 | } 103 | return -1 104 | }() 105 | if i == -1 { 106 | return 107 | } 108 | peer.Tasks = append(peer.Tasks[:i], peer.Tasks[i+1:]...) 109 | } 110 | 111 | func (peer *Peer) String() string { 112 | return fmt.Sprintf("", peer.Address, peer.Port, peer.ActiveTasks) 113 | } 114 | 115 | func (peer *Peer) GetAddress() string { 116 | return peer.Address 117 | } 118 | 119 | func (peer *Peer) send(request *Request) Response { 120 | peer.wlock.Lock() 121 | request.ReqID = peer.ReqID 122 | if request.Size != 0 { 123 | peer.sizes[request.ReqID] = request.Size 124 | } 125 | data, _ := msgpack.Marshal(request) 126 | peer.Connection.Write(data) 127 | peer.buffers[request.ReqID] = []byte{} 128 | peer.chans[request.ReqID] = make(chan Response) 129 | peer.ReqID++ 130 | peer.wlock.Unlock() 131 | // log.WithFields(log.Fields{"request": request}).Info("Sending") 132 | return <-peer.chans[request.ReqID] 133 | } 134 | 135 | func (peer *Peer) Stop() { 136 | fmt.Println(peer.Address, "stopping") 137 | peer.Listening = false 138 | if peer.Ticker != nil { 139 | peer.Ticker.Stop() 140 | } 141 | if peer.Connection != nil { 142 | peer.Connection.Close() 143 | } 144 | } 145 | 146 | func (peer *Peer) handleAnswers() { 147 | for { 148 | if !peer.Listening { 149 | return 150 | } 151 | dl := time.Now().Add(20 * time.Second) 152 | peer.Connection.SetReadDeadline(dl) 153 | // fmt.Printf("%s:%d - Set deadline: %s %v\n", peer.Address, peer.Port, dl, peer.Listening) 154 | message := make([]byte, 1024*16) 155 | peer.Lock() 156 | _, _ = peer.Connection.Read(message) 157 | peer.Unlock() 158 | answer := Response{} 159 | msgpack.Unmarshal(message, &answer) 160 | // log.WithFields(log.Fields{"answer": answer}).Info("Recv") 161 | // log.WithFields(log.Fields{"rq_id": request.ReqID, "answ_to": answer.To}).Info("Recv") 162 | if answer.StreamBytes > 0 { 163 | left := answer.StreamBytes 164 | buf := peer.buffers[answer.To] 165 | for left > 0 { 166 | peer.Lock() 167 | n, err := peer.Connection.Read(message) 168 | peer.Unlock() 169 | if err != nil { 170 | log.Warn("File streaming error: ", err) 171 | break 172 | } 173 | left = left - n 174 | buf = append(buf, message...) 175 | } 176 | peer.buffers[answer.To] = buf[0:int(math.Min(float64(answer.StreamBytes), float64(len(buf))))] 177 | } 178 | peer.Lock() 179 | answer.Buffer = peer.buffers[answer.To] 180 | peer.Unlock() 181 | peer.chans[answer.To] <- answer 182 | } 183 | } 184 | 185 | func (peer *Peer) Download(task interfaces.ITask) ([]byte, error) { 186 | var res error = nil 187 | if task.GetStarted() { 188 | res = errors.New("Parallel") 189 | } 190 | task.Start() 191 | peer.ActiveTasks++ 192 | peer.Tasks = append(peer.Tasks, task) 193 | filename := path.Join(utils.GetDataPath(), task.GetSite(), task.GetFilename()) 194 | os.MkdirAll(path.Dir(filename), 0777) 195 | location := 0 196 | request := Request{ 197 | Cmd: "streamFile", 198 | Params: RequestFile{ 199 | Site: task.GetSite(), 200 | InnerPath: task.GetFilename(), 201 | Location: location, 202 | }, 203 | Size: task.GetSize(), 204 | } 205 | message := peer.send(&request) 206 | content := message.Buffer 207 | task.AppendContent(message.Buffer, location) 208 | s := int(peer.sizes[request.ReqID]) 209 | l := len(content) 210 | if task.GetSize() != 0 && l != int(s) { 211 | for location+l < s && !task.GetDone() { 212 | log.Warn(task, len(message.Buffer), peer.sizes[request.ReqID]) 213 | l = len(content) 214 | location += l 215 | request.Params = RequestFile{ 216 | Site: task.GetSite(), 217 | InnerPath: task.GetFilename(), 218 | Location: location, 219 | } 220 | message = peer.send(&request) 221 | task.AppendContent(message.Buffer, location) 222 | } 223 | } 224 | if len(content) == 0 { 225 | if !task.Check() { 226 | log.Warnf("Hash error: %s", task) 227 | } 228 | } 229 | task.Finish() 230 | peer.RemoveTask(task) 231 | peer.ActiveTasks-- 232 | return task.GetContent(), res 233 | } 234 | 235 | func (peer *Peer) Ping() { 236 | ping := Request{ 237 | Cmd: "ping", 238 | Params: map[string]string{}, 239 | } 240 | pong := peer.send(&ping) 241 | if pong.Body == "Pong!" { 242 | // fmt.Println("Ping successfull") 243 | } 244 | } 245 | 246 | func (peer *Peer) Handshake() Response { 247 | hs := Request{ 248 | Cmd: "handshake", 249 | Params: Handshake{ 250 | Version: utils.VERSION, 251 | Rev: utils.REV, 252 | Protocol: "v2", 253 | PeerID: utils.GetPeerID(), 254 | FileserverPort: 0, 255 | PortOpened: false, 256 | TargetIP: peer.Address, 257 | CryptSupported: true, 258 | Crypt: "tls-rsa", 259 | }, 260 | } 261 | return peer.send(&hs) 262 | } 263 | 264 | func (peer *Peer) Connect() error { 265 | peer.State = Connecting 266 | certFilename := path.Join(utils.GetDataPath(), "cert-rsa.pem") 267 | keyFilename := path.Join(utils.GetDataPath(), "key-rsa.pem") 268 | cert, _ := tls.LoadX509KeyPair(certFilename, keyFilename) 269 | conn, err := tls.DialWithDialer(&net.Dialer{ 270 | Deadline: time.Now().Add(10 * time.Second), 271 | Timeout: time.Second * 5, 272 | Cancel: peer.Cancel, 273 | }, "tcp", fmt.Sprintf("%s:%d", 274 | peer.Address, 275 | peer.Port), &tls.Config{ 276 | InsecureSkipVerify: true, 277 | Certificates: []tls.Certificate{cert}, 278 | CipherSuites: []uint16{ 279 | tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 280 | tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 281 | }, 282 | }) 283 | if err == nil { 284 | peer.Connection = conn 285 | peer.State = Connected 286 | go func() { 287 | peer.handleAnswers() 288 | }() 289 | peer.Handshake() 290 | peer.Ticker = time.NewTicker(time.Second * 5) 291 | // go func() { 292 | // for _ = range peer.Ticker.C { 293 | // if peer.Listening { 294 | // peer.Ping() 295 | // } else { 296 | // return 297 | // } 298 | // } 299 | // }() 300 | } 301 | return err 302 | } 303 | 304 | func NewPeer(info io.Reader, ch chan *Peer) *Peer { 305 | addr := [4]byte{} 306 | port := [2]byte{} 307 | binary.Read(info, binary.BigEndian, &addr) 308 | binary.Read(info, binary.BigEndian, &port) 309 | if port[0] == 0 && port[1] == 0 { 310 | return nil 311 | } 312 | peer := Peer{ 313 | State: Disconnected, 314 | Cancel: make(chan struct{}), 315 | Listening: true, 316 | Address: fmt.Sprintf("%d.%d.%d.%d", addr[0], addr[1], addr[2], addr[3]), 317 | Port: binary.BigEndian.Uint64([]byte{0, 0, 0, 0, 0, 0, port[0], port[1]}), 318 | buffers: map[int][]byte{}, 319 | sizes: map[int]int64{}, 320 | chans: map[int]chan Response{}, 321 | ActiveTasks: 0, 322 | Free: ch, 323 | wlock: &sync.Mutex{}, 324 | ReqID: r.Intn(1000), 325 | } 326 | return &peer 327 | } 328 | -------------------------------------------------------------------------------- /peer_manager/peer_manager.go: -------------------------------------------------------------------------------- 1 | package peer_manager 2 | 3 | import ( 4 | "bytes" 5 | "container/heap" 6 | "crypto/sha1" 7 | "encoding/binary" 8 | "fmt" 9 | r "math/rand" 10 | "net" 11 | "net/http" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | "github.com/G1itchZero/ZeroGo/peer" 17 | "github.com/G1itchZero/ZeroGo/utils" 18 | log "github.com/Sirupsen/logrus" 19 | "github.com/google/go-querystring/query" 20 | bencode "github.com/jackpal/bencode-go" 21 | ) 22 | 23 | type Peers []*peer.Peer 24 | 25 | type PeerManager struct { 26 | Address string 27 | Peers Peers 28 | Count int 29 | Trackers []string 30 | OnPeers chan *peer.Peer 31 | OnAnnounce chan int 32 | sync.Mutex 33 | } 34 | 35 | type Announce struct { 36 | InfoHash string `url:"info_hash"` 37 | PeerID string `url:"peer_id"` 38 | Port int `url:"port"` 39 | Uploaded int `url:"uploaded"` 40 | Downloaded int `url:"downloaded"` 41 | Left int `url:"left"` 42 | Compact int `url:"compact"` 43 | NumWant int `url:"numwant"` 44 | Event string `url:"event"` 45 | } 46 | 47 | func NewPeerManager(address string) *PeerManager { 48 | pm := PeerManager{ 49 | Address: address, 50 | Peers: Peers{}, 51 | OnPeers: make(chan *peer.Peer, 100), 52 | OnAnnounce: make(chan int), 53 | } 54 | heap.Init(&pm.Peers) 55 | return &pm 56 | } 57 | 58 | func (pm *PeerManager) GetActivePeers() Peers { 59 | peers := Peers{} 60 | for _, peer := range pm.Peers { 61 | peers = append(peers, peer) 62 | } 63 | return peers 64 | } 65 | 66 | func (pm *PeerManager) Stop() { 67 | log.Println("peers stopping") 68 | log.Fatal("peers stopped") 69 | for _, peer := range pm.Peers { 70 | go peer.Stop() 71 | } 72 | pm.Peers = Peers{} 73 | log.Fatal("peers stopped") 74 | } 75 | 76 | func (pm *PeerManager) Get() *peer.Peer { 77 | for len(pm.Peers) == 0 { 78 | time.Sleep(time.Duration(time.Millisecond * 100)) 79 | } 80 | pm.Lock() 81 | p := heap.Pop(&pm.Peers) 82 | if p == nil { 83 | p = pm.Get() 84 | } 85 | pm.Count-- 86 | pm.Unlock() 87 | return p.(*peer.Peer) 88 | } 89 | 90 | func (pm *PeerManager) Announce() { 91 | pm.Trackers = utils.GetTrackers() 92 | for _, tracker := range pm.Trackers { 93 | go func(tracker string) { 94 | peers := pm.announceTracker(tracker) 95 | c := 0 96 | for _, p := range peers { 97 | if pm.peerIsKnown(p) { 98 | continue 99 | } 100 | c++ 101 | go func(p *peer.Peer) { 102 | err := pm.connectPeer(p) 103 | if err != nil { 104 | return 105 | } 106 | heap.Push(&pm.Peers, p) 107 | pm.Count++ 108 | pm.OnPeers <- p 109 | }(p) 110 | } 111 | pm.OnAnnounce <- c 112 | log.WithFields(log.Fields{ 113 | "tracker": tracker, 114 | "peers": c, 115 | }).Debug("New peers added") 116 | }(tracker) 117 | } 118 | } 119 | 120 | func (pm *PeerManager) connectPeer(peer *peer.Peer) error { 121 | err := peer.Connect() 122 | if err == nil { 123 | // log.WithFields(log.Fields{ 124 | // "peer": peer, 125 | // }).Debug("Peer connected") 126 | peer.Ping() 127 | } else { 128 | // log.WithFields(log.Fields{ 129 | // "error": err, 130 | // "peer": peer, 131 | // }).Warn("Connection error") 132 | pm.removePeer(peer) 133 | } 134 | return err 135 | } 136 | 137 | func NewAnnounce(address string, tracker string) Announce { 138 | return Announce{ 139 | InfoHash: fmt.Sprintf("%s", sha1.Sum([]byte(address))), 140 | PeerID: utils.GetPeerID(), 141 | Port: 0, //15441, 142 | Uploaded: 0, Downloaded: 0, 143 | Left: 0, Compact: 1, NumWant: 30, 144 | Event: "started", 145 | } 146 | } 147 | 148 | func (pm *PeerManager) removePeer(peer *peer.Peer) { 149 | i := func() int { 150 | i := 0 151 | for _, b := range pm.Peers { 152 | if b.Address == peer.Address { 153 | return i 154 | } 155 | i++ 156 | } 157 | return -1 158 | }() 159 | if i == -1 { 160 | return 161 | } 162 | pm.Peers = append(pm.Peers[:i], pm.Peers[i+1:]...) 163 | } 164 | 165 | func (pm *PeerManager) peerIsKnown(peer *peer.Peer) bool { 166 | if peer.Address == "0.0.0.0" || peer.Address == utils.GetExternalIP() || peer.Address == "9.12.0.6" { 167 | return true 168 | } 169 | for _, b := range pm.Peers { 170 | if b.Address == peer.Address { 171 | return true 172 | } 173 | } 174 | return false 175 | } 176 | 177 | func (pm *PeerManager) announceHTTP(tracker string) Peers { 178 | params, _ := query.Values(NewAnnounce(pm.Address, tracker)) 179 | url := fmt.Sprintf("%s?%s", tracker, params.Encode()) 180 | log.WithFields(log.Fields{ 181 | "tracker": tracker, 182 | "params": params, 183 | }).Debug("Announce HTTP tracker") 184 | resp, err := http.Get(url) 185 | if err != nil { 186 | log.Warn(fmt.Errorf("Announce error: %v", err)) 187 | return Peers{} 188 | } 189 | raw, err := bencode.Decode(resp.Body) 190 | if err != nil { 191 | log.Warn(fmt.Errorf("Announce error: %v", err)) 192 | return Peers{} 193 | } 194 | resp.Body.Close() 195 | data := raw.(map[string]interface{}) 196 | if data["peers"] == nil { 197 | return Peers{} 198 | } 199 | peerData, _ := utils.GetBytes(data["peers"]) 200 | peerReader := bytes.NewReader(peerData) 201 | peerCount := len(peerData) / 6 202 | peers := Peers{} 203 | for i := 0; i < peerCount; i++ { 204 | peer := peer.NewPeer(peerReader, pm.OnPeers) 205 | if peer == nil { 206 | continue 207 | } 208 | peers = append(peers, peer) 209 | } 210 | return peers 211 | } 212 | 213 | const ( 214 | UDP_REQUEST_CONNECT = 0 215 | UDP_REQUEST_ANNOUNCE = 1 216 | ) 217 | 218 | func (pm *PeerManager) connectUDPTracker(socket *net.UDPConn, serverAddr *net.UDPAddr, transactionID int32) (connectionID int64, err error) { 219 | connectionID = int64(0x41727101980) 220 | request := new(bytes.Buffer) 221 | var data = []interface{}{ 222 | connectionID, 223 | uint32(UDP_REQUEST_CONNECT), 224 | transactionID, 225 | } 226 | for _, v := range data { 227 | binary.Write(request, binary.BigEndian, v) 228 | } 229 | socket.WriteToUDP(request.Bytes(), serverAddr) 230 | buf := make([]byte, 16) 231 | _, _, err = socket.ReadFromUDP(buf) 232 | if err != nil { 233 | return 0, err 234 | } 235 | answer := bytes.NewReader(buf) 236 | var action int32 237 | binary.Read(answer, binary.BigEndian, &action) 238 | binary.Read(answer, binary.BigEndian, &transactionID) 239 | binary.Read(answer, binary.BigEndian, &connectionID) 240 | return connectionID, nil 241 | // fmt.Println("Received ", fmt.Sprintf("%x", buf[0:n]), " from ", addr) 242 | } 243 | 244 | func (pm *PeerManager) announceUDPTracker(socket *net.UDPConn, serverAddr *net.UDPAddr, transactionID int32, connectionID int64, port int32) (peers Peers, err error) { 245 | peerID := []byte(utils.GetPeerID()[0:20]) 246 | infoHash := sha1.Sum([]byte(pm.Address)) 247 | 248 | announce := new(bytes.Buffer) 249 | data := []interface{}{ 250 | connectionID, 251 | uint32(UDP_REQUEST_ANNOUNCE), 252 | transactionID, 253 | infoHash, 254 | peerID, 255 | int64(0), 256 | int64(0), 257 | int64(0), 258 | int32(2), 259 | int32(0), 260 | int32(0), 261 | int32(50), 262 | int16(port), 263 | } 264 | 265 | for _, v := range data { 266 | binary.Write(announce, binary.BigEndian, v) 267 | } 268 | log.WithFields(log.Fields{ 269 | "tracker": serverAddr.String(), 270 | "params": fmt.Sprintf("%x", announce.Bytes()), 271 | }).Debug("Announce UDP tracker") 272 | socket.WriteToUDP(announce.Bytes(), serverAddr) 273 | buf2 := make([]byte, 10240) 274 | n, _, err := socket.ReadFromUDP(buf2) 275 | buf2 = buf2[0:n] 276 | if err != nil { 277 | // fmt.Println("Error: ", err) 278 | return Peers{}, err 279 | } 280 | answer := bytes.NewReader(buf2) 281 | var a uint32 282 | var t uint32 283 | var i uint32 284 | var l uint32 285 | var s uint32 286 | binary.Read(answer, binary.BigEndian, &a) 287 | binary.Read(answer, binary.BigEndian, &t) 288 | binary.Read(answer, binary.BigEndian, &i) 289 | binary.Read(answer, binary.BigEndian, &l) 290 | binary.Read(answer, binary.BigEndian, &s) 291 | peers = Peers{} 292 | for answer.Len() > 0 { 293 | peer := peer.NewPeer(answer, pm.OnPeers) 294 | if peer == nil { 295 | continue 296 | } 297 | peers = append(peers, peer) 298 | } 299 | return peers, nil 300 | } 301 | 302 | func (pm *PeerManager) announceUDP(tracker string) Peers { 303 | serverAddr, _ := net.ResolveUDPAddr("udp", strings.Replace(tracker, "udp://", "", 1)) 304 | rnd := r.New(r.NewSource(time.Now().UnixNano())) 305 | transactionID := rnd.Int31() 306 | port := int32(rnd.Intn(99) + 6800) 307 | 308 | conn, err := net.ListenPacket("udp4", fmt.Sprintf(":%d", port)) 309 | if err != nil { 310 | return Peers{} 311 | } 312 | socket := conn.(*net.UDPConn) 313 | defer socket.Close() 314 | 315 | socket.SetDeadline(time.Now().Add(20 * time.Second)) 316 | 317 | connectionID, err := pm.connectUDPTracker(socket, serverAddr, transactionID) 318 | 319 | if err != nil { 320 | // fmt.Println("Error: ", err) 321 | return Peers{} 322 | } 323 | 324 | peers, err := pm.announceUDPTracker(socket, serverAddr, transactionID, connectionID, port) 325 | 326 | if err != nil { 327 | // fmt.Println("Error: ", err) 328 | return Peers{} 329 | } 330 | return peers 331 | 332 | } 333 | 334 | func (pm *PeerManager) announceTracker(tracker string) Peers { 335 | pm.Trackers = append(pm.Trackers, tracker) 336 | if strings.HasPrefix(tracker, "http://") { 337 | return pm.announceHTTP(tracker) 338 | } else if strings.HasPrefix(tracker, "udp://") { 339 | return pm.announceUDP(tracker) 340 | } 341 | return Peers{} 342 | } 343 | 344 | func (pq Peers) Len() int { return len(pq) } 345 | 346 | func (pq Peers) Less(i, j int) bool { 347 | return pq[i].ActiveTasks < pq[j].ActiveTasks 348 | } 349 | 350 | func (pq Peers) Swap(i, j int) { 351 | // pq[i], pq[j] = pq[j], pq[i] 352 | } 353 | 354 | func (pq *Peers) Push(x interface{}) { 355 | item := x.(*peer.Peer) 356 | *pq = append(*pq, item) 357 | } 358 | 359 | func (pq *Peers) Pop() interface{} { 360 | old := *pq 361 | n := len(old) 362 | if n == 0 { 363 | return nil 364 | } 365 | item := old[n-1] 366 | *pq = old[0 : n-1] 367 | return item 368 | } 369 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "html/template" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "path" 11 | "strings" 12 | "time" 13 | 14 | "github.com/G1itchZero/ZeroGo/socket" 15 | "github.com/G1itchZero/ZeroGo/utils" 16 | 17 | "github.com/G1itchZero/ZeroGo/site_manager" 18 | log "github.com/Sirupsen/logrus" 19 | "github.com/fatih/color" 20 | "github.com/gorilla/websocket" 21 | "github.com/labstack/echo" 22 | ) 23 | 24 | type Server struct { 25 | Port int 26 | Sockets map[string]*socket.UiSocket 27 | Sites *site_manager.SiteManager 28 | } 29 | 30 | func NewServer(port int, sites *site_manager.SiteManager) *Server { 31 | server := Server{ 32 | Port: port, 33 | Sockets: map[string]*socket.UiSocket{}, 34 | Sites: sites, 35 | } 36 | return &server 37 | } 38 | 39 | func NoCacheMiddleware(next echo.HandlerFunc) echo.HandlerFunc { 40 | return func(ctx echo.Context) error { 41 | header := ctx.Response().Header() 42 | header.Set("Cache-Control", "no-cache, private, max-age=0, no-store, must-revalidate") 43 | header.Set("Expires", time.Unix(0, 0).Format(http.TimeFormat)) 44 | header.Set("Pragma", "no-cache") 45 | header.Set("X-Accel-Expires", "0") 46 | return next(ctx) 47 | } 48 | } 49 | 50 | func InnerMiddleware(next echo.HandlerFunc) echo.HandlerFunc { 51 | return func(ctx echo.Context) error { 52 | if ctx.Path() == "/inner/:site" { 53 | site := ctx.Param("site") 54 | cookie := new(http.Cookie) 55 | cookie.Name = "site" 56 | cookie.Value = site 57 | cookie.Expires = time.Now().Add(24 * time.Hour) 58 | ctx.SetCookie(cookie) 59 | } 60 | return next(ctx) 61 | } 62 | } 63 | 64 | var ( 65 | upgrader = websocket.Upgrader{} 66 | ) 67 | 68 | func (s *Server) socketHandler(c echo.Context) error { 69 | ws, err := upgrader.Upgrade(c.Response(), c.Request(), nil) 70 | wrapperKey := c.QueryParam("wrapper_key") 71 | if err != nil { 72 | log.WithFields(log.Fields{ 73 | "wrapperKey": wrapperKey, 74 | "err": err, 75 | }).Error("Socket upgrade error") 76 | return err 77 | } 78 | defer ws.Close() 79 | 80 | socket, ok := s.Sockets[wrapperKey] 81 | 82 | if ws != nil && ok { 83 | socket.Serve(ws) 84 | } 85 | return errors.New("Socket closed") 86 | } 87 | 88 | func (s *Server) Serve() { 89 | e := echo.New() 90 | 91 | e.Logger.SetOutput(ioutil.Discard) 92 | e.Color.SetOutput(ioutil.Discard) 93 | e.Use(NoCacheMiddleware) 94 | e.Static("/uimedia", path.Join(utils.GetDataPath(), utils.ZN_UPDATE, utils.ZN_MEDIA)) 95 | 96 | inner := e.Group("/inner") 97 | inner.Use(InnerMiddleware) 98 | inner.Use(NoCacheMiddleware) 99 | inner.GET("/:site", s.serveInner) 100 | // inner.GET("/:site?*", s.innerLink) 101 | inner.GET("/*", s.serveInnerStatic) 102 | 103 | e.GET("/Websocket", s.socketHandler) 104 | e.GET("/:url/", s.serveWrapper) 105 | e.GET("/:url", s.serveWrapper) 106 | e.GET("/", s.serveWrapper) 107 | 108 | e.Logger.Fatal(fmt.Errorf("Serving result: %v", e.Start(fmt.Sprintf(":%d", s.Port)))) 109 | } 110 | 111 | // func (s *Server) innerLink(ctx echo.Context) error { 112 | // return ctx.Redirect(302, fmt.Sprintf("/%s?%s", ctx.Param("site"), ctx.QueryString())) 113 | // } 114 | // 115 | func (s *Server) serveWrapper(ctx echo.Context) error { 116 | yellow := color.New(color.FgYellow).SprintFunc() 117 | url := ctx.Param("url") 118 | if url == "" { 119 | url = utils.GetHomepage() 120 | } 121 | if url == "favicon.ico" { 122 | return nil 123 | } 124 | st := s.Sites.Get(url) 125 | if st == nil { 126 | ctx.HTML(404, "No .bit name found") 127 | return errors.New("No .bit name found") 128 | } 129 | log.Info(fmt.Sprintf("> %s", yellow(st.Address))) 130 | s.Sites.Sites[url] = st 131 | s.Sites.Sites[st.Address] = st 132 | wrapper := NewWrapper(st, ctx) 133 | err := wrapper.Render(ctx) 134 | if err != nil { 135 | log.Error("Wrapper rendering error", err) 136 | } 137 | socket := socket.NewUiSocket(st, s.Sites, wrapper.Key) 138 | s.Sockets[wrapper.Key] = socket 139 | return err 140 | } 141 | 142 | func (s *Server) serveInner(ctx echo.Context) error { 143 | name := ctx.Param("site") 144 | if !strings.Contains(ctx.QueryString(), "wrapper_nonce") { 145 | return ctx.Redirect(302, fmt.Sprintf("/%s?%s", ctx.Param("site"), ctx.QueryString())) 146 | } 147 | // if strings.Contains(name, "?") { 148 | // return ctx.Redirect(302, fmt.Sprintf("/%s", name)) 149 | // } 150 | root := path.Join(utils.GetDataPath(), name) 151 | filename := path.Join(root, "index.html") 152 | site := s.Sites.Sites[name] 153 | site.WaitFile("index.html") 154 | return ctx.File(filename) 155 | } 156 | 157 | func (s *Server) serveInnerStatic(ctx echo.Context) error { 158 | name, _ := ctx.Cookie("site") 159 | url := ctx.Param("*") 160 | root := path.Join(utils.GetDataPath(), name.Value) 161 | filename := path.Join(root, url) 162 | site := s.Sites.Sites[name.Value] 163 | site.WaitFile(url) 164 | return ctx.File(filename) 165 | } 166 | 167 | type Template struct { 168 | templates *template.Template 169 | } 170 | 171 | func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error { 172 | return t.templates.ExecuteTemplate(w, name, data) 173 | } 174 | -------------------------------------------------------------------------------- /server/wrapper.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "html/template" 7 | "log" 8 | 9 | "github.com/G1itchZero/ZeroGo/site" 10 | "github.com/G1itchZero/ZeroGo/utils" 11 | "github.com/labstack/echo" 12 | ) 13 | 14 | const TEMPLATE = ` 15 | 16 | 17 | 18 | 19 | {{.Title}} - ZeroNet 20 | 21 | 22 | 23 | {{.MetaTags}} 24 | 25 | 26 | 27 | 41 | 42 |
43 |
44 |
45 | 46 | 47 |
48 |
0
49 |
≡
50 | 51 |
52 | 53 | 54 | 55 |
56 |
! Test notification×
57 |
58 | 59 | 60 | 61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 | 69 | 70 | 71 | 72 | 73 | 74 | 89 | 90 | 91 | 92 | 93 | ` 94 | 95 | type Wrapper struct { 96 | Nonce string 97 | Key string 98 | Title string 99 | Rev int 100 | FileURL string 101 | Address string 102 | FileInnerPath string 103 | ShowLoadingScreen bool 104 | Lang string 105 | QueryString string 106 | Homepage string 107 | MetaTags string 108 | BodyStyle string 109 | SandboxPermissions string 110 | PostmessageNonceSecurity string 111 | ServerURL string 112 | } 113 | 114 | func NewWrapper(s *site.Site, ctx echo.Context) *Wrapper { 115 | nonce := utils.RandomString(36) 116 | wrapperKey := utils.RandomString(36) 117 | title := s.Address 118 | if s.Content != nil { 119 | title = s.Content.S("title").Data().(string) 120 | } 121 | w := Wrapper{ 122 | Rev: utils.REV, 123 | Title: title, 124 | FileURL: "/inner/" + s.Address, 125 | Address: s.Address, 126 | FileInnerPath: "index.html", 127 | ShowLoadingScreen: true, 128 | Lang: "en", 129 | QueryString: fmt.Sprintf("?wrapper_nonce=%s", nonce), 130 | Nonce: nonce, 131 | Key: wrapperKey, 132 | Homepage: "/" + utils.ZN_HOMEPAGE, 133 | } 134 | if ctx.QueryString() != "" { 135 | w.QueryString += "&" + ctx.QueryString() 136 | } 137 | return &w 138 | } 139 | 140 | func (w *Wrapper) Render(ctx echo.Context) error { 141 | tmpl, _ := template.New("wrapper").Parse(TEMPLATE) 142 | buf := new(bytes.Buffer) 143 | if err := tmpl.Execute(buf, w); err != nil { 144 | return err 145 | } 146 | ctx.Response().Header().Set(echo.HeaderContentType, echo.MIMETextHTMLCharsetUTF8) 147 | ctx.Response().WriteHeader(200) 148 | _, err := ctx.Response().Write(buf.Bytes()) 149 | if err != nil { 150 | log.Fatal(fmt.Errorf("Render error: %v", err)) 151 | } 152 | return err 153 | } 154 | -------------------------------------------------------------------------------- /site/site.go: -------------------------------------------------------------------------------- 1 | package site 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "sync" 9 | "time" 10 | 11 | "github.com/G1itchZero/ZeroGo/db" 12 | "github.com/G1itchZero/ZeroGo/downloader" 13 | "github.com/G1itchZero/ZeroGo/events" 14 | "github.com/G1itchZero/ZeroGo/utils" 15 | "github.com/Jeffail/gabs" 16 | log "github.com/Sirupsen/logrus" 17 | ) 18 | 19 | type Site struct { 20 | Address string 21 | Path string 22 | Content *gabs.Container 23 | LastContent *gabs.Container 24 | Done chan *Site 25 | Downloader *downloader.Downloader 26 | Added int 27 | Ready bool 28 | Success bool 29 | OnChanges chan events.SiteEvent 30 | Filter downloader.FilterFunc 31 | LastPeers int 32 | DB *db.DB 33 | sync.Mutex 34 | } 35 | 36 | func NewSite(address string) *Site { 37 | log.Info("Creating new site...") 38 | done := make(chan *Site, 2) 39 | site := Site{ 40 | Address: address, 41 | Path: path.Join(utils.GetDataPath(), address), 42 | Done: done, 43 | Downloader: downloader.NewDownloader(address), 44 | Ready: false, 45 | Success: true, 46 | OnChanges: make(chan events.SiteEvent), 47 | } 48 | site.Content, _ = site.Downloader.GetContent() 49 | return &site 50 | } 51 | 52 | func (site *Site) Download(ch chan *Site) { 53 | site.Done = ch 54 | if site.Downloader.TotalFiles != 0 && site.Downloader.FinishedTasks() == site.Downloader.TotalFiles { 55 | site.Ready = true 56 | site.Done <- site 57 | return 58 | } 59 | done := make(chan int) 60 | go func() { 61 | site.Lock() 62 | go site.handleEvents() 63 | modified := 0.0 64 | if site.LastContent != nil { 65 | modified = site.LastContent.S("modified").Data().(float64) 66 | } 67 | site.Success = site.Downloader.Download(done, site.Filter, modified) 68 | site.Unlock() 69 | }() 70 | <-done 71 | site.Content = site.Downloader.Content 72 | site.Content.Set(false, "cloneable") 73 | site.LastPeers = site.Downloader.Peers.Count 74 | site.initDB() 75 | site.Ready = true 76 | // site.Downloader.ProgressBar.Add(site.Downloader.TotalFiles - site.Downloader.FinishedTasks()) 77 | if site.Downloader.ProgressBar != nil { 78 | site.Downloader.ProgressBar.Update() 79 | site.Downloader.ProgressBar.Finish() 80 | } 81 | site.Done <- site 82 | } 83 | 84 | func (site *Site) initDB() { 85 | filename := path.Join(site.Path, "dbschema.json") 86 | if _, err := os.Stat(filename); err != nil { 87 | return 88 | } 89 | // log.Fatal("initDB") 90 | schema, _ := utils.LoadJSON(filename) 91 | site.DB = db.NewDB(site.Address, schema, site.Path) 92 | site.DB.Init() 93 | log.Println("DB inited") 94 | } 95 | 96 | func (site *Site) handleEvents() { 97 | a := 0 98 | for { 99 | select { 100 | case peersCount := <-site.Downloader.Peers.OnAnnounce: 101 | site.OnChanges <- events.SiteEvent{Type: "peers_added", Payload: peersCount} 102 | if len(utils.GetTrackers()) == a && site.Downloader.Peers.Count == 0 { 103 | for _, task := range site.Downloader.Files { 104 | task.Finish() 105 | task.Success = false 106 | } 107 | fmt.Println("No peers found") 108 | site.OnChanges <- events.SiteEvent{Type: "file_failed", Payload: "content.json"} 109 | site.Success = false 110 | } 111 | a++ 112 | } 113 | } 114 | } 115 | 116 | func (site *Site) GetFile(filename string) ([]byte, error) { 117 | content, err := ioutil.ReadFile(path.Join(site.Path, filename)) 118 | if err == nil { 119 | return content, nil 120 | } 121 | return nil, err 122 | } 123 | 124 | func (site *Site) Remove() { 125 | err := os.RemoveAll(site.Path) 126 | if err != nil { 127 | log.WithFields(log.Fields{ 128 | "site": site.Path, 129 | "err": err, 130 | }).Error("Error during site removing") 131 | } 132 | } 133 | 134 | func (site *Site) Wait() { 135 | for !site.Downloader.ContentRequested || site.Downloader.PendingTasksCount() > 0 { 136 | log.Info("Files left: %d", site.Downloader.PendingTasksCount()) 137 | time.Sleep(time.Millisecond * 100) 138 | } 139 | } 140 | 141 | func (site *Site) WaitFile(filename string) bool { 142 | task, ok := site.Downloader.Files[filename] 143 | for !ok && site.Success { 144 | task, ok = site.Downloader.Files[filename] 145 | // fmt.Println("waiting for", task, filename, ok, site.Downloader.Files) 146 | time.Sleep(time.Duration(time.Millisecond * 100)) 147 | } 148 | if !site.Success { 149 | return site.Success 150 | } 151 | n := 0 152 | for !task.Done && site.Success { 153 | // fmt.Println("waiting for", task) 154 | log.WithFields(log.Fields{ 155 | "task": task, 156 | }).Info("Waiting for file") 157 | time.Sleep(time.Duration(time.Millisecond * 100)) 158 | n++ 159 | task.Priority++ 160 | if n > 20 { 161 | go site.Downloader.ScheduleFile(task) 162 | n = 0 163 | } 164 | } 165 | return task.Success 166 | } 167 | 168 | func (site *Site) GetSettings() SiteSettings { 169 | size := 0.0 170 | modified := 0.0 171 | 172 | if site.Content != nil { 173 | modified = site.Content.Path("modified").Data().(float64) 174 | files, _ := site.Content.S("files").ChildrenMap() 175 | for _, file := range files { 176 | size += file.Path("size").Data().(float64) 177 | } 178 | } 179 | return SiteSettings{ 180 | 181 | Added: site.Added, 182 | BytesRecv: size, 183 | OptionalDownloaded: 0, 184 | BytesSent: 0, 185 | Peers: site.Downloader.Peers.Count, 186 | Modified: modified, 187 | SizeOptional: 0, 188 | Serving: true, 189 | Own: false, 190 | Permissions: []string{"ADMIN"}, 191 | Size: size, 192 | } 193 | } 194 | 195 | func (site *Site) GetInfo() SiteInfo { 196 | var content interface{} 197 | content = nil 198 | peers := site.Downloader.Peers.Count 199 | if site.Content != nil { 200 | content = site.Content.Data() 201 | peers = site.LastPeers 202 | } 203 | return SiteInfo{ 204 | Address: site.Address, 205 | Files: len(site.Downloader.Tasks) - 1, 206 | Peers: peers, 207 | Content: content, 208 | Workers: len(site.Downloader.Peers.GetActivePeers()), 209 | Tasks: site.Downloader.PendingTasksCount(), 210 | Settings: site.GetSettings(), 211 | 212 | SizeLimit: 100, 213 | NextSizeLimit: 120, 214 | AuthAddress: "", 215 | // AuthKeySha512: "", 216 | // AuthKey: "", 217 | BadFiles: 0, 218 | StartedTaskNum: site.Downloader.StartedTasks, 219 | ContentUpdated: 0, 220 | } 221 | } 222 | 223 | type SiteInfo struct { 224 | Address string `json:"address"` 225 | Files int `json:"files"` 226 | Peers int `json:"peers"` 227 | Content interface{} `json:"content"` 228 | Workers int `json:"workers"` 229 | Tasks int `json:"tasks"` 230 | Settings SiteSettings `json:"settings"` 231 | SizeLimit int `json:"size_limit"` 232 | NextSizeLimit int `json:"next_size_limit"` 233 | AuthAddress string `json:"auth_address"` 234 | // AuthKeySha512 string `json:"auth_key_sha512"` 235 | // AuthKey string `json:"auth_key"` 236 | BadFiles int `json:"bad_files"` 237 | CertUserID interface{} `json:"cert_user_id"` 238 | StartedTaskNum int `json:"started_task_num"` 239 | ContentUpdated float64 `json:"content_updated"` 240 | Event []interface{} `json:"event"` 241 | } 242 | 243 | type SiteSettings struct { 244 | Added int `json:"added"` 245 | BytesRecv float64 `json:"bytes_recv"` 246 | OptionalDownloaded int `json:"optional_downloaded"` 247 | Cache struct { 248 | } `json:"cache"` 249 | BytesSent int `json:"bytes_sent"` 250 | Peers int `json:"peers"` 251 | Modified float64 `json:"modified"` 252 | SizeOptional int `json:"size_optional"` 253 | Serving bool `json:"serving"` 254 | Own bool `json:"own"` 255 | Permissions []string `json:"permissions"` 256 | Size float64 `json:"size"` 257 | } 258 | 259 | // type AutoGenerated struct { 260 | // To int `json:"to"` 261 | // Cmd string `json:"cmd"` 262 | // Result []struct { 263 | // Content struct { 264 | // Files int `json:"files"` 265 | // Description string `json:"description"` 266 | // ClonedFrom string `json:"cloned_from"` 267 | // Address string `json:"address"` 268 | // Includes int `json:"includes"` 269 | // Cloneable bool `json:"cloneable"` 270 | // Optional string `json:"optional"` 271 | // InnerPath string `json:"inner_path"` 272 | // Title string `json:"title"` 273 | // FilesOptional int `json:"files_optional"` 274 | // SignsRequired int `json:"signs_required"` 275 | // Modified float64 `json:"modified"` 276 | // Ignore string `json:"ignore"` 277 | // ZeronetVersion string `json:"zeronet_version"` 278 | // PostmessageNonceSecurity bool `json:"postmessage_nonce_security"` 279 | // AddressIndex int `json:"address_index"` 280 | // BackgroundColor string `json:"background-color"` 281 | // } `json:"content"` 282 | // } `json:"result"` 283 | // ID int `json:"id"` 284 | // } 285 | -------------------------------------------------------------------------------- /site_manager/site_manager.go: -------------------------------------------------------------------------------- 1 | package site_manager 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "strings" 9 | "time" 10 | 11 | "github.com/G1itchZero/ZeroGo/downloader" 12 | "github.com/G1itchZero/ZeroGo/site" 13 | "github.com/G1itchZero/ZeroGo/utils" 14 | "github.com/Jeffail/gabs" 15 | log "github.com/Sirupsen/logrus" 16 | "gopkg.in/cheggaaa/pb.v1" 17 | ) 18 | 19 | type SiteManager struct { 20 | Sites map[string]*site.Site 21 | Names map[string]interface{} 22 | pbPool *pb.Pool 23 | } 24 | 25 | func NewSiteManager() *SiteManager { 26 | pool, _ := pb.StartPool() 27 | sm := SiteManager{ 28 | Sites: map[string]*site.Site{}, 29 | pbPool: pool, 30 | } 31 | go sm.updateSites() 32 | return &sm 33 | } 34 | 35 | func (sm *SiteManager) LoadNames() { 36 | log.Info("Loading .bit names...") 37 | names, err := utils.LoadJSON(path.Join(utils.GetDataPath(), utils.ZN_NAMES, "data/names.json")) 38 | if err != nil { 39 | log.Fatal(fmt.Errorf("Error name resolving: %v", err)) 40 | } 41 | sm.Names = names.Data().(map[string]interface{}) 42 | } 43 | 44 | func (sm *SiteManager) Remove(address string) { 45 | site := sm.Sites[address] 46 | go site.Remove() 47 | delete(sm.Sites, address) 48 | sm.SaveSites() 49 | } 50 | 51 | func (sm *SiteManager) Get(address string) *site.Site { 52 | s, ok := sm.Sites[address] 53 | if !ok { 54 | var bit string 55 | if strings.HasSuffix(address, ".bit") { 56 | bit = address 57 | address, ok = sm.Names[address].(string) 58 | if !ok { 59 | return nil 60 | } 61 | } 62 | s = site.NewSite(address) 63 | s.Added = int(time.Now().Unix()) 64 | sm.Sites[address] = s 65 | if bit != "" { 66 | sm.Sites[bit] = s 67 | } 68 | } 69 | if !utils.GetDebug() { 70 | sm.pbPool.Add(s.Downloader.ProgressBar) 71 | } 72 | go sm.processSite(s) 73 | return s 74 | } 75 | 76 | func (sm *SiteManager) GetFiles(address string, filter downloader.FilterFunc) *site.Site { 77 | s, ok := sm.Sites[address] 78 | if !ok { 79 | s = site.NewSite(address) 80 | s.Filter = filter 81 | s.Added = int(time.Now().Unix()) 82 | sm.Sites[address] = s 83 | } 84 | if !utils.GetDebug() { 85 | sm.pbPool.Add(s.Downloader.ProgressBar) 86 | } 87 | go sm.processSite(s) 88 | return s 89 | } 90 | 91 | func (sm *SiteManager) processSite(s *site.Site) { 92 | done := make(chan *site.Site, 2) 93 | s.Download(done) 94 | s.Wait() 95 | if s.Filter == nil { 96 | sm.SaveSites() 97 | } 98 | } 99 | 100 | func (sm *SiteManager) SaveSites() { 101 | sites := sm.GetSites() 102 | // log.Fatal(sites) 103 | filename := path.Join(utils.GetDataPath(), "sites.json") 104 | ioutil.WriteFile(filename, []byte(sites.StringIndent("", " ")), 0644) 105 | } 106 | 107 | func (sm *SiteManager) GetSites() *gabs.Container { 108 | sites := gabs.New() 109 | for addr, s := range sm.Sites { 110 | if s.Content != nil && s.Filter == nil && !strings.HasSuffix(addr, ".bit") { 111 | sites.Set(s.GetInfo(), addr) 112 | } 113 | } 114 | return sites 115 | } 116 | 117 | func (sm *SiteManager) updateSites() { 118 | s, _ := loadSites() 119 | if s != nil { 120 | sites, _ := s.ChildrenMap() 121 | for address, content := range sites { 122 | log.WithFields(log.Fields{ 123 | "address": address, 124 | }).Debug("Preload site") 125 | sm.Sites[address] = site.NewSite(address) 126 | sm.Sites[address].LastPeers = int(content.S("peers").Data().(float64)) 127 | sm.Sites[address].LastContent = content.S("content") 128 | } 129 | } 130 | log.Info("Sites preloaded...") 131 | } 132 | 133 | func loadSites() (*gabs.Container, error) { 134 | filename := path.Join(utils.GetDataPath(), "sites.json") 135 | if _, err := os.Stat(filename); err != nil { 136 | jsonObj := gabs.New() 137 | ioutil.WriteFile(filename, []byte(jsonObj.String()), 0644) 138 | } 139 | return utils.LoadJSON(filename) 140 | } 141 | -------------------------------------------------------------------------------- /socket/socket.go: -------------------------------------------------------------------------------- 1 | package socket 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | "github.com/G1itchZero/ZeroGo/site" 10 | "github.com/G1itchZero/ZeroGo/site_manager" 11 | "github.com/G1itchZero/ZeroGo/utils" 12 | "github.com/Jeffail/gabs" 13 | log "github.com/Sirupsen/logrus" 14 | websocket "github.com/gorilla/websocket" 15 | ) 16 | 17 | type UiSocket struct { 18 | WrapperKey string 19 | Connection *websocket.Conn 20 | Site *site.Site 21 | SiteManager *site_manager.SiteManager 22 | Disconnected chan int 23 | MsgID int 24 | sync.Mutex 25 | } 26 | 27 | func NewUiSocket(s *site.Site, sm *site_manager.SiteManager, wrapperKey string) *UiSocket { 28 | socket := UiSocket{ 29 | WrapperKey: wrapperKey, 30 | Disconnected: make(chan int), 31 | Site: s, 32 | MsgID: 1, 33 | SiteManager: sm, 34 | } 35 | return &socket 36 | } 37 | 38 | func (socket *UiSocket) Serve(ws *websocket.Conn) { 39 | socket.Connection = ws 40 | socket.Notification("done", "Hi from ZeroNet Golang client!") 41 | 42 | log.WithFields(log.Fields{ 43 | "site": socket.Site.Address, 44 | "wrapper_key": socket.WrapperKey, 45 | }).Info("New socket connection") 46 | go func() { 47 | for { 48 | select { 49 | case event := <-socket.Site.OnChanges: 50 | log.WithFields(log.Fields{ 51 | "event": event, 52 | "wrapper_key": socket.WrapperKey, 53 | }).Debug("New socket event") 54 | info := socket.Site.GetInfo() 55 | info.Event = []interface{}{event.Type, event.Payload} 56 | socket.Cmd("setSiteInfo", info) 57 | } 58 | } 59 | }() 60 | for { 61 | _, data, err := ws.ReadMessage() 62 | if err != nil { 63 | log.WithFields(log.Fields{ 64 | "site": socket.Site.Address, 65 | "wrapper_key": socket.WrapperKey, 66 | }).Warn(err) 67 | return 68 | } 69 | message := Message{} 70 | err = json.Unmarshal(data, &message) 71 | if err != nil { 72 | continue 73 | } 74 | log.WithFields(log.Fields{ 75 | "site": socket.Site.Address, 76 | "wrapper_key": socket.WrapperKey, 77 | "massage": message, 78 | }).Info("Message") 79 | 80 | switch message.Cmd { 81 | case "fileQuery": 82 | go socket.fileQuery(message) 83 | case "siteDelete": 84 | go socket.siteDelete(message) 85 | case "siteInfo": 86 | go func(message Message) { 87 | // socket.Site.Wait() 88 | info := socket.Site.GetInfo() 89 | if message.Params == "" { 90 | return 91 | } 92 | params := message.Params.(map[string]interface{}) 93 | if params["file_status"] != nil { 94 | status := "file_done" 95 | st := socket.Site.WaitFile(params["file_status"].(string)) 96 | if !st { 97 | status = "file_failed" 98 | } 99 | info.Event = []interface{}{status, params["file_status"]} 100 | } 101 | socket.Response(message.ID, info) 102 | }(message) 103 | case "siteList": 104 | go socket.siteList(message) 105 | case "serverInfo": 106 | go socket.Response(message.ID, GetServerInfo()) 107 | case "feedQuery": 108 | go socket.feedQuery(message) 109 | case "dbQuery": 110 | go socket.dbQuery(message) 111 | } 112 | } 113 | } 114 | 115 | func (socket *UiSocket) dbQuery(message Message) { 116 | var q string 117 | switch p := message.Params.(type) { 118 | case []interface{}: 119 | q = p[0].(string) 120 | case string: 121 | q = p 122 | } 123 | res, err := socket.Site.DB.Query(q) 124 | if err == nil { 125 | socket.Response(message.ID, res) 126 | } 127 | } 128 | 129 | func (socket *UiSocket) siteDelete(message Message) { 130 | socket.SiteManager.Remove(message.Params.(map[string]interface{})["address"].(string)) 131 | socket.Notification("done", "Site deleted.") 132 | } 133 | 134 | func (socket *UiSocket) siteList(message Message) { 135 | sites := []site.SiteInfo{} 136 | infos, _ := socket.SiteManager.GetSites().ChildrenMap() 137 | for _, s := range infos { 138 | sites = append(sites, s.Data().(site.SiteInfo)) 139 | } 140 | socket.Response(message.ID, sites) 141 | } 142 | 143 | func (socket *UiSocket) fileQuery(message Message) { 144 | filename := message.Params.([]interface{})[0].(string) 145 | content, _ := socket.Site.GetFile(filename) 146 | if strings.HasSuffix(filename, ".json") { 147 | jsonContent, _ := gabs.ParseJSON(content) 148 | socket.Response(message.ID, []interface{}{jsonContent.Data()}) 149 | } else { 150 | socket.Response(message.ID, []interface{}{string(content)}) 151 | } 152 | } 153 | 154 | func (socket *UiSocket) feedQuery(message Message) { 155 | socket.Response(message.ID, []Post{ 156 | { 157 | Body: "@ZeroNet: Go, go, go!", 158 | Title: "Project info", 159 | FeedName: "Golang ZeroNet", 160 | Type: "comment", 161 | DateAdded: int(time.Now().Unix()), 162 | URL: "/", 163 | Site: socket.Site.Address, 164 | }, 165 | }) 166 | } 167 | 168 | func (socket *UiSocket) Notification(notificationType string, text string) { 169 | socket.Cmd("notification", []string{notificationType, text}) 170 | } 171 | 172 | func (socket *UiSocket) Cmd(cmd string, params interface{}) { 173 | msg, _ := json.Marshal(Message{cmd, params, socket.MsgID}) 174 | socket.Lock() 175 | socket.Connection.WriteMessage(websocket.TextMessage, msg) 176 | socket.MsgID++ 177 | socket.Unlock() 178 | } 179 | 180 | func (socket *UiSocket) Response(to int, result interface{}) { 181 | msg, _ := json.Marshal(SocketResponse{"response", 1, to, result}) 182 | socket.Lock() 183 | socket.Connection.WriteMessage(websocket.TextMessage, msg) 184 | socket.Unlock() 185 | } 186 | 187 | type Message struct { 188 | Cmd string `json:"cmd"` 189 | Params interface{} `json:"params"` 190 | ID int `json:"id"` 191 | } 192 | 193 | type SocketResponse struct { 194 | Cmd string `json:"cmd"` 195 | ID int `json:"id"` 196 | To int `json:"to"` 197 | Result interface{} `json:"result"` 198 | } 199 | 200 | func GetServerInfo() ServerInfo { 201 | return ServerInfo{ 202 | IPExternal: false, 203 | FileserverIP: "*", 204 | Multiuser: false, 205 | TorEnabled: false, 206 | Plugins: []string{}, 207 | FileserverPort: 15441, 208 | MasterAddress: "15Ni39HLKXmnXHRkuh8Cpj43AtDfTwc9Gv", 209 | Language: "en", 210 | UIPort: 43111, 211 | Rev: utils.REV, 212 | UIIP: "127.0.0.1", 213 | Platform: "linux", 214 | Version: utils.VERSION, 215 | TorStatus: "Not implemented", 216 | Debug: false, 217 | } 218 | } 219 | 220 | type ServerInfo struct { 221 | IPExternal bool `json:"ip_external"` 222 | FileserverIP string `json:"fileserver_ip"` 223 | Multiuser bool `json:"multiuser"` 224 | TorEnabled bool `json:"tor_enabled"` 225 | Plugins []string `json:"plugins"` 226 | FileserverPort int `json:"fileserver_port"` 227 | MasterAddress string `json:"master_address"` 228 | Language string `json:"language"` 229 | UIPort int `json:"ui_port"` 230 | Rev int `json:"rev"` 231 | UIIP string `json:"ui_ip"` 232 | Platform string `json:"platform"` 233 | Version string `json:"version"` 234 | TorStatus string `json:"tor_status"` 235 | Debug bool `json:"debug"` 236 | } 237 | 238 | type Post struct { 239 | Body string `json:"body"` 240 | Title string `json:"title"` 241 | URL string `json:"url"` 242 | Site string `json:"site"` 243 | FeedName string `json:"feed_name"` 244 | DateAdded int `json:"date_added"` 245 | Type string `json:"type"` 246 | } 247 | -------------------------------------------------------------------------------- /tasks/file_tasks.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha512" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "os" 10 | "path" 11 | "time" 12 | 13 | "github.com/G1itchZero/ZeroGo/events" 14 | "github.com/G1itchZero/ZeroGo/interfaces" 15 | "github.com/G1itchZero/ZeroGo/utils" 16 | log "github.com/Sirupsen/logrus" 17 | ) 18 | 19 | type Tasks []*FileTask 20 | type FileTask struct { 21 | Site string 22 | Filename string 23 | Hash string `json:"sha512"` 24 | Size float64 `json:"size"` 25 | Downloaded float64 26 | Peers []interfaces.IPeer 27 | Started bool 28 | StartTime time.Time 29 | Duration time.Duration 30 | Done bool 31 | OnChanges chan events.SiteEvent 32 | Priority int 33 | Success bool 34 | FullPath string 35 | Location int 36 | Stream *os.File 37 | } 38 | 39 | func NewTask(filename string, hash string, size float64, site string, ch chan events.SiteEvent) *FileTask { 40 | p := 0 41 | if filename == "content.json" { 42 | p = 9999 43 | } else if filename == "index.html" { 44 | p = 9990 45 | } 46 | task := FileTask{ 47 | Filename: filename, 48 | Hash: hash, 49 | Size: size, 50 | Site: site, 51 | OnChanges: ch, 52 | Priority: p, 53 | FullPath: path.Join(utils.GetDataPath(), site, filename), 54 | StartTime: time.Now(), 55 | } 56 | return &task 57 | } 58 | 59 | func (task *FileTask) String() string { 60 | return fmt.Sprintf("", task.Filename, task.Priority, len(task.Peers), task.Done) 61 | } 62 | 63 | func (task *FileTask) GetFilename() string { 64 | return task.Filename 65 | } 66 | 67 | func (task *FileTask) GetContent() []byte { 68 | content, err := ioutil.ReadFile(task.FullPath) 69 | if err != nil { 70 | log.Fatal(err) 71 | } 72 | return content 73 | } 74 | 75 | func (task *FileTask) GetSize() int64 { 76 | return int64(task.Size) 77 | } 78 | 79 | func (task *FileTask) AppendContent(content []byte, location int) { 80 | if location == 0 && task.Stream != nil { 81 | return 82 | } 83 | if task.Stream == nil { 84 | var err error 85 | task.Stream, err = os.Create(task.FullPath) 86 | if err != nil { 87 | // panic? 88 | } 89 | } 90 | if (location == 0 && task.Location == 0) || location > task.Location { 91 | task.Location = location 92 | io.Copy(task.Stream, bytes.NewReader(content)) 93 | } 94 | } 95 | 96 | func (task *FileTask) Check() bool { 97 | fc, err := ioutil.ReadFile(task.FullPath) 98 | if err != nil { 99 | log.Warn(err) 100 | return false 101 | } 102 | hash := fmt.Sprintf("%x", sha512.Sum512(fc))[0:64] 103 | if task.Hash != "" && task.Hash != hash { 104 | return false 105 | // log.Fatal(fmt.Errorf("Hash error '%s': %s != %s", task.FullPath, task.Hash, hash)) 106 | } 107 | return true 108 | } 109 | 110 | func (task *FileTask) GetSite() string { 111 | return task.Site 112 | } 113 | 114 | func (task *FileTask) GetDone() bool { 115 | return task.Done 116 | } 117 | 118 | func (task *FileTask) Start() { 119 | task.Started = true 120 | } 121 | 122 | func (task *FileTask) GetStarted() bool { 123 | return task.Started 124 | } 125 | 126 | func (task *FileTask) Finish() { 127 | if !task.Done { 128 | task.Done = true 129 | task.Success = true 130 | // task.OnChanges <- events.SiteEvent{Type: "file_done", Payload: task.Filename} 131 | log.WithFields(log.Fields{ 132 | "task": task, 133 | }).Debug("Finished") 134 | task.Priority = -1 135 | task.Duration = time.Now().Sub(task.StartTime) 136 | task.Stream.Close() 137 | } 138 | } 139 | 140 | func (task *FileTask) AddPeer(p interfaces.IPeer) error { 141 | if task.Check() { 142 | task.Finish() 143 | return nil 144 | } 145 | if task.Peers == nil { 146 | task.Peers = []interfaces.IPeer{} 147 | } 148 | task.Peers = append(task.Peers, p) 149 | return p.AddTask(task) 150 | } 151 | 152 | func (a Tasks) Len() int { return len(a) } 153 | func (a Tasks) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 154 | func (a Tasks) Less(i, j int) bool { return a[i].Priority > a[j].Priority } 155 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "crypto/rsa" 7 | "crypto/x509" 8 | "crypto/x509/pkix" 9 | "encoding/gob" 10 | "encoding/pem" 11 | "errors" 12 | "fmt" 13 | "io/ioutil" 14 | "math/big" 15 | r "math/rand" 16 | "net/http" 17 | "os" 18 | "path" 19 | "strings" 20 | "time" 21 | 22 | "github.com/Jeffail/gabs" 23 | ) 24 | 25 | var PEER_ID string 26 | var DATA string 27 | 28 | // ZeroNet version mimicry 29 | const VERSION string = "0.5.1" 30 | const REV int = 1756 31 | 32 | const ZN_PATH string = "ZeroNet" 33 | const ZN_DATA string = "data" 34 | const ZN_DATA_ALT string = "data_alt" 35 | const ZN_UPDATE string = "1UPDatEDxnvHDo7TXvq6AEBARfNkyfxsp" 36 | const ZN_HOMEPAGE string = "1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D" 37 | const ZN_MEDIA string = "ZeroNet/src/Ui/media" 38 | const ZN_NAMES string = "1Name2NXVi1RDPDgf5617UoW7xA6YrhM9F" 39 | const ZN_ID string = "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" 40 | 41 | var homepage string 42 | var debug bool 43 | 44 | func SetHomepage(hp string) { 45 | homepage = hp 46 | } 47 | 48 | func SetDebug(db bool) { 49 | debug = db 50 | } 51 | 52 | func GetDebug() bool { 53 | return debug 54 | } 55 | 56 | func GetHomepage() string { 57 | return homepage 58 | } 59 | 60 | func GetDataPath() string { 61 | if DATA == "" { 62 | DATA = path.Join(".", "data") 63 | } 64 | return DATA 65 | } 66 | 67 | func GetPeerID() string { 68 | if PEER_ID == "" { 69 | PEER_ID = fmt.Sprintf("-ZN0%s-GO%s", strings.Replace(VERSION, ".", "", -1), RandomString(10)) 70 | } 71 | return PEER_ID 72 | } 73 | 74 | func RandomString(n int) string { 75 | var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 76 | b := make([]rune, n) 77 | for i := range b { 78 | b[i] = letters[r.Intn(len(letters))] 79 | } 80 | return string(b) 81 | } 82 | 83 | func CreateCerts() { 84 | template := &x509.Certificate{ 85 | IsCA: true, 86 | BasicConstraintsValid: true, 87 | SubjectKeyId: []byte{4, 8, 3}, 88 | SerialNumber: big.NewInt(9899), 89 | Subject: pkix.Name{ 90 | Country: []string{"Earth"}, 91 | Organization: []string{"Mother Nature"}, 92 | }, 93 | NotBefore: time.Now(), 94 | NotAfter: time.Now().AddDate(5, 5, 5), 95 | // see http://golang.org/pkg/crypto/x509/#KeyUsage 96 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, 97 | KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, 98 | } 99 | 100 | // generate private key 101 | privatekey, err := rsa.GenerateKey(rand.Reader, 2048) 102 | 103 | if err != nil { 104 | fmt.Println(err) 105 | } 106 | 107 | publickey := &privatekey.PublicKey 108 | 109 | // create a self-signed certificate. template = parent 110 | var parent = template 111 | cert, err := x509.CreateCertificate(rand.Reader, template, parent, publickey, privatekey) 112 | 113 | if err != nil { 114 | fmt.Println(err) 115 | } 116 | 117 | // save private key 118 | pemfile, _ := os.Create(path.Join(DATA, "key-rsa.pem")) 119 | var pemkey = &pem.Block{ 120 | Type: "PRIVATE KEY", 121 | Bytes: x509.MarshalPKCS1PrivateKey(privatekey)} 122 | pem.Encode(pemfile, pemkey) 123 | pemfile.Close() 124 | 125 | pemfile, _ = os.Create(path.Join(DATA, "cert-rsa.pem")) 126 | pemkey = &pem.Block{ 127 | Type: "CERTIFICATE", 128 | Bytes: cert} 129 | pem.Encode(pemfile, pemkey) 130 | pemfile.Close() 131 | } 132 | 133 | func loadUsers() (*gabs.Container, error) { 134 | filename := path.Join(".", ZN_PATH, ZN_DATA, "users.json") 135 | return LoadJSON(filename) 136 | } 137 | 138 | func loadContent(site string) (*gabs.Container, error) { 139 | filename := path.Join(".", ZN_PATH, ZN_DATA_ALT, site, "content.json") 140 | return LoadJSON(filename) 141 | } 142 | 143 | func LoadJSON(filename string) (*gabs.Container, error) { 144 | content, err := ioutil.ReadFile(filename) 145 | if err == nil { 146 | return gabs.ParseJSON(content) 147 | } 148 | return nil, errors.New("cant read file") 149 | } 150 | 151 | func GetBytes(key interface{}) ([]byte, error) { 152 | var buf bytes.Buffer 153 | enc := gob.NewEncoder(&buf) 154 | err := enc.Encode(key) 155 | if err != nil { 156 | return nil, err 157 | } 158 | return buf.Bytes(), nil 159 | } 160 | 161 | func GetTrackers() []string { 162 | trackers := []string{ 163 | // "zero://boot3rdez4rzn36x.onion:15441", 164 | // "zero://boot.zeronet.io#f36ca555bee6ba216b14d10f38c16f7769ff064e0e37d887603548cc2e64191d:15441", 165 | "udp://tracker.coppersurfer.tk:6969", 166 | "udp://tracker.leechers-paradise.org:6969", 167 | "udp://9.rarbg.com:2710", 168 | "http://tracker.tordb.ml:6881/announce", 169 | "http://explodie.org:6969/announce", 170 | "http://tracker1.wasabii.com.tw:6969/announce", 171 | } 172 | return trackers 173 | } 174 | 175 | func Exists(path string) (bool, error) { 176 | _, err := os.Stat(path) 177 | if err == nil { 178 | return true, nil 179 | } 180 | if os.IsNotExist(err) { 181 | return false, nil 182 | } 183 | return true, err 184 | } 185 | 186 | var IP string 187 | 188 | func GetExternalIP() string { 189 | if IP != "" { 190 | return IP 191 | } 192 | resp, _ := http.Get("http://myexternalip.com/raw") 193 | defer resp.Body.Close() 194 | ip, err := ioutil.ReadAll(resp.Body) 195 | if err != nil { 196 | return "0.0.0.0" 197 | } 198 | IP = strings.Trim(string(ip), "\n ") 199 | return IP 200 | } 201 | --------------------------------------------------------------------------------