├── stop.sh ├── start.sh ├── install.sh ├── src ├── utils │ ├── define.go │ ├── Utils.go │ ├── log4FE.go │ ├── btree_test.go │ ├── Mmap.go │ └── btree.go ├── SparrowCache │ ├── Network_test.go │ ├── MemPool_test.go │ ├── HashTable_test.go │ ├── Cache.go │ ├── MemPool.go │ ├── Network.go │ └── HashTable.go ├── SparrowContainer │ ├── Wocker │ │ └── Wocker.go │ ├── Container │ │ └── SContainer.go │ └── startContainer.go └── SparrowDB │ ├── DBService │ └── dbservice.go │ ├── DataLayer │ ├── SDatabase_test.go │ ├── SDatabase.go │ └── STable.go │ └── EngineLayer │ └── SEngine.go ├── .gitignore └── twocker.go /stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | killall FalconSearch 4 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sh install.sh 3 | ./bin/FalconSearch 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #go install SparrowDB/DBService 4 | go install SparrowContainer 5 | 6 | 7 | -------------------------------------------------------------------------------- /src/utils/define.go: -------------------------------------------------------------------------------- 1 | 2 | 3 | package utils 4 | 5 | type DocID uint64 6 | 7 | type DocIdNode struct { 8 | Docid uint32 9 | Weight uint32 10 | //Pos uint32 11 | } 12 | 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.swp 3 | *github.com* 4 | *bin/ 5 | *distribution/ 6 | *pkg/ 7 | index/* 8 | *data/ 9 | *.log* 10 | *.json* 11 | *.dat* 12 | *.sql* 13 | .DS_Store 14 | nohup.out 15 | *.log 16 | -------------------------------------------------------------------------------- /src/SparrowCache/Network_test.go: -------------------------------------------------------------------------------- 1 | package SparrowCache 2 | 3 | import ( 4 | "testing" 5 | "utils" 6 | ) 7 | 8 | func TestNetwork(t *testing.T) { 9 | 10 | logger, _ := utils.New("test_network") 11 | nn := NewNetwork(5, 7777, logger) 12 | 13 | nn.Start() 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/SparrowContainer/Wocker/Wocker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "SparrowContainer/Container" 5 | ) 6 | 7 | func main() { 8 | 9 | container := Container.NewContainerWithConfig("hello", "/root/wocker/rootfs/", "/root/start_container.sh", nil) 10 | 11 | container.RunContainer() 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/SparrowCache/MemPool_test.go: -------------------------------------------------------------------------------- 1 | package SparrowCache 2 | 3 | import "testing" 4 | 5 | func TestMemPool(t *testing.T) { 6 | 7 | //pool := NewMemPool() 8 | 9 | //pool.InitMemPool() 10 | 11 | /* 12 | for i := 0; i < 100; i++ { 13 | n:= pool.Alloc() 14 | 15 | pool.Free(n) 16 | 17 | } 18 | */ 19 | } 20 | -------------------------------------------------------------------------------- /src/SparrowCache/HashTable_test.go: -------------------------------------------------------------------------------- 1 | package SparrowCache 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestHashTable(t *testing.T) { 10 | 11 | ht := NewHashTable() 12 | 13 | ht.InitHashTable() 14 | 15 | startTime := time.Now() 16 | 17 | for i := 0; i < 1000; i++ { 18 | 19 | ht.Set(fmt.Sprintf("KEY_%v", i), fmt.Sprintf("VALUE_%v", i)) 20 | 21 | //mm, _ := ht.Get("yyy") 22 | //fmt.Printf("%v\n", mm) 23 | 24 | } 25 | 26 | endTime := time.Now() 27 | 28 | fmt.Printf("%v", endTime.Sub(startTime)) 29 | 30 | ht.Set("yyy", "mmmdddd") 31 | mm, _ := ht.Get("KEY_9843") 32 | fmt.Printf("%v\n", mm) 33 | } 34 | -------------------------------------------------------------------------------- /src/SparrowDB/DBService/dbservice.go: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * file name : dbservice.go 3 | * author : Wu Yinghao 4 | * email : wyh817@gmail.com 5 | * 6 | * file description : 7 | * 8 | ******************************************************************************/ 9 | 10 | //package DBService 11 | package main 12 | 13 | import ( 14 | "fmt" 15 | "utils" 16 | dl "SparrowDB/DataLayer" 17 | ) 18 | 19 | func main() { 20 | 21 | fmt.Printf("Start DB ....") 22 | logger,_ := utils.New("logname") 23 | 24 | logger.Info("info...") 25 | db:=dl.NewSDatabase("testdb","./",logger) 26 | db.AddData(nil) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/SparrowCache/Cache.go: -------------------------------------------------------------------------------- 1 | package SparrowCache 2 | 3 | 4 | 5 | type SCache struct { 6 | 7 | ht *SHashTable 8 | 9 | 10 | } 11 | 12 | 13 | 14 | func NewSCache() *SCache { 15 | 16 | this := &SCache{ht:NewHashTable()} 17 | 18 | this.ht.InitHashTable() 19 | 20 | 21 | return this 22 | } 23 | 24 | 25 | 26 | 27 | func (this *SCache) Set(key,value string) error { 28 | 29 | return this.ht.Set(key,value) 30 | 31 | } 32 | 33 | 34 | func (this *SCache) Get(key string) (string,bool) { 35 | 36 | 37 | value,ok:= this.ht.Get(key) 38 | 39 | if !ok { 40 | return "",false 41 | } 42 | 43 | return value,ok 44 | 45 | } 46 | 47 | -------------------------------------------------------------------------------- /src/utils/Utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | ) 9 | 10 | // Exist function description : 判断文件或者目录是否存在 11 | // params : 12 | // return : 13 | func IsExist(filename string) bool { 14 | _, err := os.Stat(filename) 15 | return err == nil || os.IsExist(err) 16 | } 17 | 18 | // WriteToJson function description : 写入json文件 19 | // params : 20 | // return : 21 | func WriteToJson(data interface{}, file_name string) error { 22 | 23 | //fmt.Printf("Writing to File [%v]...\n", file_name) 24 | info_json, err := json.Marshal(data) 25 | if err != nil { 26 | fmt.Printf("Marshal err %v\n", file_name) 27 | return err 28 | } 29 | 30 | fout, err := os.Create(file_name) 31 | defer fout.Close() 32 | if err != nil { 33 | 34 | return err 35 | } 36 | fout.Write(info_json) 37 | return nil 38 | 39 | } 40 | 41 | // ReadFromJson function description : 读取json文件 42 | // params : 43 | // return : 44 | func ReadFromJson(file_name string) ([]byte, error) { 45 | 46 | fin, err := os.Open(file_name) 47 | defer fin.Close() 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | buffer, err := ioutil.ReadAll(fin) 53 | if err != nil { 54 | return nil, err 55 | } 56 | return buffer, nil 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/SparrowDB/DataLayer/SDatabase_test.go: -------------------------------------------------------------------------------- 1 | package DataLayer 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "testing" 7 | "utils" 8 | ) 9 | 10 | const tbinfo string = `[ 11 | { 12 | "fieldname":"id", 13 | "fieldlen":10, 14 | "fieldtype":0, 15 | "makeindex":true 16 | }, 17 | { 18 | "fieldname":"name", 19 | "fieldlen":10, 20 | "fieldtype":0, 21 | "makeindex":true 22 | }, 23 | { 24 | "fieldname":"age", 25 | "fieldlen":4, 26 | "fieldtype":0, 27 | "makeindex":false 28 | } 29 | 30 | 31 | ]` 32 | 33 | func TestNewDB(t *testing.T) { 34 | 35 | logger, _ := utils.New("test_db") 36 | 37 | db := NewSDatabase("testdb", "./testdata", logger) 38 | 39 | var feilds []FieldMeta 40 | err := json.Unmarshal([]byte(tbinfo), feilds) 41 | if err != nil { 42 | fmt.Printf("error\n") 43 | } 44 | 45 | fields := make([]FieldMeta, 0) 46 | fields = append(fields, FieldMeta{FieldLen: 10, Fieldname: "id", FieldType: 0, MkIdx: true}) 47 | fields = append(fields, FieldMeta{FieldLen: 10, Fieldname: "name", FieldType: 0, MkIdx: true}) 48 | fields = append(fields, FieldMeta{FieldLen: 10, Fieldname: "age", FieldType: 0, MkIdx: false}) 49 | 50 | err = db.CreateTable("biao", fields) 51 | if err != nil { 52 | fmt.Printf("biao err\n") 53 | } 54 | 55 | content := make(map[string]string) 56 | 57 | content["id"] = "10" 58 | content["name"] = "abc" 59 | content["age"] = "20" 60 | 61 | for i := 0; i < 100000; i++ { 62 | db.AddData("biao",content) 63 | } 64 | 65 | 66 | res:=db.FindDocId("biao",3456) 67 | 68 | fmt.Printf("%v\n",res) 69 | 70 | } 71 | -------------------------------------------------------------------------------- /twocker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "syscall" 8 | ) 9 | 10 | func main() { 11 | switch os.Args[1] { 12 | case "run": 13 | parent() 14 | case "child": 15 | child() 16 | default: 17 | panic("wat should I do") 18 | } 19 | } 20 | 21 | func parent() { 22 | cmd := exec.Command("/twocker", append([]string{"child"}, os.Args[2:]...)...) 23 | cmd.SysProcAttr = &syscall.SysProcAttr{ 24 | Cloneflags: syscall.CLONE_NEWUTS | syscall.CLONE_NEWPID | syscall.CLONE_NEWNS, 25 | } 26 | cmd.Stdin = os.Stdin 27 | cmd.Stdout = os.Stdout 28 | cmd.Stderr = os.Stderr 29 | 30 | if err := cmd.Run(); err != nil { 31 | fmt.Println("ERROR", err) 32 | os.Exit(1) 33 | } 34 | } 35 | 36 | func child() { 37 | //(source string, target string, fstype string, flags uintptr, data string) 38 | //must(syscall.Mount("/root/wocker/rootfs/", "/", "rootfs", , "")) 39 | //must(os.MkdirAll("rootfs/oldrootfs", 0700)) 40 | //must(syscall.PivotRoot("rootfs", "/root/wocker/rootfs/")) 41 | //must(os.Chdir("/")) 42 | 43 | must(syscall.Mount("rootfs", "rootfs", "", syscall.MS_BIND, "")) 44 | must(os.MkdirAll("/root/wocker/rootfs/", 0700)) 45 | must(syscall.PivotRoot("rootfs", "/root/wocker/rootfs/")) 46 | must(os.Chdir("/")) 47 | 48 | cmd := exec.Command(os.Args[2], os.Args[3:]...) 49 | cmd.Stdin = os.Stdin 50 | cmd.Stdout = os.Stdout 51 | cmd.Stderr = os.Stderr 52 | 53 | if err := cmd.Run(); err != nil { 54 | fmt.Println("ERROR", err) 55 | os.Exit(1) 56 | } 57 | } 58 | 59 | func must(err error) { 60 | if err != nil { 61 | panic(err) 62 | } 63 | } -------------------------------------------------------------------------------- /src/SparrowDB/EngineLayer/SEngine.go: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * file name : SEngine.go 3 | * author : Wu Yinghao 4 | * email : wyh817@gmail.com 5 | * 6 | * file description : 引擎层，负责sql 7 | * 8 | ******************************************************************************/ 9 | 10 | package EngineLayer 11 | 12 | import ( 13 | dl "SparrowDB/DataLayer" 14 | "utils" 15 | ) 16 | 17 | type SEngine struct { 18 | Pathname string `json:"dbpathname"` 19 | db *dl.SDatabase 20 | Logger *utils.Log4FE 21 | } 22 | 23 | // NewEngine function description : 构造引擎 24 | // params : 25 | // return : 26 | func NewEngine(pathname string, logger *utils.Log4FE) *SEngine { 27 | 28 | this := &SEngine{Logger: logger, Pathname: pathname,dl:nil} 29 | 30 | return this 31 | 32 | } 33 | 34 | // OpenDB function description : 打开数据库 35 | // params : 36 | // return : 37 | func (this *SEngine) OpenDB(dbname string) error { 38 | return nil 39 | } 40 | 41 | 42 | // CreateDB function description : 创建DB 43 | // params : 44 | // return : 45 | func (this *SEngine) CreateDB(dbname string ) error { 46 | 47 | this.db = dl.NewSDatabase(dbname,this.Pathname,this.Logger) 48 | 49 | if thie.db == nil { 50 | return fmt.Errorf("Create DB Error ") 51 | } 52 | 53 | return nil 54 | 55 | } 56 | 57 | 58 | func (this *SEngine) CreateTable(tablename string, fieldinfos []FieldMeta) error { 59 | 60 | return this.db.CreateTable(tablename,fieldinfos) 61 | } 62 | 63 | 64 | 65 | func (this *SEngine) AddData(tablename string ,content map[string]string) error { 66 | 67 | return this.db.AddData(tablename,content) 68 | } 69 | 70 | 71 | 72 | 73 | func parseSql(sql string) 74 | -------------------------------------------------------------------------------- /src/SparrowCache/MemPool.go: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * file name : MemPool.go 3 | * author : Wu Yinghao 4 | * email : wyh817@gmail.com 5 | * 6 | * file description : 内存池 7 | * 8 | ******************************************************************************/ 9 | 10 | package SparrowCache 11 | 12 | import ( 13 | "fmt" 14 | "time" 15 | ) 16 | 17 | var MAX_NODE_COUNT int = 1000 18 | 19 | type Node struct { 20 | key string 21 | value string 22 | hash1 uint64 23 | hash2 uint64 24 | Next *Node 25 | } 26 | 27 | func NewNode() *Node { 28 | node := &Node{hash1: 0, hash2: 0, Next: nil} 29 | return node 30 | } 31 | 32 | // SMemPool 内存池对象 33 | type SMemPool struct { 34 | nodechanGet chan *Node 35 | nodechanGiven chan *Node 36 | nodeList []Node 37 | freeList []*Node 38 | } 39 | 40 | func (pool *SMemPool) makeNodeList() error { 41 | 42 | pool.nodeList = make([]Node, MAX_NODE_COUNT) 43 | 44 | return nil 45 | } 46 | 47 | func NewMemPool() *SMemPool { 48 | 49 | this := &SMemPool{nodechanGet: make(chan *Node, MAX_NODE_COUNT), 50 | nodechanGiven: make(chan *Node, MAX_NODE_COUNT), 51 | nodeList: make([]Node, MAX_NODE_COUNT), 52 | freeList: make([]*Node, 0)} 53 | 54 | return this 55 | } 56 | 57 | func (this *SMemPool) Alloc() *Node { 58 | 59 | return <-this.nodechanGet 60 | 61 | } 62 | 63 | func (this *SMemPool) Free(node *Node) error { 64 | 65 | this.nodechanGiven <- node 66 | 67 | return nil 68 | 69 | } 70 | 71 | func (this *SMemPool) InitMemPool() error { 72 | 73 | //初始化node 74 | 75 | go func() { 76 | //q := new(list.List) 77 | q := make([]Node, MAX_NODE_COUNT) 78 | for { 79 | 80 | if len(q) == 0 { 81 | q = append(q, make([]Node, MAX_NODE_COUNT)...) 82 | } 83 | e := q[0] 84 | timeout := time.NewTimer(time.Second) 85 | select { 86 | case b := <-this.nodechanGiven: 87 | timeout.Stop() 88 | fmt.Printf("Free Buffer...\n") 89 | //b=b[:MAX_DOCID_LEN] 90 | q = append(q, *b) 91 | case this.nodechanGet <- &e: 92 | timeout.Stop() 93 | q = q[1:] 94 | //fmt.Printf("Alloc Buffer...\n") 95 | //q.Remove(e) 96 | 97 | case <-timeout.C: 98 | fmt.Printf("remove Buffer...\n") 99 | 100 | } 101 | 102 | } 103 | 104 | }() 105 | 106 | return nil 107 | 108 | } 109 | -------------------------------------------------------------------------------- /src/SparrowContainer/Container/SContainer.go: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * file name : SContainer.go 3 | * author : Wu Yinghao 4 | * email : wyh817@gmail.com 5 | * 6 | * file description : 容器 7 | * 8 | ******************************************************************************/ 9 | 10 | package Container 11 | 12 | import ( 13 | "fmt" 14 | "os" 15 | "os/exec" 16 | "syscall" 17 | ) 18 | 19 | var CONTAINER_PROCESS string = "./SparrowContainer" 20 | 21 | type NetInfo struct { 22 | Dev string 23 | Mac string 24 | Ip string 25 | Netmask string 26 | } 27 | 28 | type SContainer struct { 29 | Name string 30 | ID string 31 | Rootfs string 32 | Net []NetInfo 33 | InitProcess string 34 | } 35 | 36 | // NewContainer function description : 新建容器 37 | // params : 38 | // return : 39 | func NewContainer(name string) *SContainer { 40 | 41 | this := &SContainer{Name: name, Net: make([]NetInfo, 0)} 42 | 43 | return this 44 | } 45 | 46 | // NewContainerWithConfig function description : 根据配置新建容器 47 | // params : 48 | // return : 49 | func NewContainerWithConfig(name, rootfs, initprocess string, netinfo []NetInfo) *SContainer { 50 | 51 | this := &SContainer{Name: name, Rootfs: rootfs, InitProcess: initprocess, Net: netinfo} 52 | 53 | return this 54 | } 55 | 56 | func (this *SContainer) SetRootFS(rootfs string) error { 57 | 58 | this.Rootfs = rootfs 59 | return nil 60 | 61 | } 62 | 63 | // RunContainer function description : 运行容器 64 | // params : 65 | // return : 66 | func (this *SContainer) RunContainer() error { 67 | 68 | args := make([]string, 0) 69 | 70 | //配置hostname 71 | hostname := fmt.Sprintf("-h=%v", this.Name) 72 | args = append(args, hostname) 73 | 74 | //配置根文件系统 75 | rootfsArg := fmt.Sprintf("-f=%v", this.Rootfs) 76 | args = append(args, rootfsArg) 77 | 78 | //配置启动进程 79 | startProcess := fmt.Sprintf("-p=%v", this.InitProcess) 80 | args = append(args, startProcess) 81 | //配置网络 82 | 83 | cmd := exec.Command(CONTAINER_PROCESS, args...) 84 | cmd.SysProcAttr = &syscall.SysProcAttr{ 85 | Cloneflags: syscall.CLONE_NEWUTS | syscall.CLONE_NEWPID | syscall.CLONE_NEWNS, // | syscall.CLONE_NEWNET, 86 | } 87 | cmd.Stdin = os.Stdin 88 | cmd.Stdout = os.Stdout 89 | cmd.Stderr = os.Stderr 90 | 91 | if err := cmd.Run(); err != nil { 92 | fmt.Println("ERROR", err) 93 | return err 94 | } 95 | 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /src/utils/log4FE.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "runtime" 8 | "github.com/apsdehal/go-logger" 9 | ) 10 | 11 | type Log4FE struct { 12 | service_name string 13 | service_env string 14 | file_handle *os.File 15 | logger_handle *logger.Logger 16 | } 17 | 18 | func New(service string) (log4FE *Log4FE, err error) { 19 | // TODO: 从配置文件里面读取：日志模式、日志路径、日志缓存 20 | filename := fmt.Sprintf("/var/log/FalconEngine/logs/%s.log", service, service) 21 | 22 | // 初始化Log4FE 23 | out, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 24 | if err != nil { 25 | filename = fmt.Sprintf("%s.log", service) 26 | out, err = os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 27 | if err != nil { 28 | return nil, err 29 | } 30 | } 31 | logtmp, _ := logger.New(filename, out) 32 | 33 | env_deploy := os.Getenv("ENV_DEPLOY") 34 | if env_deploy == "" { 35 | env_deploy = "TESTING" 36 | } 37 | 38 | log4FE = &Log4FE{ 39 | service_name: service, 40 | service_env: env_deploy, 41 | file_handle: out, 42 | logger_handle: logtmp, 43 | } 44 | 45 | return log4FE, nil 46 | } 47 | 48 | func (this *Log4FE) Close() (err error) { 49 | return this.file_handle.Close() 50 | } 51 | 52 | func (this *Log4FE) Abc(p1 string, p2 int) (err error) { 53 | fmt.Println(p1, p2) 54 | 55 | return nil 56 | } 57 | 58 | func (this *Log4FE) log(level string, format string, args ...interface{}) (err error) { 59 | msg := fmt.Sprintf(format, args...) 60 | fmt.Println(msg) 61 | 62 | _, filepath, filenum, _ := runtime.Caller(2) 63 | filename := path.Base(filepath) 64 | logmsg := fmt.Sprintf("%s %s %s %s %d - %s", this.service_name, this.service_env, level, filename, filenum, msg) 65 | this.logger_handle.Log("", logmsg) 66 | 67 | return nil 68 | } 69 | 70 | func (this *Log4FE) Fatal(format string, args ...interface{}) (err error) { 71 | return this.log("FATAL", format, args...) 72 | } 73 | 74 | func (this *Log4FE) Error(format string, args ...interface{}) (err error) { 75 | return this.log("ERROR", format, args...) 76 | } 77 | 78 | func (this *Log4FE) Warn(format string, args ...interface{}) (err error) { 79 | return this.log("WARN", format, args...) 80 | } 81 | 82 | func (this *Log4FE) Info(format string, args ...interface{}) (err error) { 83 | return this.log("INFO", format, args...) 84 | } 85 | 86 | func (this *Log4FE) Debug(format string, args ...interface{}) (err error) { 87 | return //this.log("DEBUG", format, args...) 88 | } 89 | 90 | func (this *Log4FE) Trace(format string, args ...interface{}) (err error) { 91 | return //this.log("TRACE", format, args...) 92 | } 93 | -------------------------------------------------------------------------------- /src/SparrowCache/Network.go: -------------------------------------------------------------------------------- 1 | package SparrowCache 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "net" 7 | "utils" 8 | ) 9 | 10 | type Network struct { 11 | workerChans []chan *net.TCPConn 12 | workerNum int 13 | port int 14 | cache *SCache 15 | Logger *utils.Log4FE 16 | } 17 | 18 | func NewNetwork(workernum int, mport int, logger *utils.Log4FE) *Network { 19 | 20 | this := &Network{workerNum: workernum, port: mport, Logger: logger, workerChans: make([]chan *net.TCPConn, workernum)} 21 | 22 | for idx := range this.workerChans { 23 | this.workerChans[idx] = make(chan *net.TCPConn, 100) 24 | } 25 | 26 | this.cache=NewSCache() 27 | 28 | return this 29 | 30 | } 31 | 32 | // Start function description : 启动服务器 33 | // params : 34 | // return : 35 | func (this *Network) Start() error { 36 | 37 | var tcpAddr *net.TCPAddr 38 | 39 | tcpAddr, _ = net.ResolveTCPAddr("tcp", fmt.Sprintf(":%v", this.port)) 40 | 41 | //绑定端口 42 | listener, err := net.ListenTCP("tcp", tcpAddr) 43 | if err != nil { 44 | this.Logger.Error("[ERROR] listen port [%v] fail ... %v", this.port, err) 45 | return err 46 | } 47 | 48 | //启动处理协程 49 | for i := 0; i < this.workerNum; i++ { 50 | 51 | go this.runWorker(fmt.Sprintf("worker_%v", i),i) 52 | 53 | } 54 | 55 | this.Logger.Info("[INFO] listen port[%v] , waitting for connection ", this.port) 56 | for { 57 | conn, err := listener.AcceptTCP() 58 | if err != nil { 59 | this.Logger.Error("[ERROR] Accept port [%v] fail ... %v", this.port, err) 60 | return err 61 | } 62 | 63 | select { 64 | case this.workerChans[0] <- conn : 65 | default : 66 | this.Logger.Error("[ERROR] Not Resouce ") 67 | 68 | } 69 | 70 | 71 | } 72 | } 73 | 74 | func (this *Network) runWorker(name string, num int) { 75 | 76 | for { 77 | 78 | select { 79 | case conn := <-this.workerChans[num]: 80 | requestStr, err := this.readFromConn(conn) 81 | if err != nil { 82 | continue 83 | } 84 | this.Logger.Info("[INFO] request %v",requestStr) 85 | //this.processRequst(requestStr) 86 | conn.Write([]byte(requestStr)) 87 | this.Logger.Info("[INFO] disconnected ") 88 | conn.Close() 89 | default: 90 | 91 | } 92 | 93 | } 94 | 95 | } 96 | 97 | func (this *Network) readFromConn(conn *net.TCPConn) (string, error) { 98 | 99 | //ipStr := conn.RemoteAddr().String() 100 | //defer func() { 101 | // this.Logger.Info("[INFO] disconnected %v", ipStr) 102 | // conn.Close() 103 | //}() 104 | reader := bufio.NewReader(conn) 105 | 106 | message, err := reader.ReadString('\n') 107 | if err != nil { 108 | this.Logger.Error("[ERROR] Read Error %v", err) 109 | return "", err 110 | } 111 | 112 | this.Logger.Info("[INFO] Message : [ %v ]",message) 113 | 114 | return message,nil 115 | } 116 | -------------------------------------------------------------------------------- /src/SparrowCache/HashTable.go: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * file name : HashTable.go 3 | * author : Wu Yinghao 4 | * email : wyh817@gmail.com 5 | * 6 | * file description : 哈希表结构，底层数据结构 7 | * 8 | ******************************************************************************/ 9 | 10 | package SparrowCache 11 | 12 | 13 | 14 | type LinkList struct { 15 | linklen int 16 | node *Node //NodeInterface 17 | alloc func() *Node 18 | free func(n *Node) error 19 | 20 | } 21 | 22 | func (ll *LinkList) getLen() int { 23 | return ll.linklen 24 | } 25 | 26 | func (ll *LinkList) addkv(key, value string) error { 27 | if ll.linklen == 0 { 28 | ll.node = NewNode()//ll.alloc() 29 | ll.node.key = key 30 | ll.node.value = value 31 | ll.linklen = 1 32 | return nil 33 | } 34 | 35 | p := ll.node 36 | for p.Next != nil { 37 | 38 | if p.key==key { 39 | p.value = value 40 | return nil 41 | } 42 | 43 | p = p.Next 44 | } 45 | 46 | np := NewNode()//ll.alloc() 47 | np.key = key 48 | np.value = value 49 | p.Next = np 50 | ll.linklen++ 51 | return nil 52 | 53 | } 54 | 55 | func (ll *LinkList) findKey(key string) (string, bool) { 56 | 57 | if ll.linklen == 1 && key == ll.node.key { 58 | return ll.node.value, true 59 | } 60 | 61 | if ll.linklen == 0 { 62 | return "", false 63 | } 64 | 65 | if ll.linklen > 1 { 66 | p := ll.node 67 | for p != nil { 68 | if key == p.key { 69 | return p.value, true 70 | } 71 | 72 | p = p.Next 73 | } 74 | 75 | return "", false 76 | } 77 | 78 | return "", false 79 | 80 | } 81 | 82 | // SHashTable 哈希表结构 83 | type SHashTable struct { 84 | buketNum int64 85 | HBukets []*LinkList 86 | } 87 | 88 | func NewHashTable() *SHashTable { 89 | 90 | this := &SHashTable{buketNum: 1403641} 91 | 92 | this.HBukets = make([]*LinkList, this.buketNum) 93 | 94 | return this 95 | 96 | } 97 | 98 | func (this *SHashTable) InitHashTable() error { 99 | 100 | // pool:=NewMemPool() 101 | // pool.InitMemPool() 102 | 103 | for i := range this.HBukets { 104 | 105 | this.HBukets[i] = new(LinkList) 106 | this.HBukets[i].linklen = 0 107 | this.HBukets[i].node = nil 108 | // this.HBukets[i].alloc=pool.Alloc 109 | // this.HBukets[i].free=pool.Free 110 | } 111 | 112 | return nil 113 | 114 | } 115 | 116 | func (this *SHashTable) Set(key, value string) error { 117 | 118 | index := ELFHash(key, this.buketNum) 119 | 120 | return this.HBukets[index].addkv(key, value) 121 | 122 | } 123 | 124 | func (this *SHashTable) Get(key string) (string, bool) { 125 | 126 | index := ELFHash(key, this.buketNum) 127 | 128 | return this.HBukets[index].findKey(key) 129 | 130 | } 131 | 132 | func ELFHash(str string, bukets int64) int64 { 133 | var hash int64 134 | var x int64 135 | for _, v := range str { 136 | 137 | hash = (hash << 4) + int64(v) 138 | x = hash 139 | if (x & 0xF0000000) != 0 { 140 | hash ^= (x >> 24) 141 | hash &= ^x 142 | } 143 | } 144 | return (hash & 0x7FFFFFFF) % bukets 145 | } 146 | -------------------------------------------------------------------------------- /src/SparrowContainer/startContainer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "strings" 11 | "syscall" 12 | ) 13 | 14 | func main() { 15 | 16 | var rootfs string 17 | var hostname string 18 | var process string 19 | flag.StringVar(&rootfs, "f", "", "文件系统") 20 | flag.StringVar(&hostname, "h", "hello_world", "主机名称") 21 | flag.StringVar(&process, "p", "/bin/sh", "运行进程") 22 | flag.Parse() 23 | if err := runProcess(process, rootfs, hostname); err != nil { 24 | 25 | fmt.Printf("run process error : %v", err) 26 | } 27 | 28 | } 29 | 30 | func runProcess(processname, rootfs, hostname string) error { 31 | 32 | syscall.Sethostname([]byte(hostname)) 33 | 34 | //挂载根 35 | //must(syscall.Mount("/root/wocker/rootfs", "rootfs", "", syscall.MS_BIND, "")) 36 | pivotBaseDir := "/" 37 | tmpMountPoint := "/tmp/" 38 | if err := syscall.Mount(rootfs, tmpMountPoint, "", syscall.MS_BIND, ""); err != nil { 39 | return err 40 | } 41 | 42 | procpath := filepath.Join(rootfs, "proc") 43 | tmpMountPointProc := filepath.Join(tmpMountPoint, "proc") 44 | if err := syscall.Mount(procpath, tmpMountPointProc, "proc", 0, ""); err != nil { 45 | return err 46 | } 47 | 48 | syspath := filepath.Join(rootfs, "sys") 49 | tmpMountPointSys := filepath.Join(tmpMountPoint, "sys") 50 | if err := syscall.Mount(syspath, tmpMountPointSys, "sysfs", 0, ""); err != nil { 51 | return err 52 | } 53 | 54 | tmpMountPointDev := filepath.Join(tmpMountPoint, "dev") 55 | if err := syscall.Mount("udev", tmpMountPointDev, "devtmpfs", 0, ""); err != nil { 56 | return err 57 | } 58 | 59 | tmpMountPointDevpts := filepath.Join(tmpMountPoint, "dev/pts") 60 | if err := syscall.Mount("devpts", tmpMountPointDevpts, "devpts", 0, ""); err != nil { 61 | return err 62 | } 63 | 64 | tmpDir := filepath.Join(tmpMountPoint, pivotBaseDir) 65 | os.MkdirAll(tmpDir, 0755) 66 | pivotDir, err := ioutil.TempDir(tmpDir, ".mount_root_tmp") 67 | if err != nil { 68 | return err 69 | } 70 | 71 | if err := syscall.PivotRoot(tmpMountPoint, pivotDir); err != nil { 72 | 73 | fmt.Printf("mount root err :%v\n", err) 74 | return err 75 | } 76 | 77 | if err := os.Chdir("/"); err != nil { 78 | return err 79 | } 80 | 81 | pivotDir = filepath.Join(pivotBaseDir, filepath.Base(pivotDir)) 82 | 83 | if err := syscall.Mount("", pivotDir, "", syscall.MS_PRIVATE|syscall.MS_REC, ""); err != nil { 84 | fmt.Printf("mount private err : %v \n", err) 85 | return err 86 | } 87 | 88 | if err := syscall.Unmount(pivotDir, syscall.MNT_DETACH); err != nil { 89 | fmt.Printf("unmount root_tmp error : %v \n", err) 90 | return err 91 | } 92 | if err := os.Remove(pivotDir); err != nil { 93 | fmt.Printf("remove temp mount root dir error :%v \n", err) 94 | return err 95 | } 96 | //配置网络 97 | 98 | //fmt.Printf("%v \n", os.Args[1]) 99 | 100 | fmt.Printf("command : %v ", processname) 101 | 102 | processCmd := strings.Split(processname, " ") 103 | var cmd *exec.Cmd 104 | if len(processCmd) == 1 { 105 | cmd = exec.Command(processCmd[0]) 106 | } else { 107 | cmd = exec.Command(processCmd[0], processCmd[1:]...) 108 | } 109 | 110 | cmd.Stdin = os.Stdin 111 | cmd.Stdout = os.Stdout 112 | cmd.Stderr = os.Stderr 113 | 114 | if err := cmd.Run(); err != nil { 115 | fmt.Println("ERROR", err) 116 | return err 117 | } 118 | return nil 119 | } 120 | -------------------------------------------------------------------------------- /src/SparrowDB/DataLayer/SDatabase.go: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * file name : SDatabase.go 3 | * author : Wu Yinghao 4 | * email : wyh817@gmail.com 5 | * 6 | * file description : 数据层 7 | * 8 | ******************************************************************************/ 9 | 10 | package DataLayer 11 | 12 | import ( 13 | "encoding/json" 14 | "fmt" 15 | "os" 16 | "utils" 17 | ) 18 | 19 | type TableInfo struct { 20 | Tablename string `json:"tablename"` 21 | } 22 | 23 | type SDatabase struct { 24 | Name string `json:"dbname"` 25 | Pathname string `json:"pathname"` 26 | Fullname string `json:"fullname` 27 | TableNames []string `json:"tables"` 28 | tables map[string]*STable `json:"-"` 29 | 30 | Logger *utils.Log4FE `json:"-"` 31 | } 32 | 33 | //NewSTable(tablename, pathname string, fieldsinfo map[string]FieldMeta, logger *utils.Log4FE) *STable 34 | // NewSDatabase function description : 新建数据库 35 | // params : 36 | // return : 37 | func NewSDatabase(dbname, dbpath string, logger *utils.Log4FE) *SDatabase { 38 | 39 | this := &SDatabase{Logger: logger, Pathname: dbpath, Name: dbname} 40 | 41 | this.Fullname = dbpath + "/" + dbname 42 | if utils.IsExist(this.Fullname) { 43 | this.Logger.Error("[ERROR] SDatabase[%v] is exist", dbname) 44 | return nil 45 | } 46 | 47 | os.MkdirAll(this.Fullname, 0777) 48 | 49 | if err := this.initDatabase(); err != nil { 50 | return this 51 | } 52 | 53 | this.Logger.Info("[INFO] SDatabase[%v] Create ok", dbname) 54 | 55 | return this 56 | } 57 | 58 | func OpenSDatabase(dbname, dbpath string, logger *utils.Log4FE) *SDatabase { 59 | 60 | this := &SDatabase{Logger: logger, Pathname: dbpath, Name: dbname} 61 | 62 | this.Fullname = dbpath + "/" + dbname 63 | 64 | 65 | buffer, err := utils.ReadFromJson(this.Fullname + "/_dbinfo.meta") 66 | if err != nil { 67 | return nil 68 | } 69 | 70 | err = json.Unmarshal(buffer, this) 71 | if err != nil { 72 | return nil 73 | } 74 | 75 | for _,tablename := range this.TableNames { 76 | 77 | //TODO 78 | 79 | 80 | } 81 | 82 | return this 83 | 84 | 85 | } 86 | 87 | // initDatabase function description : 初始化数据库 88 | // params : 89 | // return : 90 | func (this *SDatabase) initDatabase() error { 91 | 92 | //初始化table 93 | this.tables = make(map[string]*STable) 94 | this.TableNames = make([]string, 0) 95 | 96 | return nil 97 | 98 | } 99 | 100 | // CreateTable function description : 创建数据库表 101 | // params : 102 | // return : 103 | func (this *SDatabase) CreateTable(tablename string, fieldinfos []FieldMeta) error { 104 | 105 | if _, ok := this.tables[tablename]; ok { 106 | this.Logger.Error("[ERROR] Table[%v] Create Error", tablename) 107 | return fmt.Errorf("[ERROR] Table[%v] Create Error", tablename) 108 | 109 | } 110 | 111 | table := NewSTable(tablename, this.Fullname, fieldinfos, this.Logger) 112 | 113 | if table == nil { 114 | this.Logger.Error("[ERROR] Create Table[%v] Fail", tablename) 115 | return fmt.Errorf("[ERROR] Create Table[%v] Fail", tablename) 116 | } 117 | 118 | this.tables[tablename] = table 119 | this.TableNames = append(this.TableNames, tablename) 120 | 121 | utils.WriteToJson(this, this.Fullname+"/_dbinfo.meta") 122 | 123 | return nil 124 | 125 | } 126 | 127 | func (this *SDatabase) AddData(tablename string, content map[string]string) error { 128 | 129 | if _, ok := this.tables[tablename]; ok { 130 | 131 | return this.tables[tablename].AddData(content) 132 | 133 | } 134 | 135 | return fmt.Errorf("[ERROR] Table[%v] Not Found", tablename) 136 | 137 | } 138 | 139 | func (this *SDatabase) DeleteData(docid utils.DocID) error { 140 | 141 | return nil 142 | } 143 | 144 | func (this *SDatabase) UpdateData(docid utils.DocID, content map[string]string) error { 145 | 146 | return nil 147 | } 148 | 149 | func (this *SDatabase) FindData(field, value string) []utils.DocID { 150 | 151 | return nil 152 | 153 | } 154 | 155 | func (this *SDatabase) FindDocId(tablename string, docid int64) map[string]string { 156 | 157 | if _, ok := this.tables[tablename]; ok { 158 | 159 | return this.tables[tablename].FindDocId(docid) 160 | 161 | } 162 | 163 | this.Logger.Error("[ERROR] Table[%v] Not Found", tablename) 164 | 165 | return nil 166 | } 167 | -------------------------------------------------------------------------------- /src/SparrowDB/DataLayer/STable.go: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * file name : STable.go 3 | * author : Wu Yinghao 4 | * email : wyh817@gmail.com 5 | * 6 | * file description : 数据层 7 | * 8 | ******************************************************************************/ 9 | 10 | package DataLayer 11 | 12 | import ( 13 | "encoding/binary" 14 | "fmt" 15 | "utils" 16 | ) 17 | 18 | const BT_TAIL string = ".bt" 19 | const TB_DTL_TAIL string = ".tb.detail" 20 | 21 | // FieldMeta 字段信息 22 | type FieldMeta struct { 23 | Fieldname string `json:"fieldname"` 24 | FieldLen int `json:"fieldlen"` 25 | FieldType int `json:"fieldtype"` 26 | Default string `json:"default"` 27 | MkIdx bool `json:"makeindex"` 28 | } 29 | 30 | type STable struct { 31 | Tablename string `json:"tablename"` 32 | Fields map[string]FieldMeta `json:"fields"` 33 | FieldInfos []FieldMeta `json:"fieldinfos"` 34 | RecordLen int `json:"RecordLen"` 35 | MaxCount int64 `json:"maxcount"` 36 | Pathname string `json:"pathname"` 37 | btreeName string 38 | bt *utils.BTreedb 39 | detail *utils.Mmap 40 | Logger *utils.Log4FE 41 | } 42 | 43 | // NewSTable function description : 新建数据库表 44 | // params : 45 | // return : 46 | func NewSTable(tablename, pathname string, fieldsinfos []FieldMeta, logger *utils.Log4FE) *STable { 47 | 48 | this := &STable{MaxCount: 0, Logger: logger, Pathname: pathname, Tablename: tablename, FieldInfos: fieldsinfos, Fields: make(map[string]FieldMeta)} 49 | 50 | if utils.IsExist(pathname + "/" + tablename + TB_DTL_TAIL) { 51 | this.Logger.Error("[ERROR] STable[%v] is exist", tablename) 52 | return nil 53 | } 54 | 55 | for _, field := range fieldsinfos { 56 | if _, ok := this.Fields[field.Fieldname]; ok { 57 | this.Logger.Error("[ERROR] Field[%v] exist", field.Fieldname) 58 | return nil 59 | } 60 | this.Fields[field.Fieldname] = field 61 | } 62 | 63 | //创建表的索引，使用b+树索引 64 | if err := this.createIndex(); err != nil { 65 | this.Logger.Error("[ERROR] createIndex %v", err) 66 | return nil 67 | } 68 | 69 | //创建detail文件 70 | if err := this.createDetail(); err != nil { 71 | this.Logger.Error("[ERROR] createDetail %v", err) 72 | return nil 73 | } 74 | 75 | this.Logger.Info("[INFO] STable[%v] Create ok", tablename) 76 | 77 | return this 78 | } 79 | 80 | // createIndex function description : 创建索引 81 | // params : 82 | // return : 83 | func (this *STable) createIndex() error { 84 | 85 | //初始化b+树索引 86 | this.btreeName = this.Pathname + "/" + this.Tablename + BT_TAIL 87 | this.bt = utils.NewBTDB(this.btreeName) 88 | 89 | if this.bt == nil { 90 | this.Logger.Error("[ERROR] make b+tree error %v", this.btreeName) 91 | return fmt.Errorf("[ERROR] make b+tree error %v", this.btreeName) 92 | } 93 | 94 | for k, v := range this.Fields { 95 | 96 | this.RecordLen = this.RecordLen + v.FieldLen + 4 97 | 98 | if v.MkIdx { 99 | this.bt.AddBTree(k) 100 | } 101 | 102 | } 103 | 104 | return nil 105 | 106 | } 107 | 108 | // createDetail function description : 创建表的详情文件 109 | // params : 110 | // return : 111 | func (this *STable) createDetail() error { 112 | 113 | var err error 114 | 115 | this.detail, err = utils.NewMmap(this.Pathname+"/"+this.Tablename+TB_DTL_TAIL, utils.MODE_CREATE) 116 | 117 | if err != nil { 118 | this.Logger.Error("[ERROR] make detail error %v", err) 119 | return err 120 | } 121 | 122 | return nil 123 | 124 | } 125 | 126 | // AddData function description : 添加数据 127 | // params : 128 | // return : 129 | func (this *STable) AddData(content map[string]string) error { 130 | 131 | inbytes := make([]byte, this.RecordLen) 132 | point := uint32(0) 133 | var value string 134 | 135 | for _, fvalue := range this.FieldInfos { 136 | 137 | if _, ok := content[fvalue.Fieldname]; !ok { 138 | 139 | value = fvalue.Default 140 | 141 | } else { 142 | value = content[fvalue.Fieldname] 143 | } 144 | 145 | lens := uint32(len(value)) 146 | binary.LittleEndian.PutUint32(inbytes[point:point+4], lens) 147 | point += 4 148 | 149 | dst := inbytes[point : point+lens] 150 | copy(dst, []byte(value)) 151 | point += uint32(fvalue.FieldLen) 152 | 153 | 154 | //如果有索引要求 155 | if fvalue.MkIdx { 156 | //this.bt.A 157 | 158 | 159 | 160 | } 161 | 162 | 163 | 164 | } 165 | this.detail.AppendRecord(inbytes) 166 | this.MaxCount++ 167 | return nil 168 | } 169 | 170 | func (this *STable) DeleteData(docid utils.DocID) error { 171 | 172 | return nil 173 | } 174 | 175 | func (this *STable) UpdateData(docid utils.DocID, content map[string]string) error { 176 | 177 | return nil 178 | } 179 | 180 | func (this *STable) FindData(field, value string) []utils.DocID { 181 | return nil 182 | } 183 | 184 | func (this *STable) FindDocId(docid int64) map[string]string { 185 | 186 | if docid >= this.MaxCount { 187 | return nil 188 | } 189 | 190 | res := make(map[string]string) 191 | 192 | offset := docid * int64(this.RecordLen) 193 | 194 | outbytes := this.detail.ReadRecord(offset, uint32(this.RecordLen)) 195 | point := uint32(0) 196 | 197 | for _, fvalue := range this.FieldInfos { 198 | 199 | reallen := binary.LittleEndian.Uint32(outbytes[point : point+4]) 200 | point += 4 201 | value := string(outbytes[point : point+reallen]) 202 | point += uint32(fvalue.FieldLen) 203 | res[fvalue.Fieldname]=value 204 | 205 | } 206 | 207 | return res 208 | } 209 | -------------------------------------------------------------------------------- /src/utils/btree_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "time" 7 | //"math/rand" 8 | "testing" 9 | "unsafe" 10 | ) 11 | 12 | func TestNewTree(t *testing.T) { 13 | 14 | fmt.Printf("element len:%v\n", int(unsafe.Sizeof(element{}))) 15 | db := NewBTDB("bp.tree") 16 | ok, value := db.Search("test", "hello") 17 | fmt.Printf("found hello : %v value : %v \n===================\n", ok, value) 18 | for i := 0; i < 30; i++ { 19 | db.AddBTree(fmt.Sprintf("%v", i)) 20 | } 21 | db.AddBTree("test") 22 | //db.Set("test", "hello", 345) 23 | ok, value = db.Search("test", "hello") 24 | fmt.Printf("found hello : %v value : %v \n===================\n", ok, value) 25 | 26 | /* 27 | bt := NewEmptyBTree("bp.tree") 28 | //bt.display() 29 | 30 | //bt.put("hello",33) 31 | ok, value := bt.Search("hello") 32 | fmt.Printf("found hello : %v value : %v \n===================\n", ok, value) 33 | 34 | bt.Set("hello", 10231) 35 | fmt.Printf("SET key : hello value : %v \n===================\n", value) 36 | 37 | ok, value = bt.Search("hello") 38 | fmt.Printf("found hello : %v value : %v \n===================\n", ok, value) 39 | 40 | bt.Set("hfffd", 9932) 41 | ok, value = bt.Search("hfffd") 42 | fmt.Printf("found : %v value : %v \n===================\n", ok, value) 43 | 44 | // for i:=0;i<50;i++{ 45 | // bt.Set(fmt.Sprintf("%v",i),9932) 46 | // } 47 | for i := 0; i < 200000; i++ { 48 | bt.Set(fmt.Sprintf("%v", i), uint32(i)) 49 | } 50 | bt.Set("fmt", 444) 51 | //bt.Display() 52 | 53 | fmt.Printf("build b+tree over \n===================\n") 54 | 55 | // bt.Display() 56 | 57 | for i:=25000;i>=0;i-- { 58 | ok, value = bt.Search(fmt.Sprintf("%v", rand.Intn(200000))) 59 | if !ok{ 60 | fmt.Printf("notfound : %v value : %v \n===================\n", i, value) 61 | } 62 | 63 | } 64 | 65 | 66 | ok, value = bt.Search("248") 67 | //ok, value = bt.Search("32") 68 | //fmt.Printf("found : %v value : %v \n===================\n", ok, value) 69 | 70 | // ttttt() 71 | //bt.print() 72 | //bt.display() 73 | 74 | //bt.DisplayTree() 75 | 76 | //verifyTree(bt, testCount, t) 77 | */ 78 | } 79 | 80 | func TestLoadTree(t *testing.T) { 81 | 82 | db := NewBTDB("bp.tree") 83 | ok, value := db.Search("test", "hello") 84 | fmt.Printf("found hello : %v value : %v \n===================\n", ok, value) 85 | 86 | db.AddBTree("test") 87 | //db.Set("test", "hello", 3456) 88 | ok, value = db.Search("test", "hello") 89 | fmt.Printf("found hello : %v value : %v \n===================\n", ok, value) 90 | start := time.Now() 91 | for i := 0; i < 100000; i++ { 92 | //fmt.Printf("%04d\n", i) 93 | db.Set("test", fmt.Sprintf("%08d", rand.Intn(20000000)), uint64(i)) 94 | } 95 | //db.Set("test", "fmt", 444) 96 | //bt.Display() 97 | 98 | fmt.Printf("build b+tree over cost %v \n===================\n", time.Now().Sub(start)) 99 | start = time.Now() 100 | // bt.Display() 101 | 102 | for i := 100000; i >= 0; i-- { 103 | ii := fmt.Sprintf("%08d", rand.Intn(2000)) 104 | ok, value = db.Search("test", ii) 105 | if !ok { 106 | //fmt.Printf("notfound : %v value : %v \n===================\n", ii, value) 107 | } 108 | 109 | } 110 | fmt.Printf("query b+tree over cost %v\n===================\n", time.Now().Sub(start)) 111 | /* 112 | found, ranges := db.Range("test", "", "09999900") 113 | 114 | if found { 115 | fmt.Printf(">>>> %v\n", ranges) 116 | } 117 | 118 | key, value1, pgnum, idx, ok := db.GetFristKV("test") 119 | fmt.Printf("key:%v value:%v index:%v pagenum:%v ok:%v\n", key, value1, idx, pgnum, ok) 120 | 121 | for ok { 122 | key, value1, pgnum, idx, ok = db.GetNextKV("test", pgnum, idx) 123 | fmt.Printf("key:%v value:%v index:%v pagenum:%v ok:%v\n", key, value1, idx, pgnum, ok) 124 | 125 | } 126 | */ 127 | } 128 | 129 | /* 130 | func TestMergeTree(t *testing.T) { 131 | //start := time.Now() 132 | db := NewBTDB("bp.tree") 133 | db.AddBTree("AAA") 134 | for i := 0; i < 10; i++ { 135 | db.Set("AAA", fmt.Sprintf("%04d", i), uint64(i)) 136 | } 137 | db.AddBTree("BBB") 138 | for i := 0; i < 15; i++ { 139 | db.Set("BBB", fmt.Sprintf("%04d", i), uint64(i)) 140 | } 141 | 142 | db.AddBTree("CCC") 143 | for i := 0; i < 20; i++ { 144 | db.Set("CCC", fmt.Sprintf("%04d", rand.Intn(100)), uint64(i)) 145 | } 146 | 147 | key1, value1, pgnum1, idx1, ok1 := db.GetFristKV("AAA") 148 | fmt.Printf("key:%v value:%v index:%v pagenum:%v ok:%v\n", key1, value1, idx1, pgnum1, ok1) 149 | 150 | key2, value2, pgnum2, idx2, ok2 := db.GetFristKV("BBB") 151 | fmt.Printf("key:%v value:%v index:%v pagenum:%v ok:%v\n", key2, value2, idx2, pgnum2, ok2) 152 | 153 | key3, value3, pgnum3, idx3, ok3 := db.GetFristKV("CCC") 154 | fmt.Printf("key:%v value:%v index:%v pagenum:%v ok:%v\n", key3, value3, idx3, pgnum3, ok3) 155 | 156 | for ;ok1 || ok2 || ok3; { 157 | 158 | if ok1 { 159 | key1, value1, pgnum1, idx1, ok1 = db.GetNextKV("AAA", pgnum1, idx1) 160 | fmt.Printf("AAA:::key:%v value:%v index:%v pagenum:%v ok:%v\n", key1, value1, idx1, pgnum1, ok1) 161 | 162 | } 163 | 164 | if ok2 { 165 | key2, value2, pgnum2, idx2, ok2 = db.GetNextKV("BBB", pgnum2, idx2) 166 | fmt.Printf("BBB:::key:%v value:%v index:%v pagenum:%v ok:%v\n", key2, value2, pgnum2, idx2, ok2) 167 | } 168 | if ok3 { 169 | key3, value3, pgnum3, idx3, ok3 = db.GetNextKV("CCC", pgnum3, idx3) 170 | fmt.Printf("CCC:::key:%v value:%v index:%v pagenum:%v ok:%v\n", key3, value3, pgnum3, idx3, ok3) 171 | } 172 | 173 | } 174 | 175 | } 176 | */ 177 | -------------------------------------------------------------------------------- /src/utils/Mmap.go: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * file name : Mmap.go 3 | * author : Wu Yinghao 4 | * email : wyh817@gmail.com 5 | * 6 | * file description : mmap底层封装 7 | * 8 | ******************************************************************************/ 9 | 10 | package utils 11 | 12 | import ( 13 | //"bytes" 14 | "encoding/binary" 15 | "errors" 16 | "fmt" 17 | "os" 18 | "reflect" 19 | "syscall" 20 | "unsafe" 21 | ) 22 | 23 | type Mmap struct { 24 | MmapBytes []byte 25 | FileName string 26 | FileLen int64 27 | FilePointer int64 28 | MapType int64 29 | FileFd *os.File 30 | } 31 | 32 | const APPEND_DATA int64 = 1024 * 1024 33 | const ( 34 | MODE_APPEND = iota 35 | MODE_CREATE 36 | ) 37 | 38 | 39 | 40 | func NewMmap(file_name string, mode int) (*Mmap, error) { 41 | 42 | this := &Mmap{MmapBytes: make([]byte, 0), FileName: file_name, FileLen: 0, MapType: 0, FilePointer: 0, FileFd: nil} 43 | 44 | file_mode := os.O_RDWR 45 | file_create_mode := os.O_RDWR | os.O_CREATE | os.O_TRUNC 46 | if mode == MODE_CREATE { 47 | file_mode = os.O_RDWR | os.O_CREATE | os.O_TRUNC 48 | } 49 | 50 | f, err := os.OpenFile(file_name, file_mode, 0664) 51 | 52 | if err != nil { 53 | f, err = os.OpenFile(file_name, file_create_mode, 0664) 54 | if err != nil { 55 | return nil, err 56 | } 57 | } 58 | 59 | fi, err := f.Stat() 60 | if err != nil { 61 | fmt.Printf("ERR:%v", err) 62 | } 63 | this.FileLen = fi.Size() 64 | if mode == MODE_CREATE || this.FileLen == 0 { 65 | syscall.Ftruncate(int(f.Fd()), fi.Size()+APPEND_DATA) 66 | this.FileLen = APPEND_DATA 67 | } 68 | this.MmapBytes, err = syscall.Mmap(int(f.Fd()), 0, int(this.FileLen), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) 69 | 70 | if err != nil { 71 | fmt.Printf("MAPPING ERROR %v \n", err) 72 | return nil, err 73 | } 74 | 75 | this.FileFd = f 76 | return this, nil 77 | } 78 | 79 | func (this *Mmap) SetFileEnd(file_len int64) { 80 | this.FilePointer = file_len 81 | } 82 | 83 | func (this *Mmap) checkFilePointer(check_value int64) error { 84 | 85 | if this.FilePointer+check_value >= this.FileLen { 86 | err := syscall.Ftruncate(int(this.FileFd.Fd()), this.FileLen+APPEND_DATA) 87 | if err != nil { 88 | fmt.Printf("ftruncate error : %v\n", err) 89 | return err 90 | } 91 | this.FileLen += APPEND_DATA 92 | syscall.Munmap(this.MmapBytes) 93 | this.MmapBytes, err = syscall.Mmap(int(this.FileFd.Fd()), 0, int(this.FileLen), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) 94 | 95 | if err != nil { 96 | fmt.Printf("MAPPING ERROR %v \n", err) 97 | return err 98 | } 99 | 100 | } 101 | 102 | return nil 103 | } 104 | 105 | func (this *Mmap) checkFileCap(start, lens int64) error { 106 | 107 | if start+lens >= this.FileLen { 108 | err := syscall.Ftruncate(int(this.FileFd.Fd()), this.FileLen+APPEND_DATA) 109 | if err != nil { 110 | fmt.Printf("ftruncate error : %v\n", err) 111 | return err 112 | } 113 | 114 | this.FileLen += APPEND_DATA 115 | this.FilePointer = start + lens 116 | } 117 | 118 | return nil 119 | 120 | } 121 | 122 | func (this *Mmap) isEndOfFile(start int64) bool { 123 | 124 | if this.FilePointer == start { 125 | return true 126 | } 127 | return false 128 | 129 | } 130 | 131 | func (this *Mmap) ReadInt64(start int64) int64 { 132 | 133 | return int64(binary.LittleEndian.Uint64(this.MmapBytes[start : start+8])) 134 | } 135 | 136 | func (this *Mmap) ReadUInt64(start uint64) uint64 { 137 | 138 | return binary.LittleEndian.Uint64(this.MmapBytes[start : start+8]) 139 | } 140 | 141 | func (this *Mmap) ReadUInt64Arry(start, len uint64) []DocIdNode { 142 | 143 | arry := *(*[]DocIdNode)(unsafe.Pointer(&reflect.SliceHeader{ 144 | Data: uintptr(unsafe.Pointer(&this.MmapBytes[start])), 145 | Len: int(len), 146 | Cap: int(len), 147 | })) 148 | return arry 149 | } 150 | 151 | func (this *Mmap) ReadDocIdsArry(start, len uint64) []DocIdNode { 152 | 153 | arry := *(*[]DocIdNode)(unsafe.Pointer(&reflect.SliceHeader{ 154 | Data: uintptr(unsafe.Pointer(&this.MmapBytes[start])), 155 | Len: int(len), 156 | Cap: int(len), 157 | })) 158 | return arry 159 | } 160 | 161 | func (this *Mmap) ReadString(start, lens int64) string { 162 | 163 | return string(this.MmapBytes[start : start+lens]) 164 | } 165 | 166 | func (this *Mmap) Read(start, end int64) []byte { 167 | 168 | return this.MmapBytes[start:end] 169 | } 170 | 171 | 172 | func (this *Mmap) Write(start int64,buffer []byte) error { 173 | 174 | copy(this.MmapBytes[start:int(start)+len(buffer)],buffer) 175 | 176 | return nil//this.MmapBytes[start:end] 177 | } 178 | 179 | 180 | 181 | func (this *Mmap) WriteUInt64(start int64, value uint64) error { 182 | 183 | binary.LittleEndian.PutUint64(this.MmapBytes[start:start+8], uint64(value)) 184 | 185 | return nil //this.Sync() 186 | } 187 | 188 | func (this *Mmap) WriteInt64(start, value int64) error { 189 | binary.LittleEndian.PutUint64(this.MmapBytes[start:start+8], uint64(value)) 190 | return nil //this.Sync() 191 | } 192 | 193 | func (this *Mmap) AppendInt64(value int64) error { 194 | 195 | if err := this.checkFilePointer(8); err != nil { 196 | return err 197 | } 198 | binary.LittleEndian.PutUint64(this.MmapBytes[this.FilePointer:this.FilePointer+8], uint64(value)) 199 | this.FilePointer += 8 200 | return nil //this.Sync() 201 | } 202 | 203 | func (this *Mmap) AppendUInt64(value uint64) error { 204 | 205 | if err := this.checkFilePointer(8); err != nil { 206 | return err 207 | } 208 | 209 | binary.LittleEndian.PutUint64(this.MmapBytes[this.FilePointer:this.FilePointer+8], value) 210 | this.FilePointer += 8 211 | return nil //this.Sync() 212 | } 213 | 214 | func (this *Mmap) AppendStringWithLen(value string) error { 215 | this.AppendInt64(int64(len(value))) 216 | this.AppendString(value) 217 | return nil //this.Sync() 218 | 219 | } 220 | 221 | func (this *Mmap) AppendString(value string) error { 222 | 223 | lens := int64(len(value)) 224 | if err := this.checkFilePointer(lens); err != nil { 225 | return err 226 | } 227 | 228 | dst := this.MmapBytes[this.FilePointer : this.FilePointer+lens] 229 | copy(dst, []byte(value)) 230 | this.FilePointer += lens 231 | return nil //this.Sync() 232 | 233 | } 234 | 235 | func (this *Mmap) AppendBytes(value []byte) error { 236 | lens := int64(len(value)) 237 | if err := this.checkFilePointer(lens); err != nil { 238 | return err 239 | } 240 | dst := this.MmapBytes[this.FilePointer : this.FilePointer+lens] 241 | copy(dst, value) 242 | this.FilePointer += lens 243 | return nil //this.Sync() 244 | 245 | } 246 | 247 | func (this *Mmap) WriteBytes(start int64, value []byte) error { 248 | lens := int64(len(value)) 249 | dst := this.MmapBytes[start : start+lens] 250 | copy(dst, value) 251 | return nil //this.Sync() 252 | } 253 | 254 | func (this *Mmap) Unmap() error { 255 | 256 | syscall.Munmap(this.MmapBytes) 257 | this.FileFd.Close() 258 | return nil 259 | } 260 | 261 | func (this *Mmap) GetPointer() int64 { 262 | return this.FilePointer 263 | } 264 | 265 | func (this *Mmap) header() *reflect.SliceHeader { 266 | return (*reflect.SliceHeader)(unsafe.Pointer(&this.MmapBytes)) 267 | } 268 | 269 | func (this *Mmap) Sync() error { 270 | dh := this.header() 271 | _, _, err := syscall.Syscall(syscall.SYS_MSYNC, dh.Data, uintptr(dh.Len), syscall.MS_SYNC) 272 | if err != 0 { 273 | fmt.Printf("Sync Error ") 274 | return errors.New("Sync Error") 275 | } 276 | return nil 277 | } 278 | 279 | func (this *Mmap) AppendStringWith32Bytes(value string, lens int64) error { 280 | 281 | err := this.AppendInt64(lens) 282 | if err != nil { 283 | return err 284 | } 285 | if err := this.checkFilePointer(32); err != nil { 286 | return err 287 | } 288 | dst := this.MmapBytes[this.FilePointer : this.FilePointer+32] 289 | copy(dst, value) 290 | this.FilePointer += 32 291 | return nil //this.Sync() 292 | } 293 | 294 | func (this *Mmap) ReadStringWith32Bytes(start int64) string { 295 | 296 | lens := this.ReadInt64(start) 297 | return this.ReadString(start+8, lens) 298 | 299 | } 300 | 301 | func (this *Mmap) WriteStringWith32Bytes(start int64, value string, lens int64) error { 302 | 303 | this.WriteInt64(start, lens) 304 | this.WriteBytes(start+4, []byte(value)) 305 | return nil 306 | } 307 | 308 | 309 | 310 | 311 | func (this *Mmap) AppendUInt32(value uint32) error { 312 | 313 | if err := this.checkFilePointer(4); err != nil { 314 | return err 315 | } 316 | 317 | binary.LittleEndian.PutUint32(this.MmapBytes[this.FilePointer:this.FilePointer + 4], value) 318 | this.FilePointer += 4 319 | return nil //this.Sync() 320 | } 321 | 322 | 323 | 324 | // AppendRecord function description : 根据长度写入字符串[ | len(2bytes) | value | XXX填充 | ] 325 | // params : 326 | // return : 327 | func (this *Mmap) AppendRecord(value []byte) error { 328 | 329 | 330 | /* 331 | if err:=this.AppendUInt32(lens); err != nil { 332 | return err 333 | } 334 | 335 | reallens := uint32(len(value)) 336 | 337 | if reallens > lens { 338 | reallens = lens 339 | } 340 | 341 | if err := this.checkFilePointer(int64(lens)); err != nil { 342 | return err 343 | } 344 | dst := this.MmapBytes[this.FilePointer : this.FilePointer+int64(reallens)] 345 | copy(dst, value[:reallens]) 346 | this.FilePointer += int64(lens) 347 | return nil 348 | */ 349 | 350 | lens := int64(len(value)) 351 | 352 | 353 | if err := this.checkFilePointer(int64(lens)); err != nil { 354 | return err 355 | } 356 | dst := this.MmapBytes[this.FilePointer : this.FilePointer+lens] 357 | copy(dst, value) 358 | this.FilePointer += lens 359 | return nil 360 | 361 | 362 | } 363 | 364 | 365 | 366 | func (this *Mmap) ReadRecord(start int64,lens uint32) []byte{ 367 | 368 | return this.Read(start,start+int64(lens)) 369 | 370 | 371 | } -------------------------------------------------------------------------------- /src/utils/btree.go: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * file name : btree.go 3 | * author : Wu Yinghao 4 | * email : wyh817@gmail.com 5 | * 6 | * file description : b+tree 7 | * 8 | ******************************************************************************/ 9 | 10 | package utils 11 | 12 | //#include 13 | //import "C" 14 | import ( 15 | "errors" 16 | "fmt" 17 | "os" 18 | "reflect" 19 | "sort" 20 | //"strconv" 21 | "syscall" 22 | "unsafe" 23 | ) 24 | 25 | const ( 26 | tmeta uint8 = 1 27 | tinterior uint8 = 2 28 | tleaf uint8 = 3 29 | ) 30 | 31 | const pagesize int64 = 1024 * 4 * 2 32 | 33 | const pageheadOffset int64 = int64(unsafe.Offsetof(((*page)(nil)).elementsptr)) 34 | const elementSize int64 = int64(unsafe.Sizeof(element{})) 35 | const maxitems int64 = 100 36 | const pageheaadlen int64 = pageheadOffset + elementSize*maxitems //4*8 + 24*100 37 | const maxkeylen uint32 = 64 38 | 39 | type element struct { 40 | bkey [maxkeylen]byte 41 | ksize uint32 42 | value uint64 43 | } 44 | 45 | func (e *element) key() string { return string(e.bkey[:e.ksize]) } 46 | func (e *element) setkv(key string, value uint64) { 47 | if len(key) > int(maxkeylen) { 48 | copy(e.bkey[:], []byte(key)[:maxkeylen]) 49 | e.ksize = maxkeylen 50 | e.value = value 51 | return 52 | } 53 | copy(e.bkey[:len(key)], []byte(key)[:]) 54 | e.ksize = uint32(len(key)) 55 | e.value = value 56 | return 57 | } 58 | 59 | type sorteles []element 60 | 61 | func (s sorteles) Len() int { return len(s) } 62 | func (s sorteles) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 63 | func (s sorteles) Less(i, j int) bool { return s[i].key() > s[j].key() } 64 | 65 | // 66 | // ii, _ := strconv.Atoi(s[i].key()) 67 | // jj, _ := strconv.Atoi(s[j].key()) 68 | // return ii > jj 69 | //} 70 | 71 | // page description:页的基本单位 72 | type page struct { 73 | curid uint32 74 | preid uint32 75 | nextid uint32 76 | parentpg uint32 77 | pgtype uint8 78 | count uint32 79 | used uint32 80 | elementsptr uintptr 81 | } 82 | 83 | type pagestack struct { 84 | pageid uint32 85 | index int 86 | } 87 | 88 | func (p *page) display(bt *btree) { 89 | 90 | elements := p.getElements() 91 | fmt.Printf("[INFO]==>display :: ELEMENTS ") 92 | for i := range elements[:p.count] { 93 | e := &elements[i] 94 | 95 | if p.pgtype == tinterior { 96 | fmt.Printf("::: key[%v] %v\n", e.key(), e.value) 97 | child := bt.getpage(uint32(e.value)) 98 | child.display(bt) 99 | } else { 100 | fmt.Printf("::: key[%v] %v\t", e.key(), e.value) 101 | } 102 | } 103 | fmt.Println() 104 | 105 | } 106 | 107 | func (p *page) set(key string, value uint64, bt *btree) bool { 108 | 109 | stack := make([]pagestack, 0) 110 | 111 | found, _, idx, err := p.search(key, &stack, bt) 112 | 113 | if err != nil { 114 | fmt.Printf("[ERROR] can not set key value %v %v %v\n", found, value, idx) 115 | return false 116 | } 117 | 118 | //直接更新 119 | if found { 120 | bt.getpage(stack[len(stack)-1].pageid).getElement(idx).value = value 121 | return true 122 | } 123 | 124 | page := bt.getpage(stack[len(stack)-1].pageid) 125 | index := stack[len(stack)-1].index 126 | pg1, pg2, err := page.insertleaf(index, key, value, bt) 127 | if err != nil { 128 | return false 129 | } 130 | 131 | for idx := len(stack) - 2; idx >= 0; idx-- { 132 | 133 | page := bt.getpage(stack[idx].pageid) 134 | index := stack[idx].index 135 | pg1, pg2, _ = page.interiorinsert(index, pg1, pg2, bt) 136 | } 137 | 138 | if pg1 != 0 && pg2 != 0 { 139 | 140 | newroot, _, _ := bt.newpage(0, 0, tinterior) 141 | p1 := bt.getpage(pg1) 142 | p2 := bt.getpage(pg2) 143 | newroot.count = 2 144 | rooteles := newroot.getElements() 145 | rooteles[0].setkv(p1.getElement(int(p1.count-1)).key(), uint64(p1.curid)) 146 | rooteles[1].setkv("", uint64(p2.curid)) 147 | bt.root = newroot 148 | bt.db.setrootid(bt.name) 149 | bt.rootpgid = newroot.curid 150 | //bt.rootpg = newroot.curid 151 | fmt.Printf("new root...%v\n", newroot.curid) 152 | } 153 | 154 | return true 155 | 156 | } 157 | 158 | func (p *page) interiorinsert(index int, pg1, pg2 uint32, bt *btree) (uint32, uint32, error) { 159 | 160 | if pg1 != 0 && pg2 == 0 { 161 | child1 := bt.getpage(pg1) 162 | elements := p.getElements() 163 | child1node := child1.getElement(int(child1.count - 1)) 164 | elements[index].value = uint64(child1.curid) 165 | if elements[index].key() != "" { 166 | elements[index].setkv(child1node.key(), uint64(child1.curid)) //uintptr(unsafe.Pointer(p))) 167 | } 168 | return p.curid, 0, nil 169 | } 170 | 171 | if pg1 != 0 && pg2 != 0 { 172 | child1 := bt.getpage(pg1) 173 | child2 := bt.getpage(pg2) 174 | elements := p.getElements() 175 | child1node := child1.getElement(int(child1.count - 1)) 176 | elements[p.count].setkv(child1node.key(), uint64(child1.curid)) //uintptr(unsafe.Pointer(p))) 177 | p.count++ 178 | 179 | child2node := child2.getElement(int(child2.count - 1)) 180 | elements[index].value = uint64(child2.curid) 181 | if elements[index].key() != "" { 182 | elements[index].setkv(child2node.key(), uint64(child2.curid)) //uintptr(unsafe.Pointer(p))) 183 | } 184 | 185 | sort.Sort(sorteles(elements[:p.count])) 186 | 187 | if p.count < uint32(maxitems) { 188 | return p.curid, 0, nil 189 | } 190 | 191 | /////////parent := bt.getpage(p.parentpg) 192 | var newpage *page 193 | newpage, _, p = bt.newpage(0, p.curid, tinterior) 194 | newpage.count = 0 195 | ii := 0 196 | for i := int(p.count) / 2; i < int(p.count); i++ { 197 | pele := p.getElement(i) 198 | ele := newpage.getElement(ii) 199 | ele.setkv(pele.key(), pele.value) 200 | newpage.count++ 201 | ii++ 202 | 203 | } 204 | p.count = p.count / 2 205 | return p.curid, newpage.curid, nil 206 | 207 | } 208 | 209 | return p.curid, 0, nil 210 | 211 | } 212 | 213 | func makeBufferPage(src *page) *page { 214 | srcbuf := (*[0xFFFFFF]byte)(unsafe.Pointer(src)) 215 | buf := make([]byte, pagesize) 216 | copy(buf, srcbuf[:pagesize]) 217 | return (*page)(unsafe.Pointer(&buf[0])) 218 | 219 | } 220 | 221 | func (p *page) split(key string, value uint64, bt *btree) (uint32, uint32, error) { 222 | 223 | elements := p.getElements() 224 | elements[p.count].setkv(key, value) //uintptr(unsafe.Pointer(p))) 225 | p.count++ 226 | sort.Sort(sorteles(elements[:p.count])) 227 | 228 | ///////////parent := bt.getpage(p.parentpg) 229 | var newpage *page 230 | newpage, _, p = bt.newpage(0, p.curid, tleaf) 231 | 232 | ii := 0 233 | for i := int(p.count) / 2; i < int(p.count); i++ { 234 | pele := p.getElement(i) 235 | ele := newpage.getElement(ii) 236 | ele.setkv(pele.key(), pele.value) 237 | newpage.count++ 238 | ii++ 239 | 240 | } 241 | p.count = p.count / 2 242 | 243 | return p.curid, newpage.curid, nil 244 | 245 | } 246 | 247 | func (p *page) insertleaf(index int, key string, value uint64, bt *btree) (uint32, uint32, error) { 248 | 249 | if p.pgtype == tleaf { 250 | 251 | if p.count == uint32(maxitems) { 252 | return p.split(key, value, bt) //nil,nil,errors.New("page is full") 253 | } 254 | 255 | elements := p.getElements() 256 | elements[p.count].setkv(key, value) //uintptr(unsafe.Pointer(p))) 257 | p.count++ 258 | sort.Sort(sorteles(elements[:p.count])) 259 | return p.curid, 0, nil 260 | } 261 | return 0, 0, errors.New("insert error") 262 | 263 | } 264 | 265 | func (p *page) search(key string, stack *[]pagestack, bt *btree) (bool, uint64, int, error) { 266 | 267 | if p.pgtype == tleaf { 268 | if p.count == 0 { 269 | *stack = append(*stack, pagestack{pageid: p.curid, index: 0}) 270 | return false, 0, 0, nil 271 | } 272 | 273 | //循环查找 274 | elements := p.getElements() 275 | c := func(i int) bool { 276 | // ee,_:=strconv.Atoi(elements[i].key()) 277 | //kk,_:=strconv.Atoi(key) 278 | //return ee<=kk//elements[i].key() <= key 279 | return elements[i].key() <= key 280 | } 281 | idx := sort.Search(int(p.count), c) 282 | if idx < int(p.count) { 283 | if elements[idx].key() == key { 284 | //fmt.Printf("found : %v %v\n",key,elements[idx].value) 285 | *stack = append(*stack, pagestack{pageid: p.curid, index: idx}) 286 | return true, elements[idx].value, idx, nil 287 | } 288 | *stack = append(*stack, pagestack{pageid: p.curid, index: idx}) 289 | return false, elements[idx].value, idx, nil 290 | } 291 | 292 | *stack = append(*stack, pagestack{pageid: p.curid, index: 0}) 293 | return false, 0, 0, nil //errors.New("found error") 294 | } else if p.pgtype == tinterior { 295 | if p.count == 0 { 296 | *stack = append(*stack, pagestack{pageid: p.curid, index: 0}) 297 | return false, 0, -1, errors.New("ERROR") 298 | } 299 | 300 | //循环查找 301 | elements := p.getElements() 302 | c := func(i int) bool { 303 | //ee,_:=strconv.Atoi(elements[i].key()) 304 | //kk,_:=strconv.Atoi(key) 305 | return elements[i].key() <= key 306 | } 307 | idx := sort.Search(int(p.count), c) 308 | if idx < int(p.count) { 309 | *stack = append(*stack, pagestack{pageid: p.curid, index: idx}) 310 | sub := bt.getpage(uint32(elements[idx].value)) 311 | return sub.search(key, stack, bt) 312 | } 313 | 314 | //没有找到,需要添加 315 | *stack = append(*stack, pagestack{pageid: p.curid, index: -1}) 316 | return false, 0, -1, errors.New("found error") 317 | } 318 | fmt.Printf("[ERROR]==>SEARCH :: b+tree error \n") 319 | return false, 0, -1, errors.New("ERROR") 320 | 321 | } 322 | 323 | func (p *page) getElements() []element { 324 | 325 | return ((*[0xFFFF]element)(unsafe.Pointer(&p.elementsptr)))[:] 326 | } 327 | 328 | func (p *page) getElement(index int) *element { 329 | 330 | return &((*[0xFFFF]element)(unsafe.Pointer(&p.elementsptr)))[index] 331 | } 332 | 333 | func (p *page) getfristkv(bt *btree) (string, uint32, uint32, int, bool) { 334 | 335 | if p.pgtype == tinterior { 336 | child := bt.getpage(uint32(p.getElement(0).value)) 337 | return child.getfristkv(bt) 338 | } 339 | 340 | if p.pgtype == tleaf && p.count > 0 { 341 | res := p.getElement(0) 342 | return res.key(), uint32(res.value), uint32(p.curid), 0, true 343 | } 344 | 345 | return "", 0, 0, 0, false 346 | 347 | } 348 | 349 | // btree function description : b+树 350 | type btree struct { 351 | db *BTreedb 352 | name string 353 | root *page 354 | rootpgid uint32 355 | //cache map[uint32]*page 356 | } 357 | 358 | func loadbtree(name string, root *page, db *BTreedb) *btree { 359 | 360 | bt := &btree{db: db, name: name, root: root, rootpgid: root.curid} 361 | return bt 362 | 363 | } 364 | 365 | func newbtree(name string, db *BTreedb) *btree { 366 | bt := &btree{db: db, name: name} 367 | bt.root, _, _ = bt.newpage(0, 0, tinterior) 368 | var leaf *page 369 | leaf, bt.root, _ = bt.newpage(bt.root.curid, 0, tleaf) 370 | ele := bt.root.getElement(0) 371 | ele.value = uint64(leaf.curid) 372 | bt.rootpgid = bt.root.curid 373 | return bt 374 | } 375 | 376 | func (bt *btree) Set(key string, value uint64) error { 377 | bt.root = bt.db.getpage(bt.rootpgid) 378 | res := bt.root.set(key, value, bt) 379 | if res { 380 | //bt.db.Sync() 381 | return nil 382 | } 383 | 384 | return errors.New("update fail") 385 | } 386 | 387 | func (bt *btree) checkmmap() error { 388 | return bt.db.checkmmap() 389 | } 390 | 391 | func (bt *btree) newpage(parentid, preid uint32, pagetype uint8) (*page, *page, *page) { 392 | 393 | return bt.db.newpage(parentid, preid, pagetype) 394 | } 395 | 396 | func (bt *btree) getpage(pgid uint32) *page { 397 | 398 | // if _,ok:=bt.cache[pgid];ok{ 399 | // return bt.cache[pgid] 400 | // } 401 | //pg:= bt.db.getpage(pgid) 402 | // bt.cache[pgid]=pg 403 | // return pg 404 | return bt.db.getpage(pgid) 405 | } 406 | 407 | func (bt *btree) Search(key string) (bool, uint64) { 408 | bt.root = bt.db.getpage(bt.rootpgid) 409 | stack := make([]pagestack, 0) 410 | ok, value, _, _ := bt.root.search(key, &stack, bt) 411 | 412 | return ok, value 413 | } 414 | 415 | func (bt *btree) Range(start, end string) (bool, []uint64) { 416 | 417 | if len(start) == 0 { 418 | bt.root = bt.db.getpage(bt.rootpgid) 419 | stack1 := make([]pagestack, 0) 420 | ok, _, _, _ := bt.root.search(end, &stack1, bt) 421 | if !ok { 422 | return false, nil 423 | } 424 | startpgid := stack1[len(stack1)-1].pageid 425 | startpg := bt.db.getpage(startpgid) 426 | res := make([]uint64, 0) 427 | for idx := stack1[len(stack1)-1].index - 1; idx >= 0; idx-- { 428 | res = append(res, startpg.getElement(idx).value) 429 | } 430 | 431 | pgid := startpg.preid 432 | for pgid != 0 { 433 | pg := bt.db.getpage(pgid) 434 | for idx := int(pg.count) - 1; idx > 0; idx-- { 435 | res = append(res, pg.getElement(idx).value) 436 | } 437 | pgid = pg.preid 438 | } 439 | return true, res 440 | } 441 | 442 | if len(end) == 0 { 443 | bt.root = bt.db.getpage(bt.rootpgid) 444 | stack1 := make([]pagestack, 0) 445 | ok, _, _, _ := bt.root.search(start, &stack1, bt) 446 | if !ok { 447 | return false, nil 448 | } 449 | startpgid := stack1[len(stack1)-1].pageid 450 | startpg := bt.db.getpage(startpgid) 451 | res := make([]uint64, 0) 452 | for idx := stack1[len(stack1)-1].index; idx < int(startpg.count); idx++ { 453 | res = append(res, startpg.getElement(idx).value) 454 | } 455 | 456 | pgid := startpg.nextid 457 | for pgid != 0 { 458 | pg := bt.db.getpage(pgid) 459 | for idx := 0; idx < int(pg.count); idx++ { 460 | res = append(res, pg.getElement(idx).value) 461 | } 462 | pgid = pg.nextid 463 | } 464 | return true, res 465 | } 466 | 467 | bt.root = bt.db.getpage(bt.rootpgid) 468 | stack1 := make([]pagestack, 0) 469 | ok, _, _, _ := bt.root.search(start, &stack1, bt) 470 | if !ok { 471 | return false, nil 472 | } 473 | startpgid := stack1[len(stack1)-1].pageid 474 | 475 | stack2 := make([]pagestack, 0) 476 | ok, _, _, _ = bt.root.search(end, &stack2, bt) 477 | if !ok { 478 | return false, nil 479 | } 480 | endpgid := stack2[len(stack2)-1].pageid 481 | 482 | res := make([]uint64, 0) 483 | endpg := bt.db.getpage(endpgid) 484 | for idx := stack2[len(stack2)-1].index; idx < int(endpg.count); idx++ { 485 | res = append(res, endpg.getElement(idx).value) 486 | } 487 | 488 | pgid := endpg.nextid 489 | for pgid != startpgid && pgid != 0 { 490 | pg := bt.db.getpage(pgid) 491 | for idx := 0; idx < int(pg.count); idx++ { 492 | res = append(res, pg.getElement(idx).value) 493 | } 494 | pgid = pg.nextid 495 | } 496 | 497 | startpg := bt.db.getpage(startpgid) 498 | 499 | for idx := 0; idx < stack1[len(stack1)-1].index; idx++ { 500 | res = append(res, startpg.getElement(idx).value) 501 | } 502 | 503 | return true, res 504 | 505 | } 506 | 507 | func (bt *btree) GetFristKV() (string, uint32, uint32, int, bool) { 508 | 509 | if bt.rootpgid == 0 { 510 | return "", 0, 0, 0, false 511 | } 512 | 513 | node := bt.db.getpage(bt.rootpgid) 514 | return node.getfristkv(bt) 515 | 516 | } 517 | 518 | func (bt *btree) GetNextKV(pagenum uint32, index int) (string, uint32, uint32, int, bool) { 519 | if bt.rootpgid == 0 { 520 | return "", 0, 0, 0, false 521 | } 522 | 523 | node := bt.db.getpage(pagenum) 524 | if uint32(index+1) < node.count { 525 | ele := node.getElement(index + 1) 526 | return ele.key(), uint32(ele.value), node.curid, index + 1, true 527 | } 528 | if node.nextid == 0 { 529 | return "", 0, 0, 0, false 530 | } 531 | next := bt.db.getpage(node.nextid) 532 | if next.count > 0 { 533 | ele := node.getElement(0) 534 | return ele.key(), uint32(ele.value), next.curid, 0, true 535 | } 536 | return "", 0, 0, 0, false 537 | 538 | } 539 | 540 | func (bt *btree) Display() { 541 | bt.root.display(bt) 542 | } 543 | 544 | type metaBT struct { 545 | btname [32]byte 546 | btnamelen uint32 547 | maxkeylen uint32 548 | rootpgid uint32 549 | } 550 | 551 | func (mt *metaBT) key() string { 552 | return string(mt.btname[:mt.btnamelen]) 553 | } 554 | 555 | func (mt *metaBT) setkey(key string) { 556 | if len(key) == 0 { 557 | return 558 | } 559 | if len(key) > 32 { 560 | copy(mt.btname[:], []byte(key)[:32]) 561 | mt.btnamelen = 32 562 | return 563 | } 564 | copy(mt.btname[:len(key)], []byte(key)[:]) 565 | mt.btnamelen = uint32(len(key)) 566 | return 567 | } 568 | 569 | type metaInfo struct { 570 | magic uint32 571 | maxpgid uint32 572 | btnum uint32 573 | btinfos [64]metaBT 574 | } 575 | 576 | func (mi *metaInfo) addbt(name string, rootpgid uint32) error { 577 | 578 | mi.btinfos[mi.btnum].setkey(name) 579 | mi.btinfos[mi.btnum].rootpgid = rootpgid 580 | mi.btnum++ 581 | return nil 582 | 583 | } 584 | 585 | const magicnum uint32 = 0x9EDFEDFA 586 | 587 | type BTreedb struct { 588 | btmap map[string]*btree // btree集合 589 | filename string 590 | mmapbytes []byte 591 | //maxpgid uint32 592 | fd *os.File 593 | meta *metaInfo 594 | } 595 | 596 | func exist(filename string) bool { 597 | _, err := os.Stat(filename) 598 | return err == nil || os.IsExist(err) 599 | } 600 | 601 | func NewBTDB(dbname string) *BTreedb { 602 | 603 | //fmt.Printf("headoffset : %v \n", pageheadOffset) 604 | //fmt.Printf("elementSize: %v \n", elementSize) 605 | //fmt.Printf("pageheaadlen: %v \n", pageheaadlen) 606 | //fmt.Printf("btdbname : %v \n", dbname) 607 | file_create_mode := os.O_RDWR | os.O_CREATE | os.O_TRUNC 608 | this := &BTreedb{filename: dbname, btmap: make(map[string]*btree)} 609 | 610 | if exist(dbname) { 611 | file_create_mode = os.O_RDWR 612 | } else { 613 | file_create_mode = os.O_RDWR | os.O_CREATE | os.O_TRUNC 614 | } 615 | 616 | f, err := os.OpenFile(dbname, file_create_mode, 0664) 617 | if err != nil { 618 | return nil 619 | } 620 | 621 | fi, _ := f.Stat() 622 | filelen := fi.Size() 623 | //fmt.Printf("filelen : %v, %v \n", filelen, pagesize*2) 624 | if filelen < pagesize*2 { 625 | syscall.Ftruncate(int(f.Fd()), pagesize*2) 626 | filelen = pagesize * 2 627 | this.fd = f 628 | //var addr = unsafe.Pointer(&this.mmapbytes[0]) 629 | this.mmapbytes, err = syscall.Mmap(int(f.Fd()), 0, int(filelen), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) 630 | //ptr, err := C.mmap(addr, C.size_t(filelen), C.PROT_READ|C.PROT_WRITE, C.MAP_SHARED, C.int(f.Fd()), 0) 631 | 632 | if err != nil { 633 | fmt.Printf("MAPPING ERROR %v \n", err) 634 | return nil 635 | } 636 | //this.mmapbytes = ([]byte)(unsafe.Pointer(ptr)) 637 | this.meta = (*metaInfo)(unsafe.Pointer(&this.mmapbytes[0])) 638 | this.meta.magic = magicnum 639 | this.meta.maxpgid = 1 640 | this.meta.btnum = 0 641 | return this 642 | } 643 | this.fd = f 644 | this.mmapbytes, err = syscall.Mmap(int(f.Fd()), 0, int(filelen), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) 645 | if err != nil { 646 | fmt.Printf("MAPPING ERROR %v \n", err) 647 | return nil 648 | } 649 | this.meta = (*metaInfo)(unsafe.Pointer(&this.mmapbytes[0])) 650 | if this.meta.magic != magicnum { 651 | fmt.Printf("FILE TYPE ERROR \n") 652 | return nil 653 | } 654 | 655 | for i := uint32(0); i < this.meta.btnum; i++ { 656 | btname := this.meta.btinfos[i].key() 657 | root := this.getpage(this.meta.btinfos[i].rootpgid) 658 | this.btmap[btname] = loadbtree(btname, root, this) 659 | } 660 | 661 | return this 662 | } 663 | 664 | func (db *BTreedb) AddBTree(name string) error { 665 | 666 | if _, ok := db.btmap[name]; ok { 667 | fmt.Printf("ERROR:::%v\n", db.filename) 668 | return nil 669 | } 670 | //fmt.Printf("FILE:::%v\n", db.filename) 671 | bt := newbtree(name, db) 672 | if bt == nil { 673 | fmt.Printf("create error:::%v\n", name) 674 | return errors.New("create error") 675 | } 676 | db.btmap[name] = bt 677 | db.meta.addbt(name, bt.root.curid) 678 | db.Sync() 679 | return nil 680 | } 681 | 682 | func (db *BTreedb) header() *reflect.SliceHeader { 683 | return (*reflect.SliceHeader)(unsafe.Pointer(&db.mmapbytes)) 684 | } 685 | 686 | func (db *BTreedb) Sync() error { 687 | dh := db.header() 688 | _, _, err := syscall.Syscall(syscall.SYS_MSYNC, dh.Data, uintptr(dh.Len), syscall.MS_SYNC) 689 | if err != 0 { 690 | fmt.Printf("Sync Error ") 691 | return errors.New("Sync Error") 692 | } 693 | return nil 694 | } 695 | 696 | func (db *BTreedb) Set(btname, key string, value uint64) error { 697 | 698 | if _, ok := db.btmap[btname]; !ok { 699 | return errors.New("has one") 700 | } 701 | 702 | return db.btmap[btname].Set(key, value) 703 | 704 | } 705 | 706 | func (db *BTreedb) IncValue(btname, key string) error { 707 | if _, ok := db.btmap[btname]; !ok { 708 | return errors.New("has one") 709 | } 710 | 711 | found, value := db.btmap[btname].Search(key) 712 | if found { 713 | return db.btmap[btname].Set(key, value+1) 714 | } 715 | return db.btmap[btname].Set(key, 1) 716 | 717 | } 718 | 719 | func (db *BTreedb) Search(btname, key string) (bool, uint64) { 720 | if _, ok := db.btmap[btname]; !ok { 721 | return false, 0 722 | } 723 | 724 | return db.btmap[btname].Search(key) 725 | 726 | } 727 | 728 | func (db *BTreedb) Range(btname, start, end string) (bool, []uint64) { 729 | 730 | if _, ok := db.btmap[btname]; !ok { 731 | return false, nil 732 | } 733 | 734 | if start >= end && len(end) > 0 && len(start) > 0 { 735 | fmt.Printf("START OVER END\n") 736 | return false, nil 737 | } 738 | 739 | return db.btmap[btname].Range(start, end) 740 | 741 | } 742 | 743 | func (db *BTreedb) GetFristKV(btname string) (string, uint32, uint32, int, bool) { 744 | if _, ok := db.btmap[btname]; !ok { 745 | return "", 0, 0, 0, false 746 | } 747 | 748 | return db.btmap[btname].GetFristKV() 749 | 750 | } 751 | 752 | func (db *BTreedb) GetNextKV(btname string, pagenum uint32, index int) (string, uint32, uint32, int, bool) { 753 | if _, ok := db.btmap[btname]; !ok { 754 | return "", 0, 0, 0, false 755 | } 756 | 757 | return db.btmap[btname].GetNextKV(pagenum, index) 758 | 759 | } 760 | 761 | func (db *BTreedb) Close() error { 762 | 763 | syscall.Munmap(db.mmapbytes) 764 | db.fd.Close() 765 | return nil 766 | } 767 | 768 | func (bt *BTreedb) newpage( /*parent, pre *page*/ parentid, preid uint32, pagetype uint8) (*page, *page, *page) { 769 | 770 | if bt.checkmmap() != nil { 771 | fmt.Printf("check error \n") 772 | return nil, nil, nil 773 | } 774 | var parent *page 775 | var pre *page 776 | lpage := (*page)(unsafe.Pointer(&bt.mmapbytes[(int64(bt.meta.maxpgid) * pagesize)])) 777 | //fmt.Printf("lapge:%v\n", unsafe.Pointer(lpage)) 778 | lpage.curid = bt.meta.maxpgid 779 | lpage.pgtype = pagetype 780 | lpage.nextid = 0 781 | lpage.preid = 0 782 | if pagetype == tinterior { 783 | lpage.count = 1 784 | ele := (*[0xFFFF]element)(unsafe.Pointer(&bt.mmapbytes[(int64(bt.meta.maxpgid)*pagesize + pageheadOffset)])) 785 | lpage.used = uint32(pageheaadlen) 786 | ele[0].setkv("", 0) 787 | lpage.elementsptr = uintptr(unsafe.Pointer(ele)) 788 | 789 | } else { 790 | lpage.count = 0 791 | ele := (*[0xFFFF]element)(unsafe.Pointer(&bt.mmapbytes[(int64(bt.meta.maxpgid)*pagesize + pageheadOffset)])) 792 | lpage.elementsptr = uintptr(unsafe.Pointer(ele)) 793 | lpage.used = uint32(pageheaadlen) 794 | } 795 | //fmt.Printf("lapge:%v\n", unsafe.Pointer(lpage)) 796 | //fmt.Printf("parent:%v\n", unsafe.Pointer(parent)) 797 | if parentid != 0 { 798 | parent = bt.getpage(parentid) 799 | lpage.parentpg = parent.curid 800 | } else { 801 | lpage.parentpg = 0 802 | } 803 | 804 | if preid != 0 { 805 | pre = bt.getpage(preid) 806 | lpage.nextid = pre.nextid 807 | pre.nextid = lpage.curid 808 | lpage.preid = pre.curid 809 | } 810 | 811 | bt.meta.maxpgid++ 812 | return lpage, parent, pre 813 | } 814 | 815 | func (bt *BTreedb) checkmmap() error { 816 | if int(int64(bt.meta.maxpgid)*pagesize) >= len(bt.mmapbytes) { 817 | err := syscall.Ftruncate(int(bt.fd.Fd()), int64(bt.meta.maxpgid+1)*pagesize) 818 | if err != nil { 819 | fmt.Printf("ftruncate error : %v\n", err) 820 | return err 821 | } 822 | maxpgid := bt.meta.maxpgid 823 | syscall.Munmap(bt.mmapbytes) 824 | //fmt.Printf(".meta.maxpgid:%v\n",bt.meta.maxpgid) 825 | bt.mmapbytes, err = syscall.Mmap(int(bt.fd.Fd()), 0, int(int64( /*bt.meta.maxpgid*/ maxpgid+1)*pagesize), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) 826 | 827 | if err != nil { 828 | fmt.Printf("MAPPING ERROR %v \n", err) 829 | return err 830 | } 831 | 832 | bt.meta = (*metaInfo)(unsafe.Pointer(&bt.mmapbytes[0])) 833 | 834 | } 835 | return nil 836 | } 837 | 838 | func (bt *BTreedb) getpage(pgid uint32) *page { 839 | //fmt.Printf("pgid:%v\n",pgid) 840 | return (*page)(unsafe.Pointer(&bt.mmapbytes[(int64(pgid) * pagesize)])) 841 | 842 | } 843 | 844 | func (db *BTreedb) setrootid(btname string) error { 845 | 846 | if _, ok := db.btmap[btname]; !ok { 847 | return errors.New("no bt") 848 | } 849 | 850 | for i := uint32(0); i < db.meta.btnum; i++ { 851 | if db.meta.btinfos[i].key() == btname { 852 | db.meta.btinfos[i].rootpgid = db.btmap[btname].root.curid 853 | } 854 | 855 | } 856 | 857 | return nil 858 | } 859 | --------------------------------------------------------------------------------