├── .editorconfig ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── _legacy ├── .ayi.yml ├── cmd │ ├── README.md │ ├── xk │ │ ├── main.go │ │ └── server.go │ ├── xkctl │ │ ├── main.go │ │ └── ping.go │ ├── xkd │ │ └── main.go │ └── xkutil │ │ └── main.go ├── pkg │ ├── README.md │ ├── cmd │ │ ├── daemon │ │ │ ├── root.go │ │ │ └── schema.go │ │ ├── pkg.go │ │ ├── util │ │ │ └── root.go │ │ └── version.go │ ├── common │ │ ├── .ayi.yml │ │ ├── .gitignore │ │ ├── common.pb.go │ │ ├── common.proto │ │ ├── doc.go │ │ ├── gen │ │ │ └── main.go │ │ ├── hash.go │ │ ├── pkg.go │ │ ├── point.go │ │ ├── point_test.go │ │ ├── query.go │ │ ├── query_test.go │ │ ├── series.go │ │ ├── series_double.go │ │ ├── series_id.go │ │ ├── series_int.go │ │ └── series_test.go │ ├── config │ │ ├── bench.go │ │ ├── bench_test.go │ │ ├── daemon.go │ │ ├── daemon_test.go │ │ ├── pkg.go │ │ ├── xkb.yml │ │ └── xkd.yml │ ├── encoding │ │ ├── delta.go │ │ ├── delta_rle.go │ │ ├── delta_rle_test.go │ │ ├── delta_test.go │ │ ├── doc.go │ │ ├── pkg.go │ │ ├── pkg_test.go │ │ ├── raw.go │ │ ├── raw_test.go │ │ ├── rle.go │ │ ├── rle_test.go │ │ ├── varint.go │ │ └── varint_test.go │ ├── server │ │ ├── config.go │ │ ├── config_test.go │ │ ├── doc.go │ │ ├── grpc │ │ │ ├── config.go │ │ │ └── server.go │ │ ├── http │ │ │ ├── config.go │ │ │ ├── read.go │ │ │ ├── read_test.go │ │ │ ├── server.go │ │ │ └── write.go │ │ ├── http_server_test.go │ │ ├── payload │ │ │ ├── payload.pb.go │ │ │ └── payload.proto │ │ ├── pkg.go │ │ └── service │ │ │ ├── pkg.go │ │ │ ├── read.go │ │ │ └── write.go │ ├── storage │ │ ├── cassandra │ │ │ ├── config.go │ │ │ ├── config_test.go │ │ │ ├── doc.go │ │ │ ├── pkg.go │ │ │ ├── schema.go │ │ │ ├── stmt.go │ │ │ ├── store.go │ │ │ └── store_test.go │ │ ├── config.go │ │ ├── config_test.go │ │ ├── discard │ │ │ └── pkg.go │ │ ├── disk │ │ │ ├── .ayi.yml │ │ │ ├── config.go │ │ │ ├── config_test.go │ │ │ ├── data_reader.go │ │ │ ├── data_writer.go │ │ │ ├── data_writer_test.go │ │ │ ├── disk.pb.go │ │ │ ├── disk.proto │ │ │ ├── doc.go │ │ │ ├── encoding.go │ │ │ ├── pkg.go │ │ │ ├── pkg_test.go │ │ │ ├── reader.go │ │ │ └── store.go │ │ ├── doc.go │ │ ├── memory │ │ │ ├── .ayi.yml │ │ │ ├── config.go │ │ │ ├── config_test.go │ │ │ ├── data.go │ │ │ ├── data_test.go │ │ │ ├── doc.go │ │ │ ├── gen │ │ │ │ └── main.go │ │ │ ├── index.go │ │ │ ├── index_test.go │ │ │ ├── pkg.go │ │ │ ├── series_store_double.go │ │ │ ├── series_store_int.go │ │ │ ├── series_store_test.go │ │ │ ├── store.go │ │ │ └── store_test.go │ │ ├── pkg.go │ │ └── store.go │ ├── util │ │ ├── doc.go │ │ ├── log.go │ │ ├── log_test.go │ │ ├── pkg.go │ │ ├── test.go │ │ └── test_test.go │ └── version.go ├── playground │ ├── README.md │ ├── algo_test.go │ ├── binary_test.go │ ├── bug_server_concurrent_map_write │ │ ├── 0609_16_24.log │ │ ├── 0609_16_51_index.log │ │ └── README.md │ ├── bytes_test.go │ ├── collector_fixed_length │ │ ├── README.md │ │ ├── mem-1.json │ │ └── mem-2.json │ ├── column-vs-row.md │ ├── common │ │ ├── .gitignore │ │ ├── common.proto │ │ ├── common_test.go │ │ └── json.go │ ├── compress_test.go │ ├── disk-c │ │ └── nocomprees.c │ ├── disk │ │ └── nocompress_test.go │ ├── encoding_test.go │ ├── fixture │ │ ├── .gitignore │ │ └── xephon-no-compress │ ├── hash_test.go │ ├── map_test.go │ ├── mmap-c │ │ └── mmapdemo.c │ ├── mmap │ │ ├── demo.txt │ │ └── mmap_test.go │ ├── range_test.go │ ├── rocksdb_test.go │ ├── slice_test.go │ ├── sort_test.go │ ├── time_test.go │ ├── unsafe_test.go │ ├── xapi │ │ └── main.go │ ├── xbucket │ │ └── main.go │ └── xnaive │ │ └── main.go ├── script │ ├── cadvisor │ │ ├── README.md │ │ ├── docker-compose.yml │ │ └── up.sh │ ├── cassandra │ │ ├── cql.sh │ │ ├── pull_cassandra.sh │ │ ├── remove_cassandra.sh │ │ ├── run_cassandra.sh │ │ └── xkschema.sh │ ├── cloc_exclude.txt │ ├── gen_config.py │ ├── influxdb │ │ ├── .gitignore │ │ ├── influx.sh │ │ ├── influxdb.conf │ │ ├── pull_influxdb.sh │ │ ├── remove_influxdb.sh │ │ └── run_influxdb.sh │ ├── kairosdb │ │ ├── README.md │ │ ├── docker-compose.yml │ │ ├── node │ │ │ ├── Dockerfile │ │ │ ├── Dockerfile.h2 │ │ │ ├── kairosdb.h2.properties │ │ │ ├── kairosdb.properties │ │ │ ├── kairosdb.sh │ │ │ └── wait-for-it.sh │ │ └── up.sh │ ├── prometheus │ │ ├── .gitignore │ │ ├── README.md │ │ └── up.sh │ ├── scylladb │ │ ├── pull_scylladb.sh │ │ ├── remove_scylladb.sh │ │ ├── run_scylladb.sh │ │ └── xkschema.sh │ ├── timescaledb │ │ ├── psql.sh │ │ ├── pull_timescaledb.sh │ │ └── remove_timescaledb.sh │ ├── travis_install.sh │ └── xephon-k-cassandra │ │ ├── docker-compose.yml │ │ ├── node │ │ ├── .gitignore │ │ ├── Dockerfile │ │ ├── wait-for-it.sh │ │ └── xkdc.sh │ │ └── up.sh ├── web │ ├── README.md │ ├── app.js │ ├── index.html │ └── util.js ├── xk.yml └── xk │ ├── client │ ├── grpcclient │ │ ├── client.go │ │ └── client_test.go │ └── meta.go │ ├── config │ ├── client.go │ └── server.go │ ├── server │ ├── gommon.yml │ ├── gommon_generated.go │ ├── grpc.go │ ├── http.go │ ├── manager.go │ └── pkg.go │ ├── storage │ ├── mem │ │ ├── gommon.yml │ │ ├── gommon_generated.go │ │ ├── ring_generated.go │ │ ├── ring_generated.go.tmpl │ │ └── ring_test.go │ └── provider.go │ └── transport │ └── grpc │ ├── client.go │ ├── gommon.yml │ ├── gommon_generated.go │ ├── rpc.pb.go │ └── rpc.proto ├── doc ├── CHANGELOG.md ├── README.md ├── ROADMAP.md ├── bench │ ├── 278-legacy │ │ ├── index.html │ │ ├── influxdb.md │ │ ├── kairosdb.md │ │ ├── xephonk-cassandra.md │ │ └── xephonk-mem.md │ ├── README.md │ ├── space │ │ ├── cassandra.md │ │ ├── compression.html │ │ ├── index.html │ │ ├── influxdb.md │ │ └── xephon-k.md │ └── throughput │ │ ├── index.html │ │ ├── influx-graphite-kairosdb.html │ │ ├── influxdb.md │ │ ├── influxver.html │ │ ├── kairosdb.md │ │ ├── mysqlver.html │ │ ├── postgresver.html │ │ ├── response-time.html │ │ ├── xephon-k-cassandra.md │ │ ├── xephon-k-disk.md │ │ ├── xephon-k-mem.md │ │ └── xephon-k-null.md ├── design │ ├── data-mem.md │ └── protocol.md ├── impl │ ├── README.md │ ├── aggregation.md │ ├── api-read.md │ ├── api-write.md │ ├── api.md │ ├── collector.md │ ├── storage-cassandra.md │ ├── storage-disk.md │ └── storage-memory.md ├── legacy │ ├── benchhub-cleanup.md │ ├── roadmap.md │ └── spec │ │ ├── api.md │ │ └── schema.md ├── log │ ├── 2019 │ │ └── 2019-01 │ │ │ └── 2019-01-25-0.0.3-init-refactor.md │ └── 2020 │ │ ├── 2020-01 │ │ ├── 2020-01-17-refactor-again.md │ │ └── 2020-01-18-dep-to-mod.md │ │ └── 2020-02 │ │ └── 2020-02-03-design.md └── survey │ ├── README.md │ ├── akumuli.md │ ├── blueflood.md │ ├── btrdb.md │ ├── cassandra.md │ ├── cassandra │ ├── 1.0-about-data-model.md │ ├── 1.0-partitioning.md │ ├── 3.0-storage-engine.md │ ├── README.md │ ├── schema-in-cassandra-1-1.md │ ├── tw_data.png │ ├── tw_logical.png │ ├── tw_physical.png │ └── tw_physical_compact.png │ ├── druid │ ├── README.md │ ├── druid.png │ ├── druid_a_real_time_analytical_data_store.md │ ├── query.md │ └── the-challenges-of-building-an-anlytics-stack.md │ ├── gorilla.md │ ├── graphite-promql-influxql.md │ ├── heroic.md │ ├── influxdb.md │ ├── influxdb │ ├── query_process.md │ ├── read-path.md │ ├── tracing.md │ ├── tsm-design.md │ ├── write-path.md │ └── write_process.md │ ├── kafka.md │ ├── kairosdb.md │ ├── khronus.md │ ├── lmdb.md │ ├── newts.md │ ├── opentsdb.md │ ├── prometheus-2.md │ ├── prometheus.md │ ├── respawn.md │ ├── rrd.md │ ├── scylladb.md │ ├── template.md │ └── whisper.md ├── go.mod └── go.sum /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.md] 2 | indent_style = space 3 | indent_size = 2 4 | max_line_length = off 5 | trim_trailing_whitespace = false 6 | 7 | [Makefile] 8 | indent_style = tab -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | .idea 27 | vendor 28 | .vscode 29 | a.out 30 | xk*.zip 31 | xkd.yml 32 | xkb.yml -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: true 3 | git: 4 | depth: 3 5 | 6 | go: 7 | - "1.13" 8 | - tip 9 | 10 | # FIXME: run actual test when there is something to test ... 11 | install: 12 | - go version 13 | 14 | script: 15 | - go version -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION = 0.0.1 2 | BUILD_COMMIT = $(shell git rev-parse HEAD) 3 | BUILD_TIME = $(shell date +%Y-%m-%dT%H:%M:%S%z) 4 | CURRENT_USER = $(USER) 5 | FLAGS = -X main.version=$(VERSION) -X main.commit=$(BUILD_COMMIT) -X main.buildTime=$(BUILD_TIME) -X main.buildUser=$(CURRENT_USER) 6 | 7 | .PHONY: install 8 | install: 9 | go install -ldflags "$(FLAGS)" ./cmd/xk 10 | go install -ldflags "$(FLAGS)" ./cmd/xkctl 11 | 12 | .PHONY: generate 13 | generate: 14 | gommon generate -v 15 | 16 | .PHONY: fmt 17 | fmt: 18 | gofmt -d -l -w ./cmd/xk ./cmd/xkctl ./xk 19 | 20 | .PHONY: test 21 | test: 22 | go test -v -cover ./xk/... 23 | 24 | .PHONY: test-race 25 | test-race: 26 | go test -race ./xk/... 27 | 28 | .PHONY: loc 29 | loc: 30 | cloc --exclude-dir=vendor,.idea,playground --exclude-list-file=script/cloc_exclude.txt . -------------------------------------------------------------------------------- /_legacy/.ayi.yml: -------------------------------------------------------------------------------- 1 | install: 2 | - go install ./cmd/xkb 3 | - go install ./cmd/xkc 4 | - go install ./cmd/xkd 5 | - go install ./cmd/xkutil 6 | - sh -c "cp $(which xkd) script/xephon-k-cassandra/node" 7 | # - go install ./playground/xnaive 8 | # - go install ./playground/xbucket 9 | # - go install ./playground/xapi 10 | test: 11 | - go test -short -v -cover ./pkg/... 12 | scripts: 13 | t: 14 | - go test -v -cover ./pkg/... 15 | runc: 16 | - docker run --name tsdb-cassandra -p 9042:9042 -d cassandra:3.9 17 | start: 18 | - docker start tsdb-cassandra 19 | f: 20 | - gofmt -d -l -w ./pkg 21 | d: 22 | - sh -c "xkd" 23 | naive: 24 | # FIXME: it seems to be my Fedora has some very strict network configuration for docker 25 | - sh -c "xnaive" 26 | bucket: 27 | - sh -c "xbucket" 28 | api: 29 | - sh -c "xapi" 30 | loc: 31 | - cloc --exclude-dir=vendor,.idea,playground --exclude-list-file=script/cloc_exclude.txt . 32 | loc-play: 33 | - cloc --exclude-dir=vendor,.idea --exclude-list-file=script/cloc_exclude.txt . 34 | release: 35 | - sh -c "zip xkb-linux-amd64.zip $(which xkb)" 36 | - sh -c "zip xkc-linux-amd64.zip $(which xkc)" 37 | - sh -c "zip xkd-linux-amd64.zip $(which xkd)" 38 | 39 | -------------------------------------------------------------------------------- /_legacy/cmd/README.md: -------------------------------------------------------------------------------- 1 | # Commandline applications 2 | 3 | Current 4 | 5 | - [xk](xk) xephon-k server daemon 6 | - [xkctl](xkctl) client and util 7 | 8 | Deprecated 9 | 10 | - xkb xephon-k version of xephon-b 11 | - xkd xephon-k server daemon 12 | - xkutil used to inspect on disk file -------------------------------------------------------------------------------- /_legacy/cmd/xk/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | 8 | icli "github.com/dyweb/go.ice/cli" 9 | 10 | "github.com/xephonhq/xephon-k/xk/config" 11 | "github.com/xephonhq/xephon-k/xk/util/logutil" 12 | ) 13 | 14 | const ( 15 | myname = "xk" 16 | ) 17 | 18 | var log = logutil.Registry 19 | 20 | var ( 21 | version string 22 | commit string 23 | buildTime string 24 | buildUser string 25 | goVersion = runtime.Version() 26 | ) 27 | 28 | var buildInfo = icli.BuildInfo{Version: version, Commit: commit, BuildTime: buildTime, BuildUser: buildUser, GoVersion: goVersion} 29 | var cli *icli.Root 30 | var cfg config.ServerConfig 31 | 32 | func main() { 33 | cli = icli.New( 34 | icli.Name(myname), 35 | icli.Description("Xephon-K Server"), 36 | icli.Version(buildInfo), 37 | icli.LogRegistry(log), 38 | ) 39 | root := cli.Command() 40 | root.AddCommand(serveCmd) 41 | if err := root.Execute(); err != nil { 42 | fmt.Fprintln(os.Stderr, err) 43 | os.Exit(1) 44 | } 45 | } 46 | 47 | func mustLoadConfig() { 48 | if err := cli.LoadConfigTo(&cfg); err != nil { 49 | log.Fatal(err) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /_legacy/cmd/xk/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | "github.com/xephonhq/xephon-k/xk/server" 6 | ) 7 | 8 | var serveCmd = &cobra.Command{ 9 | Use: "serve", 10 | Short: "start Xephonk daemon", 11 | Long: "Start Xephonk daemon with gRPC and HTTP server", 12 | Run: func(cmd *cobra.Command, args []string) { 13 | mustLoadConfig() 14 | mgr, err := server.NewManager(cfg) 15 | if err != nil { 16 | log.Fatal(err) 17 | } 18 | if err := mgr.Run(); err != nil { 19 | log.Fatal(err) 20 | } 21 | }, 22 | } 23 | -------------------------------------------------------------------------------- /_legacy/cmd/xkctl/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | 8 | icli "github.com/dyweb/go.ice/cli" 9 | "google.golang.org/grpc" 10 | 11 | myrpc "github.com/xephonhq/xephon-k/xk/transport/grpc" 12 | "github.com/xephonhq/xephon-k/xk/util/logutil" 13 | ) 14 | 15 | const ( 16 | myname = "xkctl" 17 | ) 18 | 19 | var log = logutil.Registry 20 | 21 | var ( 22 | version string 23 | commit string 24 | buildTime string 25 | buildUser string 26 | goVersion = runtime.Version() 27 | ) 28 | 29 | var buildInfo = icli.BuildInfo{Version: version, Commit: commit, BuildTime: buildTime, BuildUser: buildUser, GoVersion: goVersion} 30 | var client myrpc.XephonkClient 31 | var addr = "localhost:2334" 32 | 33 | func main() { 34 | cli := icli.New( 35 | icli.Name(myname), 36 | icli.Description("Xephonk ctrl"), 37 | icli.Version(buildInfo), 38 | icli.LogRegistry(log), 39 | ) 40 | root := cli.Command() 41 | root.AddCommand(pingCmd) 42 | if err := root.Execute(); err != nil { 43 | fmt.Fprintln(os.Stderr, err) 44 | os.Exit(1) 45 | } 46 | } 47 | 48 | func mustCreateClient() { 49 | if client != nil { 50 | return 51 | } 52 | if conn, err := grpc.Dial(addr, grpc.WithInsecure()); err != nil { 53 | log.Fatalf("can't dial %v", err) 54 | } else { 55 | client = myrpc.NewClient(conn) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /_legacy/cmd/xkctl/ping.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | pb "github.com/xephonhq/xephon-k/xk/transport/grpc" 9 | ) 10 | 11 | var pingCmd = &cobra.Command{ 12 | Use: "ping", 13 | Short: "ping xephonk", 14 | Long: "Ping Xephonk server using gRPC", 15 | Run: func(cmd *cobra.Command, args []string) { 16 | mustCreateClient() 17 | if res, err := client.Ping(context.Background(), &pb.PingReq{Message: "ping from xkctl"}); err != nil { 18 | log.Fatal(err) 19 | } else { 20 | log.Infof("ping finished central response is %s", res.Message) 21 | } 22 | }, 23 | } 24 | -------------------------------------------------------------------------------- /_legacy/cmd/xkd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/cmd/daemon" 5 | ) 6 | 7 | // The daemon 8 | func main() { 9 | daemon.Execute() 10 | } 11 | -------------------------------------------------------------------------------- /_legacy/cmd/xkutil/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/cmd/util" 5 | ) 6 | 7 | func main() { 8 | util.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /_legacy/pkg/README.md: -------------------------------------------------------------------------------- 1 | # Internal Golang packages 2 | 3 | Deprecated, see [../xk](../xk) -------------------------------------------------------------------------------- /_legacy/pkg/cmd/daemon/schema.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | "github.com/xephonhq/xephon-k/pkg/storage/cassandra" 6 | ) 7 | 8 | var SchemaCmd = &cobra.Command{ 9 | Use: "schema", 10 | Short: "Create schema", 11 | Run: func(cmd *cobra.Command, args []string) { 12 | // FIXME: a dirty way to use different host when create schema 13 | cassandra.CassandraHost = config.Storage.Cassandra.Host 14 | log.Info("create schema for cassandra using default setting") 15 | cassandra.CreateSchema() 16 | log.Info("schema created!") 17 | }, 18 | } 19 | -------------------------------------------------------------------------------- /_legacy/pkg/cmd/pkg.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | "github.com/xephonhq/xephon-k/pkg/util" 6 | ) 7 | 8 | var ( 9 | debug = false 10 | ) 11 | 12 | var log = util.Logger.NewEntryWithPkg("k.cmd") 13 | 14 | var Banner = ` 15 | ▄ ▄ ▄▄▄▄▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄▄▄▄▄ ▄ ▄ ▄▄▄▄▄▄▄▄▄▄▄ ▄▄ ▄ ▄ ▄ 16 | ▐░▌ ▐░▌▐░░░░░░░░░░░▌▐░░░░░░░░░░░▌▐░▌ ▐░▌▐░░░░░░░░░░░▌▐░░▌ ▐░▌ ▐░▌ ▐░▌ 17 | ▐░▌ ▐░▌ ▐░█▀▀▀▀▀▀▀▀▀ ▐░█▀▀▀▀▀▀▀█░▌▐░▌ ▐░▌▐░█▀▀▀▀▀▀▀█░▌▐░▌░▌ ▐░▌ ▐░▌ ▐░▌ 18 | ▐░▌ ▐░▌ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░▌▐░▌ ▐░▌▐░▌▐░▌ ▐░▌ ▐░▌▐░▌ 19 | ▐░▐░▌ ▐░█▄▄▄▄▄▄▄▄▄ ▐░█▄▄▄▄▄▄▄█░▌▐░█▄▄▄▄▄▄▄█░▌▐░▌ ▐░▌▐░▌ ▐░▌ ▐░▌ ▄▄▄▄▄▄▄▄▄▄▄ ▐░▌░▌ 20 | ▐░▌ ▐░░░░░░░░░░░▌▐░░░░░░░░░░░▌▐░░░░░░░░░░░▌▐░▌ ▐░▌▐░▌ ▐░▌ ▐░▌▐░░░░░░░░░░░▌▐░░▌ 21 | ▐░▌░▌ ▐░█▀▀▀▀▀▀▀▀▀ ▐░█▀▀▀▀▀▀▀▀▀ ▐░█▀▀▀▀▀▀▀█░▌▐░▌ ▐░▌▐░▌ ▐░▌ ▐░▌ ▀▀▀▀▀▀▀▀▀▀▀ ▐░▌░▌ 22 | ▐░▌ ▐░▌ ▐░▌ ▐░▌ ▐░▌ ▐░▌▐░▌ ▐░▌▐░▌ ▐░▌▐░▌ ▐░▌▐░▌ 23 | ▐░▌ ▐░▌ ▐░█▄▄▄▄▄▄▄▄▄ ▐░▌ ▐░▌ ▐░▌▐░█▄▄▄▄▄▄▄█░▌▐░▌ ▐░▐░▌ ▐░▌ ▐░▌ 24 | ▐░▌ ▐░▌▐░░░░░░░░░░░▌▐░▌ ▐░▌ ▐░▌▐░░░░░░░░░░░▌▐░▌ ▐░░▌ ▐░▌ ▐░▌ 25 | ▀ ▀ ▀▀▀▀▀▀▀▀▀▀▀ ▀ ▀ ▀ ▀▀▀▀▀▀▀▀▀▀▀ ▀ ▀▀ ▀ ▀ 26 | ` 27 | 28 | func init() { 29 | cobra.OnInitialize(initConfig) 30 | } 31 | 32 | func initConfig() { 33 | if debug { 34 | util.UseVerboseLog() 35 | } 36 | // TODO: configuration is not supported yet 37 | // viper.AutomaticEnv() 38 | // // TODO: check file existence 39 | // viper.SetConfigFile(configFile) 40 | // err := viper.ReadInConfig() 41 | // if err != nil { 42 | // log.Warn(err) 43 | // } else { 44 | // log.Debugf("config file %s is loaded", configFile) 45 | // } 46 | } 47 | -------------------------------------------------------------------------------- /_legacy/pkg/cmd/util/root.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | "github.com/xephonhq/xephon-k/pkg/storage/disk" 6 | "github.com/xephonhq/xephon-k/pkg/util" 7 | "os" 8 | ) 9 | 10 | var ( 11 | debug = false 12 | ) 13 | 14 | var log = util.Logger.NewEntryWithPkg("k.cmd.util") 15 | 16 | var RootCmd = &cobra.Command{ 17 | Use: "xkutil", 18 | Short: "Xephon K Util", 19 | Long: "xkutil can be used for inspect and dump data file generated by Xephon K", 20 | Run: func(cmd *cobra.Command, args []string) { 21 | //log.Info(args) 22 | if len(args) < 1 { 23 | log.Info("must provide file to inspect") 24 | return 25 | } 26 | filePath := args[0] 27 | log.Infof("inspect file %s", filePath) 28 | f, err := os.OpenFile(filePath, os.O_RDONLY, 0666) 29 | assert(err) 30 | // TODO: the magic check is wrong, it will try to read glide.yaml 31 | // TODO: print too much .... 32 | r, err := disk.NewLocalDataFileReader(f) 33 | assert(err) 34 | //r.PrintAll() 35 | r.PrintAbstract() 36 | r.Close() 37 | }, 38 | } 39 | 40 | func assert(err error) { 41 | if err != nil { 42 | log.Fatal(err) 43 | } 44 | } 45 | 46 | func Execute() { 47 | if RootCmd.Execute() != nil { 48 | os.Exit(-1) 49 | } 50 | } 51 | 52 | func init() { 53 | cobra.OnInitialize(initConfig) 54 | 55 | RootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "debug") 56 | } 57 | 58 | func initConfig() { 59 | if debug { 60 | util.UseVerboseLog() 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /_legacy/pkg/cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | "github.com/xephonhq/xephon-k/pkg" 8 | ) 9 | 10 | var VersionCmd = &cobra.Command{ 11 | Use: "version", 12 | Short: "Show Xephon-K version", 13 | Run: func(cmd *cobra.Command, args []string) { 14 | fmt.Println(pkg.Version) 15 | }, 16 | } 17 | -------------------------------------------------------------------------------- /_legacy/pkg/common/.ayi.yml: -------------------------------------------------------------------------------- 1 | scripts: 2 | gen: 3 | # FIXME: use GOPATH (which need sh -c) 4 | - protoc --proto_path=/home/at15/workspace/src/:. --gogo_out=. common.proto 5 | - go run gen/main.go -------------------------------------------------------------------------------- /_legacy/pkg/common/.gitignore: -------------------------------------------------------------------------------- 1 | # temporarily disable generated files 2 | #*.pb.go 3 | #*_raw.go 4 | #*_double.go -------------------------------------------------------------------------------- /_legacy/pkg/common/common.proto: -------------------------------------------------------------------------------- 1 | // go get github.com/gogo/protobuf/protoc-gen-gogo 2 | // protoc --proto_path=/home/at15/workspace/src/:. --gogo_out=. common.proto 3 | syntax = "proto3"; 4 | 5 | package common; 6 | 7 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 8 | 9 | option (gogoproto.sizer_all) = true; 10 | option (gogoproto.marshaler_all) = true; 11 | option (gogoproto.unmarshaler_all) = true; 12 | option (gogoproto.goproto_getters_all) = false; 13 | 14 | message SeriesMeta { 15 | uint64 id = 1; 16 | int64 type = 2; 17 | int64 precision = 3; 18 | string name = 4; 19 | map tags = 5; 20 | } 21 | 22 | message IntPoint { 23 | int64 T = 1; 24 | int64 V = 2; 25 | } 26 | 27 | message IntSeries { 28 | SeriesMeta meta = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true]; 29 | repeated IntPoint points = 2 [(gogoproto.nullable) = false]; 30 | } 31 | 32 | message IntSeriesColumnar { 33 | SeriesMeta meta = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true]; 34 | repeated int64 T = 2; 35 | repeated int64 V = 3; 36 | } 37 | 38 | message DoublePoint { 39 | int64 T = 1; 40 | double V = 2; 41 | } 42 | 43 | message DoubleSeries { 44 | SeriesMeta meta = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true]; 45 | repeated DoublePoint points = 2 [(gogoproto.nullable) = false]; 46 | } 47 | 48 | message DoubleSeriesColumnar { 49 | SeriesMeta meta = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true]; 50 | repeated int64 T = 2; 51 | repeated double V = 3; 52 | } -------------------------------------------------------------------------------- /_legacy/pkg/common/doc.go: -------------------------------------------------------------------------------- 1 | package common 2 | -------------------------------------------------------------------------------- /_legacy/pkg/common/gen/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func assert(err error) { 11 | if err != nil { 12 | fmt.Println(err) 13 | os.Exit(1) 14 | } 15 | } 16 | 17 | // NOTE: it should be called from upper folder 18 | // Use `go run gen/main.go` instead of `go run main.go` 19 | func main() { 20 | fmt.Println("generate other series based on int series") 21 | 22 | // int series is used as template 23 | tplBytes, err := ioutil.ReadFile("series_int.go") 24 | assert(err) 25 | tpl := string(tplBytes) 26 | // FIXME: raw series can not have methods like GetMaxMinTime 27 | // otherSeriesTypes := map[string]string{"RawSeries": "series_raw.go", "DoubleSeries": "series_double.go"} 28 | otherSeriesTypes := map[string]string{"DoubleSeries": "series_double.go"} 29 | 30 | for newType, newFile := range otherSeriesTypes { 31 | content := strings.Replace(tpl, "IntSeries", newType, -1) 32 | data := append([]byte("// Generated from series_int.go DO NOT EDIT!\n"), []byte(content)...) 33 | err = ioutil.WriteFile(newFile, data, 0644) 34 | assert(err) 35 | } 36 | 37 | fmt.Println("finished") 38 | 39 | } 40 | -------------------------------------------------------------------------------- /_legacy/pkg/common/hash.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "sort" 4 | 5 | const ( 6 | prime64 = 1099511628211 7 | offset64 = 14695981039346656037 8 | ) 9 | 10 | // check interface 11 | var _ Hashable = (*IntSeries)(nil) 12 | var _ Hashable = (*DoubleSeries)(nil) 13 | var _ Hashable = (*Query)(nil) 14 | 15 | // TODO: hashable can be changed to get meta or generate methods for series 16 | // TODO: also gogo protobuf support generate as embedded fields 17 | type Hashable interface { 18 | GetName() string 19 | GetTags() map[string]string 20 | } 21 | 22 | // Hash returns one result for series/query have same name and tags 23 | func Hash(src Hashable) SeriesID { 24 | h := NewInlineFNV64a() 25 | h.Write([]byte(src.GetName())) 26 | tags := src.GetTags() 27 | keys := make([]string, len(tags)) 28 | // TODO: more efficient way for hashing, every time we hash, we sort it. And using []byte might be better than string 29 | // NOTE: we need to sort the key to get deterministic result 30 | i := 0 31 | for k := range tags { 32 | keys[i] = k 33 | i++ 34 | } 35 | sort.Strings(keys) 36 | for _, k := range keys { 37 | h.Write([]byte(k)) 38 | h.Write([]byte(tags[k])) 39 | } 40 | return SeriesID(h.Sum64()) 41 | } 42 | 43 | // InlineFNV64a is copied from influxdb/models, which is a alloc-free version of pkg/hash/fnv 44 | // FNV is non crypto hash, so it's faster than MD5 see playground/hash_test.go for benchmark 45 | type InlineFNV64a uint64 46 | 47 | // NewInlineFNV64a returns a new instance of InlineFNV64a. 48 | func NewInlineFNV64a() InlineFNV64a { 49 | return offset64 50 | } 51 | 52 | // Write adds data to the running hash. 53 | func (s *InlineFNV64a) Write(data []byte) (int, error) { 54 | hash := uint64(*s) 55 | for _, c := range data { 56 | hash ^= uint64(c) 57 | hash *= prime64 58 | } 59 | *s = InlineFNV64a(hash) 60 | return len(data), nil 61 | } 62 | 63 | // Sum64 returns the uint64 of the current resulting hash. 64 | func (s *InlineFNV64a) Sum64() uint64 { 65 | return uint64(*s) 66 | } 67 | -------------------------------------------------------------------------------- /_legacy/pkg/common/pkg.go: -------------------------------------------------------------------------------- 1 | package common 2 | -------------------------------------------------------------------------------- /_legacy/pkg/common/point.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strconv" 7 | ) 8 | 9 | // NOTE: The following struct for point are generated by proto in common.pb.go 10 | // IntPoint 11 | // DoublePoint 12 | 13 | // MarshalJSON implements Marshaler interface. Pointed is encoded as number array. i.e. [1359788400000,1] 14 | // https://golang.org/pkg/encoding/json/#Marshaler 15 | func (p *IntPoint) MarshalJSON() ([]byte, error) { 16 | return []byte(fmt.Sprintf("[%d,%d]", p.T, p.V)), nil 17 | } 18 | 19 | func (p *IntPoint) MarshalJSON2() ([]byte, error) { 20 | return json.Marshal([2]int64{p.T, p.V}) 21 | } 22 | 23 | func (p *IntPoint) MarshalJSON3() ([]byte, error) { 24 | return []byte("[" + strconv.FormatInt(p.T, 10) + "," + strconv.FormatInt(p.V, 10) + "]"), nil 25 | } 26 | 27 | // UnmarshalJSON implements Unmarshaler interface 28 | // https://golang.org/pkg/encoding/json/#Unmarshaler 29 | func (p *IntPoint) UnmarshalJSON(data []byte) error { 30 | var tv [2]json.Number 31 | if err := json.Unmarshal(data, &tv); err != nil { 32 | return err 33 | } 34 | // FIXME: this error checking seems to be very low efficient 35 | t, err := tv[0].Int64() 36 | p.T = t 37 | if err != nil { 38 | return err 39 | } 40 | v, err := tv[1].Int64() 41 | p.V = v 42 | if err != nil { 43 | return err 44 | } 45 | return nil 46 | } 47 | 48 | // MarshalJSON implements Marshaler interface. Point is encoded as number array. i.e. [] 49 | func (p *DoublePoint) MarshalJSON() ([]byte, error) { 50 | // TODO: precision of double value, need to copy the code in `json/encode.go` 51 | return []byte(fmt.Sprintf("[%d,%f]", p.T, p.V)), nil 52 | } 53 | 54 | // UnmarshalJSON implements Unmarshaler interface. 55 | func (p *DoublePoint) UnmarshalJSON(data []byte) error { 56 | var tv [2]json.Number 57 | if err := json.Unmarshal(data, &tv); err != nil { 58 | return err 59 | } 60 | t, err := tv[0].Int64() 61 | if err != nil { 62 | return err 63 | } 64 | p.T = t 65 | v, err := tv[1].Float64() 66 | if err != nil { 67 | return err 68 | } 69 | p.V = v 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /_legacy/pkg/common/query.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type Filter struct { 10 | Type string `json:"type"` 11 | Key string `json:"key"` 12 | Value string `json:"value,omitempty"` 13 | Values []string `json:"values,omitempty"` 14 | LeftRaw json.RawMessage `json:"l,omitempty"` 15 | RightRaw json.RawMessage `json:"r,omitempty"` 16 | LeftOperand *Filter `json:"-"` // NOTE: must use pointer to avoid invalid recursive type Filter 17 | RightOperand *Filter `json:"-"` 18 | } 19 | 20 | type filterAlias Filter 21 | 22 | type Aggregator struct { 23 | Type string `json:"type"` 24 | Window string `json:"window"` // TODO: change to time.Duration? or WindowRaw and Window with time.Duration 25 | } 26 | 27 | // Query is the query against single series if in `exact` mode, possible multiple series 28 | // in `contains` mode 29 | type Query struct { 30 | Name string `json:"name"` 31 | Tags map[string]string `json:"tags"` 32 | MatchPolicy string `json:"match_policy"` 33 | StartTime int64 `json:"start_time,omitempty"` 34 | EndTime int64 `json:"end_time,omitempty"` 35 | Filter Filter `json:"filter,omitempty"` 36 | Aggregator Aggregator `json:"aggregator,omitempty"` 37 | } 38 | 39 | // QueryResult contains the original query and number of series matched 40 | type QueryResult struct { 41 | Query 42 | Matched int `json:"matched"` 43 | } 44 | 45 | func (query *Query) GetName() string { 46 | return query.Name 47 | } 48 | 49 | func (query *Query) GetTags() map[string]string { 50 | return query.Tags 51 | } 52 | 53 | func (filter *Filter) UnmarshalJSON(data []byte) error { 54 | // NOTE: need to use Alias like readRequest in `/server/service/read.go`, otherwise stackoverflow 55 | a := (*filterAlias)(filter) 56 | err := json.Unmarshal(data, a) 57 | if err != nil { 58 | return err 59 | } 60 | if len(filter.LeftRaw) > 0 { 61 | var leftOperand Filter 62 | err := json.Unmarshal(filter.LeftRaw, &leftOperand) 63 | if err != nil { 64 | return errors.Wrap(err, "can't parse left operand") 65 | } 66 | filter.LeftOperand = &leftOperand 67 | } 68 | if len(filter.RightRaw) > 0 { 69 | var rightOperand Filter 70 | err := json.Unmarshal(filter.RightRaw, &rightOperand) 71 | if err != nil { 72 | return errors.Wrap(err, "can't parse right operand") 73 | } 74 | filter.RightOperand = &rightOperand 75 | } 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /_legacy/pkg/common/series_double.go: -------------------------------------------------------------------------------- 1 | // Generated from series_int.go DO NOT EDIT! 2 | package common 3 | 4 | import ( 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | // TODO: int series of other precision, maybe we should add millisecond to the default function as well 10 | func NewDoubleSeries(name string) *DoubleSeries { 11 | return &DoubleSeries{ 12 | SeriesMeta: SeriesMeta{ 13 | Name: name, 14 | Tags: map[string]string{}, 15 | Type: TypeDoubleSeries, 16 | Precision: time.Millisecond.Nanoseconds(), 17 | }, 18 | } 19 | } 20 | 21 | func (m *DoubleSeries) GetMinTime() int64 { 22 | if len(m.Points) == 0 { 23 | return 0 24 | } 25 | return m.Points[0].T 26 | } 27 | 28 | func (m *DoubleSeries) GetMaxTime() int64 { 29 | if len(m.Points) == 0 { 30 | return 0 31 | } 32 | return m.Points[len(m.Points)-1].T 33 | } 34 | 35 | func (m *DoubleSeries) Length() int { 36 | return len(m.Points) 37 | } 38 | 39 | func (m *DoubleSeries) PrintPoints() { 40 | for i, p := range m.Points { 41 | fmt.Printf("%d, %d, %v\n", i, p.T, p.V) 42 | } 43 | } 44 | 45 | func (m *DoubleSeriesColumnar) GetMinTime() int64 { 46 | if len(m.T) == 0 { 47 | return 0 48 | } 49 | return m.T[0] 50 | } 51 | 52 | func (m *DoubleSeriesColumnar) GetMaxTime() int64 { 53 | if len(m.T) == 0 { 54 | return 0 55 | } 56 | return m.T[len(m.T)-1] 57 | } 58 | 59 | func (m *DoubleSeriesColumnar) IsColumnar() bool { 60 | return true 61 | } 62 | 63 | func (m *DoubleSeriesColumnar) Length() int { 64 | return len(m.T) 65 | } 66 | 67 | func (m *DoubleSeriesColumnar) PrintPoints() { 68 | for i := 0; i < len(m.T); i++ { 69 | fmt.Printf("%d, %d, %v\n", i, m.T[i], m.V[i]) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /_legacy/pkg/common/series_id.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // SeriesID is hash result of metric name and (sorted) tags 4 | // TODO: 5 | // - locality sensitive hashing https://github.com/xephonhq/xephon-k/issues/25 6 | // - distributed hashing 7 | // FIXED: 8 | // - use integer instead of string https://github.com/xephonhq/xephon-k/issues/36 9 | type SeriesID uint64 10 | -------------------------------------------------------------------------------- /_legacy/pkg/common/series_int.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // TODO: int series of other precision, maybe we should add millisecond to the default function as well 9 | func NewIntSeries(name string) *IntSeries { 10 | return &IntSeries{ 11 | SeriesMeta: SeriesMeta{ 12 | Name: name, 13 | Tags: map[string]string{}, 14 | Type: TypeIntSeries, 15 | Precision: time.Millisecond.Nanoseconds(), 16 | }, 17 | } 18 | } 19 | 20 | func (m *IntSeries) GetMinTime() int64 { 21 | if len(m.Points) == 0 { 22 | return 0 23 | } 24 | return m.Points[0].T 25 | } 26 | 27 | func (m *IntSeries) GetMaxTime() int64 { 28 | if len(m.Points) == 0 { 29 | return 0 30 | } 31 | return m.Points[len(m.Points)-1].T 32 | } 33 | 34 | func (m *IntSeries) Length() int { 35 | return len(m.Points) 36 | } 37 | 38 | func (m *IntSeries) PrintPoints() { 39 | for i, p := range m.Points { 40 | fmt.Printf("%d, %d, %v\n", i, p.T, p.V) 41 | } 42 | } 43 | 44 | func (m *IntSeriesColumnar) GetMinTime() int64 { 45 | if len(m.T) == 0 { 46 | return 0 47 | } 48 | return m.T[0] 49 | } 50 | 51 | func (m *IntSeriesColumnar) GetMaxTime() int64 { 52 | if len(m.T) == 0 { 53 | return 0 54 | } 55 | return m.T[len(m.T)-1] 56 | } 57 | 58 | func (m *IntSeriesColumnar) IsColumnar() bool { 59 | return true 60 | } 61 | 62 | func (m *IntSeriesColumnar) Length() int { 63 | return len(m.T) 64 | } 65 | 66 | func (m *IntSeriesColumnar) PrintPoints() { 67 | for i := 0; i < len(m.T); i++ { 68 | fmt.Printf("%d, %d, %v\n", i, m.T[i], m.V[i]) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /_legacy/pkg/config/bench.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/xephonhq/xephon-k/pkg/bench/generator" 6 | "github.com/xephonhq/xephon-k/pkg/bench/loader" 7 | "github.com/xephonhq/xephon-k/pkg/client" 8 | "github.com/xephonhq/xephon-k/pkg/util" 9 | ) 10 | 11 | // check interface 12 | var _ Config = (*BenchConfig)(nil) 13 | 14 | type BenchConfig struct { 15 | Log util.LogConfig `yaml:"log" json:"log"` 16 | Mode string `yaml:"mode" json:"mode"` 17 | Loader loader.Config `yaml:"loader" json:"loader"` 18 | Generator generator.Config `yaml:"generator"` 19 | Targets Targets `yaml:"targets"` 20 | XXX map[string]interface{} `yaml:",inline"` 21 | } 22 | 23 | type Targets struct { 24 | InfluxDB client.Config `yaml:"influxdb"` 25 | XephonK client.Config `yaml:"xephonk"` 26 | KairosDB client.Config `yaml:"kairosdb"` 27 | } 28 | 29 | // avoid recursion in UnmarshalYAML 30 | type benchConfigAlias BenchConfig 31 | 32 | func NewBench() BenchConfig { 33 | return BenchConfig{ 34 | Log: util.NewLogConfig(), 35 | Mode: "local", 36 | Loader: loader.NewConfig(), 37 | Targets: Targets{ 38 | InfluxDB: client.Config{ 39 | Host: "localhost", 40 | Port: 8086, 41 | URL: "write?db=sb", 42 | Timeout: 30, 43 | }, XephonK: client.Config{ 44 | Host: "localhost", 45 | Port: 2333, 46 | URL: "write", 47 | Timeout: 30, 48 | }, KairosDB: client.Config{ 49 | Host: "localhost", 50 | Port: 8080, 51 | URL: "api/v1/datapoints", 52 | Timeout: 30, 53 | }}, 54 | } 55 | } 56 | 57 | func (c *BenchConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { 58 | a := (*benchConfigAlias)(c) 59 | if err := unmarshal(a); err != nil { 60 | return err 61 | } 62 | if len(c.XXX) != 0 { 63 | return errors.Errorf("undefined fields %v", c.XXX) 64 | } 65 | return nil 66 | } 67 | 68 | func (c *BenchConfig) Apply() error { 69 | if err := c.Validate(); err != nil { 70 | return err 71 | } 72 | if err := c.Log.Apply(); err != nil { 73 | return err 74 | } 75 | return nil 76 | } 77 | 78 | func (c *BenchConfig) Validate() error { 79 | if err := c.Log.Validate(); err != nil { 80 | return err 81 | } 82 | if c.Mode != "local" { 83 | return errors.Errorf("only support local mode, got %s instead", c.Mode) 84 | } 85 | if err := c.Loader.Validate(); err != nil { 86 | return err 87 | } 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /_legacy/pkg/config/bench_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "github.com/xephonhq/xephon-k/pkg/util" 6 | "gopkg.in/yaml.v2" 7 | "testing" 8 | ) 9 | 10 | func TestBenchConfig_UnmarshalYAML(t *testing.T) { 11 | assert := asst.New(t) 12 | exampleConfig := util.ReadAsBytes(t, "xkb.yml") 13 | c := NewBench() 14 | err := yaml.Unmarshal(exampleConfig, &c) 15 | assert.Nil(err) 16 | assert.Nil(c.Validate()) 17 | assert.Nil(c.Apply()) 18 | } 19 | -------------------------------------------------------------------------------- /_legacy/pkg/config/daemon.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/xephonhq/xephon-k/pkg/server" 6 | "github.com/xephonhq/xephon-k/pkg/server/grpc" 7 | "github.com/xephonhq/xephon-k/pkg/server/http" 8 | "github.com/xephonhq/xephon-k/pkg/storage" 9 | "github.com/xephonhq/xephon-k/pkg/storage/cassandra" 10 | "github.com/xephonhq/xephon-k/pkg/storage/disk" 11 | "github.com/xephonhq/xephon-k/pkg/storage/memory" 12 | "github.com/xephonhq/xephon-k/pkg/util" 13 | ) 14 | 15 | // check interface 16 | var _ Config = (*DaemonConfig)(nil) 17 | var _ Config = (*util.LogConfig)(nil) 18 | var _ Config = (*storage.Config)(nil) 19 | var _ Config = (*memory.Config)(nil) 20 | var _ Config = (*disk.Config)(nil) 21 | var _ Config = (*cassandra.Config)(nil) 22 | var _ Config = (*server.Config)(nil) 23 | var _ Config = (*http.Config)(nil) 24 | var _ Config = (*grpc.Config)(nil) 25 | 26 | type DaemonConfig struct { 27 | Log util.LogConfig `yaml:"log" json:"log"` 28 | Storage storage.Config `yaml:"storage" json:"storage"` 29 | Server server.Config `yaml:"server" json:"server"` 30 | XXX map[string]interface{} `yaml:",inline"` 31 | } 32 | 33 | // avoid recursion in UnmarshalYAML 34 | type daemonConfigAlias DaemonConfig 35 | 36 | func NewDaemon() DaemonConfig { 37 | return DaemonConfig{ 38 | Log: util.NewLogConfig(), 39 | Storage: storage.NewConfig(), 40 | Server: server.NewConfig(), 41 | } 42 | } 43 | 44 | func (c *DaemonConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { 45 | a := (*daemonConfigAlias)(c) 46 | if err := unmarshal(a); err != nil { 47 | return err 48 | } 49 | if len(c.XXX) != 0 { 50 | return errors.Errorf("undefined fields %v", c.XXX) 51 | } 52 | return nil 53 | } 54 | 55 | func (c *DaemonConfig) Apply() error { 56 | if err := c.Validate(); err != nil { 57 | return err 58 | } 59 | if err := c.Log.Apply(); err != nil { 60 | return err 61 | } 62 | if err := c.Storage.Apply(); err != nil { 63 | return err 64 | } 65 | if err := c.Server.Apply(); err != nil { 66 | return err 67 | } 68 | return nil 69 | } 70 | 71 | func (c *DaemonConfig) Validate() error { 72 | if err := c.Log.Validate(); err != nil { 73 | return err 74 | } 75 | if err := c.Storage.Validate(); err != nil { 76 | return err 77 | } 78 | if err := c.Server.Validate(); err != nil { 79 | return err 80 | } 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /_legacy/pkg/config/daemon_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "github.com/xephonhq/xephon-k/pkg/util" 6 | "gopkg.in/yaml.v2" 7 | "testing" 8 | ) 9 | 10 | func TestDaemonConfig_UnmarshalYAML(t *testing.T) { 11 | assert := asst.New(t) 12 | exampleConfig := util.ReadAsBytes(t, "xkd.yml") 13 | c := NewDaemon() 14 | err := yaml.Unmarshal(exampleConfig, &c) 15 | assert.Nil(err) 16 | assert.Nil(c.Validate()) 17 | assert.Nil(c.Apply()) 18 | } 19 | -------------------------------------------------------------------------------- /_legacy/pkg/config/pkg.go: -------------------------------------------------------------------------------- 1 | // Package config contains application config for daemon and bench 2 | package config 3 | 4 | import ( 5 | "gopkg.in/yaml.v2" 6 | ) 7 | 8 | type Config interface { 9 | yaml.Unmarshaler 10 | Apply() error 11 | Validate() error 12 | // TODO: we can't have original because YAML does not have []byte like in JSON for Unmarshaler 13 | // TODO: we call validate in apply, which trigger validate of all the children, then the apply of all the children 14 | // also trigger the validate of their own and their children, the lowest level would have its validate called n times 15 | // where n is their nested level 16 | } 17 | -------------------------------------------------------------------------------- /_legacy/pkg/config/xkb.yml: -------------------------------------------------------------------------------- 1 | # Xephon-K benchmark config example 2 | log: 3 | level: info 4 | color: true 5 | source: false 6 | # local/remote 7 | # TODO: remote is not supported, but we should, and it should also be able to spawn the needed resources 8 | mode: local 9 | loader: 10 | target: xephonk 11 | # discard | basic 12 | reporter: basic 13 | # limitBy: time | points 14 | limitBy: time 15 | points: 100000000 16 | series: 100 17 | time: 10 18 | workerNum: 10 19 | workerTimeout: 30 20 | # TODO: qps is not supported 21 | # qps: -1 22 | generator: 23 | # TODO: minimal uint is 1s 24 | timeInterval: 1 25 | timeNoise: false 26 | # int: 27 | # max: 10 28 | # min: 1 29 | # double: 30 | # max: 1.0 31 | # min: 2.2 32 | pointsPerSeries: 10000 33 | numSeries: 10 34 | # int: 0.1 35 | # double: 0.9 36 | #reporter: 37 | # report to tsdb 38 | # aggregation interval etc. 39 | # interval: 40 | targets: 41 | influxdb: 42 | host: localhost 43 | port: 8086 44 | url: write?db=xb 45 | timeout: 30 46 | xephonk: 47 | host: localhost 48 | port: 2333 49 | url: write 50 | timeout: 30 51 | kairosdb: 52 | host: localhost 53 | port: 8080 54 | url: api/v1/datapoints 55 | timeout: 30 56 | 57 | -------------------------------------------------------------------------------- /_legacy/pkg/config/xkd.yml: -------------------------------------------------------------------------------- 1 | # Xephon-K Daemon config example 2 | # NOTE: logging block is same in both daemon and bench 3 | log: 4 | level: info 5 | color: true 6 | source: false 7 | # TODO: wait until filter in gommon is usable 8 | # filter: k.server.xxx 9 | # TODO: wait until human package is added to gommon 10 | # human: true 11 | storage: 12 | memory: 13 | # row/column 14 | layout: row 15 | # TODO: custom type with custom marshaler and unmarshaler 16 | # 1 KB 1024 1 MB 1048576 17 | # 100 MB 18 | chunkSize: 104857600 19 | enableIndex: true 20 | disk: 21 | # /home/at15/workspace/ssd/lib/xephonk 22 | folder: /tmp 23 | concurrentWriteFiles: 1 24 | # 1 KB 1024 1 MB 1048576 25 | # 512 MB 26 | singleFileSize: 536870912 27 | fileBufferSize: 1048576 28 | # TODO: buffer size 29 | encoding: 30 | time: raw-big 31 | int: rle 32 | double: var 33 | cassandra: 34 | host: localhost 35 | port: 9042 36 | keyspace: xephonk 37 | # TODO: cache, concurrent write etc. 38 | server: 39 | http: 40 | host: localhost 41 | port: 2333 42 | enabled: true 43 | enablePprof: false 44 | grpc: 45 | host: localhost 46 | port: 2334 47 | enabled: true 48 | 49 | -------------------------------------------------------------------------------- /_legacy/pkg/encoding/delta_rle_test.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestDeltaRLEEncoder_WriteTime(t *testing.T) { 9 | assert := asst.New(t) 10 | var ( 11 | encoder TimeEncoder 12 | b []byte 13 | ) 14 | encoder = NewDeltaRLEEncoder() 15 | num := 100 16 | b = encodeNanoseconds(t, num, encoder) 17 | //t.Log(b) 18 | // 1 byte for codec 19 | // 9 byte for the first timestamp, 1 for its run length 20 | // 5 byte for 1,000, 000, 000 (nanosecond), num is 100, so length is 99, 1 byte 21 | assert.Equal(1+9+1+5+1, len(b)) 22 | // [6 202 172 250 165 146 253 152 200 41 1 128 168 214 185 7 99] 23 | // [6 252 209 137 246 233 253 152 200 41 1 128 168 214 185 7 99] 24 | b = encodeSeconds(t, num, encoder) 25 | //t.Log(b) 26 | // 1 byte for codec 27 | // 5 byte for first timestamp, 1 for its run length 28 | // 1 byte for 1, 29 | // FIXED: why it's 2 instead of 1, length is 99 30 | // because it is using zig-zag encoding !!! 31 | // [6 170 214 144 148 11 1 2 99] 32 | // [6 194 214 144 148 11 1 2 99] 33 | // [6 208 222 144 148 11 1 2 99] 34 | assert.Equal(1+5+1+1+1, len(b)) 35 | } 36 | 37 | func TestDeltaRLEDecoder_ReadTime(t *testing.T) { 38 | assert := asst.New(t) 39 | num := 100 40 | encoded := encodeNanoseconds(t, num, NewDeltaRLEEncoder()) 41 | decoder := NewDeltaRLEDecoder() 42 | assert.Nil(decoder.Init(encoded)) 43 | i := 0 44 | for decoder.Next() { 45 | i++ 46 | decoder.ReadTime() 47 | } 48 | assert.Equal(i, num) 49 | } 50 | -------------------------------------------------------------------------------- /_legacy/pkg/encoding/delta_test.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestDeltaEncoder_WriteTime(t *testing.T) { 9 | assert := asst.New(t) 10 | var ( 11 | encoder TimeEncoder 12 | b []byte 13 | ) 14 | encoder = NewDeltaEncoder() 15 | num := 100 16 | b = encodeNanoseconds(t, num, encoder) 17 | // delta is 1,000,000,000, which takes 5 byte 18 | // the first value is 9 byte, 1 byte larger than the actual 8 byte 19 | // 1 byte for codec 20 | assert.Equal(5*(num-1)+9+1, len(b)) 21 | b = encodeSeconds(t, num, encoder) 22 | // delta is 1 23 | // the first value is 5 byte 24 | assert.Equal(1*(num-1)+5+1, len(b)) 25 | } 26 | 27 | func TestDeltaDecoder_ReadTime(t *testing.T) { 28 | assert := asst.New(t) 29 | num := 100 30 | encoded := encodeNanoseconds(t, num, NewDeltaEncoder()) 31 | decoder := NewDeltaDecoder() 32 | assert.Nil(decoder.Init(encoded)) 33 | i := 0 34 | for decoder.Next() { 35 | i++ 36 | decoder.ReadTime() 37 | } 38 | assert.Equal(i, num) 39 | } 40 | -------------------------------------------------------------------------------- /_legacy/pkg/encoding/doc.go: -------------------------------------------------------------------------------- 1 | // Package encoding is used for compress/decompress time series data to/from raw bytes 2 | // Tracker: https://github.com/xephonhq/xephon-k/issues/47 3 | package encoding 4 | -------------------------------------------------------------------------------- /_legacy/pkg/encoding/raw_test.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | "testing" 5 | 6 | asst "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestRawBinaryEncoder_WriteTime(t *testing.T) { 10 | assert := asst.New(t) 11 | 12 | var ( 13 | encoder TimeEncoder 14 | b []byte 15 | ) 16 | encoder = NewBigEndianBinaryEncoder() 17 | num := 100 18 | b = encodeNanoseconds(t, num, encoder) 19 | // codec + value 20 | assert.Equal(8*num+1, len(b)) 21 | 22 | encoder = NewLittleEndianBinaryEncoder() 23 | b = encodeNanoseconds(t, num, encoder) 24 | assert.Equal(8*num+1, len(b)) 25 | } 26 | 27 | func TestRawBinaryDecoder_ReadTime(t *testing.T) { 28 | assert := asst.New(t) 29 | num := 100 30 | encoded := encodeNanoseconds(t, num, NewBigEndianBinaryEncoder()) 31 | decoder := NewRawBinaryDecoder() 32 | assert.Nil(decoder.Init(encoded)) 33 | i := 0 34 | for decoder.Next() { 35 | i++ 36 | decoder.ReadTime() 37 | } 38 | assert.Equal(i, num) 39 | } 40 | -------------------------------------------------------------------------------- /_legacy/pkg/encoding/rle_test.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | 6 | "testing" 7 | ) 8 | 9 | func TestRLEEncoder_WriteTime(t *testing.T) { 10 | assert := asst.New(t) 11 | var ( 12 | encoder TimeEncoder 13 | b []byte 14 | ) 15 | encoder = NewRLEEncoder() 16 | num := 100 17 | b = encodeNanoseconds(t, num, encoder) 18 | // NOTE: 9 byte for value, 1 for length, without delta, RLE is pretty useless for encoding regular interval time 19 | assert.Equal(10*num+1, len(b)) 20 | } 21 | 22 | func TestRLEDecoder_ReadTime(t *testing.T) { 23 | assert := asst.New(t) 24 | num := 100 25 | encoded := encodeNanoseconds(t, num, NewRLEEncoder()) 26 | decoder := NewRLEDecoder() 27 | assert.Nil(decoder.Init(encoded)) 28 | i := 0 29 | for decoder.Next() { 30 | i++ 31 | decoder.ReadTime() 32 | } 33 | assert.Equal(i, num) 34 | } 35 | -------------------------------------------------------------------------------- /_legacy/pkg/encoding/varint_test.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestVarIntEncoder_WriteTime(t *testing.T) { 9 | assert := asst.New(t) 10 | var ( 11 | encoder TimeEncoder 12 | b []byte 13 | ) 14 | encoder = NewVarIntEncoder() 15 | num := 100 16 | b = encodeNanoseconds(t, num, encoder) 17 | // NOTE: when use nanosecond, it takes 9 bytes to encode what is actually 8 bytes, so it becomes larger 18 | assert.Equal(9*num+1, len(b)) 19 | // when using second, it is much smaller than max of int64, so it use 5 bytes (<8) 20 | b = encodeSeconds(t, num, encoder) 21 | assert.True(len(b) < 9*num+1) // 501 22 | } 23 | 24 | func TestVarIntDecoder_ReadTime(t *testing.T) { 25 | assert := asst.New(t) 26 | num := 100 27 | encoded := encodeNanoseconds(t, num, NewVarIntEncoder()) 28 | decoder := NewVarIntDecoder() 29 | assert.Nil(decoder.Init(encoded)) 30 | i := 0 31 | for decoder.Next() { 32 | i++ 33 | decoder.ReadTime() 34 | } 35 | assert.Equal(i, num) 36 | } 37 | -------------------------------------------------------------------------------- /_legacy/pkg/server/config.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/xephonhq/xephon-k/pkg/server/grpc" 6 | "github.com/xephonhq/xephon-k/pkg/server/http" 7 | ) 8 | 9 | type Config struct { 10 | Http http.Config `yaml:"http" json:"http"` 11 | Grpc grpc.Config `yaml:"grpc" json:"grpc"` 12 | XXX map[string]interface{} `yaml:",inline"` 13 | } 14 | 15 | // avoid recursion in UnmarshalYAML 16 | type configAlias Config 17 | 18 | func NewConfig() Config { 19 | return Config{ 20 | Http: http.NewConfig(), 21 | Grpc: grpc.NewConfig(), 22 | } 23 | } 24 | 25 | func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { 26 | a := (*configAlias)(c) 27 | if err := unmarshal(a); err != nil { 28 | return err 29 | } 30 | if len(c.XXX) != 0 { 31 | return errors.Errorf("undefined fields %v", c.XXX) 32 | } 33 | return nil 34 | } 35 | 36 | func (c *Config) Apply() error { 37 | if err := c.Validate(); err != nil { 38 | return err 39 | } 40 | return nil 41 | } 42 | 43 | func (c *Config) Validate() error { 44 | if err := c.Http.Validate(); err != nil { 45 | return err 46 | } 47 | if err := c.Grpc.Validate(); err != nil { 48 | return err 49 | } 50 | if c.Http.Port == c.Grpc.Port { 51 | // FIXME: I remember go-kit can run http and grpc on same port 52 | return errors.New("can't run http and grpc service on same port") 53 | } 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /_legacy/pkg/server/config_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "github.com/xephonhq/xephon-k/pkg/server/http" 6 | "gopkg.in/yaml.v2" 7 | "testing" 8 | ) 9 | 10 | func TestConfig_UnmarshalYAML(t *testing.T) { 11 | assert := asst.New(t) 12 | empty := `` 13 | c := NewConfig() 14 | err := yaml.Unmarshal([]byte(empty), &c) 15 | assert.Nil(err) 16 | assert.Equal(http.DefaultPort, c.Http.Port) 17 | } 18 | -------------------------------------------------------------------------------- /_legacy/pkg/server/doc.go: -------------------------------------------------------------------------------- 1 | // Package server provides RESTful and gRPC API 2 | package server 3 | -------------------------------------------------------------------------------- /_legacy/pkg/server/grpc/config.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import "github.com/pkg/errors" 4 | 5 | const ( 6 | DefaultPort = 2334 7 | MinimalPort = 1024 8 | ) 9 | 10 | // TODO: test, including grpc and http packages 11 | type Config struct { 12 | Host string `yaml:"host" json:"host"` 13 | Port int `yaml:"port" json:"port"` 14 | Enabled bool `yaml:"enabled" json:"enabled"` 15 | XXX map[string]interface{} `yaml:",inline"` 16 | } 17 | 18 | // avoid recursion in UnmarshalYAML 19 | type configAlias Config 20 | 21 | func NewConfig() Config { 22 | return Config{ 23 | Host: "localhost", 24 | Port: DefaultPort, 25 | Enabled: true, 26 | } 27 | } 28 | 29 | func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { 30 | a := (*configAlias)(c) 31 | if err := unmarshal(a); err != nil { 32 | return err 33 | } 34 | if len(c.XXX) != 0 { 35 | return errors.Errorf("undefined fields %v", c.XXX) 36 | } 37 | return nil 38 | } 39 | 40 | func (c *Config) Apply() error { 41 | if err := c.Validate(); err != nil { 42 | return err 43 | } 44 | return nil 45 | } 46 | 47 | func (c *Config) Validate() error { 48 | // TODO: validate the host 49 | if c.Port < MinimalPort { 50 | return errors.Errorf("port number must be greater than %d, got %d instead", MinimalPort, c.Port) 51 | } 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /_legacy/pkg/server/grpc/server.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import ( 4 | "net" 5 | 6 | "fmt" 7 | "github.com/pkg/errors" 8 | pb "github.com/xephonhq/xephon-k/pkg/server/payload" 9 | "github.com/xephonhq/xephon-k/pkg/server/service" 10 | "github.com/xephonhq/xephon-k/pkg/util" 11 | "golang.org/x/net/context" 12 | "google.golang.org/grpc" 13 | "google.golang.org/grpc/reflection" 14 | ) 15 | 16 | var log = util.Logger.NewEntryWithPkg("k.server.grpc") 17 | 18 | type Server struct { 19 | g *grpc.Server 20 | config Config 21 | write *service.WriteService 22 | } 23 | 24 | func NewServer(config Config, write *service.WriteService) *Server { 25 | return &Server{ 26 | config: config, 27 | write: write, 28 | } 29 | } 30 | 31 | // FIXME: the new context in go 1.7? 32 | // https://github.com/grpc/grpc-go/issues/711 33 | func (s *Server) Write(ctx context.Context, req *pb.WriteRequest) (*pb.WriteResponse, error) { 34 | // TODO: is there any information I can log from its context? 35 | return s.write.Write(req) 36 | } 37 | 38 | func (s *Server) Start() error { 39 | // TODO: shutdown the server? 40 | addr := fmt.Sprintf("%s:%d", s.config.Host, s.config.Port) 41 | t, err := net.Listen("tcp", addr) 42 | if err != nil { 43 | return errors.Wrapf(err, "can't start tcp server for grpc on %s", addr) 44 | } 45 | s.g = grpc.NewServer() 46 | pb.RegisterWriteServer(s.g, s) 47 | reflection.Register(s.g) 48 | log.Infof("serve grpc on %s", addr) 49 | if err := s.g.Serve(t); err != nil { 50 | return errors.Wrapf(err, "can't start grpc server on %s", addr) 51 | } 52 | return nil 53 | } 54 | 55 | func (s *Server) Stop() { 56 | // TODO: add a timeout 57 | log.Info("stopping grpc server") 58 | // FIXME: #56 this is nil because the net.Listen has not finished, when other error, like http server failed already happened 59 | // a problem is the tcp server and the grpc server may still try to create go routine leak? ... we can have a mutex 60 | if s.g == nil { 61 | log.Warn("grpc server is not even started, but asked to stop") 62 | return 63 | } 64 | s.g.GracefulStop() 65 | log.Info("grpc server stopped") 66 | } 67 | -------------------------------------------------------------------------------- /_legacy/pkg/server/http/config.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import "github.com/pkg/errors" 4 | 5 | const ( 6 | DefaultPort = 2333 7 | MinimalPort = 1024 8 | ) 9 | 10 | type Config struct { 11 | Host string `yaml:"host" json:"host"` 12 | Port int `yaml:"port" json:"port"` 13 | Enabled bool `yaml:"enabled" json:"enabled"` 14 | EnablePProf bool `yaml:"enablePprof" json:"enablePprof"` 15 | XXX map[string]interface{} `yaml:",inline"` 16 | } 17 | 18 | // avoid recursion in UnmarshalYAML 19 | type configAlias Config 20 | 21 | func NewConfig() Config { 22 | return Config{ 23 | Host: "localhost", 24 | Port: DefaultPort, 25 | Enabled: true, 26 | EnablePProf: false, 27 | } 28 | } 29 | 30 | func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { 31 | a := (*configAlias)(c) 32 | if err := unmarshal(a); err != nil { 33 | return err 34 | } 35 | if len(c.XXX) != 0 { 36 | return errors.Errorf("undefined fields %v", c.XXX) 37 | } 38 | return nil 39 | } 40 | 41 | func (c *Config) Apply() error { 42 | if err := c.Validate(); err != nil { 43 | return err 44 | } 45 | return nil 46 | } 47 | 48 | func (c *Config) Validate() error { 49 | // TODO: validate the host 50 | if c.Port < MinimalPort { 51 | return errors.Errorf("port number must be greater than %d, got %d instead", MinimalPort, c.Port) 52 | } 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /_legacy/pkg/server/http/read_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | asst "github.com/stretchr/testify/assert" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | // TODO: rename, it's copied from old read service 11 | func TestReadServiceHTTPFactory_MakeDecode(t *testing.T) { 12 | assert := asst.New(t) 13 | queryData := `{ 14 | "start_time": 1493363958000, 15 | "end_time": 1494363958000, 16 | "queries":[ 17 | { 18 | "name":"cpi", 19 | "tags":{"machine":"machine-01","os":"ubuntu"}, 20 | "match_policy": "exact", 21 | "start_time": 1493363958000, 22 | "end_time": 1494363958000 23 | } 24 | ] 25 | }` 26 | 27 | var req readRequest 28 | err := json.NewDecoder(strings.NewReader(queryData)).Decode(&req) 29 | assert.Nil(err) 30 | // NOTE: nothing to do with decoder 31 | // FIXED: Golang can't handle nested struct array http://stackoverflow.com/questions/21268000/unmarshaling-nested-json-objects-in-golang 32 | err = json.Unmarshal([]byte(queryData), &req) 33 | assert.Nil(err) 34 | assert.Equal(1, len(req.Queries)) 35 | assert.Equal("cpi", req.Queries[0].Name) 36 | } 37 | -------------------------------------------------------------------------------- /_legacy/pkg/server/http/write.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | 6 | "encoding/json" 7 | "github.com/pkg/errors" 8 | "github.com/xephonhq/xephon-k/pkg/common" 9 | pb "github.com/xephonhq/xephon-k/pkg/server/payload" 10 | "time" 11 | ) 12 | 13 | func (s *Server) write(w http.ResponseWriter, r *http.Request) { 14 | defer func(begin time.Time) { 15 | log.Infof("POST /write %d", time.Since(begin)) 16 | }(time.Now()) 17 | // decode 18 | var rawSeries []common.RawSeries 19 | var intSeries []common.IntSeries 20 | var doubleSeries []common.DoubleSeries 21 | if err := json.NewDecoder(r.Body).Decode(&rawSeries); err != nil { 22 | invalidFormat(w, errors.Wrap(err, "can't decode write request into meta series")) 23 | return 24 | } 25 | totalSeries := len(rawSeries) 26 | log.Tracef("got %d meta series after decode ", len(rawSeries)) 27 | for i := 0; i < totalSeries; i++ { 28 | switch rawSeries[i].GetSeriesType() { 29 | case common.TypeIntSeries: 30 | // copy the meta and decode the points 31 | s := common.IntSeries{ 32 | SeriesMeta: rawSeries[i].GetMetaCopy(), 33 | } 34 | points := make([]common.IntPoint, 0) 35 | err := json.Unmarshal(rawSeries[i].Points, &points) 36 | if err != nil { 37 | invalidFormat(w, errors.Wrapf(err, "can't decode %s into int series", s.Name)) 38 | return 39 | } 40 | s.Points = points 41 | intSeries = append(intSeries, s) 42 | case common.TypeDoubleSeries: 43 | s := common.DoubleSeries{ 44 | SeriesMeta: rawSeries[i].GetMetaCopy(), 45 | } 46 | points := make([]common.DoublePoint, 0) 47 | err := json.Unmarshal(rawSeries[i].Points, &points) 48 | if err != nil { 49 | invalidFormat(w, errors.Wrapf(err, "can't decode %s into double series", s.Name)) 50 | return 51 | } 52 | s.Points = points 53 | doubleSeries = append(doubleSeries, s) 54 | default: 55 | invalidFormat(w, errors.Errorf("unsupported series type %d", rawSeries[i].GetSeriesType())) 56 | return 57 | } 58 | } 59 | // write the series 60 | log.Tracef("got %d int series after decode ", len(intSeries)) 61 | log.Tracef("got %d double series after decode ", len(doubleSeries)) 62 | req := &pb.WriteRequest{IntSeries: intSeries, DoubleSeries: doubleSeries} 63 | res, err := s.writeSvc.Write(req) 64 | if err != nil { 65 | internalError(w, err) 66 | return 67 | } 68 | // FIXME: the empty error response does not match our json format, because it has omit empty, so error: false is omitted 69 | // by pojo generated using protobuf 70 | writeJSON(w, res) 71 | } 72 | -------------------------------------------------------------------------------- /_legacy/pkg/server/payload/payload.proto: -------------------------------------------------------------------------------- 1 | // go get github.com/gogo/protobuf/protoc-gen-gogo 2 | // protoc --proto_path=/home/at15/workspace/src/:. --gogo_out=plugins=grpc:. payload.proto 3 | syntax = "proto3"; 4 | 5 | package payload; 6 | 7 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 8 | import "github.com/xephonhq/xephon-k/pkg/common/common.proto"; 9 | 10 | option (gogoproto.sizer_all) = true; 11 | option (gogoproto.marshaler_all) = true; 12 | option (gogoproto.unmarshaler_all) = true; 13 | option (gogoproto.goproto_getters_all) = false; 14 | 15 | service Write { 16 | rpc Write (WriteRequest) returns (WriteResponse) {} 17 | } 18 | 19 | message WriteRequest { 20 | repeated common.IntSeries intSeries = 1 [(gogoproto.nullable) = false]; 21 | repeated common.DoubleSeries doubleSeries = 2 [(gogoproto.nullable) = false]; 22 | } 23 | 24 | message WriteResponse { 25 | bool error = 1; 26 | string errorMsg = 2; 27 | } -------------------------------------------------------------------------------- /_legacy/pkg/server/pkg.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/util" 5 | ) 6 | 7 | var DefaultPort = 2333 8 | var log = util.Logger.NewEntryWithPkg("k.server") 9 | -------------------------------------------------------------------------------- /_legacy/pkg/server/service/pkg.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/util" 5 | ) 6 | 7 | var log = util.Logger.NewEntryWithPkg("k.s.service") 8 | -------------------------------------------------------------------------------- /_legacy/pkg/server/service/read.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/common" 5 | "github.com/xephonhq/xephon-k/pkg/storage" 6 | ) 7 | 8 | type ReadService struct { 9 | store storage.Store 10 | } 11 | 12 | func NewReadService(store storage.Store) *ReadService { 13 | return &ReadService{ 14 | store: store, 15 | } 16 | } 17 | 18 | func (r *ReadService) QuerySeries(queries []common.Query) ([]common.QueryResult, []common.Series, error) { 19 | return r.store.QuerySeries(queries) 20 | } 21 | -------------------------------------------------------------------------------- /_legacy/pkg/server/service/write.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | pb "github.com/xephonhq/xephon-k/pkg/server/payload" 5 | "github.com/xephonhq/xephon-k/pkg/storage" 6 | ) 7 | 8 | type WriteService struct { 9 | store storage.Store 10 | } 11 | 12 | func NewWriteService(store storage.Store) *WriteService { 13 | return &WriteService{ 14 | store: store, 15 | } 16 | } 17 | 18 | // TODO: support context, but protobuf is using x/net/context 19 | func (w *WriteService) Write(req *pb.WriteRequest) (*pb.WriteResponse, error) { 20 | res := &pb.WriteResponse{Error: false, ErrorMsg: ""} 21 | 22 | if len(req.IntSeries) > 0 { 23 | err := w.store.WriteIntSeries(req.IntSeries) 24 | if err != nil { 25 | res.Error = true 26 | res.ErrorMsg = err.Error() 27 | // TODO: maybe we should return error and let upper encode it into json 28 | return res, nil 29 | } 30 | } 31 | if len(req.DoubleSeries) > 0 { 32 | err := w.store.WriteDoubleSeries(req.DoubleSeries) 33 | if err != nil { 34 | res.Error = true 35 | res.ErrorMsg = err.Error() 36 | // TODO: maybe we should return error and let upper encode it into json 37 | return res, nil 38 | } 39 | } 40 | return res, nil 41 | } 42 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/cassandra/config.go: -------------------------------------------------------------------------------- 1 | package cassandra 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | ) 6 | 7 | // TODO: keyspace 8 | type Config struct { 9 | Host string `yaml:"host" json:"host"` 10 | Port int `yaml:"port" json:"port"` 11 | Keyspace string `yaml:"keyspace" json:"keyspace"` 12 | XXX map[string]interface{} `yaml:",inline"` 13 | } 14 | 15 | // avoid recursion in UnmarshalYAML 16 | type configAlias Config 17 | 18 | func NewConfig() Config { 19 | return Config{ 20 | Host: "localhost", 21 | Port: 9042, 22 | Keyspace: defaultKeySpace, 23 | } 24 | } 25 | 26 | func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { 27 | a := (*configAlias)(c) 28 | if err := unmarshal(a); err != nil { 29 | return err 30 | } 31 | if len(c.XXX) != 0 { 32 | return errors.Errorf("undefined fields %v", c.XXX) 33 | } 34 | return nil 35 | } 36 | 37 | func (c *Config) Apply() error { 38 | if err := c.Validate(); err != nil { 39 | return err 40 | } 41 | return nil 42 | } 43 | 44 | func (c *Config) Validate() error { 45 | // TODO: try to connect to cassandra? 46 | if c.Port < 0 { 47 | return errors.Errorf("port number must be positive, got %d ", c.Port) 48 | } 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/cassandra/config_test.go: -------------------------------------------------------------------------------- 1 | package cassandra 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "gopkg.in/yaml.v2" 6 | "testing" 7 | ) 8 | 9 | func TestConfig_UnmarshalYAML(t *testing.T) { 10 | assert := asst.New(t) 11 | valid := ` 12 | host: localhost 13 | port: 9042 14 | ` 15 | c := NewConfig() 16 | err := yaml.Unmarshal([]byte(valid), &c) 17 | assert.Nil(err) 18 | undefinedFields := ` 19 | haha: 123 20 | ` 21 | err = yaml.Unmarshal([]byte(undefinedFields), &c) 22 | assert.NotNil(err) 23 | wrongPort := ` 24 | port: -1 25 | ` 26 | err = yaml.Unmarshal([]byte(wrongPort), &c) 27 | assert.Nil(err) 28 | assert.NotNil(c.Validate()) 29 | assert.NotNil(c.Apply()) 30 | } 31 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/cassandra/doc.go: -------------------------------------------------------------------------------- 1 | // Package cassandra provides the functionality to use Cssandra as long term and distributed storage 2 | package cassandra 3 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/cassandra/pkg.go: -------------------------------------------------------------------------------- 1 | package cassandra 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/util" 5 | "sync" 6 | ) 7 | 8 | var log = util.Logger.NewEntryWithPkg("k.storage.cassandra") 9 | 10 | const defaultKeySpace = "xephon" 11 | 12 | var storeMap StoreMap 13 | 14 | // StoreMap is used to allow multiple cassandra session, it is also used as a singleton when you are just using the default store. 15 | // its methods use a RWMutex 16 | type StoreMap struct { 17 | mu sync.RWMutex 18 | stores map[string]*Store 19 | } 20 | 21 | func init() { 22 | storeMap.stores = make(map[string]*Store, 1) 23 | } 24 | 25 | func CreateStore(config Config) (*Store, error) { 26 | storeMap.mu.Lock() 27 | defer storeMap.mu.Unlock() 28 | if err := config.Validate(); err != nil { 29 | return nil, err 30 | } 31 | store, err := NewCassandraStore(config) 32 | if err != nil { 33 | return nil, err 34 | } 35 | storeMap.stores["default"] = store 36 | return store, nil 37 | } 38 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/cassandra/stmt.go: -------------------------------------------------------------------------------- 1 | package cassandra 2 | 3 | // read statements 4 | 5 | var selectIntStmt = ` 6 | SELECT metric_timestamp, value FROM metrics_int WHERE metric_name = ? AND tags = ? 7 | ` 8 | 9 | var selectIntByStartEndTimeStmt = ` 10 | SELECT metric_timestamp, value FROM metrics_int WHERE metric_name = ? AND tags = ? AND metric_timestamp >= ? AND metric_timestamp <= ? 11 | ` 12 | 13 | // write statements 14 | 15 | var insertIntStmt = ` 16 | INSERT INTO metrics_int (metric_name, metric_timestamp, tags, value) VALUES (?, ?, ?, ?) 17 | ` 18 | 19 | var insertDoubleStmt = ` 20 | INSERT INTO metrics_double (metric_name, metric_timestamp, tags, value) VALUES (?, ?, ?, ?) 21 | ` 22 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/cassandra/store_test.go: -------------------------------------------------------------------------------- 1 | package cassandra 2 | 3 | import ( 4 | "fmt" 5 | "github.com/xephonhq/xephon-k/pkg/common" 6 | ) 7 | 8 | // TODO: only run this test when cassandra is presented 9 | 10 | // FIXME: this code is copied from memory test 11 | func createDummySeries() []common.IntSeries { 12 | // create a bunch of series 13 | machineNumber := 10 14 | multipleSeries := make([]common.IntSeries, 0, 10) 15 | for i := 0; i < machineNumber; i++ { 16 | tags := make(map[string]string) 17 | tags["os"] = "ubuntu" 18 | tags["machine"] = fmt.Sprintf("machine-%d", i) 19 | p1 := common.IntPoint{T: 1359788400002, V: 1} 20 | p2 := common.IntPoint{T: 1359788500002, V: 2} 21 | ps1 := []common.IntPoint{p1, p2} 22 | s1 := common.IntSeries{ 23 | SeriesMeta: common.SeriesMeta{Name: "cpi", Tags: tags}, 24 | Points: ps1, 25 | } 26 | multipleSeries = append(multipleSeries, s1) 27 | } 28 | return multipleSeries 29 | } 30 | 31 | func createDummyDoubleSeries() []common.DoubleSeries { 32 | // create a bunch of series 33 | machineNumber := 10 34 | multipleSeries := make([]common.DoubleSeries, 0, 10) 35 | for i := 0; i < machineNumber; i++ { 36 | tags := make(map[string]string) 37 | tags["os"] = "ubuntu" 38 | tags["machine"] = fmt.Sprintf("machine-%d", i) 39 | p1 := common.DoublePoint{T: 1359788400002, V: 1.1} 40 | p2 := common.DoublePoint{T: 1359788500002, V: 2.2} 41 | ps1 := []common.DoublePoint{p1, p2} 42 | s1 := common.DoubleSeries{ 43 | SeriesMeta: common.SeriesMeta{Name: "cpi", Tags: tags}, 44 | Points: ps1, 45 | } 46 | multipleSeries = append(multipleSeries, s1) 47 | } 48 | return multipleSeries 49 | } 50 | 51 | //func TestStore_QueryIntSeries(t *testing.T) { 52 | // t.Skip("skip cassandra read int series test") 53 | // store := GetDefaultCassandraStore("localhost") 54 | // // FIXME: let's assume that other tests have already write it 55 | // tags := make(map[string]string) 56 | // tags["os"] = "ubuntu" 57 | // tags["machine"] = "machine-1" 58 | // qExact := common.Query{Tags: tags, Name: "cpi", MatchPolicy: "exact"} 59 | // store.QueryIntSeries(qExact) 60 | // //log.Infof("result length is %d", len()) 61 | //} 62 | // 63 | //func TestStore_WriteIntSeries(t *testing.T) { 64 | // t.Skip("skip cassandra write int series test") 65 | // store := GetDefaultCassandraStore("localhost") 66 | // store.WriteIntSeries(createDummySeries()) 67 | //} 68 | // 69 | //func TestStore_WriteDoubleSeries(t *testing.T) { 70 | // t.Skip("skip cassandra write double series test") 71 | // store := GetDefaultCassandraStore("localhost") 72 | // store.WriteDoubleSeries(createDummyDoubleSeries()) 73 | //} 74 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/config.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/xephonhq/xephon-k/pkg/storage/cassandra" 6 | "github.com/xephonhq/xephon-k/pkg/storage/disk" 7 | "github.com/xephonhq/xephon-k/pkg/storage/memory" 8 | ) 9 | 10 | type Config struct { 11 | Memory memory.Config `yaml:"memory" json:"memory"` 12 | Disk disk.Config `yaml:"disk" json:"disk"` 13 | Cassandra cassandra.Config `yaml:"cassandra" json:"cassandra"` 14 | XXX map[string]interface{} `yaml:",inline"` 15 | } 16 | 17 | // avoid recursion in UnmarshalYAML 18 | type configAlias Config 19 | 20 | func NewConfig() Config { 21 | return Config{ 22 | Memory: memory.NewConfig(), 23 | Disk: disk.NewConfig(), 24 | Cassandra: cassandra.NewConfig(), 25 | } 26 | } 27 | 28 | func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { 29 | a := (*configAlias)(c) 30 | if err := unmarshal(a); err != nil { 31 | return err 32 | } 33 | if len(c.XXX) != 0 { 34 | return errors.Errorf("undefined fields %v", c.XXX) 35 | } 36 | return nil 37 | } 38 | 39 | func (c *Config) Apply() error { 40 | if err := c.Validate(); err != nil { 41 | return err 42 | } 43 | if err := c.Memory.Apply(); err != nil { 44 | return err 45 | } 46 | if err := c.Disk.Apply(); err != nil { 47 | return err 48 | } 49 | if err := c.Cassandra.Apply(); err != nil { 50 | return err 51 | } 52 | return nil 53 | } 54 | 55 | func (c *Config) Validate() error { 56 | if err := c.Memory.Validate(); err != nil { 57 | return err 58 | } 59 | if err := c.Disk.Validate(); err != nil { 60 | return err 61 | } 62 | if err := c.Cassandra.Validate(); err != nil { 63 | return err 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/config_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "gopkg.in/yaml.v2" 6 | "testing" 7 | ) 8 | 9 | func TestConfig_UnmarshalYAML(t *testing.T) { 10 | assert := asst.New(t) 11 | valid := ` 12 | memory: 13 | layout: row 14 | chunkSize: 104857600 15 | enableIndex: true 16 | disk: 17 | folder: /tmp 18 | concurrentWriteFiles: 1 19 | singleFileSize: 536870912 20 | cassandra: 21 | host: localhost 22 | port: 9042 23 | ` 24 | c := NewConfig() 25 | err := yaml.Unmarshal([]byte(valid), &c) 26 | assert.Nil(err) 27 | assert.Nil(c.Validate()) 28 | assert.Nil(c.Apply()) 29 | undefinedFields := ` 30 | haha: 123 31 | ` 32 | err = yaml.Unmarshal([]byte(undefinedFields), &c) 33 | assert.NotNil(err) 34 | wrongLayout := ` 35 | memory: 36 | layout: hybrid 37 | ` 38 | err = yaml.Unmarshal([]byte(wrongLayout), &c) 39 | assert.Nil(err) 40 | assert.NotNil(c.Validate()) 41 | assert.NotNil(c.Apply()) 42 | } 43 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/discard/pkg.go: -------------------------------------------------------------------------------- 1 | package discard 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/common" 5 | "github.com/xephonhq/xephon-k/pkg/util" 6 | ) 7 | 8 | var log = util.Logger.NewEntryWithPkg("k.storage.discard") 9 | 10 | type Store struct { 11 | } 12 | 13 | func NewDiscardStore() *Store { 14 | return &Store{} 15 | } 16 | 17 | func (Store) StoreType() string { 18 | return "discard" 19 | } 20 | 21 | func (Store) QuerySeries([]common.Query) ([]common.QueryResult, []common.Series, error) { 22 | result := make([]common.QueryResult, 0) 23 | series := make([]common.Series, 0) 24 | return result, series, nil 25 | } 26 | 27 | func (Store) WriteIntSeries([]common.IntSeries) error { 28 | return nil 29 | } 30 | 31 | func (Store) WriteDoubleSeries([]common.DoubleSeries) error { 32 | return nil 33 | } 34 | 35 | func (Store) Shutdown() { 36 | log.Info("shutting down discard store, nothing to do") 37 | } 38 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/disk/.ayi.yml: -------------------------------------------------------------------------------- 1 | scripts: 2 | gen: 3 | # FIXME: use GOPATH (which need sh -c) 4 | - protoc --proto_path=/home/at15/workspace/src/:. --gogo_out=. disk.proto -------------------------------------------------------------------------------- /_legacy/pkg/storage/disk/config.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/xephonhq/xephon-k/pkg/encoding" 6 | "io/ioutil" 7 | "os" 8 | ) 9 | 10 | const ( 11 | MinimalSingleFileSize = 64 * 1024 * 1024 12 | ) 13 | 14 | // TODO: encoding 15 | type Config struct { 16 | Folder string `yaml:"folder" json:"folder"` 17 | ConcurrentWriteFiles int `yaml:"concurrentWriteFiles" json:"concurrentWriteFiles"` 18 | SingleFileSize int `yaml:"singleFileSize" json:"singleFileSize"` 19 | FileBufferSize int `yaml:"fileBufferSize" json:"file_buffer_size"` 20 | Encoding map[string]string `yaml:"encoding" json:"encoding"` 21 | XXX map[string]interface{} `yaml:",inline"` 22 | } 23 | 24 | // avoid recursion in UnmarshalYAML 25 | type configAlias Config 26 | 27 | func NewConfig() Config { 28 | return Config{ 29 | Folder: "/tmp", 30 | ConcurrentWriteFiles: 1, 31 | SingleFileSize: MinimalSingleFileSize, 32 | FileBufferSize: DefaultFileBufferSize, 33 | Encoding: map[string]string{ 34 | "time": "raw-big", 35 | "int": "var", 36 | "double": "var", 37 | }, 38 | } 39 | } 40 | 41 | func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { 42 | a := (*configAlias)(c) 43 | if err := unmarshal(a); err != nil { 44 | return err 45 | } 46 | if len(c.XXX) != 0 { 47 | return errors.Errorf("undefined fields %v", c.XXX) 48 | } 49 | return nil 50 | } 51 | 52 | func (c *Config) Apply() error { 53 | if err := c.Validate(); err != nil { 54 | return err 55 | } 56 | // TODO: do we really need to shrink chunk or change the layout? 57 | // this requires the config have access to the storage, which is inverse of the normal flow 58 | return nil 59 | } 60 | 61 | func (c *Config) Validate() error { 62 | f, err := ioutil.TempFile(c.Folder, "xephonk-diskconfig-probe") 63 | if err != nil { 64 | return errors.Wrap(err, "can't write file in specified folder") 65 | } 66 | f.Close() 67 | os.Remove(f.Name()) 68 | 69 | if c.ConcurrentWriteFiles != 1 { 70 | return errors.Errorf("only support write to single file, but got %d", c.ConcurrentWriteFiles) 71 | } 72 | if c.SingleFileSize < MinimalSingleFileSize { 73 | return errors.Errorf("single file size must be larger than 64MB, got %d bytes", c.SingleFileSize) 74 | } 75 | if c.FileBufferSize < DefaultFileBufferSize { 76 | return errors.Errorf("file buffer size must be larger than 1KB, got %d bytes", c.FileBufferSize) 77 | } 78 | for _, v := range c.Encoding { 79 | if _, err := encoding.Str2Codec(v); err != nil { 80 | return err 81 | } 82 | } 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/disk/config_test.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "gopkg.in/yaml.v2" 6 | "testing" 7 | ) 8 | 9 | func TestConfig_UnmarshalYAML(t *testing.T) { 10 | assert := asst.New(t) 11 | valid := ` 12 | folder: /tmp 13 | concurrentWriteFiles: 1 14 | singleFileSize: 536870912 15 | ` 16 | c := NewConfig() 17 | err := yaml.Unmarshal([]byte(valid), &c) 18 | assert.Nil(err) 19 | undefinedFields := ` 20 | haha: 123 21 | ` 22 | err = yaml.Unmarshal([]byte(undefinedFields), &c) 23 | assert.NotNil(err) 24 | wrongConcurrentWrieFiles := ` 25 | concurrentWriteFiles: 2 26 | ` 27 | err = yaml.Unmarshal([]byte(wrongConcurrentWrieFiles), &c) 28 | assert.Nil(err) 29 | assert.NotNil(c.Validate()) 30 | assert.NotNil(c.Apply()) 31 | nonExistFolder := ` 32 | folder: /whooo/hahaha 33 | ` 34 | err = yaml.Unmarshal([]byte(nonExistFolder), &c) 35 | assert.Nil(err) 36 | assert.NotNil(c.Validate()) 37 | assert.NotNil(c.Apply()) 38 | } 39 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/disk/data_writer_test.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | asst "github.com/stretchr/testify/assert" 8 | "github.com/xephonhq/xephon-k/pkg/common" 9 | "github.com/xephonhq/xephon-k/pkg/encoding" 10 | "github.com/xephonhq/xephon-k/pkg/util" 11 | ) 12 | 13 | func TestNewLocalFileIndexWriter(t *testing.T) { 14 | assert := asst.New(t) 15 | f := util.TempFile(t, "xephon") 16 | defer os.Remove(f.Name()) 17 | 18 | opt, err := NewEncodingOption() 19 | assert.Nil(err) 20 | w, err := NewLocalFileWriter(f, -1, opt) 21 | assert.Nil(err) 22 | assert.NotNil(w.Close()) 23 | } 24 | 25 | func TestLocalFileWriter_WriteSeries(t *testing.T) { 26 | assert := asst.New(t) 27 | f := util.TempFile(t, "xephon") 28 | defer os.Remove(f.Name()) 29 | 30 | opt, err := NewEncodingOption(func(o *EncodingOption) { 31 | o.TimeCodec = encoding.CodecRawBigEndian 32 | o.IntValueCodec = encoding.CodecRawBigEndian 33 | o.DoubleValueCodec = encoding.CodecRawBigEndian 34 | }) 35 | assert.Nil(err) 36 | // f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) 37 | w, err := NewLocalFileWriter(f, -1, opt) 38 | assert.Nil(err) 39 | 40 | si := common.NewIntSeries("si") 41 | si.Tags = map[string]string{"os": "ubuntu", "machine": "machine-01"} 42 | si.Points = []common.IntPoint{{T: 1359788400000, V: -1}, {T: 1359788500000, V: 2}} 43 | assert.Nil(w.WriteSeries(si)) 44 | // header + block header + time encoding + times (2) + values encoding + values(2) 45 | assert.Equal(uint64(9+4+1+16+1+16), w.n) 46 | assert.Equal(ErrNotFinalized, w.Close()) 47 | 48 | si.Points = []common.IntPoint{{T: 1359788600000, V: 3}, {T: 1359788700000, V: 4}} 49 | assert.Nil(w.WriteSeries(si)) 50 | 51 | sd := common.NewDoubleSeries("sd") 52 | sd.Tags = map[string]string{"os": "ubuntu", "machine": "machine-01"} 53 | sd.Points = []common.DoublePoint{{T: 1359788400000, V: 1.2}, {T: 1359788500000, V: -2.33}} 54 | assert.Nil(w.WriteSeries(sd)) 55 | 56 | assert.Nil(w.WriteIndex()) 57 | assert.Nil(w.Close()) 58 | 59 | // NOTE: need to re-open the file because the writer has already closed it 60 | f, err = os.OpenFile(f.Name(), os.O_RDONLY, 0666) 61 | assert.Nil(err) 62 | r, err := NewLocalDataFileReader(f) 63 | assert.Nil(err) 64 | assert.Nil(r.ReadIndexOfIndexes()) 65 | assert.Equal(2, r.SeriesCount()) 66 | assert.Nil(r.ReadAllIndexEntries()) 67 | r.PrintAll() 68 | // TODO: add more assert instead of just print 69 | assert.Nil(r.Close()) 70 | } 71 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/disk/disk.proto: -------------------------------------------------------------------------------- 1 | // go get github.com/gogo/protobuf/protoc-gen-gogo 2 | // protoc --proto_path=/home/at15/workspace/src/:. --gogo_out=. disk.proto 3 | syntax = "proto3"; 4 | 5 | package disk; 6 | 7 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 8 | import "github.com/xephonhq/xephon-k/pkg/common/common.proto"; 9 | 10 | option (gogoproto.sizer_all) = true; 11 | option (gogoproto.marshaler_all) = true; 12 | option (gogoproto.unmarshaler_all) = true; 13 | option (gogoproto.goproto_getters_all) = false; 14 | 15 | message IndexEntries { 16 | // TODO: index entries may have the overall aggregation of all the index entries, like min, max, time 17 | common.SeriesMeta Meta = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true]; 18 | repeated IndexEntry Entries = 2 [(gogoproto.nullable) = false]; 19 | } 20 | 21 | message IndexEntry { 22 | uint64 Offset = 1; 23 | uint32 BlockSize = 2; 24 | int64 minTime = 3; 25 | int64 maxTime = 4; 26 | // TODO: aggregated data 27 | } -------------------------------------------------------------------------------- /_legacy/pkg/storage/disk/doc.go: -------------------------------------------------------------------------------- 1 | // Package disk provides local disk storage 2 | package disk 3 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/disk/pkg.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "encoding/binary" 5 | 6 | "github.com/xephonhq/xephon-k/pkg/util" 7 | "sync" 8 | ) 9 | 10 | // NOTE: The following struct are generated by protobuf, see disk.proto and disk.pb.go for detail 11 | // IndexEntries 12 | // IndexEntry 13 | 14 | const ( 15 | // Version is current supported file format version 16 | Version byte = 1 17 | // MagicNumber is 'xephon-k', 8 bytes stored in big endian in uint64, used for identify file without relying on extension 18 | MagicNumber uint64 = 0x786570686F6E2D6B 19 | ) 20 | 21 | var log = util.Logger.NewEntryWithPkg("k.storage.disk") 22 | 23 | var storeMap StoreMap 24 | 25 | type StoreMap struct { 26 | mu sync.RWMutex 27 | stores map[string]*Store 28 | } 29 | 30 | func CreateStore(config Config) (*Store, error) { 31 | storeMap.mu.Lock() 32 | defer storeMap.mu.Unlock() 33 | 34 | if err := config.Validate(); err != nil { 35 | return nil, err 36 | } 37 | store, err := NewDiskStore(config) 38 | if err != nil { 39 | return nil, err 40 | } 41 | storeMap.stores["default"] = store 42 | return store, nil 43 | } 44 | 45 | func MagicBytes() []byte { 46 | return []byte("xephon-k") 47 | } 48 | 49 | func IsMagic(buf []byte) bool { 50 | // binary.BigEndian.Uint64 don't check the indexLength of the slice 51 | if len(buf) < 8 { 52 | return false 53 | } 54 | return binary.BigEndian.Uint64(buf) == MagicNumber 55 | } 56 | 57 | func IsValidFormat(buf []byte) bool { 58 | if len(buf) < 9 { 59 | return false 60 | } 61 | return buf[0] == Version && IsMagic(buf[1:]) 62 | } 63 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/disk/pkg_test.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | asst "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestMagicBytes(t *testing.T) { 11 | assert := asst.New(t) 12 | assert.True(IsMagic(MagicBytes())) 13 | } 14 | 15 | func TestIsMagic(t *testing.T) { 16 | assert := asst.New(t) 17 | var buf bytes.Buffer 18 | buf.WriteString("xephon-k") 19 | assert.True(IsMagic(buf.Bytes())) 20 | assert.False(IsMagic([]byte("x"))) 21 | } 22 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/disk/reader.go: -------------------------------------------------------------------------------- 1 | package disk 2 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/doc.go: -------------------------------------------------------------------------------- 1 | // Package storage handles storage backend 2 | package storage 3 | 4 | // TODO: other storage engine 5 | // - black hole /dev/null? or don't even do write 6 | // - json, csv? 7 | // https://dev.mysql.com/doc/refman/5.7/en/storage-engines.html 8 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/memory/.ayi.yml: -------------------------------------------------------------------------------- 1 | scripts: 2 | gen: 3 | - go run gen/main.go -------------------------------------------------------------------------------- /_legacy/pkg/storage/memory/config.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | ) 6 | 7 | const ( 8 | MinimalChunkSize = 10 * 1024 * 1024 9 | ) 10 | 11 | type Config struct { 12 | Layout string `yaml:"layout" json:"layout"` 13 | ChunkSize int `yaml:"chunkSize" json:"chunkSize"` 14 | EnableIndex bool `yaml:"enableIndex" json:"enableIndex"` 15 | XXX map[string]interface{} `yaml:",inline"` 16 | } 17 | 18 | // avoid recursion in UnmarshalYAML 19 | type configAlias Config 20 | 21 | func NewConfig() Config { 22 | return Config{ 23 | Layout: "row", 24 | ChunkSize: MinimalChunkSize, 25 | EnableIndex: true, 26 | } 27 | } 28 | 29 | func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { 30 | a := (*configAlias)(c) 31 | if err := unmarshal(a); err != nil { 32 | return err 33 | } 34 | if len(c.XXX) != 0 { 35 | return errors.Errorf("undefined fields %v", c.XXX) 36 | } 37 | return nil 38 | } 39 | 40 | func (c *Config) Apply() error { 41 | if err := c.Validate(); err != nil { 42 | return err 43 | } 44 | // TODO: do we really need to shrink chunk or change the layout? 45 | // this requires the config have access to the storage, which is inverse of the normal flow 46 | return nil 47 | } 48 | 49 | func (c *Config) Validate() error { 50 | if c.Layout != "row" && c.Layout != "column" { 51 | return errors.Errorf("unsupported memory layout %s", c.Layout) 52 | } 53 | if c.ChunkSize < MinimalChunkSize { 54 | return errors.Errorf("chunk size must be larger than 10MB, got %d bytes", c.ChunkSize) 55 | } 56 | return nil 57 | } 58 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/memory/config_test.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "gopkg.in/yaml.v2" 6 | "testing" 7 | ) 8 | 9 | func TestConfig_UnmarshalYAML(t *testing.T) { 10 | assert := asst.New(t) 11 | valid := ` 12 | layout: row 13 | chunkSize: 104857600 14 | enableIndex: true 15 | ` 16 | c := NewConfig() 17 | err := yaml.Unmarshal([]byte(valid), &c) 18 | assert.Nil(err) 19 | undefinedFields := ` 20 | haha: 123 21 | ` 22 | err = yaml.Unmarshal([]byte(undefinedFields), &c) 23 | assert.NotNil(err) 24 | wrongLayout := ` 25 | layout: hybrid` 26 | err = yaml.Unmarshal([]byte(wrongLayout), &c) 27 | assert.Nil(err) 28 | assert.NotNil(c.Validate()) 29 | assert.NotNil(c.Apply()) 30 | } 31 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/memory/data_test.go: -------------------------------------------------------------------------------- 1 | package memory 2 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/memory/doc.go: -------------------------------------------------------------------------------- 1 | // Package memory provides an in memory store, which can be used as write through or write back cache 2 | package memory 3 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/memory/gen/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func assert(err error) { 11 | if err != nil { 12 | fmt.Println(err) 13 | os.Exit(1) 14 | } 15 | } 16 | 17 | // NOTE: it should be called from upper folder 18 | // Use `go run gen/main.go` instead of `go run main.go` 19 | func main() { 20 | fmt.Println("generate other series based on int series") 21 | 22 | // int series is used as template 23 | tplBytes, err := ioutil.ReadFile("series_store_int.go") 24 | assert(err) 25 | tpl := string(tplBytes) 26 | otherSeriesTypes := map[string]map[string]string{"series_store_double.go": {"IntSeries": "DoubleSeries", "IntPoint": "DoublePoint"}} 27 | 28 | for newFile, replacements := range otherSeriesTypes { 29 | content := tpl 30 | for old, newStr := range replacements { 31 | content = strings.Replace(content, old, newStr, -1) 32 | } 33 | data := append([]byte("// Generated from series_store_int.go DO NOT EDIT!\n"), []byte(content)...) 34 | err = ioutil.WriteFile(newFile, data, 0644) 35 | assert(err) 36 | } 37 | 38 | fmt.Println("finished") 39 | 40 | } 41 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/memory/pkg.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/xephonhq/xephon-k/pkg/util" 8 | ) 9 | 10 | var log = util.Logger.NewEntryWithPkg("k.storage.memory") 11 | var initSeriesCount = 10 12 | 13 | const ( 14 | nameTagKey = "__name__" 15 | ) 16 | 17 | // singleton map for share memory store between multiple go-kit services 18 | var storeMap StoreMap 19 | 20 | // StoreMap protects underlying mem store with a RWMutex 21 | type StoreMap struct { 22 | mu sync.RWMutex 23 | stores map[string]*Store 24 | } 25 | 26 | func init() { 27 | storeMap.stores = make(map[string]*Store, 1) 28 | } 29 | 30 | func CreateStore(config Config) (*Store, error) { 31 | storeMap.mu.Lock() 32 | defer storeMap.mu.Unlock() 33 | 34 | if err := config.Validate(); err != nil { 35 | return nil, err 36 | } 37 | store := NewMemStore(config) 38 | storeMap.stores["default"] = store 39 | return store, nil 40 | } 41 | 42 | func GetStore() (*Store, error) { 43 | storeMap.mu.RLock() 44 | defer storeMap.mu.RUnlock() 45 | s, ok := storeMap.stores["default"] 46 | if !ok { 47 | return nil, errors.New("default store is not created! call CreateStore first") 48 | } 49 | return s, nil 50 | } 51 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/memory/series_store_test.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/xephonhq/xephon-k/pkg/common" 8 | ) 9 | 10 | func TestIntSeriesStore_WriteSeries(t *testing.T) { 11 | asst := assert.New(t) 12 | store := NewIntSeriesStore(common.IntSeries{}) 13 | asst.Equal(0, store.length) 14 | tags := map[string]string{"os": "ubuntu"} 15 | p1 := common.IntPoint{T: 1359788400002, V: 1} 16 | p2 := common.IntPoint{T: 1359788400003, V: 2} 17 | s1 := common.IntSeries{SeriesMeta: common.SeriesMeta{Name: "cpi", Tags: tags}, Points: []common.IntPoint{p1, p2}} 18 | store.WriteSeries(s1) 19 | asst.Equal(2, store.length) 20 | p3 := common.IntPoint{T: 1359788400001, V: 3} 21 | p4 := common.IntPoint{T: 1359788400004, V: 4} 22 | s2 := common.IntSeries{SeriesMeta: common.SeriesMeta{Name: "cpi", Tags: tags}, Points: []common.IntPoint{p3, p4}} 23 | store.WriteSeries(s2) 24 | asst.Equal(4, store.length) 25 | p5 := common.IntPoint{T: 1359788400005, V: 5} 26 | p6 := common.IntPoint{T: 1359788400006, V: 6} 27 | s3 := common.IntSeries{SeriesMeta: common.SeriesMeta{Name: "cpi", Tags: tags}, Points: []common.IntPoint{p5, p6}} 28 | store.WriteSeries(s3) 29 | asst.Equal(6, store.length) 30 | } 31 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/memory/store_test.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | "fmt" 5 | "github.com/xephonhq/xephon-k/pkg/common" 6 | "testing" 7 | ) 8 | 9 | func createDummySeries() []common.IntSeries { 10 | // create a bunch of series 11 | machineNumber := 10 12 | multipleSeries := make([]common.IntSeries, 0, 10) 13 | for i := 0; i < machineNumber; i++ { 14 | tags := make(map[string]string) 15 | tags["os"] = "ubuntu" 16 | tags["machine"] = fmt.Sprintf("machine-%d", i) 17 | p1 := common.IntPoint{T: 1359788400002, V: 1} 18 | ps1 := []common.IntPoint{p1} 19 | s1 := common.IntSeries{SeriesMeta: common.SeriesMeta{Name: "cpi", Tags: tags}, Points: ps1} 20 | multipleSeries = append(multipleSeries, s1) 21 | } 22 | return multipleSeries 23 | } 24 | 25 | // TODO: validate the write 26 | func TestStore_WriteIntSeries(t *testing.T) { 27 | t.Skip() 28 | //store := NewMemStore() 29 | //store.WriteIntSeries(createDummySeries()) 30 | } 31 | 32 | // TODO: test query in batch 33 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/pkg.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/util" 5 | ) 6 | 7 | var log = util.Logger.NewEntryWithPkg("k.storage") 8 | -------------------------------------------------------------------------------- /_legacy/pkg/storage/store.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/common" 5 | "github.com/xephonhq/xephon-k/pkg/storage/cassandra" 6 | "github.com/xephonhq/xephon-k/pkg/storage/discard" 7 | "github.com/xephonhq/xephon-k/pkg/storage/disk" 8 | "github.com/xephonhq/xephon-k/pkg/storage/memory" 9 | ) 10 | 11 | // check interface 12 | var _ Store = (*discard.Store)(nil) 13 | var _ Store = (*memory.Store)(nil) 14 | var _ Store = (*disk.Store)(nil) 15 | var _ Store = (*cassandra.Store)(nil) 16 | 17 | // Store is the base interface for all type of storages 18 | // TODO: each store should maintains some counter for internal metrics 19 | type Store interface { 20 | StoreType() string 21 | QuerySeries([]common.Query) ([]common.QueryResult, []common.Series, error) 22 | WriteIntSeries([]common.IntSeries) error 23 | WriteDoubleSeries([]common.DoubleSeries) error 24 | Shutdown() 25 | } 26 | 27 | func CreateStore(engine string, config Config) (Store, error) { 28 | switch engine { 29 | case "discard", "null": 30 | return discard.NewDiscardStore(), nil 31 | case "memory": 32 | return memory.CreateStore(config.Memory) 33 | case "disk": 34 | return disk.CreateStore(config.Disk) 35 | case "cassandra": 36 | return cassandra.CreateStore(config.Cassandra) 37 | default: 38 | log.Fatalf("unknown storage %s", engine) 39 | return nil, nil 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /_legacy/pkg/util/doc.go: -------------------------------------------------------------------------------- 1 | // Package util provide wrappers and hacks for standard libraries and third party libraries 2 | package util 3 | -------------------------------------------------------------------------------- /_legacy/pkg/util/log_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "gopkg.in/yaml.v2" 6 | "testing" 7 | ) 8 | 9 | func TestLogConfig_UnmarshalYAML(t *testing.T) { 10 | assert := asst.New(t) 11 | valid := ` 12 | level: info 13 | source: false 14 | color: false 15 | ` 16 | c := NewLogConfig() 17 | // FIXED: fatal error: stack overflow, use alias, like filter and read service 18 | err := yaml.Unmarshal([]byte(valid), &c) 19 | assert.Nil(err) 20 | undefinedFields := ` 21 | haha: 123 22 | ` 23 | err = yaml.Unmarshal([]byte(undefinedFields), &c) 24 | //t.Log(err) 25 | assert.NotNil(err) 26 | wrongLevel := ` 27 | level: haha 28 | ` 29 | err = yaml.Unmarshal([]byte(wrongLevel), &c) 30 | assert.Nil(err) 31 | assert.NotNil(c.Validate()) 32 | assert.NotNil(c.Apply()) 33 | } 34 | -------------------------------------------------------------------------------- /_legacy/pkg/util/pkg.go: -------------------------------------------------------------------------------- 1 | package util 2 | -------------------------------------------------------------------------------- /_legacy/pkg/util/test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/pkg/common" 5 | "io/ioutil" 6 | "os" 7 | "sync/atomic" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | var dummyVar = "dummy" 13 | var mockID int32 14 | var mockedVars map[int32]interface{} 15 | var mockedVarsValue map[int32]interface{} 16 | 17 | // FIXME: these won't work 18 | //func MockStringVar(v *interface{}, val interface{}) int32{ 19 | //func RecoverMockedVar(mockID int32) { 20 | // *mockedVars[mockID] = mockedVarsValue[mockID] 21 | //} 22 | 23 | func MockStringVar(v *string, val string) int32 { 24 | atomic.AddInt32(&mockID, 1) 25 | mockedVars[mockID] = v 26 | mockedVarsValue[mockID] = *v 27 | *v = val 28 | return mockID 29 | } 30 | 31 | func RecoverMockedStringVar(mockID int32) { 32 | //log.Info(mockedVarsValue[mockID]) 33 | *mockedVars[mockID].(*string) = mockedVarsValue[mockID].(string) 34 | } 35 | 36 | // TODO: use bench/generator to generate fake series for testing 37 | 38 | // TempFile wraps ioutil.TempFile to avoid checking return value and pass dir during test 39 | func TempFile(t *testing.T, prefix string) *os.File { 40 | f, err := ioutil.TempFile("", prefix) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | return f 45 | } 46 | 47 | func ReadAsBytes(t *testing.T, p string) []byte { 48 | f, err := os.OpenFile(p, os.O_RDONLY, 0666) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | b, err := ioutil.ReadAll(f) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | return b 57 | } 58 | 59 | func CreateDummyIntPoints() common.IntSeries { 60 | tags := make(map[string]string) 61 | tags["os"] = "ubuntu" 62 | tags["machine"] = "machine-01" 63 | startTime := time.Now().Unix() * 1000 64 | s := common.IntSeries{ 65 | SeriesMeta: common.SeriesMeta{Name: "cpi", Tags: tags}, 66 | } 67 | for i := 0; i < 5; i++ { 68 | s.Points = append(s.Points, common.IntPoint{T: startTime + int64(i*1000), V: int64(i)}) 69 | } 70 | return s 71 | } 72 | 73 | func init() { 74 | mockedVars = make(map[int32]interface{}, 5) 75 | mockedVarsValue = make(map[int32]interface{}, 5) 76 | } 77 | -------------------------------------------------------------------------------- /_legacy/pkg/util/test_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | asst "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestMockVar(t *testing.T) { 9 | assert := asst.New(t) 10 | id := MockStringVar(&dummyVar, "ha") 11 | assert.Equal(dummyVar, "ha") 12 | RecoverMockedStringVar(id) 13 | assert.Equal("dummy", dummyVar) 14 | } 15 | -------------------------------------------------------------------------------- /_legacy/pkg/version.go: -------------------------------------------------------------------------------- 1 | package pkg 2 | 3 | // Version is not semver 4 | var Version = "0.0.3-dev" 5 | -------------------------------------------------------------------------------- /_legacy/playground/README.md: -------------------------------------------------------------------------------- 1 | # Playground 2 | 3 | For testing language semantics and do microbenchmark. 4 | Prototype and library examples are also put here. 5 | 6 | ## Language 7 | 8 | - https://github.com/golang/go/wiki/SliceTricks 9 | 10 | ## Prototype 11 | 12 | - API: xapi 13 | - Cassandra Bucket: xbucket 14 | - Not implemented, only create key space 15 | - Cassandra Naive: xnaive 16 | - put all data of one series in a single physical row 17 | - a.k.a use series name as partition key 18 | - Local disk: disk 19 | - write header, points and part of index 20 | - can't read them out due to incomplete index 21 | - Local disk C: disk-c 22 | - only read header -------------------------------------------------------------------------------- /_legacy/playground/algo_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import ( 4 | "testing" 5 | "fmt" 6 | ) 7 | 8 | func TestSort_Selection(t *testing.T) { 9 | // selection sort 10 | // TODO: bechmark with stl's sort 11 | arr := []int{3, 1, 8, 9, 2} 12 | var smallestVal int 13 | var smallestIndex int 14 | for i := 0; i < len(arr)-1; i++ { 15 | smallestVal = arr[i] 16 | smallestIndex = i 17 | for j := i + 1; j < len(arr); j++ { 18 | if arr[j] < smallestVal { 19 | smallestIndex = j 20 | smallestVal = arr[j] 21 | } 22 | } 23 | // swap 24 | if i != smallestIndex { 25 | arr[i], arr[smallestIndex] = arr[smallestIndex], arr[i] 26 | } 27 | } 28 | fmt.Println(arr) 29 | } 30 | 31 | // https://www.cs.cmu.edu/~adamchik/15-121/lectures/Binary%20Heaps/heaps.html 32 | type heap struct { 33 | data []int 34 | } 35 | 36 | // TODO: percolation down 37 | 38 | func (h *heap) insert(x int) { 39 | h.data = append(h.data, x) 40 | // percolation up 41 | pos := len(h.data) - 1 42 | for ; pos > 1 && x < h.data[pos/2]; pos = pos / 2 { 43 | h.data[pos] = h.data[pos/2] 44 | } 45 | h.data[pos] = x 46 | } 47 | 48 | func TestHeap_Insert(t *testing.T) { 49 | h := heap{data: []int{0, 6, 7, 12, 10, 15, 17}} 50 | h.insert(5) 51 | fmt.Println(h.data) 52 | } 53 | -------------------------------------------------------------------------------- /_legacy/playground/binary_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import ( 4 | "testing" 5 | "encoding/binary" 6 | "math" 7 | ) 8 | 9 | func TestBinary_Int64(t *testing.T) { 10 | v := int64(-1) 11 | var b [8]byte 12 | binary.BigEndian.PutUint64(b[:], uint64(v)) 13 | vd := binary.BigEndian.Uint64(b[:]) 14 | t.Log(vd) 15 | t.Log(int64(vd)) 16 | } 17 | 18 | func TestBinary_Int64Uint64(t *testing.T) { 19 | a := int64(1) 20 | b := uint64(a) 21 | t.Log(a, b) 22 | } 23 | 24 | func TestBinary_Float64(t *testing.T) { 25 | var f float64 26 | f = 3.1415926 27 | u := math.Float64bits(f) 28 | var b [8]byte 29 | binary.BigEndian.PutUint64(b[:], u) 30 | ud := binary.BigEndian.Uint64(b[:]) 31 | f2 := math.Float64frombits(ud) 32 | t.Log(f == f2) 33 | } 34 | 35 | func TestBinary_Uvarint(t *testing.T) { 36 | buf := make([]byte, 20) 37 | t.Log(binary.PutUvarint(buf, 1)) // 1 38 | t.Log(binary.PutVarint(buf, -1)) // 1 39 | t.Log(binary.PutUvarint(buf, 256)) // 2 40 | 41 | t.Log(binary.PutVarint(buf, -256)) // 2 42 | ux, n := binary.Uvarint(buf) 43 | t.Log(ux, n) 44 | t.Log(int64(ux)) 45 | // zig zag encoding 46 | x := int64(ux >> 1) 47 | if ux&1 != 0 { 48 | x = ^x 49 | } 50 | t.Log(x) 51 | } 52 | -------------------------------------------------------------------------------- /_legacy/playground/bug_server_concurrent_map_write/README.md: -------------------------------------------------------------------------------- 1 | # Concurrent map writes 2 | 3 | https://github.com/xephonhq/xephon-k/issues/57 -------------------------------------------------------------------------------- /_legacy/playground/bytes_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import "testing" 4 | 5 | func TestBytes_NegativeIndex(t *testing.T) { 6 | b := make([]byte, 10) 7 | b[9] = 1 8 | 9 | // this won't compile 10 | // b[-1] 11 | 12 | // this will panic: runtime error: index out of range 13 | //i := -1 14 | //_ = b[i] 15 | } -------------------------------------------------------------------------------- /_legacy/playground/collector_fixed_length/README.md: -------------------------------------------------------------------------------- 1 | # Collector have fixed length series 2 | 3 | https://github.com/xephonhq/xephon-k/issues/33 4 | 5 | - [mem-1.json](mem-1.json) is the first payload after the collector is started 6 | - [mem-2.json](mem-2.json) shows the serializer is not reset between each send 7 | - [ ] but the server does not report any error? Yeah ... it seems they drop that part silently -------------------------------------------------------------------------------- /_legacy/playground/collector_fixed_length/mem-1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "mem.total", 4 | "tags": { 5 | "host": "at15-PC4073" 6 | }, 7 | "points": [ 8 | [ 9 | 1494832196000, 10 | 32858548 11 | ], 12 | [ 13 | 1494832197000, 14 | 32858548 15 | ], 16 | [ 17 | 1494832198000, 18 | 32858548 19 | ], 20 | [ 21 | 1494832199000, 22 | 32858548 23 | ], 24 | [ 25 | 1494832200000, 26 | 32858548 27 | ], 28 | [ 29 | 1494832201000, 30 | 32858548 31 | ], 32 | [ 33 | 1494832202000, 34 | 32858548 35 | ], 36 | [ 37 | 1494832203000, 38 | 32858548 39 | ], 40 | [ 41 | 1494832204000, 42 | 32858548 43 | ], 44 | [ 45 | 1494832205000, 46 | 32858548 47 | ] 48 | ] 49 | }, 50 | { 51 | "name": "mem.free", 52 | "tags": { 53 | "host": "at15-PC4073" 54 | }, 55 | "points": [ 56 | [ 57 | 1494832196000, 58 | 20316588 59 | ], 60 | [ 61 | 1494832197000, 62 | 20317844 63 | ], 64 | [ 65 | 1494832198000, 66 | 20344596 67 | ], 68 | [ 69 | 1494832199000, 70 | 20345128 71 | ], 72 | [ 73 | 1494832200000, 74 | 20339624 75 | ], 76 | [ 77 | 1494832201000, 78 | 20338972 79 | ], 80 | [ 81 | 1494832202000, 82 | 20338108 83 | ], 84 | [ 85 | 1494832203000, 86 | 20338100 87 | ], 88 | [ 89 | 1494832204000, 90 | 20338960 91 | ], 92 | [ 93 | 1494832205000, 94 | 20338804 95 | ] 96 | ] 97 | } 98 | ] -------------------------------------------------------------------------------- /_legacy/playground/column-vs-row.md: -------------------------------------------------------------------------------- 1 | # Column vs Row format 2 | 3 | | Time | Value | 4 | | :--- | :---- | 5 | | 123 | 2.0 | 6 | | 124 | 1.2 | 7 | | 125 | 2.1 | 8 | 9 | ````js 10 | // row format 11 | [ 12 | {t: 123, v: 2.0}, 13 | {t: 124, v: 1.2}, 14 | {t: 125, v: 2.1} 15 | ] 16 | // columnar format 17 | { 18 | t: [123, 124, 125], 19 | v: [2.0, 1.2, 2.1] 20 | } 21 | ```` 22 | -------------------------------------------------------------------------------- /_legacy/playground/common/.gitignore: -------------------------------------------------------------------------------- 1 | *.pb.go -------------------------------------------------------------------------------- /_legacy/playground/common/common.proto: -------------------------------------------------------------------------------- 1 | // go get github.com/gogo/protobuf/protoc-gen-gogo 2 | // protoc --proto_path=/home/at15/workspace/src/:. --gogo_out=. common.proto 3 | syntax = "proto3"; 4 | 5 | package common; 6 | 7 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 8 | 9 | option (gogoproto.sizer_all) = true; 10 | option (gogoproto.marshaler_all) = true; 11 | option (gogoproto.unmarshaler_all) = true; 12 | option (gogoproto.goproto_getters_all) = false; 13 | 14 | message Meta { 15 | uint64 id = 1; 16 | int64 type = 2; 17 | string name = 3; 18 | map tags = 4; 19 | } 20 | 21 | message IntPoint { 22 | int64 T = 1; 23 | int64 V = 2; 24 | } 25 | 26 | message IntSeries { 27 | Meta meta = 1 [(gogoproto.nullable) = false]; 28 | repeated IntPoint points = 2 [(gogoproto.nullable) = false]; 29 | } 30 | 31 | message DoublePoint { 32 | int64 T = 1; 33 | double V = 2; 34 | } 35 | 36 | message DoubleSeries { 37 | Meta meta = 1 [(gogoproto.nullable) = false]; 38 | repeated DoublePoint points = 2 [(gogoproto.nullable) = false]; 39 | } -------------------------------------------------------------------------------- /_legacy/playground/common/common_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "testing" 5 | "encoding/json" 6 | ) 7 | 8 | func TestIntPoint_Marshal(t *testing.T) { 9 | p := IntPoint{T: 123, V: 1} 10 | b, err := p.Marshal() 11 | if err != nil { 12 | t.Fatal("cant marshal to bytes") 13 | } 14 | p2 := IntPoint{} 15 | err = p2.Unmarshal(b) 16 | if err != nil { 17 | t.Fatal("can't unmarshal from bytes") 18 | } 19 | // the generated type can co-exists with custom JSON 20 | bj, err := json.Marshal(p) 21 | t.Log(string(bj)) 22 | } 23 | 24 | func TestMeta_Marshal(t *testing.T) { 25 | m := Meta{ 26 | Id: 123, 27 | Type: 1, 28 | Name: "haha", 29 | Tags: map[string]string{"os": "ubuntu"}, 30 | } 31 | b, err := m.Marshal() 32 | if err != nil { 33 | t.Fatal("can't marshal to bytes") 34 | } 35 | m2 := Meta{} 36 | err = m2.Unmarshal(b) 37 | if err != nil { 38 | t.Fatal("can't unmarshal from bytes") 39 | } 40 | if m.Id != m2.Id { 41 | t.Fatal("should match") 42 | } 43 | } 44 | 45 | func TestIntSeries_Marshal(t *testing.T) { 46 | intS := IntSeries{ 47 | Meta: Meta{ 48 | Id: 123, 49 | Type: 1, 50 | Name: "haha", 51 | Tags: map[string]string{"os": "ubuntu"}, 52 | }, 53 | Points: []IntPoint{{T: 1, V: 1}, {T: 1, V: 2}}, 54 | } 55 | b, err := intS.Marshal() 56 | if err != nil { 57 | t.Fatal("can't marshal int series") 58 | } 59 | intS2 := IntSeries{} 60 | err = intS2.Unmarshal(b) 61 | if err != nil { 62 | t.Fatal("can't unmarshal int series") 63 | } 64 | if intS.Meta.Id != intS2.Meta.Id { 65 | t.Fatal("should match") 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /_legacy/playground/common/json.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "encoding/json" 6 | ) 7 | 8 | func (m *IntPoint) MarshalJSON() ([]byte, error) { 9 | return []byte(fmt.Sprintf("[%d,%d]", m.T, m.V)), nil 10 | } 11 | 12 | func (m *IntPoint) UnmarshalJSON(data []byte) error { 13 | var tv [2]json.Number 14 | if err := json.Unmarshal(data, &tv); err != nil { 15 | return err 16 | } 17 | // FIXME: this error checking seems to be very low efficient 18 | t, err := tv[0].Int64() 19 | m.T = t 20 | if err != nil { 21 | return err 22 | } 23 | v, err := tv[1].Int64() 24 | m.V = v 25 | if err != nil { 26 | return err 27 | } 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /_legacy/playground/compress_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "compress/zlib" 7 | "io" 8 | "os" 9 | "testing" 10 | //"time" 11 | ) 12 | 13 | // https://golang.org/pkg/compress/gzip/#example__writerReader 14 | func TestGzip(t *testing.T) { 15 | var buf bytes.Buffer 16 | 17 | data := "I am going to duplicate I am I am I am I am I am\n" 18 | 19 | t.Logf("data length %d\n", len(data)) 20 | 21 | zw := gzip.NewWriter(&buf) 22 | //zw.Name = "a-new-hope.txt" 23 | //zw.Comment = "an epic space opera by George Lucas" 24 | //zw.ModTime = time.Date(1977, time.May, 25, 0, 0, 0, 0, time.UTC) 25 | 26 | _, err := zw.Write([]byte(data)) 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | if err := zw.Close(); err != nil { 31 | t.Fatal(err) 32 | } 33 | // the size actually increased since the data is too small 34 | t.Logf("after gzip %d\n", buf.Len()) 35 | 36 | zr, err := gzip.NewReader(&buf) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | // NOTE: output comes from here 42 | if _, err := io.Copy(os.Stdout, zr); err != nil { 43 | t.Fatal(err) 44 | } 45 | 46 | if err := zr.Close(); err != nil { 47 | t.Fatal(err) 48 | } 49 | } 50 | 51 | func TestZlib(t *testing.T) { 52 | var buf bytes.Buffer 53 | 54 | data := "I am going to duplicate I am I am I am I am I am\n" 55 | 56 | t.Logf("data length %d\n", len(data)) 57 | 58 | zw := zlib.NewWriter(&buf) 59 | 60 | _, err := zw.Write([]byte(data)) 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | if err := zw.Close(); err != nil { 65 | t.Fatal(err) 66 | } 67 | // the size actually increased since the data is too small 68 | t.Logf("after zlib %d\n", buf.Len()) 69 | 70 | zr, err := zlib.NewReader(&buf) 71 | if err != nil { 72 | t.Fatal(err) 73 | } 74 | 75 | // NOTE: output comes from here 76 | if _, err := io.Copy(os.Stdout, zr); err != nil { 77 | t.Fatal(err) 78 | } 79 | 80 | if err := zr.Close(); err != nil { 81 | t.Fatal(err) 82 | } 83 | } -------------------------------------------------------------------------------- /_legacy/playground/disk-c/nocomprees.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Read no compressed data block generated from go 3 | * mmap code based on http://beej.us/guide/bgipc/output/html/multipage/mmap.html 4 | * 5 | * And there are arguments about mmap and read http://stackoverflow.com/questions/45972/mmap-vs-reading-blocks 6 | * I guess it might perform better when you mmap file in many processes 7 | */ 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | int main(int argc, char *argv[]) 18 | { 19 | int fd, offset; 20 | char *data; 21 | struct stat sbuf; 22 | 23 | char *file = "../fixture/xephon-no-compress"; 24 | 25 | if ((fd = open(file, O_RDONLY)) == -1) 26 | { 27 | perror("open"); 28 | exit(1); 29 | } 30 | 31 | if (stat(file, &sbuf) == -1) 32 | { 33 | perror("stat"); 34 | exit(1); 35 | } 36 | 37 | // http://stackoverflow.com/questions/38561/what-is-the-argument-for-printf-that-formats-a-long 38 | printf("file size %ld\n", sbuf.st_size); 39 | 40 | data = mmap((caddr_t)0, sbuf.st_size, PROT_READ, MAP_SHARED, fd, 0); 41 | 42 | if (data == (caddr_t)(-1)) 43 | { 44 | perror("mmap"); 45 | exit(1); 46 | } 47 | 48 | // TODO: it seems I don't need to care about byte order for single byte (since there is no order problem for single byte?) 49 | printf("magic number:"); 50 | for (int i = 0; i < 8; i++) { 51 | printf("%c", data[i]); 52 | } 53 | printf("\n"); // the magic number should be xephon-k when printed out 54 | printf("version is %d\n", data[8]); 55 | printf("time compression is %d\n", data[9]); 56 | printf("value compression is %d\n", data[10]); 57 | 58 | munmap(data, sbuf.st_size); 59 | close(fd); 60 | 61 | return 0; 62 | } -------------------------------------------------------------------------------- /_legacy/playground/encoding_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import ( 4 | "testing" 5 | "encoding/json" 6 | "os" 7 | ) 8 | 9 | // https://golang.org/pkg/encoding/json/#Encoder.Encode 10 | func TestEncoding_JSON(t *testing.T) { 11 | encoder := json.NewEncoder(os.Stdout) 12 | encoder.Encode(1) 13 | encoder.Encode(map[string]string{"foo": "bar"}) 14 | } 15 | -------------------------------------------------------------------------------- /_legacy/playground/fixture/.gitignore: -------------------------------------------------------------------------------- 1 | # This folder is used to share file between the Golang code and C code 2 | # normally written by Golang and read in C via mmap 3 | # 4 | # Actually non of the fixtures should be put into git repository, but some are there as a momento -------------------------------------------------------------------------------- /_legacy/playground/fixture/xephon-no-compress: -------------------------------------------------------------------------------- 1 | xephon-k -------------------------------------------------------------------------------- /_legacy/playground/map_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | type index struct { 9 | m map[string]string 10 | } 11 | 12 | func TestNestedMap(t *testing.T) { 13 | var m map[string]map[string]bool 14 | m = make(map[string]map[string]bool, 10) 15 | m["app"] = make(map[string]bool, 10) 16 | fmt.Println(len(m["app"])) 17 | // NOTE: you can't use cap on map http://devs.cloudimmunity.com/gotchas-and-common-mistakes-in-go-golang/index.html#map_cap 18 | // fmt.Println(cap(m["app"])) 19 | } 20 | 21 | func TestMap_Init(t *testing.T) { 22 | i := index{} 23 | // i.m["ha"] = "hahah" // assignment to entry in nil map [recovered] 24 | i.m = make(map[string]string) 25 | i.m["ha"] = "hahah" 26 | } 27 | 28 | type foo struct { 29 | bar string 30 | } 31 | 32 | func TestMap_Range(t *testing.T) { 33 | m := make(map[string]foo) 34 | m["a"] = foo{bar: "a"} 35 | // this can't modify the value 36 | for k := range m { 37 | t := m[k] 38 | t.bar = "t" 39 | } 40 | t.Log(m) 41 | // you have to assign the modified value 42 | for k := range m { 43 | m[k] = foo{bar: "t"} 44 | } 45 | t.Log(m) 46 | } 47 | -------------------------------------------------------------------------------- /_legacy/playground/mmap-c/mmapdemo.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | // http://beej.us/guide/bgipc/output/html/multipage/mmap.html 10 | 11 | int main(int argc, char *argv[]) 12 | { 13 | int fd, offset; 14 | char *data; 15 | struct stat sbuf; 16 | 17 | if (argc != 2) { 18 | fprintf(stderr, "usage: mmapdemo offset\n"); 19 | exit(1); 20 | } 21 | 22 | if ((fd = open("mmapdemo.c", O_RDONLY)) == -1) { 23 | perror("open"); 24 | exit(1); 25 | } 26 | 27 | if (stat("mmapdemo.c", &sbuf) == -1) { 28 | perror("stat"); 29 | exit(1); 30 | } 31 | 32 | offset = atoi(argv[1]); 33 | if (offset < 0 || offset > sbuf.st_size-1) { 34 | fprintf(stderr, "mmapdemo: offset must be in the range 0-%d\n", sbuf.st_size-1); 35 | exit(1); 36 | } 37 | 38 | data = mmap((caddr_t)0, sbuf.st_size, PROT_READ, MAP_SHARED, fd, 0); 39 | 40 | if (data == (caddr_t)(-1)) { 41 | perror("mmap"); 42 | exit(1); 43 | } 44 | 45 | printf("byte at offset %d is '%c'\n", offset, data[offset]); 46 | 47 | return 0; 48 | } -------------------------------------------------------------------------------- /_legacy/playground/mmap/demo.txt: -------------------------------------------------------------------------------- 1 | abc 2 | 123 3 | 456 4 | hahaha -------------------------------------------------------------------------------- /_legacy/playground/mmap/mmap_test.go: -------------------------------------------------------------------------------- 1 | package mmap 2 | 3 | import ( 4 | "testing" 5 | "os" 6 | "syscall" 7 | ) 8 | 9 | // TODO: 10 | // - [ ] write 11 | // - [ ] write & read struct 12 | // - [x] write in golang, read in C/C++ 13 | // - [ ] support larger than 4GB 14 | 15 | func TestMmap_Read(t *testing.T) { 16 | // C source files not allowed when not using cgo or SWIG 17 | f, err := os.Open("demo.txt") 18 | if err != nil { 19 | t.Fatal(err) 20 | return 21 | } 22 | stat, err := f.Stat() 23 | size := stat.Size() 24 | // NOTE: from https://github.com/google/codesearch/blob/master/index/mmap_linux.go#L19 25 | // we can't handle single file larger than 4GB, due to the len function does not work with slice longer than 2^32 26 | // NOTE: bolt use unsafe to convert to byte array to support 256TB ... https://github.com/boltdb/bolt/blob/master/bolt_unix.go#L48 27 | // TODO: why not use int64(int(size)) == size 28 | if int64(int(size+4095)) != size+4095 { 29 | t.Fatalf("%s: too large for mmap", f.Name()) 30 | return 31 | } 32 | n := int(size) 33 | if n == 0 { 34 | t.Fatal("file is empty!") 35 | return 36 | } 37 | // TODO: why codesearch is using (n+4095)&^4095? align to 4KB? I think so 38 | t.Log((n+4095)&^4095) 39 | data, err := syscall.Mmap(int(f.Fd()),0, n, syscall.PROT_READ, syscall.MAP_SHARED) 40 | if err != nil { 41 | t.Fatalf("mmap %s: %v", f.Name(), err) 42 | } 43 | t.Log(len(data)) 44 | t.Log(string(data[0:10])) 45 | } 46 | -------------------------------------------------------------------------------- /_legacy/playground/range_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import "testing" 4 | 5 | // TODO: test if using range on objects would have extra copy 6 | 7 | type Query struct { 8 | StartTime int 9 | } 10 | 11 | func TestRange_Modify(t *testing.T) { 12 | queries := []Query{{StartTime: 0}, {StartTime: 1}} 13 | // NOTE: you can't, it makes a copy of the value 14 | for _, q := range queries { 15 | q.StartTime = 1 16 | } 17 | t.Log(queries) 18 | 19 | for i := 0; i < len(queries); i++ { 20 | // NOTE: this still makes a copy 21 | q := queries[i] 22 | q.StartTime = 1 23 | } 24 | t.Log(queries) 25 | 26 | // NOTE: this the only working way 27 | for i := 0; i < len(queries); i++ { 28 | q := &queries[i] 29 | q.StartTime = 1 30 | } 31 | t.Log(queries) 32 | } 33 | -------------------------------------------------------------------------------- /_legacy/playground/rocksdb_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | // Need to compile RocksDB 4 | // https://github.com/tecbot/gorocksdb 5 | // https://github.com/siddontang/ledisdb/tree/master/store/rocksdb -------------------------------------------------------------------------------- /_legacy/playground/slice_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import ( 4 | "testing" 5 | "fmt" 6 | ) 7 | 8 | // Golang support swap element like python, amazing /w\ 9 | func TestSlice_SwapElement(t *testing.T) { 10 | s := []string{"a", "b"} 11 | s[0], s[1] = s[1], s[0] 12 | fmt.Println(s) 13 | } 14 | 15 | func TestSlice_Equal(t *testing.T) { 16 | s := []int{1, 2} 17 | // can't compare two slice 18 | //s2 := s 19 | //fmt.Println(s == s2) 20 | s1 := &s 21 | s2 := &s 22 | // we can compare pointer to slice though 23 | fmt.Println(s1 == s2) 24 | } 25 | 26 | func TestSlice_Sub(t *testing.T) { 27 | s := make([]byte, 10) 28 | s2 := s[1:10] 29 | t.Log(s2) 30 | } 31 | 32 | 33 | // it's quite strange, I used to think range is slower, but the result is opposite 34 | func BenchmarkSlice_IteratePrimitive(b *testing.B) { 35 | var t int 36 | arr := make([]int, 1000) 37 | // 5000000 373 ns/op 38 | b.Run("range", func(b *testing.B) { 39 | for i := 0; i < b.N; i++ { 40 | for _, j := range arr { 41 | t = j 42 | } 43 | } 44 | }) 45 | // 3000000 467 ns/op 46 | b.Run("index", func(b *testing.B) { 47 | for i := 0; i < b.N; i++ { 48 | for j := 0; j < len(arr); j++ { 49 | t = arr[j] 50 | } 51 | } 52 | }) 53 | // 3000000 557 ns/op 54 | b.Run("index without len", func(b *testing.B) { 55 | for i := 0; i < b.N; i++ { 56 | for j := 0; j < 1000; j++ { 57 | t = arr[j] 58 | } 59 | } 60 | }) 61 | _ = t 62 | } 63 | 64 | // for struct it seems using index is faster due to copy 65 | func BenchmarkSlice_IterateStruct(b *testing.B) { 66 | type ts struct { 67 | a int 68 | b string 69 | } 70 | var t ts 71 | arr := make([]ts, 1000) 72 | for i := 0; i < 1000; i++ { 73 | arr[i] = ts{a: 1, b: "I am a short string"} 74 | } 75 | // 1000000 1421 ns/op 76 | b.Run("range", func(b *testing.B) { 77 | for i := 0; i < b.N; i++ { 78 | for _, j := range arr { 79 | t = j 80 | } 81 | } 82 | }) 83 | // 1000000 1132 ns/op 84 | b.Run("index", func(b *testing.B) { 85 | for i := 0; i < b.N; i++ { 86 | for j := 0; j < len(arr); j++ { 87 | t = arr[j] 88 | } 89 | } 90 | }) 91 | // 1000000 1177 ns/op 92 | b.Run("index without len", func(b *testing.B) { 93 | for i := 0; i < b.N; i++ { 94 | for j := 0; j < 1000; j++ { 95 | t = arr[j] 96 | } 97 | } 98 | }) 99 | _ = t 100 | } 101 | -------------------------------------------------------------------------------- /_legacy/playground/sort_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import ( 4 | "testing" 5 | "fmt" 6 | "sort" 7 | ) 8 | 9 | type StringID string 10 | 11 | func TestString_Compare(t *testing.T) { 12 | s1 := "abc" 13 | s2 := "bbc" 14 | fmt.Println(s1 < s2) // true 15 | } 16 | 17 | func TestStringID_Compare(t *testing.T) { 18 | i1 := StringID("a") 19 | i2 := StringID("b") 20 | i3 := StringID("a1") 21 | fmt.Println(i1 < i2) // true 22 | fmt.Println(i1 < i3) // true 23 | } 24 | 25 | func TestStringID_Search(t *testing.T) { 26 | s := []StringID{"a", "b", "c"} 27 | i := sort.Search(len(s), func(i int) bool { return s[i] >= "b" }) 28 | fmt.Println(i) 29 | } 30 | 31 | func TestStringID_Sort(t *testing.T) { 32 | //s := []StringID{"a", "b", "c"} 33 | //sort.Strings(s) // cannot use s (type []StringID) as type []string in argument to sort.Strings 34 | } 35 | -------------------------------------------------------------------------------- /_legacy/playground/time_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | "math" 7 | "reflect" 8 | "github.com/xephonhq/xephon-k/pkg/bench/generator" 9 | ) 10 | 11 | func TestTime_Unix(t *testing.T) { 12 | // https://github.com/xephonhq/xephon-k/issues/35 13 | // Different time stamp precision 14 | t.Log(math.MaxInt32) 15 | t.Log(time.Now().Unix()) 16 | t.Log(time.Now().Unix() * 1000) 17 | t.Log(time.Now().UnixNano()) 18 | t.Log(math.MaxInt64) 19 | } 20 | 21 | func TestTime_Type(t *testing.T) { 22 | t.Log(reflect.TypeOf(time.Millisecond)) 23 | } 24 | 25 | func TestTime_Match(t *testing.T) { 26 | var tm time.Duration 27 | 28 | //tm = time.Second 29 | //tm = time.Millisecond 30 | tm = generator.DefaultOption.GetPrecision() 31 | 32 | switch tm { 33 | case time.Second: 34 | t.Log("second") 35 | case time.Millisecond: 36 | t.Log("millisecond") 37 | case time.Nanosecond: 38 | t.Log("nanosecond") 39 | } 40 | 41 | table := []struct { 42 | opt generator.Option 43 | }{ 44 | {opt: generator.DefaultOption}, 45 | } 46 | 47 | for _, tt := range table { 48 | t.Log(tt.opt.GetPrecision()) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /_legacy/playground/unsafe_test.go: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import ( 4 | "testing" 5 | "unsafe" 6 | "bytes" 7 | "encoding/binary" 8 | ) 9 | 10 | func TestUnsafe_Ptr(t *testing.T) { 11 | // covert int to a byte slice 12 | i := int64(1) 13 | b := *(*[4]byte)(unsafe.Pointer(&i)) // unsafe_test.go:12: [1 0 0 0] 14 | t.Log(b) 15 | } 16 | 17 | // go test -benchmem -bench=. unsafe_test.go 18 | //BenchmarkUnsafe_EncodeInt64-8 200000 12061 ns/op 9440 B/op 7 allocs/op 19 | //BenchmarkBinary_EncodeInt64-8 100000 15835 ns/op 18912 B/op 8 allocs/op 20 | 21 | func BenchmarkUnsafe_EncodeInt64(b *testing.B) { 22 | for i := 0; i < b.N; i++ { 23 | j := uint64(0) 24 | b := bytes.NewBuffer(nil) 25 | for ; j < 1000; j++ { 26 | b.Write((*(*[4]byte)(unsafe.Pointer(&i)))[:]) 27 | } 28 | } 29 | } 30 | 31 | func BenchmarkBinary_EncodeInt64(b *testing.B) { 32 | for i := 0; i < b.N; i++ { 33 | j := uint64(0) 34 | b := bytes.NewBuffer(nil) 35 | b8 := make([]byte, 8) 36 | for ; j < 1000; j++ { 37 | binary.BigEndian.PutUint64(b8, j) 38 | b.Write(b8) 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /_legacy/playground/xapi/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | ) 9 | 10 | // This demostrate the API from spec 11 | 12 | func main() { 13 | log.Print("Let's start API server") 14 | mux := http.NewServeMux() 15 | mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { 16 | // The "/" pattern matches everything, need to check if it's really the root request 17 | if req.URL.Path != "/" { 18 | http.NotFound(w, req) 19 | return 20 | } 21 | // TODO: should return API doc etc. 22 | fmt.Fprintf(w, "welcom to home page!") 23 | }) 24 | mux.HandleFunc("/info/version", func(w http.ResponseWriter, req *http.Request) { 25 | v := map[string]string{"version": "0.0.1"} 26 | b, err := json.Marshal(v) 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | // FIXED: this would result in an array being printed out in the browser 31 | // fmt.Fprint(w, b) 32 | // TODO: is there a way to use the bytes directly without converting it to string? 33 | fmt.Fprint(w, string(b)) 34 | }) 35 | // TODO: the real logic for writing into Cassandra 36 | mux.HandleFunc("/w", func(w http.ResponseWriter, req *http.Request) { 37 | // TODO: check if the input body is valid 38 | if req.Method != "POST" { 39 | fmt.Fprintf(w, "you should use post!") 40 | } else { 41 | fmt.Fprintf(w, "nice post, but I don't know what to do /w\\") 42 | } 43 | }) 44 | // TODO: the real logic for reading from Cassandra 45 | mux.HandleFunc("/q", func(w http.ResponseWriter, req *http.Request) { 46 | // TODO: check if the input body is valid 47 | if req.Method != "POST" { 48 | fmt.Fprintf(w, "you should use post!") 49 | } else { 50 | fmt.Fprintf(w, "nice post, but I don't know what to do /w\\") 51 | } 52 | }) 53 | err := http.ListenAndServe(":8080", mux) 54 | log.Fatal(err) 55 | } 56 | -------------------------------------------------------------------------------- /_legacy/playground/xbucket/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | 6 | "fmt" 7 | 8 | "github.com/gocql/gocql" 9 | ) 10 | 11 | // This demostrate how to use the bucket schema 12 | 13 | var defaultKeyspace = "system" 14 | var keyspace = "xephonbucket" 15 | 16 | func main() { 17 | log.Print("Let's use bucket schema") 18 | cluster := gocql.NewCluster("127.0.0.1") 19 | // https://github.com/gocql/gocql/blob/master/common_test.go#L98 20 | // It seems you can connect to system keyspace to create new keyspace 21 | log.Print("Connect use system namespace") 22 | cluster.Keyspace = defaultKeyspace 23 | session, err := cluster.CreateSession() 24 | defer session.Close() 25 | if err != nil { 26 | log.Fatal(err) 27 | } 28 | log.Print("connected using system namespace") 29 | 30 | createKeyspaceStmt := fmt.Sprintf("CREATE KEYSPACE IF NOT EXISTS %s WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':1}", keyspace) 31 | if err = session.Query(createKeyspaceStmt).Exec(); err != nil { 32 | log.Fatal(err) 33 | } 34 | log.Printf("keyspace %s created", keyspace) 35 | 36 | // TODO: can I create a new session using existing cluster variable, or do I need to create a new one 37 | } 38 | -------------------------------------------------------------------------------- /_legacy/script/cadvisor/README.md: -------------------------------------------------------------------------------- 1 | # Local monitor 2 | 3 | - Use [cadvisor](https://github.com/google/cadvisor) to monitor both host and container in current local machine 4 | - Use [grafana](http://grafana.org/) for visualization 5 | 6 | ## Usage 7 | 8 | - `up.sh` 9 | - browse `http://localhost:8080` 10 | - FIXME: 8080:9999 can't map the port out, strange 11 | 12 | ## Ref 13 | 14 | - https://github.com/stefanprodan/dockprom Docker hosts and containers monitoring with Prometheus, Grafana, cAdvisor, NodeExporter and AlertManager -------------------------------------------------------------------------------- /_legacy/script/cadvisor/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | cadvisor: 4 | image: google/cadvisor:v0.25.0 5 | container_name: cadvisor 6 | volumes: 7 | - /:/rootfs:ro 8 | - /var/run:/var/run:rw 9 | - /sys:/sys:ro 10 | - /var/lib/docker/:/var/lib/docker:ro 11 | ports: 12 | - "8080:8080" -------------------------------------------------------------------------------- /_legacy/script/cadvisor/up.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Thanks to https://github.com/jepsen-io/jepsen/blob/master/docker/up.sh 3 | 4 | set -e # exit on an error 5 | 6 | ERROR(){ 7 | /bin/echo -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@" 8 | } 9 | 10 | WARNING(){ 11 | /bin/echo -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@" 12 | } 13 | 14 | INFO(){ 15 | /bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@" 16 | } 17 | 18 | exists() { 19 | type $1 > /dev/null 2>&1 20 | } 21 | 22 | exists docker || { ERROR "Please install docker (https://docs.docker.com/engine/installation/)"; exit 1; } 23 | exists docker-compose || { ERROR "Please install docker-compose (https://docs.docker.com/compose/install/)"; exit 1; } 24 | 25 | INFO "Running \`docker-compose build\`" 26 | # docker-compose build --no-cache 27 | docker-compose build 28 | 29 | INFO "Running \`docker-compose up\`" 30 | docker-compose up -------------------------------------------------------------------------------- /_legacy/script/cassandra/cql.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker exec -it tsdb-cassandra bash -------------------------------------------------------------------------------- /_legacy/script/cassandra/pull_cassandra.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # TODO: merge the scripts 4 | # TODO: use not hard coded path 5 | docker run --name tsdb-cassandra -v /home/at15/workspace/ssd/lib/cassandra:/var/lib/cassandra -p 9042:9042 -d cassandra:3.10 6 | sleep 10 7 | echo "create cassandra schema for xephon-k" 8 | xkd schema --config ../../xkd.yml 9 | xkd schema --config ../../xkd.yml 10 | xkd schema --config ../../xkd.yml 11 | echo "schema created" -------------------------------------------------------------------------------- /_legacy/script/cassandra/remove_cassandra.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker stop tsdb-cassandra 4 | docker rm tsdb-cassandra 5 | sudo rm -rf /home/at15/workspace/ssd/lib/cassandra -------------------------------------------------------------------------------- /_legacy/script/cassandra/run_cassandra.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # TODO: if docker start fail, pull the image and name it tsdb-cassandra 4 | 5 | docker start tsdb-cassandra 6 | -------------------------------------------------------------------------------- /_legacy/script/cassandra/xkschema.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "create cassandra schema for xephon-k" 4 | xkd schema --config ../../xkd.yml 5 | xkd schema --config ../../xkd.yml 6 | xkd schema --config ../../xkd.yml 7 | echo "schema created" -------------------------------------------------------------------------------- /_legacy/script/cloc_exclude.txt: -------------------------------------------------------------------------------- 1 | pkg/common/common.pb.go 2 | pkg/storage/disk/disk.pb.go 3 | xk/transport/grpc/rpc.pb.go -------------------------------------------------------------------------------- /_legacy/script/gen_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # TBD 4 | 5 | type Config struct { 6 | XXX map[string]interface{} `yaml:",inline"` 7 | } 8 | 9 | // avoid recursion in UnmarshalYAML 10 | type configAlias Config 11 | 12 | func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { 13 | a := (*configAlias)(c) 14 | if err := unmarshal(a); err != nil { 15 | return err 16 | } 17 | if len(c.XXX) != 0 { 18 | return errors.Errorf("undefined fields %v", c.XXX) 19 | } 20 | return nil 21 | } 22 | 23 | 24 | func (c *Config) Apply() error { 25 | if err := c.Validate(); err != nil { 26 | return err 27 | } 28 | return nil 29 | } 30 | 31 | func (c *Config) Validate() error { 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /_legacy/script/influxdb/.gitignore: -------------------------------------------------------------------------------- 1 | influxdb-* 2 | -------------------------------------------------------------------------------- /_legacy/script/influxdb/influx.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "influx" 4 | echo "create database xb" 5 | echo "show field keys;" 6 | docker exec -it tsdb-influxdb bash -------------------------------------------------------------------------------- /_legacy/script/influxdb/pull_influxdb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # TODO: change configuration 4 | # https://docs.influxdata.com/influxdb/v1.2/administration/config/ 5 | # default 6 | # cache-max-memory-size = 1G 7 | # cache-snapshot-memory-size = 24MB 8 | # The cache snapshot memory size is the size at which the engine will snapshot the cache and write it to a TSM file, freeing up memor 9 | # INFLUXDB_DATA_CACHE_SNAPSHOT_MEMORY_SIZE 10 | # TODO: use not hard coded path 11 | docker run --name tsdb-influxdb -v /home/at15/workspace/ssd/lib/influxdb:/var/lib/influxdb -p 8083:8083 -p 8086:8086 -d influxdb:1.2.4 12 | sleep 5 13 | curl -XPOST 'http://localhost:8086/query?u=myusername&p=mypassword' --data-urlencode 'q=CREATE DATABASE "xb"' -------------------------------------------------------------------------------- /_legacy/script/influxdb/remove_influxdb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker stop tsdb-influxdb 4 | docker rm tsdb-influxdb 5 | sudo rm -rf /home/at15/workspace/ssd/lib/influxdb -------------------------------------------------------------------------------- /_legacy/script/influxdb/run_influxdb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker start tsdb-influxdb 4 | sleep 5 5 | curl -XPOST 'http://localhost:8086/query?u=myusername&p=mypassword' --data-urlencode 'q=CREATE DATABASE "xb"' -------------------------------------------------------------------------------- /_legacy/script/kairosdb/README.md: -------------------------------------------------------------------------------- 1 | # KairosDB 2 | 3 | - http://kairosdb.github.io/ 4 | 5 | ## Usage 6 | 7 | Use docker-compose to run KairosDB with Cassandra 8 | 9 | - `./up.sh` or `docker-compose build`, `docker-compose up` 10 | 11 | Build and run a single KairosDB node with H2 12 | 13 | - rename `Dockerfile.h2` to `Dockerfile` in `node` folder 14 | - `docker build -t xephonhq/kairosdb ./node` 15 | - `docker run -p 8080:8080 --name xephonhq-kairosdb xephonhq/kairosdb` 16 | 17 | ## NOTE 18 | 19 | - [wait-for-it](https://github.com/vishnubob/wait-for-it) does not support alpine, some pr do help https://github.com/vishnubob/wait-for-it/pull/6 20 | 21 | ## Requirement 22 | 23 | - [official docker support for Kairosdb?](https://github.com/kairosdb/kairosdb/issues/288) 24 | - JDK7/8 25 | - Cassandra 2.2 with Thrift enabled 26 | - Cassnadra start and listen on thrift port before KairosDB start 27 | 28 | ## TODO 29 | 30 | - [ ] switch to Oracle JDK for Cassandra 31 | - [ ] switch to Oracle JDK for KairosDB 32 | - [ ] support 3 Node Cassandra cluster (TODO: does C* has 2n+1 requirement?) 33 | - [ ] mount volume 34 | 35 | ## Docker images 36 | 37 | They are listed for reference but only Cassandra image is used 38 | 39 | - https://github.com/cit-lab/kairosdb/tree/feature/alpine 40 | -------------------------------------------------------------------------------- /_legacy/script/kairosdb/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | kairosdb: 4 | container_name: xkb-kairosdb 5 | hostname: kairosdb 6 | build: ./node 7 | ports: 8 | # Web UI and HTTP API 9 | - "8080:8080" 10 | # NOTE: depends_on is not enough, wait-for-it is used in kairosdb.sh 11 | # in order to let cassandra running before KairosDB start 12 | depends_on: 13 | - xkbkairosdbcassandra 14 | links: 15 | - xkbkairosdbcassandra 16 | xkbkairosdbcassandra: 17 | image: cassandra:2.2 18 | environment: 19 | - CASSANDRA_START_RPC=true 20 | -------------------------------------------------------------------------------- /_legacy/script/kairosdb/node/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:8-alpine 2 | 3 | MAINTAINER at15 at15@dongyue.io 4 | 5 | # Thanks to @zbintliff in https://github.com/kairosdb/kairosdb/issues/288 6 | RUN apk upgrade libssl1.0 --update-cache && \ 7 | apk add curl ca-certificates bash 8 | 9 | RUN mkdir /opt; \ 10 | cd /opt; \ 11 | curl -L https://github.com/kairosdb/kairosdb/releases/download/v1.1.2/kairosdb-1.1.2-1.tar.gz | \ 12 | tar zxvfp - 13 | 14 | # Use Cassandra 15 | COPY kairosdb.properties /opt/kairosdb/conf/kairosdb.properties 16 | 17 | # KairosDB must start after Cassandra is running and thrift protocol is enabled 18 | COPY wait-for-it.sh /usr/bin/wait-for-it 19 | COPY kairosdb.sh /usr/bin/kairosdb 20 | 21 | EXPOSE 4242 8080 2003 2004 22 | ENTRYPOINT [] 23 | CMD [ "/usr/bin/kairosdb" ] 24 | -------------------------------------------------------------------------------- /_legacy/script/kairosdb/node/Dockerfile.h2: -------------------------------------------------------------------------------- 1 | FROM java:8-alpine 2 | 3 | MAINTAINER at15 at15@dongyue.io 4 | 5 | # Thanks to @zbintliff in https://github.com/kairosdb/kairosdb/issues/288 6 | RUN apk upgrade libssl1.0 --update-cache && \ 7 | apk add curl ca-certificates bash 8 | 9 | RUN mkdir /opt; \ 10 | cd /opt; \ 11 | curl -L https://github.com/kairosdb/kairosdb/releases/download/v1.1.2/kairosdb-1.1.2-1.tar.gz | \ 12 | tar zxvfp - 13 | 14 | # Use H2 15 | COPY kairosdb.h2.properties /opt/kairosdb/conf/kairosdb.properties 16 | 17 | EXPOSE 4242 8080 2003 2004 18 | ENTRYPOINT [ "/opt/kairosdb/bin/kairosdb.sh" ] 19 | CMD [ "run" ] 20 | -------------------------------------------------------------------------------- /_legacy/script/kairosdb/node/kairosdb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "waiting for cassandra to start" 4 | # FIXME： timeout is not included 5 | wait-for-it xkbkairosdbcassandra:9042 6 | echo "cassandra started" 7 | /opt/kairosdb/bin/kairosdb.sh run 8 | -------------------------------------------------------------------------------- /_legacy/script/kairosdb/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Thanks to https://github.com/jepsen-io/jepsen/blob/master/docker/up.sh 4 | 5 | set -e # exit on an error 6 | 7 | ERROR(){ 8 | /bin/echo -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@" 9 | } 10 | 11 | WARNING(){ 12 | /bin/echo -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@" 13 | } 14 | 15 | INFO(){ 16 | /bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@" 17 | } 18 | 19 | exists() { 20 | type $1 > /dev/null 2>&1 21 | } 22 | 23 | exists docker || { ERROR "Please install docker (https://docs.docker.com/engine/installation/)"; exit 1; } 24 | exists docker-compose || { ERROR "Please install docker-compose (https://docs.docker.com/compose/install/)"; exit 1; } 25 | 26 | INFO "Running \`docker-compose build\`" 27 | # docker-compose build --no-cache 28 | docker-compose build 29 | 30 | INFO "Running \`docker-compose up\`" 31 | docker-compose up 32 | -------------------------------------------------------------------------------- /_legacy/script/prometheus/.gitignore: -------------------------------------------------------------------------------- 1 | prometheus-* -------------------------------------------------------------------------------- /_legacy/script/prometheus/README.md: -------------------------------------------------------------------------------- 1 | # Prometheus 2 | 3 | ## Ref 4 | 5 | https://github.com/at15/hadoop-spark-perf/tree/master/provision/monitor -------------------------------------------------------------------------------- /_legacy/script/prometheus/up.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # get the script path http://stackoverflow.com/questions/4774054/reliable-way-for-a-bash-script-to-get-the-full-path-to-itself 4 | pushd `dirname $0` > /dev/null 5 | SCRIPTPATH=`pwd -P` 6 | popd > /dev/null 7 | ORIGINAL_WD=${PWD} 8 | cd ${SCRIPTPATH} 9 | 10 | cd prometheus-1.5.2.linux-amd64 11 | ./prometheus -config.file=prometheus.yml -------------------------------------------------------------------------------- /_legacy/script/scylladb/pull_scylladb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # TODO: merge the scripts 4 | # TODO: mount volume 5 | docker run --name tsdb-scylladb -p 9042:9042 -d scylladb/scylla:1.7.0 -------------------------------------------------------------------------------- /_legacy/script/scylladb/remove_scylladb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker stop tsdb-scylladb 4 | docker rm tsdb-scylladb -------------------------------------------------------------------------------- /_legacy/script/scylladb/run_scylladb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker start tsdb-scylladb -------------------------------------------------------------------------------- /_legacy/script/scylladb/xkschema.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "create cassandra schema for xephon-k" 4 | xkd schema --cassandra-host localhost 5 | xkd schema --cassandra-host localhost 6 | xkd schema --cassandra-host localhost 7 | echo "schema created" -------------------------------------------------------------------------------- /_legacy/script/timescaledb/psql.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "use psql" 4 | docker exec -it tsdb-timescaledb bash -------------------------------------------------------------------------------- /_legacy/script/timescaledb/pull_timescaledb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # https://docs.timescale.com/other-sample-datasets 4 | # psql -U postgres 5 | # create database devices_small; 6 | # \c devices_small; 7 | # CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; 8 | # psql -U postgres -d devices_small < devices.sql 9 | # psql -U postgres -d devices_small -c "\COPY readings FROM devices_small_readings.csv CSV" 10 | # psql -U postgres -d devices_small -c "\COPY device_info FROM devices_small_device_info.csv CSV" 11 | docker run -d --name tsdb-timescaledb -p 5432:5432 \ 12 | -v /home/at15/workspace/data:/opt/xephonk \ 13 | -e PGDATA=/var/lib/postgresql/data/timescaledb \ 14 | timescale/timescaledb postgres \ 15 | -cshared_preload_libraries=timescaledb -------------------------------------------------------------------------------- /_legacy/script/timescaledb/remove_timescaledb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker stop tsdb-timescaledb 4 | docker rm tsdb-timescaledb -------------------------------------------------------------------------------- /_legacy/script/travis_install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # install glide 4 | 5 | # switch folder 6 | # get the script path http://stackoverflow.com/questions/4774054/reliable-way-for-a-bash-script-to-get-the-full-path-to-itself 7 | pushd `dirname $0` > /dev/null 8 | SCRIPTPATH=`pwd -P` 9 | popd > /dev/null 10 | # get current working directory 11 | ORIGINAL_WD=${PWD} 12 | # switch to script directory 13 | cd ${SCRIPTPATH} 14 | 15 | # download and extract 16 | wget https://github.com/Masterminds/glide/releases/download/v0.12.3/glide-v0.12.3-linux-amd64.tar.gz 17 | tar -zxvf glide-v0.12.3-linux-amd64.tar.gz 18 | # add glide to path 19 | export PATH=$PATH:${SCRIPTPATH}/linux-amd64 20 | # show it is working 21 | glide -v 22 | 23 | # install dependencies 24 | cd .. 25 | glide install 26 | 27 | # go back to the old working directory 28 | cd ${ORIGINAL_WD} -------------------------------------------------------------------------------- /_legacy/script/xephon-k-cassandra/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | xephon-k-cassandra: 4 | container_name: xkb-xephon-k-cassandra 5 | hostname: xephon-k 6 | build: ./node 7 | ports: 8 | - "23333:23333" 9 | # NOTE: depends_on is not enough, wait-for-it is used in kairosdb.sh 10 | # in order to let cassandra running before KairosDB start 11 | depends_on: 12 | - xkbxephonkcassandracassandra 13 | links: 14 | - xkbxephonkcassandracassandra 15 | xkbxephonkcassandracassandra: 16 | image: cassandra:3.10 17 | # environment: 18 | # - CASSANDRA_START_RPC=true 19 | -------------------------------------------------------------------------------- /_legacy/script/xephon-k-cassandra/node/.gitignore: -------------------------------------------------------------------------------- 1 | xkd -------------------------------------------------------------------------------- /_legacy/script/xephon-k-cassandra/node/Dockerfile: -------------------------------------------------------------------------------- 1 | #FROM alpine:3.5 2 | FROM ubuntu:16.04 3 | 4 | MAINTAINER at15 at15@dongyue.io 5 | 6 | # Thanks to @zbintliff in https://github.com/kairosdb/kairosdb/issues/288 7 | #RUN apk upgrade libssl1.0 --update-cache && \ 8 | # apk add curl ca-certificates bash 9 | 10 | # KairosDB must start after Cassandra is running and thrift protocol is enabled 11 | COPY wait-for-it.sh /usr/bin/wait-for-it 12 | COPY xkdc.sh /usr/bin/xkdc 13 | COPY xkd /usr/bin/xkd 14 | 15 | EXPOSE 23333 16 | ENTRYPOINT [] 17 | CMD [ "/usr/bin/xkdc" ] 18 | -------------------------------------------------------------------------------- /_legacy/script/xephon-k-cassandra/node/xkdc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "waiting for cassandra to start" 4 | # FIXME： timeout is not included 5 | wait-for-it -t 60 xkbxephonkcassandracassandra:9042 6 | #sleep 10 7 | echo "cassandra started" 8 | # FIXME: copy config file and use new command line flags 9 | xkd schema --cassandra-host xkbxephonkcassandracassandra 10 | xkd schema --cassandra-host xkbxephonkcassandracassandra 11 | xkd schema --cassandra-host xkbxephonkcassandracassandra 12 | xkd -b c --cassandra-host xkbxephonkcassandracassandra 13 | -------------------------------------------------------------------------------- /_legacy/script/xephon-k-cassandra/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Thanks to https://github.com/jepsen-io/jepsen/blob/master/docker/up.sh 4 | 5 | set -e # exit on an error 6 | 7 | ERROR(){ 8 | /bin/echo -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@" 9 | } 10 | 11 | WARNING(){ 12 | /bin/echo -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@" 13 | } 14 | 15 | INFO(){ 16 | /bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@" 17 | } 18 | 19 | exists() { 20 | type $1 > /dev/null 2>&1 21 | } 22 | 23 | exists docker || { ERROR "Please install docker (https://docs.docker.com/engine/installation/)"; exit 1; } 24 | exists docker-compose || { ERROR "Please install docker-compose (https://docs.docker.com/compose/install/)"; exit 1; } 25 | 26 | INFO "Running \`docker-compose build\`" 27 | # docker-compose build --no-cache 28 | docker-compose build 29 | 30 | INFO "Running \`docker-compose up\`" 31 | docker-compose up 32 | -------------------------------------------------------------------------------- /_legacy/web/README.md: -------------------------------------------------------------------------------- 1 | # Web UI 2 | 3 | A simple UI using echarts 4 | 5 | ## Vue 6 | 7 | Share state 8 | 9 | - https://vuejs.org/v2/guide/state-management.html 10 | - https://vuejs.org/v2/guide/components.html#Non-Parent-Child-Communication 11 | - Vuex https://vuex.vuejs.org/en/getting-started.html 12 | 13 | Code editor 14 | 15 | - Ace fiddle https://jsfiddle.net/bc_rikko/gbpw2q9x/3/ 16 | - CodeMirror https://surmon-china.github.io/vue-codemirror/ 17 | - Brace (browserify Ace) https://admin.vuebulma.com/#/components/brace 18 | 19 | Moment 20 | 21 | - https://momentjs.com/docs/#/parsing/string/ 22 | - https://momentjs.com/docs/#/manipulating/add/ 23 | 24 | ## Notes 25 | 26 | - Ace editor need to specify element size, otherwise it would disappear 27 | - FireFox does not support `date` at all http://stackoverflow.com/questions/22983013/how-to-get-html-5-input-type-date-working-in-firefox-and-or-ie-10 28 | - Chrome does not support `datetime`, but support `datetime-local` 29 | 30 | ## Ref 31 | 32 | - Line stack http://gallery.echartsjs.com/editor.html?c=line-stack 33 | - Dynamic time series http://gallery.echartsjs.com/editor.html?c=dynamic-data2 34 | - Heat map (like GitHub strike) http://gallery.echartsjs.com/editor.html?c=calendar-heatmap 35 | - http://gallery.echartsjs.com/editor.html?c=mix-timeline-finance 36 | - http://gallery.echartsjs.com/editor.html?c=candlestick-sh 37 | -------------------------------------------------------------------------------- /_legacy/web/util.js: -------------------------------------------------------------------------------- 1 | // util functions 2 | 3 | function str2Timestamp(str) { 4 | return parseInt(moment(str).format('x')); 5 | } 6 | 7 | // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/random 8 | function randomInt(min, max) { 9 | min = Math.ceil(min); 10 | max = Math.floor(max); 11 | return Math.floor(Math.random() * (max - min)) + min; 12 | } 13 | -------------------------------------------------------------------------------- /_legacy/xk.yml: -------------------------------------------------------------------------------- 1 | http: 2 | addr: :2333 3 | secure: false 4 | cert: xk.crt 5 | key: xk.key 6 | enableTracing: false 7 | grpc: 8 | addr: :2334 9 | secure: false 10 | cert: xk.crt 11 | key: xk.key 12 | enableTracing: false -------------------------------------------------------------------------------- /_legacy/xk/client/grpcclient/client_test.go: -------------------------------------------------------------------------------- 1 | package grpcclient 2 | 3 | import "testing" 4 | 5 | func TestClient_WriteIntPoint(t *testing.T) { 6 | t.Skipf("TODO: wait until server is implemented") 7 | } 8 | -------------------------------------------------------------------------------- /_legacy/xk/client/meta.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/libtsdb/libtsdb-go/libtsdb" 7 | ) 8 | 9 | const ( 10 | name = "xephonk" 11 | precision = time.Nanosecond 12 | ) 13 | 14 | var meta = libtsdb.Meta{ 15 | Name: name, 16 | TimePrecision: precision, 17 | SupportTag: true, 18 | SupportInt: true, 19 | SupportDouble: true, 20 | SupportBatchSeries: true, 21 | SupportBatchPoints: true, 22 | } 23 | 24 | func Meta() libtsdb.Meta { 25 | return meta 26 | } 27 | 28 | func init() { 29 | libtsdb.RegisterMeta(name, meta) 30 | } 31 | -------------------------------------------------------------------------------- /_legacy/xk/config/client.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type ClientConfig struct { 4 | Addr string `yaml:"addr"` 5 | Prepare bool `yaml:"prepare"` 6 | Columnar bool `yaml:"columnar"` 7 | } 8 | -------------------------------------------------------------------------------- /_legacy/xk/config/server.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | iconfig "github.com/dyweb/go.ice/config" 5 | ) 6 | 7 | type ServerConfig struct { 8 | Http iconfig.HttpServerConfig `yaml:"http"` 9 | Grpc iconfig.GrpcServerConfig `yaml:"grpc"` 10 | } 11 | -------------------------------------------------------------------------------- /_legacy/xk/server/gommon.yml: -------------------------------------------------------------------------------- 1 | loggers: 2 | - struct: "*GrpcServer" 3 | receiver: "srv" 4 | - struct: "*HttpServer" 5 | receiver: "srv" 6 | - struct: "*Manager" 7 | receiver: "mgr" -------------------------------------------------------------------------------- /_legacy/xk/server/gommon_generated.go: -------------------------------------------------------------------------------- 1 | // Code generated by gommon from xk/server/gommon.yml DO NOT EDIT. 2 | 3 | package server 4 | 5 | import dlog "github.com/dyweb/gommon/log" 6 | 7 | func (srv *GrpcServer) SetLogger(logger *dlog.Logger) { 8 | srv.log = logger 9 | } 10 | 11 | func (srv *GrpcServer) GetLogger() *dlog.Logger { 12 | return srv.log 13 | } 14 | 15 | func (srv *GrpcServer) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { 16 | return justCallMe() 17 | } 18 | 19 | func (srv *HttpServer) SetLogger(logger *dlog.Logger) { 20 | srv.log = logger 21 | } 22 | 23 | func (srv *HttpServer) GetLogger() *dlog.Logger { 24 | return srv.log 25 | } 26 | 27 | func (srv *HttpServer) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { 28 | return justCallMe() 29 | } 30 | 31 | func (mgr *Manager) SetLogger(logger *dlog.Logger) { 32 | mgr.log = logger 33 | } 34 | 35 | func (mgr *Manager) GetLogger() *dlog.Logger { 36 | return mgr.log 37 | } 38 | 39 | func (mgr *Manager) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { 40 | return justCallMe() 41 | } 42 | -------------------------------------------------------------------------------- /_legacy/xk/server/grpc.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | 6 | dlog "github.com/dyweb/gommon/log" 7 | 8 | pb "github.com/xephonhq/xephon-k/xk/transport/grpc" 9 | "google.golang.org/grpc/codes" 10 | "google.golang.org/grpc/status" 11 | ) 12 | 13 | var _ pb.XephonkServer = (*GrpcServer)(nil) 14 | 15 | type GrpcServer struct { 16 | log *dlog.Logger 17 | } 18 | 19 | func NewGrpcServer() (*GrpcServer, error) { 20 | srv := &GrpcServer{} 21 | dlog.NewStructLogger(log, srv) 22 | return srv, nil 23 | } 24 | 25 | func (srv *GrpcServer) Ping(ctx context.Context, ping *pb.PingReq) (*pb.PingRes, error) { 26 | return &pb.PingRes{Message: "pong from xephonk your message is " + ping.Message}, nil 27 | } 28 | 29 | func (srv *GrpcServer) WritePoints(ctx context.Context, req *pb.WritePointsReq) (*pb.WritePointsRes, error) { 30 | return nil, status.Error(codes.Unimplemented, "not implemented") 31 | } 32 | 33 | func (srv *GrpcServer) WriteSeries(ctx context.Context, req *pb.WriteSeriesReq) (*pb.WriteSeriesRes, error) { 34 | return nil, status.Error(codes.Unimplemented, "not implemented") 35 | } 36 | 37 | func (srv *GrpcServer) PrepareSeries(ctx context.Context, req *pb.PrepareSeriesReq) (*pb.PrepareSeriesRes, error) { 38 | return nil, status.Error(codes.Unimplemented, "not implemented") 39 | } 40 | 41 | func (srv *GrpcServer) WritePreparedPoints(ctx context.Context, req *pb.WritePreparedPointsReq) (*pb.WritePreparedPointsRes, error) { 42 | return nil, status.Error(codes.Unimplemented, "not implemented") 43 | } 44 | 45 | func (srv *GrpcServer) WritePreparedSeries(ctx context.Context, req *pb.WritePreparedSeriesReq) (*pb.WritePreparedSeriesRes, error) { 46 | return nil, status.Error(codes.Unimplemented, "not implemented") 47 | } 48 | 49 | func (srv *GrpcServer) WritePreparedSeriesColumnar(ctx context.Context, req *pb.WritePreparedSeriesColumnarReq) (*pb.WritePreparedSeriesColumnarRes, error) { 50 | return nil, status.Error(codes.Unimplemented, "not implemented") 51 | } 52 | -------------------------------------------------------------------------------- /_legacy/xk/server/http.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | dlog "github.com/dyweb/gommon/log" 5 | "net/http" 6 | ) 7 | 8 | type HttpServer struct { 9 | log *dlog.Logger 10 | } 11 | 12 | func NewHttpServer() (*HttpServer, error) { 13 | s := &HttpServer{} 14 | dlog.NewStructLogger(log, s) 15 | return s, nil 16 | } 17 | 18 | func (srv *HttpServer) Handler() http.Handler { 19 | mux := http.NewServeMux() 20 | mux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) { 21 | w.Write([]byte("pong")) 22 | }) 23 | return mux 24 | } 25 | -------------------------------------------------------------------------------- /_legacy/xk/server/pkg.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/xephonhq/xephon-k/xk/util/logutil" 5 | ) 6 | 7 | var log = logutil.NewPackageLogger() 8 | -------------------------------------------------------------------------------- /_legacy/xk/storage/mem/gommon.yml: -------------------------------------------------------------------------------- 1 | gotmpls: 2 | - src: ring_generated.go.tmpl 3 | dst: ring_generated.go 4 | go: true 5 | data: 6 | - Name: Int 7 | Type: int64 8 | - Name: Double 9 | Type: float64 -------------------------------------------------------------------------------- /_legacy/xk/storage/mem/gommon_generated.go: -------------------------------------------------------------------------------- 1 | // Code generated by gommon from xk/storage/mem/gommon.yml DO NOT EDIT. 2 | 3 | package mem 4 | -------------------------------------------------------------------------------- /_legacy/xk/storage/mem/ring_generated.go.tmpl: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import "sync" 4 | 5 | {{ range .}} 6 | // {{.Name}}Ring is a simple cache without compression 7 | type {{.Name}}Ring struct { 8 | numPartitions uint64 9 | // TODO: benchmark performance of using pointer and struct https://segment.com/blog/allocation-efficiency-in-high-performance-go-services/ 10 | partitions []*{{.Name}}Partition 11 | } 12 | 13 | // TODO: default value for n? 14 | func New{{.Name}}Ring(n int) *{{.Name}}Ring { 15 | r := &{{.Name}}Ring{ 16 | numPartitions: uint64(n), 17 | partitions: make([]*{{.Name}}Partition, n), 18 | } 19 | for i := 0; i < n; i++ { 20 | r.partitions[i] = &{{.Name}}Partition{ 21 | store: make(map[uint64]*{{.Name}}Store), 22 | } 23 | } 24 | return r 25 | } 26 | 27 | func (r *{{.Name}}Ring) getPartition(hash uint64) *{{.Name}}Partition { 28 | return r.partitions[int(hash%r.numPartitions)] 29 | } 30 | 31 | type {{.Name}}Partition struct { 32 | mu sync.RWMutex 33 | store map[uint64]*{{.Name}}Store 34 | } 35 | 36 | type {{.Name}}Store struct { 37 | // TODO: metrics tags 38 | mu sync.RWMutex 39 | times []int64 40 | values []{{.Type}} 41 | size int 42 | } 43 | 44 | func New{{.Name}}Store(len int) *{{.Name}}Store { 45 | return &{{.Name}}Store{ 46 | times: make([]int64, 0, len), 47 | values: make([]{{.Type}}, 0, len), 48 | } 49 | } 50 | 51 | func (p *{{.Name}}Partition) WritePoints(hash uint64, times []int64, values []{{.Type}}) { 52 | p.mu.RLock() 53 | s := p.store[hash] 54 | if s != nil { 55 | s.mu.Lock() 56 | p.mu.RUnlock() 57 | s.size += len(times) 58 | s.times = append(s.times, times...) 59 | s.values = append(s.values, values...) 60 | s.mu.Unlock() 61 | return 62 | } 63 | p.mu.RUnlock() 64 | p.mu.Lock() 65 | s = New{{.Name}}Store(len(times)) 66 | s.size += len(times) 67 | s.times = append(s.times, times...) 68 | s.values = append(s.values, values...) 69 | p.store[hash] = s 70 | p.mu.Unlock() 71 | return 72 | } 73 | {{ end }} -------------------------------------------------------------------------------- /_legacy/xk/storage/mem/ring_test.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "runtime" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/dyweb/gommon/util/testutil" 10 | ) 11 | 12 | func TestNewDoubleRing(t *testing.T) { 13 | r := NewDoubleRing(runtime.NumCPU()) 14 | var i uint64 15 | var j int64 16 | var batchSize int64 = 10 17 | for i = 0; i < 100; i++ { 18 | now := time.Now().UnixNano() 19 | times := make([]int64, batchSize) 20 | values := make([]float64, batchSize) 21 | for j = 0; j < batchSize; j++ { 22 | times[j] = now + j 23 | values[j] = float64(j) 24 | } 25 | r.getPartition(i).WritePoints(i, times[:], values[:]) 26 | } 27 | if testutil.Dump().B() { 28 | for i := 0; i < runtime.NumCPU(); i++ { 29 | t.Logf("partition %d size %d", i, len(r.partitions[i].store)) 30 | for j, s := range r.partitions[i].store { 31 | t.Logf("hash %d size %d", j, s.size) 32 | } 33 | } 34 | } 35 | } 36 | 37 | // NOTE: race 38 | func TestDoublePartition_WritePoints(t *testing.T) { 39 | var wg sync.WaitGroup 40 | var batchSize int64 = 10 41 | concurrency := runtime.NumCPU() 42 | r := NewDoubleRing(runtime.NumCPU()) 43 | wg.Add(concurrency) 44 | for c := 0; c < concurrency; c++ { 45 | go func() { 46 | var i uint64 47 | var j int64 48 | 49 | for i = 0; i < 100; i++ { 50 | now := time.Now().UnixNano() 51 | times := make([]int64, batchSize) 52 | values := make([]float64, batchSize) 53 | for j = 0; j < batchSize; j++ { 54 | times[j] = now + j 55 | values[j] = float64(j) 56 | } 57 | r.getPartition(i).WritePoints(i, times[:], values[:]) 58 | } 59 | wg.Done() 60 | }() 61 | } 62 | wg.Wait() 63 | } 64 | -------------------------------------------------------------------------------- /_legacy/xk/storage/provider.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | type MetaProvider interface { 4 | } 5 | 6 | type DataProvider interface { 7 | } 8 | 9 | type Provider interface { 10 | MetaProvider 11 | DataProvider 12 | } 13 | 14 | // TODO: code for register provider etc. 15 | -------------------------------------------------------------------------------- /_legacy/xk/transport/grpc/client.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import "google.golang.org/grpc" 4 | 5 | func NewClient(conn *grpc.ClientConn) XephonkClient { 6 | return NewXephonkClient(conn) 7 | } 8 | -------------------------------------------------------------------------------- /_legacy/xk/transport/grpc/gommon.yml: -------------------------------------------------------------------------------- 1 | shells: 2 | - code: protoc --proto_path=$GOPATH/src/:. --gogo_out=plugins=grpc:. rpc.proto 3 | shell: true 4 | cd: true -------------------------------------------------------------------------------- /_legacy/xk/transport/grpc/gommon_generated.go: -------------------------------------------------------------------------------- 1 | // Code generated by gommon from xk/transport/grpc/gommon.yml DO NOT EDIT. 2 | 3 | package grpc 4 | -------------------------------------------------------------------------------- /doc/CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xephonhq/xephon-k/b6a213ea9b529207205e882977047960920b4e30/doc/CHANGELOG.md -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | - [Survey on existing Cassandra based TSDBs](survey) 4 | - [Benchmark](bench) 5 | - [Roadmap](roadmap.md) -------------------------------------------------------------------------------- /doc/ROADMAP.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xephonhq/xephon-k/b6a213ea9b529207205e882977047960920b4e30/doc/ROADMAP.md -------------------------------------------------------------------------------- /doc/bench/278-legacy/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Bechmark Result 6 | 7 | 8 | 9 |
10 | 66 | 67 | -------------------------------------------------------------------------------- /doc/bench/278-legacy/influxdb.md: -------------------------------------------------------------------------------- 1 | # Benchmark Result: InfluxDB 2 | 3 | version: 1.2.2 4 | 5 | ````bash 6 | xkb --db i 7 | ```` 8 | 9 | - mem: 70 MB 10 | 11 | ```` 12 | INFO[0033] basic report finished via context pkg=k.b.reporter 13 | INFO[0033] total request 118 pkg=k.b.reporter 14 | INFO[0033] fastest 96604875 pkg=k.b.reporter 15 | INFO[0033] slowest 572493221 pkg=k.b.reporter 16 | INFO[0033] total request size 613600 pkg=k.b.reporter 17 | INFO[0033] toatl response size 0 pkg=k.b.reporter 18 | INFO[0033] 204: 118 pkg=k.b.reporter 19 | INFO[0033] loader finished pkg=k.b.loader 20 | 21 | Duration: 5 22 | Worker number: 10 23 | Batch size: 100 24 | Timeout: 30 25 | TargetDB: influxdb 26 | ```` 27 | 28 | ````bash 29 | xkb --db i -c 100 30 | ```` 31 | 32 | - mem: 90 MB 33 | 34 | ```` 35 | INFO[0015] total request 139 pkg=k.b.reporter 36 | INFO[0015] fastest 102784459 pkg=k.b.reporter 37 | INFO[0015] slowest 4299014783 pkg=k.b.reporter 38 | INFO[0015] total request size 722800 pkg=k.b.reporter 39 | INFO[0015] toatl response size 0 pkg=k.b.reporter 40 | INFO[0015] 204: 139 pkg=k.b.reporter 41 | INFO[0015] loader finished pkg=k.b.loader 42 | 43 | Duration: 5 44 | Worker number: 100 45 | Batch size: 100 46 | Timeout: 30 47 | TargetDB: influxdb 48 | ```` 49 | 50 | ````bash 51 | xkb --db i -c 1000 52 | ```` 53 | 54 | - mem: 450 MB 55 | 56 | ```` 57 | INFO[0017] total request 1026 pkg=k.b.reporter 58 | INFO[0017] fastest 26741 pkg=k.b.reporter 59 | INFO[0017] slowest 5936632875 pkg=k.b.reporter 60 | INFO[0017] total request size 5335200 pkg=k.b.reporter 61 | INFO[0017] toatl response size 0 pkg=k.b.reporter 62 | INFO[0017] 204: 131 pkg=k.b.reporter 63 | INFO[0017] 0: 895 pkg=k.b.reporter 64 | INFO[0017] loader finished pkg=k.b.loader 65 | 66 | Duration: 5 67 | Worker number: 1000 68 | Batch size: 100 69 | Timeout: 30 70 | TargetDB: influxdb 71 | ```` 72 | 73 | ````bash 74 | xkb --db i -c 100 -d 60 75 | ```` 76 | 77 | - mem: 121 MB 78 | - [ ] TODO: virtual size is the disk size? 790 MB? 79 | 80 | ```` 81 | INFO[0068] total request 1332 pkg=k.b.reporter 82 | INFO[0068] fastest 87702586 pkg=k.b.reporter 83 | INFO[0068] slowest 5231069450 pkg=k.b.reporter 84 | INFO[0068] total request size 6926400 pkg=k.b.reporter 85 | INFO[0068] toatl response size 0 pkg=k.b.reporter 86 | INFO[0068] 204: 1332 pkg=k.b.reporter 87 | INFO[0068] loader finished pkg=k.b.loader 88 | 89 | Duration: 60 90 | Worker number: 100 91 | Batch size: 100 92 | Timeout: 30 93 | TargetDB: influxdb 94 | ```` -------------------------------------------------------------------------------- /doc/bench/README.md: -------------------------------------------------------------------------------- 1 | # Benchmark 2 | 3 | TBD 4 | 5 | ## Known issues 6 | 7 | - [ ] InfluxDB has extremely poor performance [#15](https://github.com/xephonhq/xephon-k/issues/15) -------------------------------------------------------------------------------- /doc/bench/space/compression.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Space usage after compression 6 | 7 | 8 | 9 |
10 | 63 | 64 | -------------------------------------------------------------------------------- /doc/bench/space/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Space usage 6 | 7 | 8 | 9 |
10 | 72 | 73 | -------------------------------------------------------------------------------- /doc/bench/throughput/influx-graphite-kairosdb.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Benchmark Result 6 | 7 | 8 | 9 |
10 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /doc/bench/throughput/influxver.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Bechmark Result 6 | 7 | 8 | 9 | 10 |
11 | 47 | 48 | -------------------------------------------------------------------------------- /doc/bench/throughput/mysqlver.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Benchmark Result 6 | 7 | 8 | 9 | 10 |
11 | 47 | 48 | -------------------------------------------------------------------------------- /doc/bench/throughput/postgresver.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Benchmark Result 6 | 7 | 8 | 9 | 10 |
11 | 47 | 48 | -------------------------------------------------------------------------------- /doc/design/data-mem.md: -------------------------------------------------------------------------------- 1 | # In memory data store 2 | 3 | - issue: https://github.com/xephonhq/xephon-k/issues/65 4 | 5 | ## Previously, on Xephon-K 6 | 7 | Old in memory data store, just a map with mutex, 8 | 9 | ````go 10 | // Data is a map using SeriesID as key 11 | type Data struct { 12 | mu sync.Mutex 13 | series map[common.SeriesID]SeriesStore 14 | } 15 | ```` 16 | 17 | ## Problems 18 | 19 | - reduce lock contention, need benchmark to see what happens to map with a lock with many go routine try to access it 20 | - #57 https://github.com/xephonhq/xephon-k/issues/57 storage memory concurrent map write 21 | - sharding could be a way 22 | - [ ] TODO: see how prometheus and influxdb handle this 23 | - reduce gc overhead 24 | - [ ] https://github.com/allegro/bigcache https://allegro.tech/2016/03/writing-fast-cache-service-in-go.html 25 | - test with larger memory usage 26 | 27 | 28 | ## Go specific 29 | 30 | - https://github.com/allegro/bigcache 31 | - https://github.com/coocood/freecache 32 | 33 | ## Other databases 34 | 35 | ### InfluxDB 36 | 37 | - see [doc/survey/influxdb/write-path.md](../survey/influxdb/write-path.md) and [doc/survey/influxdb/read-path.md](../survey/influxdb/read-path.md) 38 | - use a ring shard (its robin hood hashing is used for meta index), it has # of cores partitions 39 | - each partition has a `map[string]*entry` where key is measurement + tags + field key, partition has rwmutex 40 | - each entry contains a slice of values, entry has rwmutex 41 | - data in memory is not compressed 42 | - default cache max memory is 1GB 43 | - default cache snapshot memory size is 25MB 44 | - default cache snapshot duration is 10 minute -------------------------------------------------------------------------------- /doc/design/protocol.md: -------------------------------------------------------------------------------- 1 | # Protocol 2 | 3 | Framework independent 4 | 5 | - TCP 6 | - HTTP/JSON 7 | - HTTP/Line 8 | 9 | Framework 10 | 11 | - GRPC -------------------------------------------------------------------------------- /doc/impl/README.md: -------------------------------------------------------------------------------- 1 | # Implementation 2 | 3 | A general overview of the implementation. Highly likely to be outdated. 4 | 5 | ## Write 6 | 7 | - client 8 | - HTTP Server `pkg/server/http_server.go` 9 | - JSON decoded `pkg/server/service/write.go` `MakeDecode` 10 | - use `MetaSeries` to decode meta first (TODO: this might introduce extra copy of the points part in payload) 11 | - switch `SeriesType` to decode into `IntSeries` and `DoubleSeries` 12 | - FIXME: return error if meet un matched `SeriesType`, (should allow others to pass?) 13 | - EndPoint `pkg/server/service/write.go` `MakeEndpoint` 14 | - `WriteServiceServerImpl` call `WriteInt` of its `Store` (memory, disk, cassandra etc.) 15 | - Store `pkg/storage/store.go` defines the interface 16 | - `pkg/storage/memory/store.go` `WriteIntSeries` write both the data and index 17 | - `pkg/storage/cassandra/store.go` `WriteIntSeries` only write data 18 | - `pkg/storage/disk/store.go` TODO: nothing is implemented 19 | - Memory: WriteIntSeries Data 20 | - `pkg/storage/memory/data.go` `WriteIntSeries` write to different maps for different types 21 | - TODO: this introduce problem for lookup, should use a single map, otherwise for each series ID we need two lookup 22 | - the two maps contain pointer to `IntSeriesStore`, `DoubleSeriesStore` separately. 23 | - Memory: IntSeriesStore 24 | - `pkg/storage/memory/series_store.go` protect a `common.IntSeries` with a RWMutex 25 | - TODO: should use several (link list) blocks instead of a huge slice 26 | - `WriteSeries` sort the input and do a merge sort 27 | - TODO: the sort can be avoided if we allow user to specify it when input 28 | - TODO: there are many possible speed ups, like append it directly 29 | - TODO: compute the common aggregation (allow user to configure it) 30 | - Memory: WriteIntSeries Index 31 | - `pkg/storage/memory/index.go` `Index Add(id, tagKey, tagValue)` 32 | - create new inverted index if `tagKeytagValue` does not exists FIXME: currently direct concat the two string 33 | - call `InvertedIndex Add(id)`, use quick search to find if the id already exists, otherwise the position to insert it (to keep the order) 34 | - TODO: could add a map as extra layer to speed up insert and merge it in in batch 35 | 36 | ## Read 37 | 38 | TBD -------------------------------------------------------------------------------- /doc/impl/aggregation.md: -------------------------------------------------------------------------------- 1 | # Aggregation 2 | 3 | ## Ref 4 | 5 | Average 6 | 7 | - http://www.datastax.com/dev/blog/metric-collection-and-storage-with-cassandra 8 | - https://en.wikipedia.org/wiki/Moving_average#Cumulative_moving_average -------------------------------------------------------------------------------- /doc/impl/api.md: -------------------------------------------------------------------------------- 1 | # API Implementation Notes 2 | 3 | Won't use 4 | 5 | - ~~existing web framework~~ now using go-kit 6 | 7 | Won't support 8 | 9 | - HTTP2 10 | - HTTPS 11 | 12 | Libs may help 13 | 14 | - https://github.com/asaskevich/govalidator got functions like `IsJSON` 15 | - https://github.com/throttled/throttled a rate limiter support multiple store backend, though I didn't find how it identify client 16 | 17 | ## Basic 18 | 19 | ### Use net/http 20 | 21 | - https://cryptic.io/go-http/ 22 | - use server mux 23 | - [ ] test without listening to port 24 | - [ ] it also mentioned middleware 25 | 26 | ### Build middleware 27 | 28 | - [ ] http://www.alexedwards.net/blog/making-and-using-middleware 29 | 30 | ### Inner Representation 31 | 32 | - [ ] The payload from KairosDB may not work 33 | 34 | ## Advanced 35 | 36 | Some a little bit advanced topic 37 | 38 | ### Context and Concurrency 39 | 40 | - [ ] https://blog.golang.org/concurrency-is-not-parallelism 41 | - [ ] https://blog.golang.org/pipelines 42 | 43 | Context 44 | 45 | It seems context already exists before 1.7, it's just it is added to http in 1.7 46 | 47 | - [ ] https://medium.com/@matryer/context-has-arrived-per-request-state-in-go-1-7-4d095be83bd8#.v7y4nj9kp 48 | - [ ] https://medium.com/@cep21/how-to-correctly-use-context-context-in-go-1-7-8f2c0fafdf39#.h4ij95tdz 49 | - [ ] https://blog.golang.org/context 50 | -------------------------------------------------------------------------------- /doc/impl/collector.md: -------------------------------------------------------------------------------- 1 | # Collector 2 | 3 | For Linux systems, all use `/proc` 4 | 5 | ## Issues 6 | 7 | - [System Metrics Collector #21](https://github.com/xephonhq/xephon-k/issues/21) 8 | - [Process Metrics Collector #22](https://github.com/xephonhq/xephon-k/issues/22) 9 | - [Container metrics collector #31](https://github.com/xephonhq/xephon-k/issues/31) 10 | - using cAdvisor 11 | - [ ] Handle the many duplicate code, really annoying 12 | 13 | ## Ref 14 | 15 | - [gopsutil](https://github.com/shirou/gopsutil) 16 | - [Prometheus node exporter system](https://github.com/prometheus/node_exporter/blob/master/collector/stat_linux.go#L87) 17 | - [Elastic Search Metric Beat System](https://github.com/elastic/beats/tree/master/metricbeat/module/system) 18 | - it is also using gopsutil like InfluxDB's telegraf -------------------------------------------------------------------------------- /doc/impl/storage-cassandra.md: -------------------------------------------------------------------------------- 1 | # Storage: Cassandra 2 | 3 | ## v0 - v0.1.0 4 | 5 | v0.0.2 6 | 7 | - assume schema is already created by external tools 8 | - use naive schema 9 | - no tags 10 | - don't support avg, sum, moving avg -------------------------------------------------------------------------------- /doc/legacy/roadmap.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | - [ ] TODO: this doc is outdated 4 | 5 | ## Current 6 | 7 | - support query 8 | - built in graphing web service [#19](https://github.com/xephonhq/xephon-k/issues/19) 9 | - using echarts 10 | - pack using gulp 11 | - (optional) bind assets to the binary (may need to rewrite gorice, I don't like gobinddata) 12 | - (optional) watch and reload 13 | - support aggregation on query 14 | - various window size 15 | - avg 16 | - sum 17 | - min 18 | - max 19 | - support gRPC 20 | - [Survey](survey) 21 | - [x] [existing TSDBs using C*](https://github.com/xephonhq/awesome-time-series-database#cassandra) 22 | - [x] categorize schemas 23 | - [Specification](spec-draft.md) 24 | - [x] naive schema 25 | - [x] [naive schema's naive implementation](../pkg/bin/xnaive/main.go) 26 | - storing metrics 27 | - [x] memory without index tag 28 | - [x] cassandra without index tag 29 | - query metrics as it is 30 | - [x] without using tag 31 | - query aggregation 32 | - index text without using external search engine 33 | - i.e. https://github.com/balzaczyy/golucene 34 | 35 | ## Outline 36 | 37 | Version 0.1.0 38 | 39 | - single tsdb node 40 | - use advanced Cassandra schema (bucket, partition etc.) 41 | - provide RESTful API for querying and insert data 42 | - support `max`, `sum`, `avg` etc. 43 | 44 | Version 0.2.0 45 | 46 | - multiple tsdb nodes for sharding 47 | - split big time range into small time range and aggregate result 48 | - limit resources for each docker node 49 | - filter by tag (no full text search on tag value, just exact match) 50 | - provide gRPC server for inserting data 51 | 52 | Version 0.3.0 53 | 54 | - full text search on tags using Elasticsearch 55 | - ~~support visualization using Grafana~~ we will use own graphing system 56 | 57 | Version 0.4.0 58 | 59 | - integrate [tsdb-ql](https://github.com/xephonhq/tsdb-ql) as query language 60 | 61 | Version 0.5.0 62 | 63 | - benchmark against Heroic, KairosDB, InfluxDB 64 | - use [tsdb-proxy](https://github.com/xephonhq/tsdb-proxy) for support multiple input protocol 65 | 66 | Version 0.6.0 67 | 68 | - replace Cassandra with custom storage engine, may based on RocksDB 69 | -------------------------------------------------------------------------------- /doc/log/2019/2019-01/2019-01-25-0.0.3-init-refactor.md: -------------------------------------------------------------------------------- 1 | # 2019-01-25 Init Refactor 0.3.0 2 | 3 | This is the doc following [#69](https://github.com/xephonhq/xephon-k/issues/69) to restart Xephon-K, 4 | it's been a year ... Same as other refactor, go.ice, ayi, need to come up with a basic plan. 5 | 6 | The rough plan for the refactor is the following: 7 | 8 | - what has been done in xephon-k (supported backends, supported queries) 9 | - new goals (likely most existing implementation will be abandoned) 10 | - need implementation in other languages, for practice other language, also the design is no longer limit to the implementation of one language 11 | - different scopes, embed, single node server, distributed 12 | - different model, time series with id, time series with tags, traces, common tabular analytical (column) store 13 | - a initial embed implementation in Go, can reuse existing implementation (which have a lot of wrong optimization, i.e. pass pointer of slice ....) 14 | - can be used for sundial, or just collect my local machine metrics 15 | - might favor java or cpp cause at least the fist one won't cause me a day to setup dependency and compile things -------------------------------------------------------------------------------- /doc/log/2020/2020-01/2020-01-17-refactor-again.md: -------------------------------------------------------------------------------- 1 | # 2020-01-17 Refactor again 2 | 3 | It's almost (exactly) 1 year after [previous effort for refactoring](../../2019/2019-01/2019-01-25-0.0.3-init-refactor.md) (well rewrite). 4 | I have to say although I didn't write any code, I did write a detailed plan in [#69](https://github.com/xephonhq/xephon-k/issues/69). 5 | 6 | ## Design 7 | 8 | I'd start with the simplest data model and single node 9 | 10 | Data model 11 | 12 | - tagged time series like Prometheus and InfluxDB 13 | 14 | Storage engine 15 | 16 | - assuming tag index can fit into memory (obviously it may not be the case) 17 | - use simple columnar store, i.e. just blocks. 18 | - support WAL but can be disabled (mainly used to see the effect of having both WAL and data on single disk) 19 | - support compaction on single node 20 | 21 | Compression 22 | 23 | - relies on libtsdb-go 24 | 25 | Protocol 26 | 27 | Each protocol should have a prepared statement style variant (because is might be much faster? though gzip entire payload could beat it?) 28 | 29 | - http json (kairosdb like) 30 | - tcp, text and binary 31 | - grpc 32 | - prometheus 33 | 34 | Query 35 | 36 | - filter by tag and returns as it is, i.e. no aggregation, no group by. Mainly used for verify correctness 37 | 38 | ## Dependencies 39 | 40 | - compression in libtsdb 41 | - need to move existing simple compression code to libtsdb, and implement gorilla tsz 42 | - error and logging from gommon 43 | - tracing? 44 | - I want to have a simple in process tracing, it may not be that feature rich [tokio-rs/tracing](https://github.com/tokio-rs/tracing) might be a good reference. 45 | - end to end benchmark from xephon-b -------------------------------------------------------------------------------- /doc/log/2020/2020-01/2020-01-18-dep-to-mod.md: -------------------------------------------------------------------------------- 1 | # 2020-01-18 Dep to Mod 2 | 3 | Decided to convert from dep to mod and realized the code is too old to be reused (unlike Xephon-B). 4 | Most code is very old, using pkg/errors and x/net/context ... All of my newer code has switched to gommon/errors long time ago. 5 | 6 | So there is no need to keep existing code ... (any more) ... 7 | 8 | -------------------------------------------------------------------------------- /doc/log/2020/2020-02/2020-02-03-design.md: -------------------------------------------------------------------------------- 1 | # 2020-02-03 Design 2 | 3 | Focus on single node implementation that can be used for benchhub. 4 | -------------------------------------------------------------------------------- /doc/survey/README.md: -------------------------------------------------------------------------------- 1 | # Survey 2 | 3 | Survey of existing Time series databases design and implementation details, schema, data structure, 4 | storage engine, on disk format, index format etc. Also includes some column database and K-V store. 5 | Contents are merged with [Xephon-S](https://github.com/xephonhq/xephon-s/tree/master/doc/survey), 6 | [tsdb-proxy-java](https://github.com/xephonhq/tsdb-proxy-java/blob/master/doc/survey) 7 | 8 | ## Cassandra based 9 | 10 | Cassandra 11 | 12 | - [Cassandra](cassandra.md) 13 | - [ScyllaDB](scylladb.md) 14 | 15 | [Time Series Databases using Cassandra as Backend](https://github.com/xephonhq/awesome-time-series-database#cassandra) 16 | 17 | - [KairosDB](kairosdb.md) 18 | - [OpenTSDB](opentsdb.md) 19 | - [Heroic](heroic.md) 20 | - [Newts](newts.md) 21 | - [Khronus](khronus.md) 22 | - [Blueflood](blueflood.md) 23 | 24 | ## Memory 25 | 26 | - [Gorilla (Beringei)](gorilla.md) 27 | 28 | ## RRD 29 | 30 | - [RRD Tool](rrd.md) 31 | - [Whisper](whisper.md) 32 | - [ ] Ceres 33 | 34 | ## Tree 35 | 36 | - [Akumuli](akumuli.md) 37 | - [BtrDB](btrdb.md) 38 | - [ ] LMDB 39 | 40 | ## LSM 41 | 42 | - [InfluxDB TSM Tree](influxdb.md) 43 | - [ ] LevelDB 44 | - [ ] RocksDB 45 | 46 | ## Column 47 | 48 | - [Druid](druid/README.md) 49 | - [ ] Protobuf, Dremel 50 | - [ ] Parquet (open source implementation of Dremel) 51 | - [ ] ClickHouse 52 | - [ ] Apache Kudu ?or Impla 53 | - [ ] linkedin pinot 54 | 55 | ## Others 56 | 57 | - [Respawn](respawn.md) 58 | - [Prometheus](prometheus.md) 59 | - [ ] TODO: merge the document about Prometheus 60 | 61 | ## Query language 62 | 63 | - [graphite-promql-influxql](graphite-promql-influxql.md) 64 | -------------------------------------------------------------------------------- /doc/survey/blueflood.md: -------------------------------------------------------------------------------- 1 | # Blueflood 2 | 3 | - GitHub: https://github.com/rackerlabs/blueflood/ 4 | 5 | ## Protocol 6 | 7 | ## Schema 8 | 9 | - https://github.com/rackerlabs/blueflood/blob/master/src/cassandra/cli/load.cdl 10 | - it has separated rollup table 11 | 12 | ### Metrics Schema 13 | 14 | full, 5m, 20m, 60m, 240m, 1440m, 15 | 16 | ````sql 17 | CREATE TABLE IF NOT EXISTS "DATA".metrics_full ( 18 | key text, 19 | column1 bigint, 20 | value blob, 21 | PRIMARY KEY (key, column1) 22 | ) WITH COMPACT STORAGE AND speculative_retry = 'NONE'; 23 | 24 | CREATE TABLE IF NOT EXISTS "DATA".metrics_5m ( 25 | key text, 26 | column1 bigint, 27 | value blob, 28 | PRIMARY KEY (key, column1) 29 | ) WITH COMPACT STORAGE AND speculative_retry = 'NONE'; 30 | ```` 31 | 32 | preaggreated full, 5m, 20m, 60m, 240m, 1440m, 33 | 34 | ````sql 35 | CREATE TABLE IF NOT EXISTS "DATA".metrics_preaggregated_full ( 36 | key text, 37 | column1 bigint, 38 | value blob, 39 | PRIMARY KEY (key, column1) 40 | ) WITH COMPACT STORAGE AND speculative_retry = 'NONE'; 41 | 42 | CREATE TABLE IF NOT EXISTS "DATA".metrics_preaggregated_5m ( 43 | key text, 44 | column1 bigint, 45 | value blob, 46 | PRIMARY KEY (key, column1) 47 | ) WITH COMPACT STORAGE AND speculative_retry = 'NONE'; 48 | ```` 49 | 50 | ### Meta Schema 51 | 52 | ````sql 53 | CREATE TABLE IF NOT EXISTS "DATA".metrics_metadata ( 54 | key text, 55 | column1 text, 56 | value blob, 57 | PRIMARY KEY (key, column1) 58 | ) WITH COMPACT STORAGE AND speculative_retry = 'NONE'; 59 | ```` 60 | 61 | ### Extra Schema 62 | -------------------------------------------------------------------------------- /doc/survey/btrdb.md: -------------------------------------------------------------------------------- 1 | # BTrDB 2 | 3 | - https://www.usenix.org/sites/default/files/conference/protected-files/fast16_sildes_andersen.pdf 4 | 5 | ## TODO 6 | 7 | - [ ] No timestamp in internal nodes? 8 | - it's time partitioned, so each block has fixed time range? 9 | - [ ] does it mentioned hole? 10 | - [ ] How the version number is set? 11 | - [ ] How is the out of order handled? 12 | - [ ] Crash recovery? seems not mentioned at all? 13 | 14 | ## Meta 15 | 16 | - 53 million values / second insert (four-node) 17 | - 119 million queried values / second read (four-node) 18 | 19 | ## Special of telemetry 20 | 21 | - arrives out of order 22 | - delayed and duplicated 23 | - nanosecond timestamp 24 | 25 | ## Time partitioned Tree 26 | 27 | - time partitioning 28 | - multi resolution 29 | - version annotated 30 | - [ ] copy on write 31 | - k-ary tree 32 | 33 | Internal Node 34 | 35 | - Each internal node holds scalar summaries of the sub-trees below it, along with the links to the subtrees. 36 | - min 37 | - mean 38 | - max 39 | - count 40 | - When querying a stream for statistical records, the tree need only be traversed to the depth corresponding to the desired resolution. 41 | - 2 (address + version) * 8 (64 bit) * K (K = 64) = 1024 KB 42 | - 4 (min,max,mean,count) * 8 (64 bit) * K (K = 64) = 2048 KB 43 | - total 3 KB 44 | - [ ] No timestamp in internal nodes? 45 | 46 | Leaf 47 | 48 | - 16KB (1024 points) 49 | 50 | Block Store 51 | 52 | - [ ] Fields such as a block’s address, UUID, resolution (tree depth) and time extent are useful for traversing the 53 | tree, but can be deduced from context when a block is read from disk, so are stripped before the block enters the compression engine 54 | - a pool of pre-created initial addresses 55 | 56 | Delta Delta compression 57 | -------------------------------------------------------------------------------- /doc/survey/cassandra.md: -------------------------------------------------------------------------------- 1 | # Cassandra 2 | 3 | NOTE: moved to [separated folder](cassandra) 4 | 5 | ## Usage 6 | 7 | For Fedora `sudo systemctl start docker.service` might be needed 8 | 9 | - `docker run --name tsdb-cassandra -p 9042:9042 -d cassandra:3.9` 10 | - `docker stop tsdb-cassandra` 11 | - `docker start tsdb-cassandra` 12 | - `docker exec -it tsdb-cassandra bash` 13 | 14 | ## TODO 15 | 16 | ## DONE 17 | 18 | - [x] keys http://stackoverflow.com/questions/24949676/difference-between-partition-key-composite-key-and-clustering-key-in-cassandra 19 | - [x] Thrift, CQL and the underlying storage (I think I got a bit confused when trying to use 20 | CQL to understand KairosDB's schema design), see [Cassandra](cassandra/README.md) 21 | -------------------------------------------------------------------------------- /doc/survey/cassandra/1.0-about-data-model.md: -------------------------------------------------------------------------------- 1 | # The Cassandra Data Model 2 | 3 | http://docs.datastax.com/en/archived/cassandra/1.0/docs/ddl/about-data-model.html 4 | 5 | The doc is for 1.0 but I think it covers a lot of detail that is omitted by 3.x documentations 6 | 7 | ## Take away 8 | 9 | | Cassandra | RDBMS | 10 | | :------------- | :------------- | 11 | | Keyspace | Database | 12 | | Column Family | Table | 13 | 14 | - Each row in a column family is not required to have the same set of columns. 15 | 16 | ## Detail 17 | 18 | - the keyspace is the container for your application data, similar to a database or schema in a relational database 19 | - Inside the keyspace are one or more column family objects, which are analogous to tables 20 | - Column families contain columns, and a set of related columns is identified by an application-supplied row key. Each row in a column family is not required to have the same set of columns. 21 | -------------------------------------------------------------------------------- /doc/survey/cassandra/1.0-partitioning.md: -------------------------------------------------------------------------------- 1 | # Partitioning 2 | 3 | http://docs.datastax.com/en/archived/cassandra/1.0/docs/cluster_architecture/partitioning.html 4 | 5 | The doc is for 1.0 but I think it covers a lot of detail that is omitted by 3.x documentations 6 | 7 | ## Take away 8 | 9 | - Column family data is partitioned across the nodes based on the row key 10 | - random partitioner (most cases) and byte order partitioner 11 | - [ ] Sequential writes can cause hot spots, although they mentioned it in byte order partitioner, 12 | but write to a wide row will also only hit a single node? 13 | 14 | ## Detail 15 | 16 | 'Data partitioning determines how data is distributed across the nodes in the cluster. Three factors are involved with data distribution:' 17 | 18 | - A partitioner that determines which node to store the data on 19 | - The number of copies of data, 20 | 21 | ### RandomPartitioner 22 | 23 | - The RandomPartitioner (org.apache.cassandra.dht.RandomPartitioner) is the default partitioning strategy for a Cassandra cluster, 24 | and **in almost all cases is the right choice** 25 | - The RandomPartitioner uses consistent hashing to determine which node stores which row. Unlike naive modulus-by-node-count, consistent hashing ensures that when nodes are added to the cluster, the minimum possible set of data is effected 26 | 27 | ### ByteOrderPartitioner 28 | 29 | - The ByteOrderPartitioner orders rows lexically by key bytes 30 | - Using the ordered partitioner allows range scans over rows 31 | 32 | It is not recommended due to 33 | 34 | - Sequential writes can cause hot spots: If your application tends to write or update a sequential block of rows at a time, then these writes are not distributed across the cluster; they all go to one node. This is frequently a problem for applications dealing with timestamped data 35 | - [ ] TODO: this is similar to Bigtable's tall and narrow suggestion? 36 | - More administrative overhead to load balance the cluster 37 | - Uneven load balancing for multiple column families 38 | -------------------------------------------------------------------------------- /doc/survey/cassandra/tw_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xephonhq/xephon-k/b6a213ea9b529207205e882977047960920b4e30/doc/survey/cassandra/tw_data.png -------------------------------------------------------------------------------- /doc/survey/cassandra/tw_logical.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xephonhq/xephon-k/b6a213ea9b529207205e882977047960920b4e30/doc/survey/cassandra/tw_logical.png -------------------------------------------------------------------------------- /doc/survey/cassandra/tw_physical.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xephonhq/xephon-k/b6a213ea9b529207205e882977047960920b4e30/doc/survey/cassandra/tw_physical.png -------------------------------------------------------------------------------- /doc/survey/cassandra/tw_physical_compact.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xephonhq/xephon-k/b6a213ea9b529207205e882977047960920b4e30/doc/survey/cassandra/tw_physical_compact.png -------------------------------------------------------------------------------- /doc/survey/druid/README.md: -------------------------------------------------------------------------------- 1 | # Druid 2 | 3 | - http://druid.io/ 4 | - https://github.com/druid-io/druid 5 | 6 | ## Paper 7 | 8 | - [Druid: A Real-time Analytical Data Store](druid_a_real_time_analytical_data_store.md) SIGMOD 2014 9 | 10 | ## Talks 11 | 12 | - [Stories from the Trenches – The Challenges of Building an Analytics Stack](the-challenges-of-building-an-anlytics-stack.md) 13 | 14 | ## API 15 | 16 | - [Query](query.md) 17 | 18 | ## Internal 19 | -------------------------------------------------------------------------------- /doc/survey/druid/druid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xephonhq/xephon-k/b6a213ea9b529207205e882977047960920b4e30/doc/survey/druid/druid.png -------------------------------------------------------------------------------- /doc/survey/druid/the-challenges-of-building-an-anlytics-stack.md: -------------------------------------------------------------------------------- 1 | # Stories from the Trenches – The Challenges of Building an Analytics Stack 2 | 3 | - https://www.youtube.com/watch?v=Sz4w75xRrYM 4 | 5 | ## Take away 6 | 7 | - https://dev.twitter.com/streaming/public 8 | 9 | ## Notes 10 | 11 | ![druid](druid.png) 12 | 13 | - Immutable data 14 | - In Memory is Overrated 15 | - mmap + SSD 16 | - cost of scaling CPU << cost of adding RAM 17 | - decompression on the fly (LZF, Snappy, LZ4) 18 | - Low Latency vs High throughput 19 | - combine batch + streaming 20 | - immutable made it easy to combine the two ingestion methods 21 | - Makes for easy backfill and re-processing 22 | - Historical Node 23 | - Real-time Node 24 | - Not All Data is Created Equal 25 | - user really care about recent data 26 | - user still want to run quarterly report 27 | - large queries create bottlenecks and resource contention 28 | - Smarter Rebalancing 29 | - Create Data Tiers 30 | - Addressing Multitenancy 31 | - HyperLogLog sketches 32 | - Approximate top-k 33 | - Approximate histograms (monitoring) 34 | - Monitoring 35 | - Use Druid to monitor Druid 36 | - **Use cases should define engineering** 37 | -------------------------------------------------------------------------------- /doc/survey/heroic.md: -------------------------------------------------------------------------------- 1 | # Heroic 2 | 3 | - GitHub: https://github.com/spotify/heroic 4 | 5 | ## Protocol 6 | 7 | - CQL 8 | 9 | ## Schema 10 | 11 | - [legacy](https://github.com/spotify/heroic/tree/master/metric/datastax/src/main/resources/com.spotify.heroic.metric.datastax.schema.legacy) 12 | - [next generation](https://github.com/spotify/heroic/tree/master/metric/datastax/src/main/resources/com.spotify.heroic.metric.datastax.schema.ng) 13 | 14 | ### Metrics Schema 15 | 16 | Keyspace 17 | 18 | ````sql 19 | CREATE KEYSPACE IF NOT EXISTS {{keyspace}} 20 | WITH REPLICATION = { 21 | 'class' : 'SimpleStrategy', 22 | 'replication_factor' : 3 23 | }; 24 | ```` 25 | 26 | Metrics 27 | 28 | ````sql 29 | CREATE TABLE IF NOT EXISTS {{keyspace}}.metrics ( 30 | metric_key blob, 31 | data_timestamp_offset int, 32 | data_value double, 33 | PRIMARY KEY(metric_key, data_timestamp_offset) 34 | ) WITH COMPACT STORAGE; 35 | ```` 36 | 37 | ### Meta Schema 38 | 39 | ### Extra Schema 40 | -------------------------------------------------------------------------------- /doc/survey/influxdb.md: -------------------------------------------------------------------------------- 1 | # InfluxDB 2 | 3 | - index https://www.influxdata.com/path-1-billion-time-series-influxdb-high-cardinality-indexing-ready-testing/ 4 | - https://github.com/influxdata/influxdb/tree/tsi 5 | - current implementation https://github.com/influxdata/influxdb/tree/master/tsdb/index/tsi1 6 | - new proposal https://github.com/influxdata/influxdb/blob/er-tsi-prop/docs/tsm/TSI_CARDINALITY_PROPOSAL.md 7 | - the original PR of this new proposal? https://github.com/influxdata/influxdb/pull/7186 8 | - store https://docs.influxdata.com/influxdb/v1.2/concepts/storage_engine/ 9 | 10 | ## Path to 1 Billion Time Series: InfluxDB high cardinality indexing ready for testing 11 | 12 | InfluxDB actually looks like two databases in one: 13 | 14 | - a time series data store 15 | - an inverted index for the measurement, tag, and field metadata 16 | 17 | Old: in memory, build when start 18 | New: TSI file, mmap 19 | 20 | - WAL 21 | - LSM 22 | - In Memory 23 | - mmap index 24 | - compact file 25 | - [Robin Hood Hashing](https://github.com/influxdata/influxdb/blob/tsi/pkg/rhh/rhh.go) 26 | - https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf 27 | - [HyperLogLog++](https://github.com/influxdata/influxdb/blob/tsi/pkg/estimator/hll/hll.go) 28 | - https://github.com/clarkduvall/hyperloglog 29 | 30 | To Solve 31 | 32 | similar to [Prometheus's Series Churn](prometheus.md), they also try to solve Kubernetes problem 33 | for https://github.com/kubernetes/heapster I think heapster is using cadvisor, but they don't share storage? 34 | 35 | To be Solved 36 | 37 | - limit 38 | - having all these series hot for reads and writes 39 | - [ ] scale-out clustering? 40 | - queries that hit all series in the DB could potentially blow out the memory usage 41 | - add guard rails and limits into the language and eventually spill-to-disk query processing 42 | 43 | ## The InfluxDB Storage Engine and the Time-Structured Merge Tree (TSM) 44 | 45 | - [ ] shard in retention policy, auto roll up? 46 | - The timestamps and values are compressed and stored separately using encodings dependent on the data type and its shape 47 | - Timestamp 48 | - delta encoding 49 | - run-length encoding 50 | - simple8b https://github.com/jwilder/encoding/tree/master/simple8b 51 | - Float 52 | - XOR like Gorilla 53 | - Integer 54 | - ZigZag Protobuf? 55 | - simple8b 56 | - Strings 57 | - Snappy 58 | -------------------------------------------------------------------------------- /doc/survey/influxdb/query_process.md: -------------------------------------------------------------------------------- 1 | # Query Process 2 | 3 | Deprecated, see [read-path](read-path.md) 4 | 5 | a6c543039763c0f08253d71a43aefe3b570ecf37 6 | 7 | - services/httpd/handler.go 8 | - L431 `results := h.QueryExecutor.ExecuteQuery(query, opts, closing)` 9 | - influxql/query_executor.go 10 | - L328 `err := e.StatementExecutor.ExecuteStatement(stmt, ctx)` 11 | - influxdb/coordinator/statement_executor.go 12 | - L54 `func (e *StatementExecutor) ExecuteStatement(stmt influxql.Statement, ctx influxql.ExecutionContext) error` 13 | - L57 `e.executeSelectStatement(stmt, &ctx)` 14 | - L432 `func (e *StatementExecutor) executeSelectStatement(stmt *influxql.SelectStatement, ctx *influxql.ExecutionContext)` 15 | - L433 `itrs, stmt, err := e.createIterators(stmt, ctx)` 16 | - L439 `em := influxql.NewEmitter(itrs, stmt.TimeAscending(), ctx.ChunkSize)` 17 | - L457 `row, partial, err := em.Emit()` 18 | - [ ] the emitter queries from iterator, does the iterator query from cache first and then query from disk? 19 | -------------------------------------------------------------------------------- /doc/survey/influxdb/tracing.md: -------------------------------------------------------------------------------- 1 | # Tracing 2 | 3 | - InfluxDB has its own tracing package in `pkg/tracing`, it is not using open tracing, and has protobuf definition for wire -------------------------------------------------------------------------------- /doc/survey/influxdb/tsm-design.md: -------------------------------------------------------------------------------- 1 | # Time Structured Merge Tree 2 | 3 | - https://docs.influxdata.com/influxdb/v1.2/concepts/storage_engine/ 4 | - https://github.com/influxdata/influxdb/blob/master/tsdb/engine/tsm1/DESIGN.md 5 | 6 | - [ ] The index is composed of a sequence of index entries ordered lexicographically by key and then by time 7 | - If we we limit file sizes to 4GB, we would use 4 bytes for each pointer -------------------------------------------------------------------------------- /doc/survey/influxdb/write_process.md: -------------------------------------------------------------------------------- 1 | # Write Process 2 | 3 | Deprecated, see [write-path](write-path.md) 4 | 5 | a6c543039763c0f08253d71a43aefe3b570ecf37 6 | 7 | - services/httpd/handler.go 8 | - L581 `func (h *Handler) serveWrite` 9 | - L673 `if err := h.PointsWriter.WritePoints(database, r.URL.Query().Get("rp"), consistency, points)` 10 | - coordinator/points_writer.go 11 | - L288 `func (w *PointsWriter) WritePoints(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point` 12 | - L309 `ch <- w.writeToShard(shard, database, retentionPolicy, points)` 13 | - L353 `func (w *PointsWriter) writeToShard(shard *meta.ShardInfo, database, retentionPolicy string, points []models.Point)` 14 | - tsdb/store.go 15 | - L888 `func (s *Store) WriteToShard(shardID uint64, points []models.Point)` 16 | - tsdb/shard.go 17 | - L433 `func (s *Shard) WritePoints(points []models.Point) error` 18 | - L462 `if err := s.engine.WritePoints(points);` 19 | - tsdb/engine/tsm1/engine.go 20 | - L804 `func (e *Engine) WritePoints(points []models.Point) error` 21 | - L854 `err := e.Cache.WriteMulti(values)` 22 | - L859 `_, err = e.WAL.WriteMulti(values)` 23 | - tsdb/engine/tsm1/cache.go 24 | - L258 `func (c *Cache) Write(key string, values []Value) error` 25 | - L270 `if err := c.store.write(key, values); ` 26 | - L178 `type storer interface` 27 | - tsdb/engine/tsm1/ring.go` implements storer 28 | - 29 | -------------------------------------------------------------------------------- /doc/survey/kafka.md: -------------------------------------------------------------------------------- 1 | # Kafka 2 | 3 | Kafka is used by Druid, Spotify Heroic to improve performance. 4 | 5 | ## Design 6 | 7 | - use mmap 8 | - they quote an article from Varnish author 9 | - use sendfile to avoid extra copy 10 | - use own binary format 11 | -------------------------------------------------------------------------------- /doc/survey/khronus.md: -------------------------------------------------------------------------------- 1 | # Khronus 2 | 3 | 4 | - GitHub: https://github.com/Searchlight/khronus 5 | - It uses https://github.com/HdrHistogram/HdrHistogram for Histogram 6 | 7 | ## Protocol 8 | 9 | - CQL 10 | 11 | ## Schema 12 | 13 | - `create keyspace if not exists $keyspacePlusSuffix with replication = {'class':'SimpleStrategy', 'replication_factor': $getRF};` 14 | - https://github.com/Searchlight/khronus/blob/master/khronus-core/src/main/scala/com/searchlight/khronus/store/Cassandra.scala#L76 15 | 16 | ### Metrics Schema 17 | 18 | - `create table if not exists ${tableName(window)} (metric text, timestamp bigint, buckets ${getBucketsCollectionType(window)}, primary key (metric, timestamp)) with gc_grace_seconds = 0 and compaction = {'class': 'LeveledCompactionStrategy' }` 19 | - https://github.com/Searchlight/khronus/blob/master/khronus-core/src/main/scala/com/searchlight/khronus/store/BucketStore.scala#L69 20 | 21 | ### Meta Schema 22 | 23 | - `create table if not exists meta (key text, metric text, timestamp bigint, active boolean, primary key (key, metric))` 24 | - https://github.com/Searchlight/khronus/blob/master/khronus-core/src/main/scala/com/searchlight/khronus/store/MetaStore.scala#L65 25 | 26 | ### Extra Schema 27 | 28 | - summary store (seems to be rollup or preaggreate) 29 | - https://github.com/Searchlight/khronus/blob/master/khronus-core/src/main/scala/com/searchlight/khronus/store/SummaryStore.scala 30 | -------------------------------------------------------------------------------- /doc/survey/lmdb.md: -------------------------------------------------------------------------------- 1 | # LMDB: 2 | 3 | ## Ref 4 | 5 | - [Howard Chu - LMDB The Databaseology Lectures - CMU Fall 2015](https://youtu.be/tEa5sAh-kVk?t=24m36s) 6 | 7 | ## Design 8 | 9 | - similar to append only B+ 10 | - the root noode is always written last 11 | - this really **likes Akumuli** 12 | - but disk space grows without bound 13 | - WAL 14 | - only have two fixed root nodes 15 | - pick the one is newest when cold start 16 | - [ ] because they just need two versions? 17 | - two B+ trees per root node 18 | - one store user data 19 | - one keep track of free space 20 | - single writer concurrent with many readers 21 | - transaction handling 22 | - skipped 23 | - a memory mapped reader table 24 | 25 | Dark Underside 26 | 27 | - assume uniform record size 28 | - can't use sibling pointers 29 | 30 | Page Pointers 31 | 32 | - naive code mallocs pointer structs to point to pages and their parents 33 | - LMDB: fixed size stack 34 | 35 | Cursor Tracking 36 | 37 | - BerkeleyDB sets a tombstone 38 | - SQLite3 invalidates all open cursors 39 | - LMDB updates all open cursors 40 | 41 | Turn read ahead off? 42 | 43 | - shadow paging? 44 | 45 | Multi thread, faster than Memached because no lock? 46 | -------------------------------------------------------------------------------- /doc/survey/newts.md: -------------------------------------------------------------------------------- 1 | # Newts 2 | 3 | - GitHub: https://github.com/OpenNMS/newts 4 | - They use Dropwizard for REST API https://github.com/OpenNMS/newts/tree/master/rest 5 | 6 | ## Protocol 7 | 8 | - CQL 9 | 10 | ## Schema 11 | 12 | - [Samples Schema CQL](https://github.com/OpenNMS/newts/blob/master/cassandra/storage/src/main/resources/samples_schema.cql) 13 | - [x] ~~TODO: is this just the sample, or they are using it?~~ 14 | - It's **samples** not sample ... 15 | 16 | Keyspace 17 | 18 | ````sql 19 | CREATE KEYSPACE $KEYSPACE$ 20 | WITH replication = {'class': 'SimpleStrategy', 'replication_factor': $REPLICATION_FACTOR$}; 21 | ```` 22 | 23 | Metrics 24 | 25 | - [ ] TODO: the attributes field is the `tags`? 26 | - [ ] TODO: is this a tall and narrow table? 27 | - [ ] what are the `context`, `partition`, `resource` 28 | 29 | ````sql 30 | CREATE TABLE $KEYSPACE$.samples ( 31 | context text, 32 | partition int, 33 | resource text, 34 | collected_at timestamp, 35 | metric_name text, 36 | value blob, 37 | attributes map, 38 | PRIMARY KEY((context, partition, resource), collected_at, metric_name) 39 | ); 40 | ```` 41 | -------------------------------------------------------------------------------- /doc/survey/opentsdb.md: -------------------------------------------------------------------------------- 1 | # OpenTSDB 2 | 3 | - GitHub: {{ url }} 4 | 5 | ## Protocol 6 | 7 | - Thrift 8 | 9 | It uses a [shim](https://github.com/OpenTSDB/asynccassandra) to connect Cassandra using Thrift protocol in a HBase way. 10 | Detail can be found [here](http://opentsdb.net/docs/build/html/user_guide/backends/cassandra.html) 11 | 12 | ## Schema 13 | 14 | ### Metrics Schema 15 | 16 | ### Meta Schema 17 | 18 | ### Extra Schema 19 | 20 | ## Query 21 | 22 | ````json 23 | { 24 | "start": 1356998400, 25 | "end": 1356998460, 26 | "queries": [ 27 | { 28 | "aggregator": "sum", 29 | "metric": "sys.cpu.0", 30 | "rate": "true", 31 | "filters": [ 32 | { 33 | "type":"wildcard", 34 | "tagk":"host", 35 | "filter":"*", 36 | "groupBy":true 37 | }, 38 | { 39 | "type":"literal_or", 40 | "tagk":"dc", 41 | "filter":"lga|lga1|lga2", 42 | "groupBy":false 43 | }, 44 | ] 45 | }, 46 | { 47 | "aggregator": "sum", 48 | "tsuids": [ 49 | "000001000002000042", 50 | "000001000002000043" 51 | ] 52 | } 53 | ] 54 | } 55 | ```` -------------------------------------------------------------------------------- /doc/survey/prometheus.md: -------------------------------------------------------------------------------- 1 | # Prometheus 2 | 3 | ## In memory representation 4 | 5 | `common/model/value.go` 6 | 7 | ````go 8 | // SamplePair pairs a SampleValue with a Timestamp. 9 | type SamplePair struct { 10 | Timestamp Time 11 | Value SampleValue 12 | } 13 | 14 | // Sample is a sample pair associated with a metric. 15 | type Sample struct { 16 | Metric Metric `json:"metric"` 17 | Value SampleValue `json:"value"` 18 | Timestamp Time `json:"timestamp"` 19 | } 20 | 21 | // Samples is a sortable Sample slice. It implements sort.Interface. 22 | type Samples []*Sample 23 | ```` 24 | 25 | `common/model/metric.go` 26 | 27 | ````go 28 | type Metric LabelSet 29 | ```` 30 | 31 | ## Aggregation 32 | 33 | `promql/engine.go` 34 | 35 | ````go 36 | // vector is basically only an alias for model.Samples, but the 37 | // contract is that in a Vector, all Samples have the same timestamp. 38 | type vector []*sample 39 | ```` 40 | 41 | `promql/functions.go` 42 | 43 | ````go 44 | func aggrOverTime(ev *evaluator, args Expressions, aggrFn func([]model.SamplePair) model.SampleValue) model.Value { 45 | mat := ev.evalMatrix(args[0]) 46 | resultVector := vector{} 47 | 48 | for _, el := range mat { 49 | if len(el.Values) == 0 { 50 | continue 51 | } 52 | 53 | el.Metric.Del(model.MetricNameLabel) 54 | resultVector = append(resultVector, &sample{ 55 | Metric: el.Metric, 56 | Value: aggrFn(el.Values), 57 | Timestamp: ev.Timestamp, 58 | }) 59 | } 60 | return resultVector 61 | } 62 | 63 | // === avg_over_time(matrix model.ValMatrix) Vector === 64 | func funcAvgOverTime(ev *evaluator, args Expressions) model.Value { 65 | return aggrOverTime(ev, args, func(values []model.SamplePair) model.SampleValue { 66 | var sum model.SampleValue 67 | for _, v := range values { 68 | sum += v.Value 69 | } 70 | return sum / model.SampleValue(len(values)) 71 | }) 72 | } 73 | 74 | // === count_over_time(matrix model.ValMatrix) Vector === 75 | func funcCountOverTime(ev *evaluator, args Expressions) model.Value { 76 | return aggrOverTime(ev, args, func(values []model.SamplePair) model.SampleValue { 77 | return model.SampleValue(len(values)) 78 | }) 79 | } 80 | ```` 81 | 82 | I don't like how they implement aggregation, 83 | it has too many function calls and can't be optimized much by compiler, 84 | it's kind of the volcano type, but this is really overkill for time series, and using generator would make things much faster. 85 | -------------------------------------------------------------------------------- /doc/survey/respawn.md: -------------------------------------------------------------------------------- 1 | # Respawn 2 | 3 | - https://users.ece.cmu.edu/~agr/resources/publications/respawn-rtss-13.pdf 4 | 5 | ## Take away 6 | 7 | - http://www.cs.cmu.edu/~ishafer/ is a guy interested in time series 8 | - 'The availability of fresh (recent) data often conflicts with the throughput gains achievable by batching data.' 9 | - https://github.com/BodyTrack/datastore 10 | - Dataseries: An efficient, flexible data format for structured serial data 11 | - https://github.com/stevedh/readingdb (BTDS) (by the same group of guys from UCB, software defined buildings) 12 | - Typical operations performed on time series data 13 | - plotting 14 | - zooming 15 | - correlation 16 | - clustering 17 | - prediction 18 | - pattern matching 19 | - summarization 20 | - Predictive caching 21 | - Periodic Migration: when browsing, clients will frequently request tiles with the lowest resolution first 22 | - Proactive Migration: based on standard deviation, especially effective at accelerating access time for sparse data feeds 23 | - SQLite > BTDS > MySQL > OpenTSDB 24 | 25 | ## Meta 26 | 27 | - edge device (i.e. ARM) 28 | - cloud-to-edge partitioning 29 | - multi-resolution storage 30 | - high resolution at edge 31 | - low resolution at cloud backend 32 | - dispatcher + bloom filter 33 | - Low-level (higher resolution) tiles are migrated based on both client access patterns and based on data metrics like standard deviation. 34 | -------------------------------------------------------------------------------- /doc/survey/scylladb.md: -------------------------------------------------------------------------------- 1 | # ScyllaDB 2 | 3 | - http://www.scylladb.com/ 4 | - https://github.com/scylladb/scylla 5 | - https://github.com/scylladb/scylla-grafana-monitoring it's quite funny they use prometheus for monitoring 6 | 7 | ## Usage 8 | 9 | - `docker run --name tsdb-scylla -p 9042:9042 -d scylladb/scylla:1.6.0` 10 | - `docker stop tsdb-scylla` 11 | - `docker start tsdb-scylla` 12 | -------------------------------------------------------------------------------- /doc/survey/template.md: -------------------------------------------------------------------------------- 1 | # {{ name }} 2 | 3 | TODO: the template need a lot of change 4 | 5 | - GitHub: {{ url }} 6 | 7 | ## Protocol 8 | 9 | ## Schema 10 | 11 | ### Metrics Schema 12 | 13 | ### Meta Schema 14 | 15 | ### Extra Schema 16 | -------------------------------------------------------------------------------- /doc/survey/whisper.md: -------------------------------------------------------------------------------- 1 | # The Whisper Database 2 | 3 | ## TODO 4 | 5 | - [ ] the incoming data point is written to all archives at once 6 | - [ ] so for lower resolution, you move the windows in higher resolution and calculate the new value? 7 | 8 | ## Ref 9 | 10 | - http://graphite.readthedocs.io/en/latest/whisper.html 11 | - RRD can't back fill http://graphite.readthedocs.io/en/latest/whisper.html#differences-between-whisper-and-rrd 12 | - https://github.com/graphite-project/whisper 13 | 14 | ## Meta 15 | 16 | - On Disk Size: fixed 17 | - Time Interval: constant 18 | - Time precision: Unix timestamp in seconds 19 | - Value precision: python `float` 20 | - which is pretty big, how is it serialized on disk, and it can be compressed using general compression algorithm 21 | - Store Timestamp: Yes 22 | - [ ] full timestamp? 23 | 24 | ## Architecture 25 | 26 | - **'To support accurate aggregation from higher to lower resolution archives, the precision of a longer retention archive must be divisible by precision of next lower retention archive'** 27 | - 'Whisper will not do partial point interpolation' (插值) 28 | 29 | > For example, an archive with 1 data point every 60 seconds can have a lower-resolution archive following it with a resolution of 1 data point every 300 seconds because 60 cleanly divides 300. In contrast, a 180 second precision (3 minutes) could not be followed by a 600 second precision (10 minutes) because the ratio of points to be propagated from the first archive to the next would be 3 1/3 and Whisper will not do partial point interpolation. 30 | 31 | - Rollup Aggregation 32 | - avg 33 | - sum 34 | - last 35 | - max 36 | - min 37 | 38 | - the incoming data point is written to all archives at once 39 | - **Each data point is stored with its timestamp** 40 | 41 | > Rather than a timestamp being inferred from its position in the archive, timestamps are stored with each point. The timestamps are used during data retrieval to check the validity of the data point. If a timestamp does not match the expected value for its position relative to the beginning of the requested series, it is known to be out of date and a null value is returned 42 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/xephonhq/xephon-k 2 | 3 | go 1.13 4 | 5 | replace github.com/dyweb/go.ice => ../../dyweb/go.ice 6 | 7 | replace github.com/libtsdb/libtsdb-go => ../../libtsdb/libtsdb-go 8 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xephonhq/xephon-k/b6a213ea9b529207205e882977047960920b4e30/go.sum --------------------------------------------------------------------------------