├── .gitignore ├── go.mod ├── pkg ├── model │ └── model.go ├── store │ ├── schema.go │ └── store.go └── web │ └── server.go ├── static └── index.html ├── go.sum ├── Makefile ├── cmd └── simpleton │ └── main.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | simpleton.db 3 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/borud/simpleton 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/gorilla/mux v1.7.4 7 | github.com/jessevdk/go-flags v1.4.0 8 | github.com/jmoiron/sqlx v1.2.0 9 | github.com/mattn/go-sqlite3 v2.0.3+incompatible 10 | ) 11 | -------------------------------------------------------------------------------- /pkg/model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "time" 4 | 5 | type Data struct { 6 | ID int64 `db:"id" json:"id"` 7 | Timestamp time.Time `db:"timestamp" json:"timestamp"` 8 | FromAddr string `db:"from_addr" json:"fromAddr"` 9 | PacketSize int `db:"packet_size" json:"packetSize"` 10 | Payload []byte `db:"payload" json:"payload"` 11 | } 12 | -------------------------------------------------------------------------------- /static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Simpleton 7 | 8 | 9 |

Simpleton

10 | 11 | Some examples 12 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /pkg/store/schema.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "strings" 7 | 8 | "github.com/jmoiron/sqlx" 9 | _ "github.com/mattn/go-sqlite3" // Load sqlite3 driver 10 | ) 11 | 12 | const schema = ` 13 | CREATE TABLE IF NOT EXISTS data ( 14 | id INTEGER PRIMARY KEY AUTOINCREMENT, 15 | timestamp DATETIME NOT NULL, 16 | from_addr STRING NOT NULL, 17 | packet_size INT NOT NULL, 18 | payload BLOB NOT NULL 19 | ); 20 | ` 21 | 22 | func createSchema(db *sqlx.DB, fileName string) { 23 | log.Printf("Creating database schema in %s", fileName) 24 | 25 | for n, statement := range strings.Split(schema, ";") { 26 | if _, err := db.Exec(statement); err != nil { 27 | panic(fmt.Sprintf("Statement %d failed: \"%s\" : %s", n+1, statement, err)) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 2 | github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= 3 | github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 4 | github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= 5 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 6 | github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= 7 | github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= 8 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 9 | github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 10 | github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= 11 | github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 12 | -------------------------------------------------------------------------------- /pkg/store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "os" 5 | "sync" 6 | 7 | "github.com/borud/simpleton/pkg/model" 8 | "github.com/jmoiron/sqlx" 9 | _ "github.com/mattn/go-sqlite3" // Load sqlite3 driver 10 | ) 11 | 12 | // SqliteStore implements the store interface with Sqlite 13 | type SqliteStore struct { 14 | mu sync.Mutex 15 | db *sqlx.DB 16 | } 17 | 18 | // New creates new Store backed by SQLite3 19 | func New(dbFile string) (*SqliteStore, error) { 20 | var databaseFileExisted = false 21 | if _, err := os.Stat(dbFile); err == nil { 22 | databaseFileExisted = true 23 | } 24 | 25 | d, err := sqlx.Open("sqlite3", dbFile) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | if err = d.Ping(); err != nil { 31 | return nil, err 32 | } 33 | 34 | if !databaseFileExisted { 35 | createSchema(d, dbFile) 36 | } 37 | 38 | return &SqliteStore{db: d}, nil 39 | } 40 | 41 | // PutData stores one row of data in the database 42 | func (s *SqliteStore) PutData(data *model.Data) (int64, error) { 43 | s.mu.Lock() 44 | defer s.mu.Unlock() 45 | 46 | r, err := s.db.NamedExec("INSERT INTO data (timestamp, from_addr, packet_size, payload) VALUES(:timestamp,:from_addr,:packet_size,:payload)", data) 47 | if err != nil { 48 | return 0, err 49 | } 50 | return r.LastInsertId() 51 | } 52 | 53 | // ListData returns a list of the data from the database sorted by ID 54 | // in descending order (newest first) 55 | func (s *SqliteStore) ListData(offset int, limit int) ([]model.Data, error) { 56 | s.mu.Lock() 57 | defer s.mu.Unlock() 58 | 59 | var data []model.Data 60 | err := s.db.Select(&data, "SELECT * FROM data ORDER BY id DESC LIMIT ? OFFSET ?", limit, offset) 61 | return data, err 62 | } 63 | 64 | // Get fetches a single datapoint by id 65 | func (s *SqliteStore) Get(id int64) (*model.Data, error) { 66 | s.mu.Lock() 67 | defer s.mu.Unlock() 68 | 69 | var row model.Data 70 | err := s.db.QueryRowx("SELECT * FROM data WHERE id = ?", id).StructScan(&row) 71 | return &row, err 72 | } 73 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CGO_ENABLED ?= 1 2 | CGO_CFLAGS ?= 3 | CGO_LDFLAGS ?= 4 | BUILD_TAGS ?= 5 | APP_NAME ?= simpleton 6 | VERSION ?= 7 | BIN_EXT ?= 8 | 9 | GO := GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO_ENABLED) CGO_CFLAGS=$(CGO_CFLAGS) CGO_LDFLAGS=$(CGO_LDFLAGS) GO111MODULE=auto go 10 | PACKAGES = $(shell $(GO) list ./... | grep -v '/vendor/') 11 | PROTOBUFS = $(shell find . -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq | grep -v /vendor/) 12 | TARGET_PACKAGES = $(shell find . -name 'main.go' -print0 | xargs -0 -n1 dirname | sort | uniq | grep -v /vendor/) 13 | 14 | ifeq ($(GOOS),windows) 15 | BIN_EXT = .exe 16 | endif 17 | 18 | ifeq ($(VERSION),) 19 | VERSION = latest 20 | endif 21 | 22 | .DEFAULT_GOAL := build 23 | 24 | .PHONY: help 25 | help: 26 | @echo " GOOS = $(GOOS)" 27 | @echo " GOARCH = $(GOARCH)" 28 | @echo " CGO_ENABLED = $(CGO_ENABLED)" 29 | @echo " CGO_CFLAGS = $(CGO_CFLAGS)" 30 | @echo " CGO_LDFLAGS = $(CGO_LDFLAGS)" 31 | @echo " BUILD_TAGS = $(BUILD_TAGS)" 32 | @echo " VERSION = $(VERSION)" 33 | 34 | 35 | .PHONY: protoc 36 | protoc: 37 | @for proto_dir in $(PROTOBUFS); do echo $$proto_dir; protoc --proto_path=. --proto_path=$$proto_dir --go_out=plugins=grpc:$(GOPATH)/src $$proto_dir/*.proto || exit 1; done 38 | 39 | .PHONY: format 40 | format: 41 | @$(GO) fmt $(PACKAGES) 42 | 43 | .PHONY: test 44 | test: 45 | @$(GO) test -v -tags="$(BUILD_TAGS)" $(PACKAGES) 46 | 47 | .PHONY: build 48 | build: 49 | @for target_pkg in $(TARGET_PACKAGES); do $(GO) build -tags="$(BUILD_TAGS)" $(LDFLAGS) -o ./bin/`basename $$target_pkg`$(BIN_EXT) $$target_pkg || exit 1; done 50 | 51 | .PHONY: install 52 | install: 53 | @for target_pkg in $(TARGET_PACKAGES); do $(GO) install -tags="$(BUILD_TAGS)" $(LDFLAGS) $$target_pkg || exit 1; done 54 | 55 | .PHONY: dist 56 | dist: build 57 | @mkdir -p ./dist/$(GOOS)-$(GOARCH)/bin 58 | @(cd ./dist/$(GOOS)-$(GOARCH); tar cfz ../$(APP_NAME)-${VERSION}.$(GOOS)-$(GOARCH).tar.gz .) 59 | 60 | .PHONY: git-tag 61 | git-tag: 62 | ifeq ($(VERSION),$(filter $(VERSION),latest master "")) 63 | @echo "please specify VERSION" 64 | else 65 | @git tag -a $(VERSION) -m "Release $(VERSION)" 66 | @git push origin $(VERSION) 67 | endif 68 | 69 | .PHONY: clean 70 | clean: 71 | @rm -rf ./bin 72 | @rm -rf ./data 73 | @rm -rf ./dist 74 | -------------------------------------------------------------------------------- /pkg/web/server.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "strconv" 9 | 10 | "github.com/borud/simpleton/pkg/store" 11 | "github.com/gorilla/mux" 12 | ) 13 | 14 | // Server implements the system's webserver. 15 | type Server struct { 16 | listenAddr string 17 | staticDir string 18 | maxUploadSize int64 19 | db *store.SqliteStore 20 | } 21 | 22 | const ( 23 | defaultOffset = 0 24 | defaultLimit = 10 25 | ) 26 | 27 | // New creates a new webserver instance 28 | func New(store *store.SqliteStore, listen string, staticDir string) *Server { 29 | return &Server{ 30 | listenAddr: listen, 31 | staticDir: staticDir, 32 | db: store, 33 | } 34 | } 35 | 36 | // dataHandler lists the data in the database 37 | func (s *Server) dataHandler(w http.ResponseWriter, r *http.Request) { 38 | offset, err := strconv.Atoi(r.URL.Query().Get("offset")) 39 | if err != nil { 40 | offset = defaultOffset 41 | } 42 | 43 | limit, err := strconv.Atoi(r.URL.Query().Get("limit")) 44 | if err != nil { 45 | limit = defaultLimit 46 | } 47 | 48 | dataArray, err := s.db.ListData(offset, limit) 49 | if err != nil { 50 | http.Error(w, "Error listing documents", http.StatusNotFound) 51 | log.Printf("Unable to list docs offset = %d, limit = %d: %v", offset, limit, err) 52 | return 53 | } 54 | 55 | json, err := json.MarshalIndent(dataArray, "", " ") 56 | if err != nil { 57 | http.Error(w, fmt.Sprintf("Error converting data to JSON: %v", err), http.StatusBadRequest) 58 | return 59 | } 60 | 61 | w.Header().Add("Content-Type", "application/json") 62 | w.Write(json) 63 | } 64 | 65 | // dataPayloadHandler returns just the payload and sets the MIME type 66 | // to application/octet-stream 67 | func (s *Server) dataPayloadHandler(w http.ResponseWriter, r *http.Request) { 68 | vars := mux.Vars(r) 69 | idString := vars["id"] 70 | 71 | id, err := strconv.Atoi(idString) 72 | if err != nil { 73 | http.Error(w, "Invalid id", http.StatusBadRequest) 74 | return 75 | } 76 | 77 | data, err := s.db.Get(int64(id)) 78 | if err != nil { 79 | http.Error(w, fmt.Sprintf("Data %s not found", idString), http.StatusNotFound) 80 | log.Printf("Data '%s' not found: %v", idString, err) 81 | return 82 | } 83 | 84 | w.Header().Add("Content-Type", "application/octet-stream") 85 | w.Write(data.Payload) 86 | } 87 | 88 | // ListenAndServe ... 89 | func (s *Server) ListenAndServe() { 90 | m := mux.NewRouter().StrictSlash(true) 91 | 92 | // Data access 93 | m.HandleFunc("/data", s.dataHandler).Methods("GET") 94 | m.HandleFunc("/data/{id}", s.dataPayloadHandler).Methods("GET") 95 | 96 | // Serve static files 97 | m.PathPrefix("/").Handler(http.FileServer(http.Dir(s.staticDir))) 98 | 99 | server := &http.Server{ 100 | Handler: m, 101 | Addr: s.listenAddr, 102 | } 103 | 104 | log.Printf("Webserver listening to '%s'", s.listenAddr) 105 | log.Printf("Webserver terminated: '%v'", server.ListenAndServe()) 106 | } 107 | -------------------------------------------------------------------------------- /cmd/simpleton/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net" 7 | "os" 8 | "time" 9 | 10 | "github.com/borud/simpleton/pkg/model" 11 | "github.com/borud/simpleton/pkg/store" 12 | "github.com/borud/simpleton/pkg/web" 13 | "github.com/jessevdk/go-flags" 14 | ) 15 | 16 | // Options contains the command line options 17 | // 18 | type Options struct { 19 | // Webserver options 20 | WebServerListenAddress string `short:"w" long:"webserver-listen-address" description:"Listen address for webserver" default:":8008" value-name:"[]:"` 21 | WebServerStaticDir string `short:"s" long:"webserver-static-dir" description:"Static dir for files served through webserver" default:"static" value-name:""` 22 | 23 | // UDP listener 24 | UDPListenAddress string `short:"u" long:"udp-listener" description:"Listen address for UDP listener" default:":7000" value-name:"<[host]:port>"` 25 | UDPBufferSize int `short:"b" long:"udp-buffer-size" description:"Size of UDP read buffer" default:"1024" value-name:""` 26 | 27 | // Database options 28 | DBFilename string `short:"d" long:"db" description:"Data storage file" default:"simpleton.db" value-name:""` 29 | 30 | // Verbose 31 | Verbose bool `short:"v" long:"verbose" description:"Turn on verbose logging"` 32 | } 33 | 34 | var parsedOptions Options 35 | 36 | // listenUDP listens for incoming UDP packets and passes them off to 37 | // the database storage. 38 | // 39 | func listenUDP(db *store.SqliteStore) { 40 | pc, err := net.ListenPacket("udp", parsedOptions.UDPListenAddress) 41 | if err != nil { 42 | log.Fatalf("Failed to listen to %s: %v", parsedOptions.UDPListenAddress, err) 43 | } 44 | 45 | go func() { 46 | buffer := make([]byte, parsedOptions.UDPBufferSize) 47 | for { 48 | n, addr, err := pc.ReadFrom(buffer) 49 | if err != nil { 50 | log.Printf("Error reading, exiting: %v", err) 51 | } 52 | 53 | data := model.Data{ 54 | Timestamp: time.Now(), 55 | FromAddr: addr.String(), 56 | PacketSize: n, 57 | Payload: buffer[:n], 58 | } 59 | 60 | id, err := db.PutData(&data) 61 | if err != nil { 62 | log.Printf("Error storing data: %v", err) 63 | continue 64 | } 65 | 66 | // Update the assigned id 67 | data.ID = id 68 | 69 | if parsedOptions.Verbose { 70 | json, err := json.Marshal(data) 71 | if err != nil { 72 | log.Printf("Error marshalling to JSON: %v", err) 73 | continue 74 | } 75 | log.Printf("DATA> %s", json) 76 | } 77 | } 78 | }() 79 | log.Printf("Started UDP listener on %s", parsedOptions.UDPListenAddress) 80 | } 81 | 82 | func main() { 83 | // Parse command line options 84 | parser := flags.NewParser(&parsedOptions, flags.Default) 85 | _, err := parser.Parse() 86 | if err != nil { 87 | if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp { 88 | os.Exit(0) 89 | } 90 | log.Fatalf("Error parsing flags: %v", err) 91 | } 92 | 93 | // Open database 94 | db, err := store.New(parsedOptions.DBFilename) 95 | if err != nil { 96 | log.Fatalf("Unable to open or create database: %v", err) 97 | } 98 | 99 | // Listen to UDP socket 100 | listenUDP(db) 101 | 102 | // Set up webserver 103 | webServer := web.New(db, parsedOptions.WebServerListenAddress, parsedOptions.WebServerStaticDir) 104 | webServer.ListenAndServe() 105 | } 106 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Simpleton 2 | 3 | Simpleton is a dead simple UDP to database logging solution that just 4 | accepts UDP packets and stores them in an SQLite3 database. Per 5 | default it stores them in `simpleton.db` in the current directory, but 6 | you can override this with command line options. 7 | 8 | This isn't terribly useful for anything but really simple testing, but 9 | you can expand on it. 10 | 11 | ## Building 12 | 13 | In order to build simpleton you just run `make` and the binary will 14 | turn up in the `bin` directory. Per default it will build for OSX. 15 | 16 | make 17 | 18 | ## Building for other platforms 19 | 20 | To build for other platforms please edit the `GOOS` and `GOARCH` 21 | variables in the `Makefile`. You can also enter these parameters on 22 | the command line when you run `make`, like this. 23 | 24 | GOOS=linux GOARCH=amd64 make 25 | 26 | You can find the values for these variables for different platforms in 27 | [syslist.go](https://github.com/golang/go/blob/master/src/go/build/syslist.go), 28 | but the most common values are: 29 | 30 | | OS | GOOS | GOARCH | 31 | |---------|----------|--------| 32 | | OSX | darwin | amd64 | 33 | | Linux | linux | amd64 | 34 | | Windows | windows | amd64 | 35 | 36 | Of course, you can cross compile (eg compile Linux binaries on OSX 37 | machines) by just setting the right combination of GOOS and GOARCH, 38 | though Windows you might run into trouble. (I haven't built this for 39 | Windows). 40 | 41 | ## Running 42 | 43 | The binary will turn up in `bin`, so you can run it from the main 44 | directory with: 45 | 46 | bin/simpleton 47 | 48 | To list the command line options, you use the `-h` flag: 49 | 50 | bin/simpleton -h 51 | 52 | Here is an example of running Simpleton with options to make it listen 53 | to a particular interface (10.1.0.3 in the example) and port (7788) 54 | and store the database in `/tmp/simpleton.db`: 55 | 56 | bin/simpleton -u 10.1.0.3:7788 -d /tmp/simpleton.db 57 | 58 | ## Poking around the database 59 | 60 | If you want to poke around the resulting database you can install 61 | SQLite3 on your machine and inspect the database using the `sqlite3` 62 | command. To open the database in the previous example just run: 63 | 64 | sqlite3 /tmp/simpleton.db 65 | 66 | Type `.schema` to see the very simple database schema. You can now 67 | perform SQL statements on the data. 68 | 69 | Note: I'm not entirely sure about the concurrency of SQLite3 so I 70 | wouldn't use the database as an integration point (you never should). 71 | 72 | This is also why the code has a mutex lock around database accesses. 73 | The code was taken from a project that has multiple goroutines 74 | accessing the database. This program doesn't have that, but I left 75 | the mutex locking in just as a reminder. 76 | 77 | For production uses you should use a PostgreSQL database or similar, 78 | that is built for concurrency. But for small experiments and when you 79 | have limited concurrency, SQLite3 is a surprisingly capable little 80 | beast. 81 | 82 | 83 | ## Accessing via HTTP interface 84 | 85 | Note that the HTTP interface has **no authentication or security 86 | mechanisms** so don't use this for anything other than testing. The 87 | default address of the web interface is: 88 | 89 | http://localhost:8008/ 90 | 91 | The web interface is quite simple. You have two URLs that access 92 | data: 93 | 94 | /data 95 | /data/{id} 96 | 97 | The first returns a JSON array, the second only returns the payload of 98 | the data entry given by ID. The `/data` path will be limited to just 99 | the 20 newest entries in the database, but you can page through the 100 | database by setting `offset` and `limit` URL parameters: 101 | 102 | /data?offset=10&limit=10 103 | 104 | 105 | Simpleton supports having a directory with static files so that you 106 | can make some HTML pages with useful links to the content or perhaps 107 | to host JS-frontend applications. 108 | 109 | Check the command line help to see the parameters you can fiddle with. 110 | --------------------------------------------------------------------------------