├── .bazelrc ├── .test-bazelrc ├── .travis.yml ├── BUILD.bazel ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── WORKSPACE ├── go.mod ├── go.sum └── source ├── filestore ├── BUILD.bazel ├── cloudstorage.go ├── filestore.go ├── filestore_test_suite.go ├── memory.go └── memory_test.go ├── resources ├── BUILD.bazel ├── conversions.go ├── conversions_test.go └── resources.go ├── sensor ├── BUILD.bazel ├── client │ ├── BUILD.bazel │ ├── client.go │ └── client_test.go ├── fleetspeak │ ├── BUILD.bazel │ ├── config │ │ ├── labels │ │ └── socket_service.txt │ ├── fleetspeak.go │ └── fleetspeak_test.go ├── host │ ├── BUILD.bazel │ ├── host.go │ └── host_test.go ├── main.go ├── proto │ ├── BUILD.bazel │ ├── sensor.pb.go │ └── sensor.proto └── suricata │ ├── BUILD.bazel │ ├── proto │ ├── BUILD.bazel │ ├── suricata_eve.pb.go │ └── suricata_eve.proto │ ├── socket │ ├── BUILD.bazel │ ├── socket.go │ └── socket_test.go │ ├── suricata.go │ └── suricata_test.go └── server ├── BUILD.bazel ├── client ├── BUILD.bazel ├── client.go └── client_test.go ├── fleetspeak ├── BUILD.bazel └── fleetspeak.go ├── main.go ├── proto ├── BUILD.bazel ├── service.pb.go └── service.proto ├── service ├── BUILD.bazel ├── service.go ├── service_helpers.go ├── service_helpers_test.go └── service_test.go └── store ├── BUILD.bazel ├── conversions.go ├── conversions_test.go ├── datastore.go ├── datastore_test.go ├── memory.go ├── memory_test.go ├── store.go └── store_test_suite.go /.bazelrc: -------------------------------------------------------------------------------- 1 | # Docker image build requirements. 2 | build --host_force_python=PY2 3 | test --host_force_python=PY2 4 | run --host_force_python=PY2 -------------------------------------------------------------------------------- /.test-bazelrc: -------------------------------------------------------------------------------- 1 | # Test settings according to: https://github.com/bazelbuild/rules_go#how-do-i-run-bazel-on-travis-ci 2 | # These settings limit resource consumption and log clutter. 3 | startup --host_jvm_args=-Xmx2500m 4 | startup --host_jvm_args=-Xms2500m 5 | 6 | --local_resources=1536,1.5,0.5 7 | --noshow_progress 8 | --noshow_loading_progress 9 | 10 | build --spawn_strategy=standalone --genrule_strategy=standalone 11 | build --verbose_failures 12 | 13 | test --test_strategy=standalone 14 | test --ram_utilization_factor=10 15 | test --test_output=errors -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | 3 | addons: 4 | apt: 5 | sources: 6 | - ubuntu-toolchain-r-test 7 | packages: 8 | - wget 9 | - pkg-config 10 | 11 | before_install: 12 | # Install gcloud & start datastore emulator: 13 | - export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" 14 | - echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list 15 | - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - 16 | - sudo apt-get update && sudo apt-get install google-cloud-sdk google-cloud-sdk-datastore-emulator 17 | - gcloud beta emulators datastore start --no-legacy --project test-project-name --host-port localhost:9999 --no-store-on-disk --consistency=1 & 18 | # Install Bazel: 19 | - wget https://github.com/bazelbuild/bazel/releases/download/0.27.1/bazel_0.27.1-linux-x86_64.deb 20 | - echo "10b4dc5dd25b6372e03553713a29705afef9e03a6e1169983e77c69b2ca993ed bazel_0.27.1-linux-x86_64.deb" | sha256sum -c 21 | - sudo dpkg -i bazel_0.27.1-linux-x86_64.deb 22 | 23 | script: 24 | - bazel --bazelrc=.test-bazelrc test //... -------------------------------------------------------------------------------- /BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@bazel_gazelle//:def.bzl", "gazelle") 2 | 3 | # gazelle:prefix github.com/google/emitto 4 | gazelle(name = "gazelle") 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows 28 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). 29 | 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Emitto 2 | 3 | [![Travis Build Status](https://api.travis-ci.org/google/emitto.svg?branch=master)](https://travis-ci.org/google/emitto) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/google/emitto)](https://goreportcard.com/report/github.com/google/emitto) 5 | 6 | ## About 7 | 8 | Emitto is a service which provides a robust and targeted way to manage, store, 9 | and administer [Suricata](https://suricata-ids.org/) intrusion detection system 10 | (IDS) rules to a distributed network sensor monitoring deployment. 11 | 12 | ## Building 13 | 14 | 1) Install [Bazel](https://bazel.build/) 15 | 16 | 2) Download the source code 17 | 18 | 3) Build the project: 19 | 20 | ```bash 21 | cd emitto/ 22 | bazel build //... 23 | ``` 24 | 25 | ### Docker 26 | 27 | Both the sever and sensor client can be built with the Bazel Docker rules located in their respective BUILD files. 28 | 29 | For example, to build and run the sensor client locally in Docker: 30 | 31 | ```bash 32 | bazel --bazelrc=.bazelrc build --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 source/sensor:sensor_image.tar 33 | docker load -i bazel-bin/source/sensor/sensor_image.tar 34 | docker run bazel/source/sensor:sensor_image 35 | ``` 36 | 37 | The current `go_image` rule uses a [distroless](https://github.com/GoogleContainerTools/distroless) 38 | runtime base image, and will run using the default Go binary flag values. The `container_image` rule can be extended to support a custom `env` and other image attributes. 39 | 40 | Bazel supports the ability to push an image to a remote repository, and also to pull a remote image to be 41 | used as a custom base dependency. More information about Bazel Docker rules can be found [here](https://github.com/bazelbuild/rules_docker). 42 | 43 | ### Prerequisites 44 | 45 | The following services and products must be established and configured before 46 | running Emitto. 47 | 48 | #### Fleetspeak 49 | 50 | Emitto leverages [Fleetspeak](https://github.com/google/fleetspeak) for 51 | reliable, multi-homed communication between the server and network sensors. 52 | 53 | While this code is working internally with an internal deployment of 54 | Fleetspeak, 55 | the open source version is still in development. See the Fleetspeak 56 | [status](https://github.com/google/fleetspeak#status) section for more 57 | information and updates. 58 | 59 | Please read the Fleetspeak 60 | [disclaimer](https://github.com/google/fleetspeak#disclaimer). 61 | 62 | #### Google Cloud Platform 63 | 64 | By default, Emitto uses [Google Cloud Datastore](https://cloud.google.com/datastore/) 65 | and [Google Cloud Storage](https://cloud.google.com/storage/) for object and rule file 66 | storage, respectively. 67 | 68 | ## Discussions & Announcements 69 | 70 | The [Emitto](https://groups.google.com/forum/#!forum/emitto) Google Groups 71 | forum 72 | will be used for community discussions and announcements. 73 | 74 | ## DISCLAIMER 75 | 76 | This is not an officially supported Google product. 77 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 2 | 3 | http_archive( 4 | name = "io_bazel_rules_go", 5 | urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.18.5/rules_go-0.18.5.tar.gz"], 6 | sha256 = "a82a352bffae6bee4e95f68a8d80a70e87f42c4741e6a448bec11998fcc82329", 7 | ) 8 | 9 | http_archive( 10 | name = "bazel_gazelle", 11 | urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.17.0/bazel-gazelle-0.17.0.tar.gz"], 12 | sha256 = "3c681998538231a2d24d0c07ed5a7658cb72bfb5fd4bf9911157c0e9ac6a2687", 13 | ) 14 | 15 | http_archive( 16 | name = "io_bazel_rules_docker", 17 | sha256 = "87fc6a2b128147a0a3039a2fd0b53cc1f2ed5adb8716f50756544a572999ae9a", 18 | strip_prefix = "rules_docker-0.8.1", 19 | urls = ["https://github.com/bazelbuild/rules_docker/archive/v0.8.1.tar.gz"], 20 | ) 21 | 22 | # Go 23 | load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains") 24 | go_rules_dependencies() 25 | go_register_toolchains() 26 | 27 | # Docker 28 | load( 29 | "@io_bazel_rules_docker//repositories:repositories.bzl", 30 | container_repositories = "repositories", 31 | ) 32 | container_repositories() 33 | 34 | load( 35 | "@io_bazel_rules_docker//go:image.bzl", 36 | _go_image_repos = "repositories", 37 | ) 38 | _go_image_repos() 39 | 40 | # Gazelle 41 | load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") 42 | gazelle_dependencies() 43 | 44 | # Dependencies 45 | go_repository( 46 | name = "com_github_google_uuid", 47 | commit = "c2e93f3ae59f2904160ceaab466009f965df46d6", 48 | importpath = "github.com/google/uuid", 49 | ) 50 | 51 | go_repository( 52 | name = "com_github_google_fleetspeak", 53 | commit = "bc95dd6941494461d2e5dff0a7f4c78a07ff724d", 54 | importpath = "github.com/google/fleetspeak", 55 | ) 56 | 57 | go_repository( 58 | name = "com_github_spf13_afero", 59 | commit = "588a75ec4f32903aa5e39a2619ba6a4631e28424", 60 | importpath = "github.com/spf13/afero", 61 | ) 62 | 63 | go_repository( 64 | name = "com_github_google_go_cmp", 65 | commit = "917e382dab80060fd1f094402bfbb5137ec3c4ff", 66 | importpath = "github.com/google/go-cmp", 67 | ) 68 | 69 | go_repository( 70 | name = "com_github_fatih_camelcase", 71 | commit = "9db1b65eb38bb28986b93b521af1b7891ee1b04d", 72 | importpath = "github.com/fatih/camelcase", 73 | ) 74 | 75 | go_repository( 76 | name = "org_golang_google_api", 77 | commit = "b50168921e183c95644f04416271d702a730550c", 78 | importpath = "google.golang.org/api", 79 | ) 80 | 81 | go_repository( 82 | name = "com_google_cloud_go", 83 | commit = "5ea6847e42e62cd77aab6fb7047b15132174a634", 84 | importpath = "cloud.google.com/go", 85 | ) 86 | 87 | go_repository( 88 | name = "org_golang_x_oauth2", 89 | commit = "aaccbc9213b0974828f81aaac109d194880e3014", 90 | importpath = "golang.org/x/oauth2", 91 | ) 92 | 93 | go_repository( 94 | name = "com_github_googleapis_gax_go", 95 | commit = "bd5b16380fd03dc758d11cef74ba2e3bc8b0e8c2", 96 | importpath = "github.com/googleapis/gax-go", 97 | ) 98 | 99 | go_repository( 100 | name = "io_opencensus_go", 101 | commit = "6325d764b2d4a66576c5623aa1e6010b4148a429", 102 | importpath = "go.opencensus.io", 103 | ) 104 | 105 | go_repository( 106 | name = "com_github_hashicorp_golang_lru", 107 | commit = "59383c442f7d7b190497e9bb8fc17a48d06cd03f", 108 | importpath = "github.com/hashicorp/golang-lru", 109 | ) 110 | 111 | go_repository( 112 | name = "org_golang_google_grpc", 113 | commit = "532a0b98cb9580f72fd376b539f9eb984f92e054", 114 | importpath = "google.golang.org/grpc", 115 | ) 116 | 117 | go_repository( 118 | name = "com_github_golang_glog", 119 | commit = "23def4e6c14b4da8ac2ed8007337bc5eb5007998", 120 | importpath = "github.com/golang/glog", 121 | ) 122 | 123 | go_repository( 124 | name = "org_golang_google_genproto", 125 | commit = "eb0b1bdb6ae60fcfc41b8d907b50dfb346112301", 126 | importpath = "google.golang.org/genproto", 127 | ) 128 | 129 | go_repository( 130 | name = "com_github_google_emitto", 131 | commit = "0c93e985f54f1fedf41251553458150c12642e5a", 132 | importpath = "github.com/google/emitto", 133 | ) 134 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/google/emitto 2 | 3 | go 1.12 4 | 5 | require ( 6 | cloud.google.com/go v0.40.0 7 | github.com/fatih/camelcase v1.0.0 8 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b 9 | github.com/golang/protobuf v1.3.1 10 | github.com/google/fleetspeak v0.0.0-20190621113530-9faf6757a79a 11 | github.com/google/go-cmp v0.3.0 12 | github.com/google/uuid v1.1.1 13 | github.com/spf13/afero v1.2.2 14 | google.golang.org/api v0.7.0 15 | google.golang.org/genproto v0.0.0-20190627203621-eb59cef1c072 16 | google.golang.org/grpc v1.21.1 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.40.0 h1:FjSY7bOj+WzJe6TZRVtXI2b9kAYvtNg4lMbcH2+MUkk= 5 | cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro= 6 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 7 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 8 | github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= 9 | github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= 10 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 11 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 12 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 13 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 14 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 15 | github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= 16 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 17 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 18 | github.com/google/fleetspeak v0.0.0-20190621113530-9faf6757a79a h1:54As1hKUKuuzM0TbzckS74+m+xqOxLIfeyC8JNEpKi8= 19 | github.com/google/fleetspeak v0.0.0-20190621113530-9faf6757a79a/go.mod h1:7JqQDy/4/FsmOVFmKz2n8NKWglLmIWueoxfWv1ed7JQ= 20 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 21 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 22 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 23 | github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= 24 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 25 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 26 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 27 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 28 | github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc= 29 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 30 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 31 | github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= 32 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 33 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 34 | github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= 35 | github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= 36 | go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= 37 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 38 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 39 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 40 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 41 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 42 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 43 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 44 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 45 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 46 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 47 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 48 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 49 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 50 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= 51 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 52 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 53 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 54 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= 55 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 56 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 57 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 58 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 59 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 60 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 61 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 62 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 63 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= 64 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 65 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 66 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 67 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 68 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 69 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 70 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 71 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 72 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 73 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 74 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 75 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 76 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 77 | google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= 78 | google.golang.org/api v0.7.0 h1:9sdfJOzWlkqPltHAuzT2Cp+yrBeY1KRVYgms8soxMwM= 79 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 80 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 81 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 82 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 83 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 84 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 85 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 86 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 87 | google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= 88 | google.golang.org/genproto v0.0.0-20190627203621-eb59cef1c072 h1:Ct/ZXYnRnqFsiN9c89ZgAXESkBg3eZFDW11KuF+Koz0= 89 | google.golang.org/genproto v0.0.0-20190627203621-eb59cef1c072/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= 90 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 91 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 92 | google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= 93 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 94 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 95 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 96 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 97 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 98 | -------------------------------------------------------------------------------- /source/filestore/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "cloudstorage.go", 7 | "filestore.go", 8 | "filestore_test_suite.go", 9 | "memory.go", 10 | ], 11 | importpath = "github.com/google/emitto/source/filestore", 12 | visibility = ["//visibility:public"], 13 | deps = [ 14 | "@com_github_golang_glog//:go_default_library", 15 | "@com_github_google_go_cmp//cmp:go_default_library", 16 | "@com_github_spf13_afero//:go_default_library", 17 | "@com_google_cloud_go//storage:go_default_library", 18 | "@org_golang_google_api//option:go_default_library", 19 | "@org_golang_google_grpc//codes:go_default_library", 20 | "@org_golang_google_grpc//status:go_default_library", 21 | ], 22 | ) 23 | 24 | go_test( 25 | name = "go_default_test", 26 | srcs = ["memory_test.go"], 27 | embed = [":go_default_library"], 28 | ) 29 | -------------------------------------------------------------------------------- /source/filestore/cloudstorage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package filestore 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "io/ioutil" 21 | 22 | "cloud.google.com/go/storage" 23 | "google.golang.org/api/option" 24 | ) 25 | 26 | // GCSFileStore is a Google Cloud Storage implementation of a FileStore. 27 | type GCSFileStore struct { 28 | client *storage.Client 29 | bucket *storage.BucketHandle 30 | } 31 | 32 | // NewGCSFileStore returns a new GCSFileStore. 33 | func NewGCSFileStore(bucket string, client *storage.Client) *GCSFileStore { 34 | return &GCSFileStore{ 35 | client: client, 36 | bucket: client.Bucket(bucket), 37 | } 38 | } 39 | 40 | // NewGCSClient initializes a new Google Cloud Storage Client. 41 | // Follow these instructions to set up application credentials: 42 | // https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually. 43 | func NewGCSClient(ctx context.Context, credFile string, scopes []string) (*storage.Client, error) { 44 | c, err := storage.NewClient(ctx, option.WithCredentialsFile(credFile)) 45 | if err != nil { 46 | return nil, fmt.Errorf("GCS client creation failed: %v", err) 47 | } 48 | return c, nil 49 | } 50 | 51 | // AddRuleFile uploads a rule file to GCS. 52 | func (s *GCSFileStore) AddRuleFile(ctx context.Context, path string, rules []byte) error { 53 | writer := s.bucket.Object(path).NewWriter(ctx) 54 | defer writer.Close() 55 | if _, err := writer.Write(rules); err != nil { 56 | return fmt.Errorf("writing to rule file %q failed: %v", path, err) 57 | } 58 | return nil 59 | } 60 | 61 | // GetRuleFile returns a rule file from GCS. 62 | func (s *GCSFileStore) GetRuleFile(ctx context.Context, path string) ([]byte, error) { 63 | reader, err := s.bucket.Object(path).NewReader(ctx) 64 | if err != nil { 65 | return nil, fmt.Errorf("reader creation failed: %v", err) 66 | } 67 | defer reader.Close() 68 | data, err := ioutil.ReadAll(reader) 69 | if err != nil { 70 | return nil, fmt.Errorf("rule file %q download failed: %v", path, err) 71 | } 72 | return data, nil 73 | } 74 | 75 | // DeleteRuleFile removes a rule file from GCS. 76 | func (s *GCSFileStore) DeleteRuleFile(ctx context.Context, ruleFile string) error { 77 | obj := s.bucket.Object(ruleFile) 78 | if err := obj.Delete(ctx); err != nil { 79 | return fmt.Errorf("object deletion failed: %v", err) 80 | } 81 | return nil 82 | } 83 | 84 | // Close GCS client connection. 85 | func (s *GCSFileStore) Close() error { 86 | return s.client.Close() 87 | } 88 | -------------------------------------------------------------------------------- /source/filestore/filestore.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package filestore contains functionality to store Emitto rule files. 16 | package filestore 17 | 18 | import ( 19 | "context" 20 | ) 21 | 22 | // FileStore represents a Emitto file store. 23 | type FileStore interface { 24 | // AddRuleFile creates a new rule file at the specified path. 25 | AddRuleFile(ctx context.Context, path string, rules []byte) error 26 | // GetRuleFile retrieves an existing rule file by path. 27 | GetRuleFile(ctx context.Context, path string) ([]byte, error) 28 | // DeleteRuleFile removes an existing rule file by path. 29 | DeleteRuleFile(ctx context.Context, path string) error 30 | } 31 | -------------------------------------------------------------------------------- /source/filestore/filestore_test_suite.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package filestore 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "path/filepath" 21 | "reflect" 22 | "strings" 23 | "testing" 24 | 25 | "github.com/google/go-cmp/cmp" 26 | 27 | log "github.com/golang/glog" 28 | ) 29 | 30 | // suite contains store tests and the underlying FileStore implementation. 31 | type suite struct { 32 | builder func() FileStore 33 | } 34 | 35 | // RunTestSuite runs all generic store tests. 36 | // 37 | // The tests use the provided builder to instantiate a FileStore. 38 | // The builder is expected to always return a valid Store. 39 | func RunTestSuite(t *testing.T, builder func() FileStore) { 40 | s := &suite{builder} 41 | s.Run(t) 42 | } 43 | 44 | // Run runs Test* methods of the suite as subtests. 45 | func (s *suite) Run(t *testing.T) { 46 | sv := reflect.ValueOf(s) 47 | st := reflect.TypeOf(s) 48 | for i := 0; i < sv.NumMethod(); i++ { 49 | n := st.Method(i).Name 50 | if strings.HasPrefix(n, "Test") { 51 | mv := sv.MethodByName(n) 52 | mt := mv.Type() 53 | if mt.NumIn() != 1 || !reflect.TypeOf(t).AssignableTo(mt.In(0)) { 54 | log.Fatalf("Method %q of the test suite must have 1 argument of type *testing.T", n) 55 | } 56 | if mt.NumOut() != 0 { 57 | log.Fatalf("Method %q of the test suite must have no return value", n) 58 | } 59 | m := mv.Interface().(func(t *testing.T)) 60 | t.Run(n, m) 61 | } 62 | } 63 | } 64 | 65 | var ( 66 | dir1 = "/location1/zone1/" 67 | path1 = filepath.Join(dir1, "rulefile1") 68 | rules1 = []byte("rule 1\nrule 2\nrule3") 69 | 70 | path2 = filepath.Join(dir1, "rulefile2") 71 | rules2 = []byte("rule 4\nrule 5\nrule6") 72 | 73 | dir2 = "/location2/zone2/" 74 | path3 = filepath.Join(dir2, "rulefile1") 75 | rules3 = []byte("rule 7\nrule 8\nrule9") 76 | 77 | dir3 = "/location3/zone3/" 78 | path4 = filepath.Join(dir3, "rulefile1") 79 | rules4 = []byte("rule 1\nrule 2\nrule3") 80 | ) 81 | 82 | func (s *suite) TestAddRuleFile(t *testing.T) { 83 | st := s.builder() 84 | ctx := context.Background() 85 | 86 | if err := st.AddRuleFile(ctx, path1, rules1); err != nil { 87 | t.Error(err) 88 | } 89 | } 90 | 91 | func (s *suite) TestGetRuleFile(t *testing.T) { 92 | st := s.builder() 93 | ctx := context.Background() 94 | 95 | if err := st.AddRuleFile(ctx, path1, rules1); err != nil { 96 | t.Error(err) 97 | } 98 | got, err := st.GetRuleFile(ctx, path1) 99 | if err != nil { 100 | t.Error(err) 101 | } 102 | if diff := cmp.Diff(rules1, got); diff != "" { 103 | t.Errorf("expectation mismatch:\n%s", diff) 104 | } 105 | } 106 | 107 | func (s *suite) TestDeleteRuleFile(t *testing.T) { 108 | st := s.builder() 109 | ctx := context.Background() 110 | 111 | if err := st.AddRuleFile(ctx, path1, rules1); err != nil { 112 | t.Error(err) 113 | } 114 | if err := st.DeleteRuleFile(ctx, path1); err != nil { 115 | t.Error(err) 116 | } 117 | if _, err := st.GetRuleFile(ctx, path1); err == nil { 118 | t.Error(errors.New("returning a non-existent rule file should have raised an error")) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /source/filestore/memory.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package filestore 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | 22 | "github.com/spf13/afero" 23 | "google.golang.org/grpc/codes" 24 | "google.golang.org/grpc/status" 25 | ) 26 | 27 | // MemoryFileStore a memory implementation of a FileStore. 28 | type MemoryFileStore struct { 29 | store afero.Fs 30 | } 31 | 32 | // NewMemoryFileStore returns a MemoryFileStore. 33 | func NewMemoryFileStore() *MemoryFileStore { 34 | return &MemoryFileStore{ 35 | store: afero.NewMemMapFs(), 36 | } 37 | } 38 | 39 | // AddRuleFile stores a rule file in the filestore. 40 | func (s *MemoryFileStore) AddRuleFile(ctx context.Context, path string, rules []byte) error { 41 | f, err := s.store.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666) 42 | if err != nil { 43 | return status.Error(codes.Internal, fmt.Sprintf("failed to open rule file %q: %v", path, err)) 44 | } 45 | defer f.Close() 46 | 47 | if _, err = f.Write(rules); err != nil { 48 | return status.Error(codes.Internal, fmt.Sprintf("failed to write rules to %q: %v", path, err)) 49 | } 50 | return nil 51 | } 52 | 53 | // GetRuleFile retrieves a rule file from the filestore. 54 | func (s *MemoryFileStore) GetRuleFile(ctx context.Context, path string) ([]byte, error) { 55 | return afero.ReadFile(s.store, path) 56 | } 57 | 58 | // DeleteRuleFile removes a rule file from the filestore. 59 | func (s *MemoryFileStore) DeleteRuleFile(ctx context.Context, path string) error { 60 | return s.store.Remove(path) 61 | } 62 | -------------------------------------------------------------------------------- /source/filestore/memory_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package filestore 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestMemoryFileStore(t *testing.T) { 22 | RunTestSuite(t, func() FileStore { return NewMemoryFileStore() }) 23 | } 24 | -------------------------------------------------------------------------------- /source/resources/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "conversions.go", 7 | "resources.go", 8 | ], 9 | importpath = "github.com/google/emitto/source/resources", 10 | visibility = ["//visibility:public"], 11 | deps = [ 12 | "//source/sensor/proto:go_default_library", 13 | "//source/server/proto:go_default_library", 14 | "@com_github_fatih_camelcase//:go_default_library", 15 | "@com_github_golang_glog//:go_default_library", 16 | ], 17 | ) 18 | 19 | go_test( 20 | name = "go_default_test", 21 | srcs = ["conversions_test.go"], 22 | embed = [":go_default_library"], 23 | deps = [ 24 | "//source/sensor/proto:go_default_library", 25 | "//source/server/proto:go_default_library", 26 | "@com_github_golang_protobuf//proto:go_default_library", 27 | "@com_github_google_go_cmp//cmp:go_default_library", 28 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 29 | ], 30 | ) 31 | -------------------------------------------------------------------------------- /source/resources/conversions.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package resources 16 | 17 | import ( 18 | "bytes" 19 | "errors" 20 | "fmt" 21 | "reflect" 22 | "strconv" 23 | "strings" 24 | "time" 25 | 26 | "github.com/fatih/camelcase" 27 | 28 | log "github.com/golang/glog" 29 | spb "github.com/google/emitto/source/sensor/proto" 30 | pb "github.com/google/emitto/source/server/proto" 31 | ) 32 | 33 | // ProtoToLocation converts a proto Location to an internal Location. 34 | func ProtoToLocation(l *pb.Location) *Location { 35 | var zones []string 36 | for _, z := range l.Zones { 37 | zones = append(zones, z) 38 | } 39 | return &Location{ 40 | Name: l.Name, 41 | Zones: zones, 42 | } 43 | } 44 | 45 | // LocationToProto converts an internal Location to proto Location. 46 | func LocationToProto(l *Location) *pb.Location { 47 | var zones []string 48 | for _, z := range l.Zones { 49 | zones = append(zones, z) 50 | } 51 | return &pb.Location{ 52 | Name: l.Name, 53 | Zones: zones, 54 | } 55 | } 56 | 57 | // ProtoToRule converts a proto Rule to an internal Rule. 58 | func ProtoToRule(r *pb.Rule) *Rule { 59 | var zones []string 60 | for _, z := range r.LocationZones { 61 | zones = append(zones, z) 62 | } 63 | return &Rule{ 64 | ID: r.Id, 65 | Body: r.Body, 66 | LocZones: zones, 67 | } 68 | } 69 | 70 | // RuleToProto converts an internal Rule to a proto Rule. 71 | func RuleToProto(r *Rule) *pb.Rule { 72 | var zones []string 73 | for _, z := range r.LocZones { 74 | zones = append(zones, z) 75 | } 76 | return &pb.Rule{ 77 | Id: r.ID, 78 | Body: r.Body, 79 | LocationZones: zones, 80 | } 81 | } 82 | 83 | // MakeRuleFile builds a rule file given Rule objects. 84 | func MakeRuleFile(rules []*Rule) []byte { 85 | var buf bytes.Buffer 86 | for _, r := range rules { 87 | buf.WriteString(fmt.Sprintf("%s\n", r.Body)) 88 | } 89 | return buf.Bytes() 90 | } 91 | 92 | // MutationsMapping returns a map of fields and their mutability for Rule, Location, 93 | // and SensorMessage objects. 94 | // 95 | // Fields are in the form "field_name" where "struct.FieldName" = "field_name". 96 | // obj must not be a pointer. 97 | func MutationsMapping(obj interface{}) (map[string]bool, error) { 98 | v := reflect.ValueOf(obj) 99 | if v.Kind() == reflect.Ptr { 100 | return nil, errors.New("object must not be a pointer") 101 | } 102 | m := make(map[string]bool) 103 | for i := 0; i < v.NumField(); i++ { 104 | tag := v.Type().Field(i).Tag.Get("mutable") 105 | if tag == "" { 106 | return nil, errors.New(`all Rule fields must contain a "mutable" tag`) 107 | } 108 | mutable, err := strconv.ParseBool(tag) 109 | if err != nil { 110 | return nil, err 111 | } 112 | m[strings.ToLower(strings.Join(camelcase.Split(v.Type().Field(i).Name), "_"))] = mutable 113 | } 114 | return m, nil 115 | } 116 | 117 | // ProtoToSensorMessage converts a proto sensor message to an internal SensorMessage. 118 | func ProtoToSensorMessage(m *spb.SensorMessage) *SensorMessage { 119 | msg := &SensorMessage{ 120 | ID: m.GetId(), 121 | } 122 | switch t := m.Type.(type) { 123 | case *spb.SensorMessage_Alert: 124 | msg.Time = time.Unix(m.GetAlert().GetTime().GetSeconds(), 0).Format(time.RFC1123Z) 125 | msg.Host = m.GetAlert().GetHost().String() 126 | case *spb.SensorMessage_Heartbeat: 127 | msg.Time = time.Unix(m.GetHeartbeat().GetTime().GetSeconds(), 0).Format(time.RFC1123Z) 128 | msg.Host = m.GetHeartbeat().GetHost().String() 129 | default: 130 | log.Errorf("Unknown sensor message type (%T)", t) 131 | } 132 | return msg 133 | } 134 | 135 | // ProtoToSensorRequest converts a proto SensorMessage to an internal SensorRequest. 136 | func ProtoToSensorRequest(m *spb.SensorMessage) *SensorRequest { 137 | return &SensorRequest{ 138 | ID: m.GetResponse().GetId(), 139 | Status: m.GetResponse().GetStatus().String(), 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /source/resources/conversions_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package resources 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/golang/protobuf/proto" 21 | "github.com/google/go-cmp/cmp" 22 | 23 | tpb "github.com/golang/protobuf/ptypes/timestamp" 24 | spb "github.com/google/emitto/source/sensor/proto" 25 | pb "github.com/google/emitto/source/server/proto" 26 | ) 27 | 28 | func TestProtoToLocation(t *testing.T) { 29 | p := &pb.Location{ 30 | Name: "test", 31 | Zones: []string{"dmz", "prod"}, 32 | } 33 | want := &Location{ 34 | Name: "test", 35 | Zones: []string{"dmz", "prod"}, 36 | } 37 | if diff := cmp.Diff(want, ProtoToLocation(p)); diff != "" { 38 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 39 | } 40 | } 41 | 42 | func TestLocationToProto(t *testing.T) { 43 | l := &Location{ 44 | Name: "test", 45 | Zones: []string{"dmz", "prod"}, 46 | } 47 | want := &pb.Location{ 48 | Name: "test", 49 | Zones: []string{"dmz", "prod"}, 50 | } 51 | if diff := cmp.Diff(want, LocationToProto(l), cmp.Comparer(proto.Equal)); diff != "" { 52 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 53 | } 54 | } 55 | 56 | func TestProtoToRule(t *testing.T) { 57 | p := &pb.Rule{ 58 | Id: 1234567890, 59 | Body: "test rule", 60 | LocationZones: []string{"test:dmz", "test:corp"}, 61 | } 62 | want := &Rule{ 63 | ID: 1234567890, 64 | Body: "test rule", 65 | LocZones: []string{"test:dmz", "test:corp"}, 66 | } 67 | if diff := cmp.Diff(want, ProtoToRule(p)); diff != "" { 68 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 69 | } 70 | } 71 | 72 | func TestRuleToProto(t *testing.T) { 73 | r := &Rule{ 74 | ID: 1234567890, 75 | Body: "test rule", 76 | LocZones: []string{"test:dmz", "test:corp"}, 77 | } 78 | want := &pb.Rule{ 79 | Id: 1234567890, 80 | Body: "test rule", 81 | LocationZones: []string{"test:dmz", "test:corp"}, 82 | } 83 | if diff := cmp.Diff(want, RuleToProto(r), cmp.Comparer(proto.Equal)); diff != "" { 84 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 85 | } 86 | } 87 | 88 | func TestMakeRuleFile(t *testing.T) { 89 | rules := []*Rule{ 90 | { 91 | ID: 123, 92 | Body: "test rule", 93 | LocZones: []string{"test:dmz", "test:corp"}, 94 | }, 95 | { 96 | ID: 1234, 97 | Body: "test rule", 98 | LocZones: []string{"test:dmz", "test:corp"}, 99 | }, 100 | { 101 | ID: 12345, 102 | Body: "test rule", 103 | LocZones: []string{"test:dmz", "test:corp"}, 104 | }, 105 | } 106 | want := []byte("test rule\ntest rule\ntest rule\n") 107 | if diff := cmp.Diff(string(want), string(MakeRuleFile(rules))); diff != "" { 108 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 109 | } 110 | } 111 | 112 | func TestMutationsMapping(t *testing.T) { 113 | for _, tt := range []struct { 114 | desc string 115 | object interface{} 116 | want map[string]bool 117 | wantErr bool 118 | }{ 119 | { 120 | desc: "valid mappings", 121 | object: struct { 122 | f1 string `mutable:"false"` 123 | f2, f3 string `mutable:"true"` 124 | }{}, 125 | want: map[string]bool{"f_1": false, "f_2": true, "f_3": true}, 126 | }, 127 | { 128 | desc: "missing tags", 129 | object: struct{ f1 string }{}, 130 | wantErr: true, 131 | }, 132 | { 133 | desc: "pointer object", 134 | object: &struct{ f1 string }{}, 135 | wantErr: true, 136 | }, 137 | } { 138 | got, err := MutationsMapping(tt.object) 139 | if (err != nil) != tt.wantErr { 140 | t.Errorf("%s: got err=%v, wantErr=%v", tt.desc, err, tt.wantErr) 141 | } 142 | if err != nil { 143 | continue 144 | } 145 | if diff := cmp.Diff(tt.want, got); diff != "" { 146 | t.Errorf("%s: expectation mismatch (-want +got):\n%s", tt.desc, diff) 147 | } 148 | } 149 | } 150 | 151 | func TestProtoToSensorMessage(t *testing.T) { 152 | for _, tt := range []struct { 153 | desc string 154 | p *spb.SensorMessage 155 | want *SensorMessage 156 | }{ 157 | { 158 | desc: "heartbeat mismatch", 159 | p: &spb.SensorMessage{ 160 | Id: "test_id", 161 | Type: &spb.SensorMessage_Heartbeat{ 162 | Heartbeat: &spb.Heartbeat{ 163 | Time: &tpb.Timestamp{Seconds: 123}, 164 | Host: &spb.Host{Fqdn: "id1", Ip: "id2", Uuid: "id3", Org: "org", Zone: "zone"}, 165 | }, 166 | }, 167 | }, 168 | want: &SensorMessage{ 169 | ID: "test_id", 170 | Time: "Thu, 01 Jan 1970 00:02:03 +0000", 171 | Host: `fqdn:"id1" ip:"id2" uuid:"id3" org:"org" zone:"zone" `, 172 | }, 173 | }, 174 | { 175 | desc: "alert mistmatch", 176 | p: &spb.SensorMessage{ 177 | Id: "test_id", 178 | Type: &spb.SensorMessage_Alert{ 179 | Alert: &spb.SensorAlert{ 180 | Time: &tpb.Timestamp{Seconds: 123}, 181 | Host: &spb.Host{Fqdn: "id1", Ip: "id2", Uuid: "id3", Org: "org", Zone: "zone"}, 182 | }, 183 | }, 184 | }, 185 | want: &SensorMessage{ 186 | ID: "test_id", 187 | Time: "Thu, 01 Jan 1970 00:02:03 +0000", 188 | Host: `fqdn:"id1" ip:"id2" uuid:"id3" org:"org" zone:"zone" `, 189 | }, 190 | }, 191 | } { 192 | if diff := cmp.Diff(tt.want, ProtoToSensorMessage(tt.p)); diff != "" { 193 | t.Errorf("%s (-want +got):\n%s", tt.desc, diff) 194 | } 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /source/resources/resources.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package resources contains common objects and conversion functions. 16 | package resources 17 | 18 | const ( 19 | // fleetspeakPrefix is the default label prefix prepended to all client labels. 20 | fleetspeakPrefix = "alphabet-" 21 | // LocationNamePrefix is the Fleetspeak label prefix for sensor location name. 22 | LocationNamePrefix = fleetspeakPrefix + "location-name-" 23 | // LocationZonePrefix is the Fleetspeak label prefix for sensor location zone. 24 | LocationZonePrefix = fleetspeakPrefix + "location-zone-" 25 | ) 26 | 27 | // Location defines an arbirary organization of sensors, segmented into a least one zone. 28 | type Location struct { 29 | // The unique name of the location, e.g. "company1". 30 | Name string `mutable:"false"` 31 | // The list of zones or "segments" to organize sensors, e.g. {"dmz", "prod"}. 32 | Zones []string `mutable:"true"` 33 | // Last modified time of the message. Applied by the Store. 34 | LastModified string `mutable:"true"` 35 | } 36 | 37 | // ZoneFilterMode defines how the location zones will be selected. 38 | type ZoneFilterMode string 39 | 40 | const ( 41 | // All is to select all zones. 42 | All ZoneFilterMode = "all" 43 | // Include is to select only a specific subset of zones. 44 | Include ZoneFilterMode = "include" 45 | // Exclude is to select all zones except a specific subset of zones. 46 | Exclude ZoneFilterMode = "exclude" 47 | ) 48 | 49 | // LocationSelector represents a way to select zones from a given location. 50 | type LocationSelector struct { 51 | // The unique name of the location. 52 | Name string 53 | // Define how the location zones will be selected. 54 | Mode ZoneFilterMode 55 | // List of zones which to be filtered in or out of the location zones, depending on the Mode. 56 | Zones []string 57 | } 58 | 59 | // Rule is an IDS rule, e.g. Snort or Suricata. 60 | type Rule struct { 61 | // The unique rule ID. 62 | ID int64 `mutable:"false"` 63 | // The rule itself. 64 | Body string `mutable:"true"` 65 | // Select in which organization and zone the rule is enabled, e.g. "google:dmz". 66 | LocZones []string `mutable:"true"` 67 | // Last modified time of the message. Applied by the Store. 68 | LastModified string `mutable:"true"` 69 | } 70 | 71 | // SensorRequestType represents the type of sensor request message. 72 | type SensorRequestType string 73 | 74 | // Sensor request types as described in the sensor proto. 75 | const ( 76 | DeployRules SensorRequestType = "DeployRules" 77 | ReloadRules SensorRequestType = "ReloadRules" 78 | ) 79 | 80 | // SensorRequest contains the details and state of a sensor request message. 81 | type SensorRequest struct { 82 | // The request message ID. 83 | ID string `mutable:"false"` 84 | // The creation time of the message. 85 | Time string `mutable:"false"` 86 | // Fleetspeak client ID (Hex-encoded bytes). 87 | ClientID string `mutable:"false"` 88 | // Type of message. 89 | Type SensorRequestType `mutable:"false"` 90 | // Status of the request. 91 | Status string `mutable:"true"` 92 | // Last modified time of the message. Applied by the Store. 93 | LastModified string `mutable:"true"` 94 | } 95 | 96 | // SensorMessageType represents the type of message issued from a sensor. 97 | type SensorMessageType string 98 | 99 | const ( 100 | // Response represents a sensor response to a sensor request. 101 | Response SensorMessageType = "Response" 102 | // Alert represents a sensor alert. 103 | Alert SensorMessageType = "Alert" 104 | // Heartbeat represents a sensor heartbeat. 105 | Heartbeat SensorMessageType = "Heartbeat" 106 | ) 107 | 108 | // SensorMessage contains the details and state of a sensor message. 109 | type SensorMessage struct { 110 | // The message ID. 111 | ID string `mutable:"false"` 112 | // The creation time of the message. 113 | Time string `mutable:"false"` 114 | // Fleetspeak client ID (Hex-encoded bytes). 115 | ClientID string `mutable:"false"` 116 | // Type of message. 117 | Type SensorMessageType `mutable:"false"` 118 | // Host information of sender. 119 | Host string `mutable:"false"` 120 | // Status of the request. 121 | Status string `mutable:"false"` 122 | } 123 | -------------------------------------------------------------------------------- /source/sensor/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["main.go"], 6 | importpath = "github.com/google/emitto/source/sensor", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//source/filestore:go_default_library", 10 | "//source/sensor/client:go_default_library", 11 | "@com_github_golang_glog//:go_default_library", 12 | "@com_google_cloud_go//storage:go_default_library", 13 | ], 14 | ) 15 | 16 | go_binary( 17 | name = "sensor", 18 | embed = [":go_default_library"], 19 | visibility = ["//visibility:public"], 20 | ) 21 | 22 | load("@io_bazel_rules_docker//go:image.bzl", "go_image") 23 | 24 | go_image( 25 | name = "sensor_image_base", 26 | embed = [":go_default_library"], 27 | ) 28 | 29 | load("@io_bazel_rules_docker//container:container.bzl", "container_image") 30 | 31 | container_image( 32 | name = "sensor_image", 33 | base = ":sensor_image_base", 34 | stamp = True, 35 | ) 36 | -------------------------------------------------------------------------------- /source/sensor/client/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["client.go"], 6 | importpath = "github.com/google/emitto/source/sensor/client", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//source/filestore:go_default_library", 10 | "//source/sensor/fleetspeak:go_default_library", 11 | "//source/sensor/host:go_default_library", 12 | "//source/sensor/proto:go_default_library", 13 | "//source/sensor/suricata:go_default_library", 14 | "//source/sensor/suricata/proto:go_default_library", 15 | "@com_github_golang_glog//:go_default_library", 16 | "@com_github_golang_protobuf//ptypes:go_default_library_gen", 17 | "@com_github_google_fleetspeak//fleetspeak/src/common/proto/fleetspeak:go_default_library", 18 | "@com_github_google_uuid//:go_default_library", 19 | "@org_golang_google_grpc//codes:go_default_library", 20 | "@org_golang_google_grpc//status:go_default_library", 21 | ], 22 | ) 23 | 24 | go_test( 25 | name = "go_default_test", 26 | srcs = ["client_test.go"], 27 | embed = [":go_default_library"], 28 | deps = [ 29 | "//source/sensor/host:go_default_library", 30 | "//source/sensor/proto:go_default_library", 31 | "@com_github_golang_protobuf//proto:go_default_library", 32 | "@com_github_google_go_cmp//cmp:go_default_library", 33 | "@org_golang_google_grpc//codes:go_default_library", 34 | "@org_golang_google_grpc//status:go_default_library", 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /source/sensor/client/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package client contains sensor client functionality. 16 | package client 17 | 18 | import ( 19 | "bufio" 20 | "context" 21 | "encoding/json" 22 | "fmt" 23 | "io/ioutil" 24 | "os" 25 | "path/filepath" 26 | "time" 27 | 28 | "github.com/golang/protobuf/ptypes" 29 | "github.com/google/emitto/source/filestore" 30 | "github.com/google/emitto/source/sensor/fleetspeak" 31 | "github.com/google/emitto/source/sensor/host" 32 | "github.com/google/emitto/source/sensor/suricata" 33 | "github.com/google/uuid" 34 | "google.golang.org/grpc/codes" 35 | "google.golang.org/grpc/status" 36 | 37 | log "github.com/golang/glog" 38 | pb "github.com/google/emitto/source/sensor/proto" 39 | evepb "github.com/google/emitto/source/sensor/suricata/proto" 40 | fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak" 41 | ) 42 | 43 | // SuricataController represents a Suricata controller. 44 | type SuricataController interface { 45 | // ReloadRules reloads Suricata rules. 46 | ReloadRules() error 47 | } 48 | 49 | // FleetspeakClient represents a Fleetspeak client. 50 | type FleetspeakClient interface { 51 | // SendMessage sends a message to Fleetspeak. 52 | SendMessage(m *pb.SensorMessage) (string, error) 53 | // Receive initiates receiving messages from Fleetspeak. 54 | Receive(done <-chan struct{}) 55 | // Messages provides access to the messages received from Fleetspeak. 56 | Messages() chan *fspb.Message 57 | } 58 | 59 | // Client represents a Emitto sensor client. 60 | type Client struct { 61 | FSClient FleetspeakClient 62 | ctrl SuricataController 63 | ruleStore filestore.FileStore 64 | host *host.Host 65 | org string 66 | zone string 67 | ruleFile string 68 | } 69 | 70 | // New creates a new Emitto sensor client. 71 | func New(ctx context.Context, fleetspeakSocket, org, zone, ruleFile, suricataSocket string, filestore filestore.FileStore) (*Client, error) { 72 | h, err := host.New() 73 | if err != nil { 74 | return nil, fmt.Errorf("failed to created new Host: %v", err) 75 | } 76 | return &Client{ 77 | FSClient: fleetspeak.New(fleetspeakSocket), 78 | ctrl: suricata.NewController(suricataSocket), 79 | ruleStore: filestore, 80 | host: h, 81 | org: org, 82 | zone: zone, 83 | ruleFile: ruleFile, 84 | }, nil 85 | } 86 | 87 | // ProcessMessage handles a Fleetspeak message from Emitto. 88 | func (c *Client) ProcessMessage(ctx context.Context, m *fspb.Message) error { 89 | var req pb.SensorRequest 90 | if err := ptypes.UnmarshalAny(m.Data, &req); err != nil { 91 | return fmt.Errorf("failed to unmarshal Fleetspeak message: %v", err) 92 | } 93 | switch t := req.Type.(type) { 94 | case *pb.SensorRequest_DeployRules: 95 | log.Infof("Received DeployRules request %q", req.GetId()) 96 | return c.sendResponse(req.GetId(), c.deployRules(ctx, t.DeployRules.GetRuleFile())) 97 | case *pb.SensorRequest_ReloadRules: 98 | log.Infof("Received ReloadRules request %q", req.GetId()) 99 | return c.sendResponse(req.GetId(), c.reloadRules()) 100 | default: 101 | return c.sendResponse(req.GetId(), status.New(codes.InvalidArgument, fmt.Sprintf("unknown request type: %T", t))) 102 | } 103 | } 104 | 105 | // sendResponse sends a SensorResponse to the Fleetspeak client. 106 | func (c *Client) sendResponse(id string, s *status.Status) error { 107 | if err := c.host.Update(); err != nil { 108 | return fmt.Errorf("failed to update host info: %v", err) 109 | } 110 | resp := &pb.SensorResponse{ 111 | Id: id, 112 | Time: ptypes.TimestampNow(), 113 | Host: c.getHostInfo(), 114 | Status: s.Proto(), 115 | } 116 | msg := &pb.SensorMessage{ 117 | Id: uuid.New().String(), 118 | Type: &pb.SensorMessage_Response{ 119 | Response: resp, 120 | }, 121 | } 122 | _, err := c.FSClient.SendMessage(msg) 123 | if err != nil { 124 | return fmt.Errorf("failed to send response: %+v", resp) 125 | } 126 | log.V(1).Infof("Sent response: %+v", resp) 127 | return nil 128 | } 129 | 130 | // deployRules fetches an updated rule file, replaces the existing rule file with the updated 131 | // version, and then issues a command for Suricata to reload the rule engine. 132 | func (c *Client) deployRules(ctx context.Context, ruleFile string) *status.Status { 133 | rules, err := c.ruleStore.GetRuleFile(ctx, ruleFile) 134 | if err != nil { 135 | return status.New(codes.NotFound, fmt.Sprintf("failed to download rules from Cloud Storage: %v", err)) 136 | } 137 | if _, err := os.Stat(c.ruleFile); os.IsNotExist(err) { 138 | return status.New(codes.NotFound, fmt.Sprintf("rule file does not exist %q", c.ruleFile)) 139 | } 140 | // Backup existing rule before writing. 141 | backup, err := createBackup(c.ruleFile) 142 | if err != nil { 143 | return status.New(codes.Internal, fmt.Sprintf("failed to create backup for the rule file %q: %v", c.ruleFile, err)) 144 | } 145 | 146 | if err := ioutil.WriteFile(c.ruleFile, rules, 0644); err != nil { 147 | // Restore rule from backup file. 148 | if err := copyFile(backup, c.ruleFile); err != nil { 149 | log.Errorf("failed to restore rule file %q from backup: %q", c.ruleFile, backup) 150 | } 151 | return status.New(codes.Internal, fmt.Sprintf("failed to write rule file to disk: %v", err)) 152 | } 153 | log.Infof("Successfully wrote new rules to %q", c.ruleFile) 154 | if err := c.reloadRules(); err != nil { 155 | return status.New(codes.FailedPrecondition, fmt.Sprintf("failed to reload Suricata rules: %v", err)) 156 | } 157 | log.Info("Successfully reloaded Suricata rules") 158 | return status.New(codes.OK, "OK") 159 | } 160 | 161 | // reloadRules reloads rules via the Suricata socket. 162 | func (c *Client) reloadRules() *status.Status { 163 | if err := c.ctrl.ReloadRules(); err != nil { 164 | return status.New(codes.FailedPrecondition, fmt.Sprintf("failed to issue socket command: %v", err)) 165 | } 166 | return status.New(codes.OK, "OK") 167 | } 168 | 169 | func (c *Client) getHostInfo() *pb.Host { 170 | return &pb.Host{ 171 | Fqdn: c.host.FQDN(), 172 | Ip: c.host.IP().String(), 173 | Org: c.org, 174 | Zone: c.zone, 175 | } 176 | } 177 | 178 | // MonitorSurcataEVELog reads logs from last `d` duration of time, 179 | // and sends a notification alert if the threshold is exceeded 180 | func (c *Client) MonitorSurcataEVELog(d time.Duration, threshold int, logFile string) { 181 | lines, err := readLines(logFile) 182 | if err != nil { 183 | log.Errorf("Error reading lines from log file %v: %v", logFile, err) 184 | return 185 | } 186 | 187 | now := time.Now().UTC() 188 | alerts := 0 189 | for i := len(lines) - 1; i >= 0; i-- { 190 | if alerts > threshold { 191 | break 192 | } 193 | eve, err := parseLogLine(lines[i]) 194 | if err != nil { 195 | log.Error(err) 196 | c.sendAlertNotification(err.Error()) 197 | return 198 | } 199 | 200 | // https://suricata.readthedocs.io/en/suricata-4.1.4/output/eve/eve-json-format.html#event-type-alert 201 | if eve.EventType != "alert" { 202 | continue 203 | } 204 | 205 | t, err := time.Parse("2006-01-02T15:04:05.999999+0000", eve.Timestamp) 206 | if err != nil { 207 | e := fmt.Errorf("Error parsing timestamp %q: %v", eve.Timestamp, err) 208 | log.Error(e) 209 | c.sendAlertNotification(e.Error()) 210 | } 211 | 212 | // Only check last logs within specified duration. 213 | if now.Sub(t) > d { 214 | break 215 | } 216 | 217 | alerts++ 218 | } 219 | // If the threshold is exceeded, send notification. 220 | if alerts > threshold { 221 | c.sendAlertNotification(fmt.Sprintf("Sensor has detected: %v alerts in last: %v", alerts, d)) 222 | } 223 | } 224 | 225 | // SendHeartbeat sends a heartbeat message to the server. 226 | func (c *Client) SendHeartbeat() { 227 | c.FSClient.SendMessage(&pb.SensorMessage{ 228 | Id: uuid.New().String(), 229 | Type: &pb.SensorMessage_Heartbeat{ 230 | Heartbeat: &pb.Heartbeat{ 231 | Time: ptypes.TimestampNow(), 232 | Host: c.getHostInfo(), 233 | }, 234 | }, 235 | }) 236 | } 237 | 238 | func (c *Client) sendAlertNotification(alert string) { 239 | c.FSClient.SendMessage(&pb.SensorMessage{ 240 | Id: uuid.New().String(), 241 | Type: &pb.SensorMessage_Alert{ 242 | Alert: &pb.SensorAlert{ 243 | Time: ptypes.TimestampNow(), 244 | Host: c.getHostInfo(), 245 | Status: status.New(codes.Internal, alert).Proto(), 246 | }, 247 | }, 248 | }) 249 | } 250 | 251 | // createBackup creates copy of filename to filename_YYMMDD_number, with number randomly 252 | // chosen such that the file name is unique and returns the chosen file name. 253 | func createBackup(filename string) (string, error) { 254 | // create backup file. 255 | f, err := ioutil.TempFile(filepath.Dir(filename), fmt.Sprintf("%v_%v_", filepath.Base(filename), time.Now().Format("060102"))) 256 | if err != nil { 257 | return "", err 258 | } 259 | backup := f.Name() 260 | f.Close() 261 | if err := copyFile(filename, backup); err != nil { 262 | return "", err 263 | } 264 | return backup, nil 265 | } 266 | 267 | func copyFile(src, dest string) error { 268 | data, err := ioutil.ReadFile(src) 269 | if err != nil { 270 | return err 271 | } 272 | return ioutil.WriteFile(dest, data, 0644) 273 | } 274 | 275 | // Note: currently suricata eve.json file is ~1.5GB, reading ~1GB file with 400k lines 276 | // takes less than a second, so this should not become performance bottleneck in near future. 277 | func readLines(path string) ([]string, error) { 278 | file, err := os.Open(path) 279 | if err != nil { 280 | return nil, err 281 | } 282 | defer file.Close() 283 | 284 | var lines []string 285 | scanner := bufio.NewScanner(file) 286 | for scanner.Scan() { 287 | lines = append(lines, scanner.Text()) 288 | } 289 | return lines, scanner.Err() 290 | } 291 | 292 | // parseLogLine unmarshals json and returns as EVE protobuf message. 293 | func parseLogLine(line string) (*evepb.EVE, error) { 294 | var eve evepb.EVE 295 | if err := json.Unmarshal([]byte(line), &eve); err != nil { 296 | return nil, fmt.Errorf("cannot unmarshal json string %q: %v", line, err) 297 | } 298 | return &eve, nil 299 | } 300 | -------------------------------------------------------------------------------- /source/sensor/client/client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package client 16 | 17 | import ( 18 | "fmt" 19 | "io/ioutil" 20 | "os" 21 | "path/filepath" 22 | "strings" 23 | "testing" 24 | "time" 25 | 26 | "github.com/golang/protobuf/proto" 27 | "github.com/google/emitto/source/sensor/host" 28 | "github.com/google/go-cmp/cmp" 29 | "google.golang.org/grpc/codes" 30 | "google.golang.org/grpc/status" 31 | 32 | pb "github.com/google/emitto/source/sensor/proto" 33 | ) 34 | 35 | type fakeSuricataController struct{} 36 | 37 | func (s *fakeSuricataController) ReloadRules() error { return nil } 38 | 39 | func TestSocketReloadRules(t *testing.T) { 40 | c := &Client{ 41 | ctrl: &fakeSuricataController{}, 42 | } 43 | 44 | got := c.reloadRules() 45 | want := status.New(codes.OK, "OK") 46 | if diff := cmp.Diff(want, got, cmp.Comparer(proto.Equal), cmp.AllowUnexported(*want, *got)); diff != "" { 47 | t.Fatalf("expectation mismatch:\n%s", diff) 48 | } 49 | } 50 | 51 | func TestParseLogLine(t *testing.T) { 52 | for _, tt := range []struct { 53 | desc string 54 | line string 55 | want string 56 | wantErr bool 57 | }{ 58 | { 59 | desc: "well-formed JSON", 60 | line: `{"timestamp": "2019-05-13T14:12:19.384640+0000", "event_type": "alert"}`, 61 | want: `timestamp:"2019-05-13T14:12:19.384640+0000" event_type:"alert" `, 62 | }, { 63 | desc: "malformed JSON; missing parenthesis", 64 | line: `{"dest_port":234234`, 65 | wantErr: true, 66 | }, { 67 | desc: "malformed JSON", 68 | line: ` -- [ERRCODE: SC_ERR_EVENT_ENGINE(210)]`, 69 | wantErr: true, 70 | }, 71 | } { 72 | eve, err := parseLogLine(tt.line) 73 | 74 | if (err != nil) != tt.wantErr { 75 | t.Errorf("%s: got err=%v, wantErr=%t", tt.desc, err, tt.wantErr) 76 | } 77 | if err != nil { 78 | continue 79 | } 80 | if diff := cmp.Diff(tt.want, eve.String()); diff != "" { 81 | t.Errorf("parseLogLine(%v) expectation mismatch (+want -got)\n%s", tt.line, diff) 82 | } 83 | } 84 | } 85 | 86 | // Creates temporary log file with 10 logs 1 second apart, checks if the sensor emits the alert. 87 | func TestMonitorSurcataEVELog(t *testing.T) { 88 | 89 | f := &fakeFleetspeakClient{} 90 | client := &Client{ 91 | FSClient: f, 92 | host: &host.Host{}, 93 | } 94 | 95 | d, err := ioutil.TempDir("/tmp", "eve-logs") 96 | if err != nil { 97 | t.Fatalf("failed to create temp dir: %v", err) 98 | } 99 | defer os.RemoveAll(d) 100 | 101 | var lines []string 102 | now := time.Now().UTC() 103 | for i := 0; i < 10; i++ { 104 | timestamp := now.Add(time.Duration(-i) * time.Second).Format("2006-01-02T15:04:05.999999+0000") 105 | lines = append(lines, fmt.Sprintf(`{"timestamp": %q, "event_type": "alert"}`, timestamp)) 106 | } 107 | 108 | logFile := filepath.Join(d, "eve.json") 109 | if err := ioutil.WriteFile(logFile, []byte(strings.Join(lines, "\n")), 0666); err != nil { 110 | t.Fatalf("failed to write in file %v: %v", logFile, err) 111 | } 112 | client.MonitorSurcataEVELog(time.Minute, 9, logFile) 113 | if len(f.Msgs) != 1 { 114 | t.Errorf("TestMonitorSurcataEVELog(%v, %v, %v), expected to emit alert message", time.Minute, 9, logFile) 115 | } 116 | f.Msgs = f.Msgs[:0] 117 | client.MonitorSurcataEVELog(time.Second, 1, logFile) 118 | if len(f.Msgs) != 0 { 119 | t.Errorf("TestMonitorSurcataEVELog(%v, %v, %v), emitted unexpected alert messages: %+v", time.Minute, 1, logFile, f.Msgs) 120 | } 121 | } 122 | 123 | // Creates temporary file, calls createBackup and compares content of returned file to original file. 124 | func TestCreateBackup(t *testing.T) { 125 | d, err := ioutil.TempDir("/tmp", "test") 126 | if err != nil { 127 | t.Fatalf("failed to create temp dir: %v", err) 128 | } 129 | defer os.RemoveAll(d) 130 | 131 | srcData := []byte{1, 2, 3} 132 | srcFile := filepath.Join(d, "src.txt") 133 | if err := ioutil.WriteFile(srcFile, srcData, 0644); err != nil { 134 | t.Fatalf("failed to write in file %v: %v", srcFile, err) 135 | } 136 | backup, err := createBackup(srcFile) 137 | if err != nil { 138 | t.Errorf("createBackup(%v) returned an error: %v", srcFile, err) 139 | } 140 | backupData, err := ioutil.ReadFile(backup) 141 | if err != nil { 142 | t.Errorf("error reading file %v: %v", srcFile, err) 143 | } 144 | if diff := cmp.Diff(srcData, backupData); diff != "" { 145 | t.Errorf("backup file does not match source (-want +got):\n%s", diff) 146 | } 147 | } 148 | 149 | type fakeFleetspeakClient struct { 150 | FleetspeakClient 151 | Msgs []*pb.SensorMessage 152 | } 153 | 154 | func (c *fakeFleetspeakClient) SendMessage(m *pb.SensorMessage) (string, error) { 155 | c.Msgs = append(c.Msgs, m) 156 | return "ok", nil 157 | } 158 | -------------------------------------------------------------------------------- /source/sensor/fleetspeak/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["fleetspeak.go"], 6 | importpath = "github.com/google/emitto/source/sensor/fleetspeak", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//source/sensor/proto:go_default_library", 10 | "@com_github_golang_glog//:go_default_library", 11 | "@com_github_golang_protobuf//ptypes:go_default_library_gen", 12 | "@com_github_google_fleetspeak//fleetspeak/src/client/channel:go_default_library", 13 | "@com_github_google_fleetspeak//fleetspeak/src/client/service:go_default_library", 14 | "@com_github_google_fleetspeak//fleetspeak/src/client/socketservice/client:go_default_library", 15 | "@com_github_google_fleetspeak//fleetspeak/src/common/proto/fleetspeak:go_default_library", 16 | ], 17 | ) 18 | 19 | go_test( 20 | name = "go_default_test", 21 | srcs = ["fleetspeak_test.go"], 22 | embed = [":go_default_library"], 23 | deps = [ 24 | "@com_github_golang_glog//:go_default_library", 25 | "@com_github_golang_protobuf//proto:go_default_library", 26 | "@com_github_google_fleetspeak//fleetspeak/src/client/channel:go_default_library", 27 | "@com_github_google_fleetspeak//fleetspeak/src/client/service:go_default_library", 28 | "@com_github_google_fleetspeak//fleetspeak/src/common/proto/fleetspeak:go_default_library", 29 | "@com_github_google_go_cmp//cmp:go_default_library", 30 | "@io_bazel_rules_go//proto/wkt:any_go_proto", 31 | ], 32 | ) 33 | -------------------------------------------------------------------------------- /source/sensor/fleetspeak/config/labels: -------------------------------------------------------------------------------- 1 | # Labels for Fleetspeak sensor clients. 2 | # 3 | # Primary identification label. 4 | emitto 5 | # Name of the location of the sensor. 6 | location-name- 7 | # Zone of the specified location. 8 | location-zone- 9 | -------------------------------------------------------------------------------- /source/sensor/fleetspeak/config/socket_service.txt: -------------------------------------------------------------------------------- 1 | name: "Emitto" 2 | factory: "Socket" 3 | config { 4 | [type.googleapis.com/fleetspeak.socketservice.Config] { 5 | api_proxy_path: "/var/run/emitto.sock" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /source/sensor/fleetspeak/fleetspeak.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package fleetspeak provides functionality for network sensors to communicate with the Emitto 16 | // service via Fleetspeak. 17 | package fleetspeak 18 | 19 | import ( 20 | "math/rand" 21 | "time" 22 | 23 | "github.com/golang/protobuf/ptypes" 24 | "github.com/google/fleetspeak/fleetspeak/src/client/channel" 25 | "github.com/google/fleetspeak/fleetspeak/src/client/service" 26 | "github.com/google/fleetspeak/fleetspeak/src/client/socketservice/client" 27 | 28 | log "github.com/golang/glog" 29 | pb "github.com/google/emitto/source/sensor/proto" 30 | fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak" 31 | ) 32 | 33 | const ( 34 | // Service name for Fleetspeak messages. 35 | serviceName = "Emitto" 36 | // Maximum size of Messages channel used to receive Fleetspeak client messages. 37 | maxMessages = 1 38 | ) 39 | 40 | // Client contains functionality to send and receive messages to/from a Fleetspeak Client. 41 | type Client struct { 42 | // Channel used for sending messages to the Fleetspeak client. 43 | fsChan *channel.RelentlessChannel 44 | // Callback for Fleetspeak client send acknowledgements. 45 | callbackChan chan string 46 | // Messages is used to queue received Fleetspeak client messages for sensor client consumption. 47 | messages chan *fspb.Message 48 | } 49 | 50 | // New initializes a Client. 51 | func New(socket string) *Client { 52 | rc := client.OpenChannel(socket, time.Now().Format(time.RFC1123Z)) 53 | 54 | return &Client{ 55 | fsChan: rc, 56 | callbackChan: make(chan string, 5), // To prevent potential locking. 57 | messages: make(chan *fspb.Message, maxMessages), 58 | } 59 | } 60 | 61 | // SendMessage a message to the Fleetspeak client. This call blocks until Fleetspeak has 62 | // acknowledged the message. 63 | func (c *Client) SendMessage(m *pb.SensorMessage) (string, error) { 64 | req, err := c.createRequest(m) 65 | if err != nil { 66 | return "", err 67 | } 68 | return c.sendAndWait(req), nil 69 | } 70 | 71 | // sendAndWait sends a message to the Fleetspeak client and waits indefinitely. Only one sendAndWait 72 | // should be called by the Client at a time to avoid non-chronological request ID logging from the 73 | // Fleetspeak client callback channel. 74 | func (c *Client) sendAndWait(msg *service.AckMessage) string { 75 | c.fsChan.Out <- *msg 76 | log.Infof("Sent message (%X) to Fleetspeak; awaiting acknowledgement...", msg.M.GetSourceMessageId()) 77 | ack := <-c.callbackChan 78 | log.Infof("Received ack %q from Fleetspeak", ack) 79 | return ack 80 | } 81 | 82 | // createRequest composes a Fleetspeak AckMessage. 83 | // Fleetspeak is optimized to handle messages sizes < 2MB. 84 | func (c *Client) createRequest(m *pb.SensorMessage) (*service.AckMessage, error) { 85 | data, err := ptypes.MarshalAny(m) 86 | if err != nil { 87 | return nil, err 88 | } 89 | id := make([]byte, 16) 90 | rand.Read(id) 91 | return &service.AckMessage{ 92 | M: &fspb.Message{ 93 | SourceMessageId: id, 94 | Destination: &fspb.Address{ 95 | ServiceName: serviceName, 96 | }, 97 | Data: data, 98 | Background: true, 99 | }, 100 | Ack: func() { 101 | c.callbackChan <- m.Id 102 | }, 103 | }, nil 104 | } 105 | 106 | // Receive continuously receives new messages from the Fleetspeak client's In channel. Once it 107 | // receives a message, it will send it to the Messages channel for the sensor client to process. 108 | func (c *Client) Receive(done <-chan struct{}) { 109 | for { 110 | select { 111 | case m := <-c.fsChan.In: 112 | log.Infof("Received message (%X) from Fleetspeak", m.GetSourceMessageId()) 113 | c.messages <- m 114 | case <-done: 115 | log.Warning("Stopped receiving messages from Fleetspeak") 116 | close(c.messages) 117 | return 118 | } 119 | } 120 | } 121 | 122 | // Messages returns the channel containing incoming Fleetspeak messages. 123 | func (c *Client) Messages() chan *fspb.Message { 124 | return c.messages 125 | } 126 | -------------------------------------------------------------------------------- /source/sensor/fleetspeak/fleetspeak_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fleetspeak 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/golang/protobuf/proto" 22 | "github.com/golang/protobuf/ptypes/any" 23 | "github.com/google/fleetspeak/fleetspeak/src/client/channel" 24 | "github.com/google/fleetspeak/fleetspeak/src/client/service" 25 | "github.com/google/go-cmp/cmp" 26 | 27 | log "github.com/golang/glog" 28 | fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak" 29 | ) 30 | 31 | func TestSendAndWait(t *testing.T) { 32 | o := make(chan service.AckMessage) 33 | callbackChan := make(chan string) 34 | defer close(o) 35 | 36 | // Client with fake relentless channel to Fleetspeak client. 37 | c := &Client{ 38 | fsChan: &channel.RelentlessChannel{ 39 | Out: o, 40 | }, 41 | callbackChan: callbackChan, 42 | } 43 | 44 | // Fleetspeak client acknowledgement message. 45 | ackMsg := &service.AckMessage{ 46 | Ack: func() { 47 | c.callbackChan <- "TEST_OP_ID" 48 | }, 49 | } 50 | go c.sendAndWait(ackMsg) 51 | 52 | // Fake Fleetspeak client message handling. 53 | go func() { 54 | for { 55 | select { 56 | case <-o: 57 | log.Info("TestSendAndWait() Fleetspeak client received message") 58 | time.Sleep(5 * time.Second) // Simulate work. 59 | callbackChan <- "TEST_OP_ID" 60 | return 61 | default: 62 | log.Info("No new messages") 63 | } 64 | } 65 | }() 66 | 67 | want := "TEST_OP_ID" 68 | got := <-c.callbackChan 69 | if diff := cmp.Diff(got, want); diff != "" { 70 | t.Errorf("TestSendAndWait() expectation mismatch:\n%s", diff) 71 | } 72 | } 73 | 74 | func TestReceive(t *testing.T) { 75 | in := make(chan *fspb.Message) 76 | 77 | // Client with fake relentless channel to Fleetspeak client. 78 | c := &Client{ 79 | fsChan: &channel.RelentlessChannel{ 80 | In: in, 81 | }, 82 | messages: make(chan *fspb.Message, maxMessages), 83 | } 84 | 85 | // Fake Fleetspeak client message sending. 86 | // Send 3 messages. 87 | m := &fspb.Message{ 88 | MessageId: []byte{1, 2, 3}, 89 | Data: &any.Any{ 90 | Value: []byte{1, 2, 3}, 91 | }, 92 | } 93 | 94 | go func() { 95 | for i := 0; i < 3; i++ { 96 | in <- m 97 | } 98 | }() 99 | 100 | // Receive messages. 101 | done := make(chan struct{}) 102 | go func() { 103 | c.Receive(done) 104 | }() 105 | 106 | // Simulate work, then close receiving loop. 107 | time.Sleep(5 * time.Second) 108 | close(done) 109 | 110 | // Verify received messages. 111 | want := &fspb.Message{ 112 | MessageId: []byte{1, 2, 3}, 113 | Data: &any.Any{ 114 | Value: []byte{1, 2, 3}, 115 | }, 116 | } 117 | 118 | for got := range c.Messages() { 119 | if diff := cmp.Diff(want, got, cmp.Comparer(proto.Equal)); diff != "" { 120 | t.Errorf("TestReceive() expectation mismatch (-want +got):\n%s", diff) 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /source/sensor/host/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["host.go"], 6 | importpath = "github.com/google/emitto/source/sensor/host", 7 | visibility = ["//visibility:public"], 8 | deps = ["@com_github_golang_glog//:go_default_library"], 9 | ) 10 | 11 | go_test( 12 | name = "go_default_test", 13 | srcs = ["host_test.go"], 14 | embed = [":go_default_library"], 15 | deps = ["@com_github_google_go_cmp//cmp:go_default_library"], 16 | ) 17 | -------------------------------------------------------------------------------- /source/sensor/host/host.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package host provides functionality to obtain basic host information for a host. 16 | package host 17 | 18 | import ( 19 | "fmt" 20 | "net" 21 | "os" 22 | "time" 23 | 24 | log "github.com/golang/glog" 25 | ) 26 | 27 | var ( 28 | // To facilitate unit testing. 29 | netDial = net.Dial 30 | osHostname = os.Hostname 31 | ) 32 | 33 | // Host contains basic host information. 34 | type Host struct { 35 | fqdn string 36 | ip net.IP 37 | } 38 | 39 | // FQDN returns the Host FQDN. 40 | func (h *Host) FQDN() string { 41 | return h.fqdn 42 | } 43 | 44 | // IP returns the Host IP address. 45 | func (h *Host) IP() net.IP { 46 | return h.ip 47 | } 48 | 49 | // New performs an initial update and returns a Host. 50 | func New() (*Host, error) { 51 | h := new(Host) 52 | 53 | // Wait indefinitely for the host to come online. 54 | for { 55 | if err := h.Update(); err != nil { 56 | log.Errorf("host update failed: %v; retrying after 5 seconds", err) 57 | time.Sleep(5 * time.Second) 58 | } else { 59 | break 60 | } 61 | } 62 | return h, nil 63 | } 64 | 65 | // Update updates the host. 66 | func (h *Host) Update() error { 67 | if err := h.updateFQDN(); err != nil { 68 | return err 69 | } 70 | if err := h.updateIP(); err != nil { 71 | return err 72 | } 73 | log.Infof("Host info: hostname: %s, ip: %s", h.fqdn, h.ip) 74 | return nil 75 | } 76 | 77 | // updateFQDN updates the host fqdn. 78 | func (h *Host) updateFQDN() error { 79 | fqdn, err := osHostname() 80 | if err != nil { 81 | return fmt.Errorf("failed to retrieve fqdn: %v", err) 82 | } 83 | h.fqdn = fqdn 84 | return nil 85 | } 86 | 87 | // updateIP updates the host IP address. 88 | func (h *Host) updateIP() error { 89 | conn, err := netDial("udp", "8.8.8.8:9") // RFC863. 90 | if err != nil { 91 | return fmt.Errorf("failed to retrieve ip address: %v", err) 92 | } 93 | defer conn.Close() 94 | h.ip = conn.LocalAddr().(*net.UDPAddr).IP 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /source/sensor/host/host_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package host 16 | 17 | import ( 18 | "net" 19 | "testing" 20 | 21 | "github.com/google/go-cmp/cmp" 22 | ) 23 | 24 | type fakeConn struct { 25 | net.Conn 26 | localAddr *net.UDPAddr 27 | } 28 | 29 | func (c fakeConn) LocalAddr() net.Addr { return c.localAddr } 30 | func (fakeConn) Close() error { return nil } 31 | 32 | func fakeHost() *Host { 33 | return &Host{ 34 | fqdn: "test_1_host_name", 35 | ip: net.ParseIP("100.97.26.27"), 36 | } 37 | } 38 | 39 | func TestUpdate(t *testing.T) { 40 | for _, tt := range []struct { 41 | desc string 42 | osHostname func() (string, error) 43 | netDial func(string, string) (net.Conn, error) 44 | want *Host 45 | }{ 46 | { 47 | desc: "no change", 48 | osHostname: func() (string, error) { return "test_1_host_name", nil }, 49 | netDial: func(string, string) (net.Conn, error) { 50 | updAddr, _ := net.ResolveUDPAddr("udp", "[100.97.26.27]:5688") 51 | return &fakeConn{localAddr: updAddr}, nil 52 | }, 53 | want: &Host{ 54 | fqdn: "test_1_host_name", 55 | ip: net.ParseIP("100.97.26.27"), 56 | }, 57 | }, 58 | { 59 | desc: "fqdn changed", 60 | osHostname: func() (string, error) { return "test_2_host_name", nil }, 61 | netDial: nil, 62 | want: &Host{ 63 | fqdn: "test_2_host_name", 64 | ip: net.ParseIP("100.97.26.27"), 65 | }, 66 | }, 67 | { 68 | desc: "ip changed", 69 | osHostname: nil, 70 | netDial: func(string, string) (net.Conn, error) { 71 | updAddr, _ := net.ResolveUDPAddr("udp", "[10.10.10.10]:5688") 72 | return &fakeConn{localAddr: updAddr}, nil 73 | }, 74 | want: &Host{ 75 | fqdn: "test_2_host_name", 76 | ip: net.ParseIP("10.10.10.10"), 77 | }, 78 | }, 79 | } { 80 | h := fakeHost() 81 | 82 | // Override functions, if applicable. 83 | if tt.osHostname != nil { 84 | osHostname = tt.osHostname 85 | } 86 | if tt.netDial != nil { 87 | netDial = tt.netDial 88 | } 89 | 90 | if err := h.Update(); err != nil { 91 | t.Errorf("TestUpdate(%s): got unexpected error: %v", tt.desc, err) 92 | } 93 | if diff := cmp.Diff(tt.want, h, cmp.AllowUnexported(*tt.want, *h)); diff != "" { 94 | t.Errorf("TestUpdate(%s): expectation mismatch (-want +got):\n%s", tt.desc, diff) 95 | } 96 | } 97 | } 98 | 99 | func TestGetters(t *testing.T) { 100 | h := fakeHost() 101 | 102 | if diff := cmp.Diff("test_1_host_name", h.FQDN()); diff != "" { 103 | t.Errorf("TestGetters: expectation mismatch (-want +got):\n%s", diff) 104 | } 105 | if diff := cmp.Diff("100.97.26.27", h.IP().String()); diff != "" { 106 | t.Errorf("TestGetters: expectation mismatch (-want +got):\n%s", diff) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /source/sensor/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package main initializes and starts the Emitto sensor client. 16 | package main 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "time" 22 | 23 | "cloud.google.com/go/storage" 24 | "github.com/google/emitto/source/filestore" 25 | "github.com/google/emitto/source/sensor/client" 26 | 27 | log "github.com/golang/glog" 28 | ) 29 | 30 | var ( 31 | // Suricata flags. 32 | fsSocket = flag.String("fleetspeak_socket", "", "Fleetspeak client socket") 33 | suricataSocket = flag.String("suricata_socket", "", "Suricata Unix socket") 34 | ruleFile = flag.String("rule_file", "", "Suricata rule file path") 35 | memoryStorage = flag.Bool("memory_storage", false, "Use memory store and filestore") 36 | 37 | // Sensor identity flags. 38 | org = flag.String("org", "", "Sensor organization") 39 | zone = flag.String("zone", "", "Sensor zone") 40 | 41 | // Google Cloud Project flags. 42 | projectID = flag.String("project_id", "", "Google Cloud project ID") 43 | storageBucket = flag.String("storage_bucket", "", "Google Cloud Storage bucket for storing rule files") 44 | credFile = flag.String("cred_file", "", "Path of the JSON application credential file") 45 | 46 | // Suricata EVE monitoring flags. 47 | monitorEVE = flag.Bool("monitor_eve", false, "Monitor EVE logs") 48 | alertPollingPeriod = flag.Duration("alerts_polling", 10*time.Minute, "Polling interval for checking alerts") 49 | alertThreshold = flag.Int("alerts_threshold", 100, "Alerting threshold for Suricata alerts") 50 | // https://suricata.readthedocs.io/en/suricata-4.1.4/output/eve/eve-json-output.html 51 | suricataEVELog = flag.String("eve_log", "", "Path of the eve.json file") 52 | 53 | // Heartbeat flags. 54 | heartbeat = flag.Bool("heartbeat", false, "Send heartbeat to server") 55 | heartbeatPollingPeriod = flag.Duration("heartbeat_polling", 10*time.Minute, "Polling interval for sending heartbeats") 56 | ) 57 | 58 | func main() { 59 | ctx := context.Background() 60 | 61 | fs, closeFStore := mustGetFileStore(ctx) 62 | defer closeFStore() 63 | sc, err := client.New(ctx, *fsSocket, *org, *zone, *ruleFile, *suricataSocket, fs) 64 | if err != nil { 65 | log.Exitf("failed to create sensor client: %v", err) 66 | } 67 | 68 | done := make(chan struct{}) 69 | go func() { 70 | sc.FSClient.Receive(done) 71 | }() 72 | defer close(done) 73 | 74 | if *monitorEVE { 75 | go monitorEVELog(sc) 76 | } 77 | 78 | if *heartbeat { 79 | go heartbeatPolling(sc) 80 | } 81 | 82 | for msg := range sc.FSClient.Messages() { 83 | if err := sc.ProcessMessage(ctx, msg); err != nil { 84 | log.Error(err.Error()) 85 | } 86 | } 87 | } 88 | 89 | func mustGetFileStore(ctx context.Context) (filestore.FileStore, func() error) { 90 | if *memoryStorage { 91 | return filestore.NewMemoryFileStore(), func() error { return nil } 92 | } 93 | c, err := filestore.NewGCSClient(ctx, *credFile, []string{storage.ScopeReadOnly}) 94 | if err != nil { 95 | log.Exitf("failed to create Google Cloud Storage client: %v", err) 96 | } 97 | return filestore.NewGCSFileStore(*storageBucket, c), c.Close 98 | } 99 | 100 | func monitorEVELog(sc *client.Client) { 101 | for range time.Tick(*alertPollingPeriod) { 102 | sc.MonitorSurcataEVELog(*alertPollingPeriod, *alertThreshold, *suricataEVELog) 103 | } 104 | } 105 | 106 | func heartbeatPolling(sc *client.Client) { 107 | for range time.Tick(*heartbeatPollingPeriod) { 108 | sc.SendHeartbeat() 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /source/sensor/proto/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 3 | 4 | proto_library( 5 | name = "emitto_sensor_proto", 6 | srcs = ["sensor.proto"], 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "@com_google_protobuf//:timestamp_proto", 10 | "@go_googleapis//google/rpc:status_proto", 11 | ], 12 | ) 13 | 14 | go_proto_library( 15 | name = "emitto_sensor_go_proto", 16 | importpath = "github.com/google/emitto/source/sensor/proto", 17 | proto = ":emitto_sensor_proto", 18 | visibility = ["//visibility:public"], 19 | deps = ["@go_googleapis//google/rpc:status_go_proto"], 20 | ) 21 | 22 | go_library( 23 | name = "go_default_library", 24 | embed = [":emitto_sensor_go_proto"], 25 | importpath = "github.com/google/emitto/source/sensor/proto", 26 | visibility = ["//visibility:public"], 27 | ) 28 | -------------------------------------------------------------------------------- /source/sensor/proto/sensor.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package emitto.sensor; 18 | 19 | import "google/protobuf/timestamp.proto"; 20 | import "google/rpc/status.proto"; 21 | 22 | // DeployRules instructs a sensor to fetch an updated rules file and 23 | // to reload the rules engine. 24 | message DeployRules { 25 | // Updated rule file. 26 | string rule_file = 1; 27 | } 28 | 29 | // ReloadRules instructs a sensor to reload the rules engine. 30 | message ReloadRules {} 31 | 32 | // SensorRequest represents an operation for a sensor to perform. 33 | message SensorRequest { 34 | // Unique request ID. 35 | string id = 1; 36 | 37 | // Request time. 38 | google.protobuf.Timestamp time = 2; 39 | 40 | // Type is the type of operation for a sensor to perform. 41 | oneof type { 42 | DeployRules deploy_rules = 3; 43 | ReloadRules reload_rules = 4; 44 | } 45 | } 46 | 47 | // Sensor host information. 48 | message Host { 49 | // FQDN of the sensor. 50 | string fqdn = 1; 51 | // IP address of the sensor. 52 | string ip = 2; 53 | // UUID of the sensor. 54 | string uuid = 3; 55 | // Org of the sensor. 56 | string org = 4; 57 | // Zone of the sensor. 58 | string zone = 5; 59 | } 60 | 61 | // SensorMessage contains a type of sensor message to be sent to the server. 62 | message SensorMessage { 63 | // Unique ID of the message. 64 | string id = 1; 65 | oneof type { 66 | SensorResponse response = 2; 67 | SensorAlert alert = 3; 68 | Heartbeat heartbeat = 4; 69 | } 70 | } 71 | 72 | // SensorResponse is a response to a SensorRequest. 73 | message SensorResponse { 74 | // SensorRequest ID. 75 | string id = 1; 76 | 77 | // Response time. 78 | google.protobuf.Timestamp time = 2; 79 | 80 | // Status of the corresponding request. 81 | google.rpc.Status status = 3; 82 | 83 | // Sensor host information. 84 | Host host = 4; 85 | } 86 | 87 | // SensorAlert is an alert originating from the sensor. 88 | message SensorAlert { 89 | // Notification time. 90 | google.protobuf.Timestamp time = 1; 91 | 92 | // Notification status details. 93 | google.rpc.Status status = 2; 94 | 95 | // Sensor host information. 96 | Host host = 3; 97 | } 98 | 99 | 100 | // Heartbeat is a heartbeat message from the sensor. 101 | message Heartbeat { 102 | // Heartbeat time. 103 | google.protobuf.Timestamp time = 1; 104 | 105 | // Sensor host information. 106 | Host host = 2; 107 | } 108 | -------------------------------------------------------------------------------- /source/sensor/suricata/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["suricata.go"], 6 | importpath = "github.com/google/emitto/source/sensor/suricata", 7 | visibility = ["//visibility:public"], 8 | deps = ["//source/sensor/suricata/socket:go_default_library"], 9 | ) 10 | 11 | go_test( 12 | name = "go_default_test", 13 | srcs = ["suricata_test.go"], 14 | embed = [":go_default_library"], 15 | deps = ["//source/sensor/suricata/socket:go_default_library"], 16 | ) 17 | -------------------------------------------------------------------------------- /source/sensor/suricata/proto/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 3 | 4 | proto_library( 5 | name = "emitto_sensor_proto", 6 | srcs = ["suricata_eve.proto"], 7 | visibility = ["//visibility:public"], 8 | ) 9 | 10 | go_proto_library( 11 | name = "emitto_sensor_go_proto", 12 | importpath = "github.com/google/emitto/source/sensor/suricata/proto", 13 | proto = ":emitto_sensor_proto", 14 | visibility = ["//visibility:public"], 15 | ) 16 | 17 | go_library( 18 | name = "go_default_library", 19 | embed = [":emitto_sensor_go_proto"], 20 | importpath = "github.com/google/emitto/source/sensor/suricata/proto", 21 | visibility = ["//visibility:public"], 22 | ) 23 | -------------------------------------------------------------------------------- /source/sensor/suricata/proto/suricata_eve.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package emitto.sensor; 18 | 19 | // EVE represents Suricata EVE json output. 20 | // http://suricata.readthedocs.io/en/latest/output/eve/eve-json-output.html (not 21 | // all structures are documented). 22 | message EVE { 23 | string timestamp = 1; 24 | string event_type = 2; 25 | string src_ip = 3; 26 | int32 src_port = 4; 27 | string dest_ip = 5; 28 | int32 dest_port = 6; 29 | string proto = 7; 30 | int32 pcap_cnt = 8; 31 | string app_proto = 9; 32 | string app_proto_tc = 10; 33 | string app_proto_ts = 11; 34 | int64 flow_id = 12; 35 | int32 vlan = 13; 36 | int32 tx_id = 14; 37 | string packet = 15; 38 | int32 icmp_type = 16; 39 | int32 icmp_code = 17; 40 | int32 response_icmp_code = 18; 41 | int32 response_icmp_type = 19; 42 | Vars vars = 20; 43 | Alert alert = 21; 44 | HTTP http = 22; 45 | FileInfo fileinfo = 23; 46 | TCP tcp = 24; 47 | DNS dns = 25; 48 | TLS tls = 26; 49 | Flow flow = 27; 50 | PacketInfo packet_info = 28; 51 | SSH ssh = 29; 52 | SMTP smtp = 30; 53 | Email email = 31; 54 | } 55 | 56 | // Vars from the rule metadata field. 57 | message Vars { 58 | map flowbits = 1; 59 | } 60 | 61 | // Alert EVE data. 62 | message Alert { 63 | string action = 1; 64 | int32 gid = 2; 65 | int32 signature_id = 3; 66 | int32 rev = 4; 67 | string signature = 5; 68 | string category = 6; 69 | int32 severity = 7; 70 | int32 tenant_id = 8; 71 | Metadata metadata = 9; 72 | } 73 | 74 | // Metadata EVE data. 75 | message Metadata { 76 | repeated string updated_at = 1; 77 | repeated string created_at = 2; 78 | } 79 | 80 | // HTTP EVE data. 81 | message HTTP { 82 | string hostname = 1; 83 | string url = 2; 84 | string http_user_agent = 3; 85 | string http_content_type = 4; 86 | string http_refer = 5; 87 | string http_method = 6; 88 | string protocol = 7; 89 | int32 status = 8; 90 | int32 length = 9; 91 | string redirect = 10; 92 | string xff = 11; 93 | string http_request_body = 12; 94 | string http_response_body = 13; 95 | int32 http_port = 14; 96 | } 97 | 98 | // FileInfo EVE data. 99 | message FileInfo { 100 | string filename = 1; 101 | string state = 2; 102 | bool stored = 3; 103 | int32 size = 4; 104 | int32 tx_id = 5; 105 | bool gaps = 6; 106 | } 107 | 108 | // TCP EVE data. 109 | message TCP { 110 | string tcp_flags = 1; 111 | string tcp_flags_ts = 2; 112 | string tcp_flags_tc = 3; 113 | bool syn = 4; 114 | bool rst = 5; 115 | bool psh = 6; 116 | bool ack = 7; 117 | bool ecn = 8; 118 | bool cwr = 9; 119 | bool fin = 10; 120 | bool urg = 11; 121 | string state = 12; 122 | } 123 | 124 | // Flow EVE data. 125 | message Flow { 126 | int32 pkts_toserver = 1; 127 | int32 pkts_toclient = 2; 128 | int32 bytes_toserver = 3; 129 | int32 bytes_toclient = 4; 130 | string start = 5; 131 | string end = 6; 132 | int32 age = 7; 133 | string state = 8; 134 | string reason = 9; 135 | bool alerted = 10; 136 | } 137 | 138 | // DNS EVE data. 139 | message DNS { 140 | string type = 1; 141 | int32 id = 2; 142 | string rrname = 3; 143 | string rrtype = 4; 144 | string rdata = 5; 145 | string rcode = 8; 146 | int32 ttl = 6; 147 | int32 tx_id = 7; 148 | bool aa = 9; 149 | bool qr = 10; 150 | bool rd = 11; 151 | bool ra = 12; 152 | string flags = 13; 153 | } 154 | 155 | // TLS EVE data. 156 | message TLS { 157 | string subject = 1; 158 | string issuerdn = 2; 159 | bool session_resumed = 3; 160 | string serial = 4; 161 | string fingerprint = 5; 162 | string sni = 6; 163 | string version = 7; 164 | string notbefore = 8; 165 | string notafter = 9; 166 | string certificate = 10; 167 | string chain = 11; 168 | JA3 ja3 = 12; 169 | } 170 | 171 | // JA3 TLS EVE data. 172 | message JA3 { 173 | string hash = 1; 174 | string data = 2; 175 | string string = 3; 176 | } 177 | 178 | // PacketInfo EVE data. 179 | message PacketInfo { 180 | int32 linktype = 1; 181 | } 182 | 183 | // SSH EVE data. 184 | message SSH { 185 | Server server = 1; 186 | Client client = 2; 187 | } 188 | 189 | // Client SSH EVE data. 190 | message Client { 191 | string proto_version = 1; 192 | string software_version = 2; 193 | } 194 | 195 | // Server SSH EVE data. 196 | message Server { 197 | string proto_version = 1; 198 | string software_version = 2; 199 | } 200 | 201 | // SMTP EVE data. 202 | message SMTP { 203 | string helo = 1; 204 | string mail_from = 2; 205 | repeated string rcpt_to = 3; 206 | } 207 | 208 | // Email EVE data. 209 | message Email { 210 | string status = 1; 211 | } 212 | -------------------------------------------------------------------------------- /source/sensor/suricata/socket/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["socket.go"], 6 | importpath = "github.com/google/emitto/source/sensor/suricata/socket", 7 | visibility = ["//visibility:public"], 8 | deps = ["@com_github_golang_glog//:go_default_library"], 9 | ) 10 | 11 | go_test( 12 | name = "go_default_test", 13 | srcs = ["socket_test.go"], 14 | embed = [":go_default_library"], 15 | ) 16 | -------------------------------------------------------------------------------- /source/sensor/suricata/socket/socket.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package socket contains functionality to send commands to Suricata via its Unix socket. 16 | // 17 | // Proper usage of the socket: 18 | // 1. Connect() 19 | // 2. Send() 20 | // 3. Close() 21 | package socket 22 | 23 | import ( 24 | "encoding/json" 25 | "fmt" 26 | "io/ioutil" 27 | "net" 28 | "time" 29 | 30 | log "github.com/golang/glog" 31 | ) 32 | 33 | var ( 34 | retryAttempts = 3 35 | retryInterval = 5 * time.Second 36 | ) 37 | 38 | const versionID = "1.0" 39 | 40 | // Version represents a version message, which must be sent and return "OK" before sending commands. 41 | type Version struct { 42 | ID string `json:"version,"` 43 | } 44 | 45 | // CommandName represents a Suricata Unix socket command name. 46 | // 47 | // https://suricata.readthedocs.io/en/suricata-4.0.5/unix-socket.html 48 | type CommandName string 49 | 50 | // Suricata socket commands. 51 | const ( 52 | ReloadRules CommandName = "reload-rules" 53 | ) 54 | 55 | // validCommands contains the currently supported socket commands. 56 | var validCommands = map[CommandName]bool{ 57 | ReloadRules: true, 58 | } 59 | 60 | // Command represents a Suricata Unix socket command. 61 | // 62 | // Protocol: https://redmine.openinfosecfoundation.org/projects/suricata/wiki/Unix_Socket#Protocol 63 | type Command struct { 64 | Name CommandName `json:"command,"` 65 | Args map[string]string `json:"arguments,omitempty"` 66 | } 67 | 68 | // Response represents a Suricata Unix socket command response. 69 | type Response struct { 70 | Return string `json:"return,"` 71 | Message string `json:"message,string"` 72 | } 73 | 74 | // Socket represents a Suricata Unix socket server connection. 75 | type Socket struct { 76 | addr string 77 | conn net.Conn 78 | } 79 | 80 | // New creates a new Socket. 81 | func New(addr string) *Socket { 82 | return &Socket{addr: addr} 83 | } 84 | 85 | // Connect dials the Suricata Unix socket and prepares the connection for receiving commands. 86 | func (s *Socket) Connect() error { 87 | if err := retry(retryAttempts, retryInterval, func() error { 88 | c, err := net.Dial("unix", s.addr) 89 | if err != nil { 90 | return fmt.Errorf("failed to connect to Suricata socket (%s): %v", s.addr, err) 91 | } 92 | s.conn = c 93 | return nil 94 | }); err != nil { 95 | return err 96 | } 97 | 98 | return s.version() 99 | } 100 | 101 | // Close closes the Suricata Unix socket connection. 102 | func (s *Socket) Close() error { 103 | return s.conn.Close() 104 | } 105 | 106 | // version sends a Suricata version message to establish the communication protocol. 107 | // The protocol is defined as the following: 108 | // 1. Client connects to the socket. 109 | // 2. Client sends a version message: { "version": "$VERSION_ID" }. 110 | // 3. Server answers with { "return": "OK|NOK" }. 111 | func (s *Socket) version() error { 112 | buf, err := json.Marshal(&Version{versionID}) 113 | if err != nil { 114 | return err 115 | } 116 | if _, err := s.conn.Write(buf); err != nil { 117 | return err 118 | } 119 | 120 | resp, err := ioutil.ReadAll(s.conn) 121 | if err != nil { 122 | return err 123 | } 124 | r := new(Response) 125 | if err := json.Unmarshal(resp, r); err != nil { 126 | return err 127 | } 128 | if r.Return != "OK" { 129 | return fmt.Errorf("failed to establish communication protocol with Suricata: %s: %s", r.Return, r.Message) 130 | } 131 | return nil 132 | } 133 | 134 | // Send sends a command to Suricata and returns its response. 135 | func (s *Socket) Send(cmd *Command) (*Response, error) { 136 | if _, ok := validCommands[cmd.Name]; !ok { 137 | return nil, fmt.Errorf("unsupported command type: %s", cmd.Name) 138 | } 139 | 140 | buf, err := json.Marshal(cmd) 141 | if err != nil { 142 | return nil, err 143 | } 144 | if _, err := s.conn.Write(buf); err != nil { 145 | return nil, err 146 | } 147 | 148 | // The read blocks until it receives an EOF or returns an error. 149 | resp, err := ioutil.ReadAll(s.conn) 150 | if err != nil { 151 | return nil, err 152 | } 153 | r := new(Response) 154 | if err := json.Unmarshal(resp, r); err != nil { 155 | return nil, err 156 | } 157 | if r.Return != "OK" { 158 | return nil, fmt.Errorf("received an error response from command (%+v): %s: %s", cmd, r.Return, r.Message) 159 | } 160 | 161 | return r, nil 162 | } 163 | 164 | // retry calls a function up to the specified number of attempts if the call encounters an error. 165 | // Each retry will sleep for a specified duration. 166 | func retry(attempts int, sleep time.Duration, do func() error) error { 167 | var err error 168 | for i := 0; i < attempts; i++ { 169 | if err = do(); err == nil { 170 | return nil 171 | } 172 | time.Sleep(sleep) 173 | log.Infof("retrying after error: %v", err) 174 | } 175 | return fmt.Errorf("failed after %d attempts; last error: %v", attempts, err) 176 | } 177 | -------------------------------------------------------------------------------- /source/sensor/suricata/socket/socket_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package socket 16 | 17 | import ( 18 | "encoding/json" 19 | "io" 20 | "net" 21 | "strings" 22 | "testing" 23 | ) 24 | 25 | // fakeSocketServer mimiks a Suricata Unix socket server. It handles receiving a version message 26 | // and a subsequent command. 27 | // cmdSize is the expected Command message size for exiting the read loop. 28 | func fakeSocketServer(t *testing.T, conn net.Conn, resp *Response, cmdSize int) { 29 | // Receive command. 30 | in := make([]byte, 0, 128) 31 | for len(in) < cmdSize { 32 | tmp := make([]byte, 64) 33 | n, err := conn.Read(tmp) 34 | if err != nil && err != io.EOF { 35 | t.Fatal(err) 36 | } 37 | in = append(in, tmp[:n]...) 38 | } 39 | 40 | // Send response. 41 | out, err := json.Marshal(resp) 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | conn.Write(out) 46 | conn.Close() 47 | } 48 | 49 | func TestVersion(t *testing.T) { 50 | buf, err := json.Marshal(&Version{versionID}) 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | 55 | server, client := net.Pipe() 56 | defer client.Close() 57 | go fakeSocketServer(t, server, &Response{ 58 | Return: "OK", 59 | Message: "1.0", 60 | }, len(buf)) 61 | 62 | s := &Socket{conn: client} 63 | if err := s.version(); err != nil { 64 | t.Fatal(err) 65 | } 66 | } 67 | 68 | func TestInvalidCommandName(t *testing.T) { 69 | _, client := net.Pipe() 70 | defer client.Close() 71 | s := &Socket{conn: client} 72 | _, err := s.Send(&Command{Name: "list-iface"}) 73 | if err == nil { 74 | t.Fatal(err) 75 | } 76 | } 77 | 78 | func TestInvalidResponse(t *testing.T) { 79 | cmd := &Command{Name: ReloadRules} 80 | buf, err := json.Marshal(cmd) 81 | if err != nil { 82 | t.Fatal(err) 83 | } 84 | 85 | server, client := net.Pipe() 86 | defer client.Close() 87 | go fakeSocketServer(t, server, &Response{ 88 | Return: "NOK", 89 | Message: "error", 90 | }, len(buf)) 91 | 92 | s := &Socket{conn: client} 93 | if _, err := s.Send(cmd); err == nil { 94 | t.Fatal(err) 95 | } 96 | } 97 | 98 | func TestConnectTimeout(t *testing.T) { 99 | p := "/tmp/nonexistent.socket" 100 | s := New(p) 101 | if err := s.Connect(); !strings.Contains(err.Error(), "connect: no such file or directory") { 102 | t.Fatalf("expected connection timeout from: %s", p) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /source/sensor/suricata/suricata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package suricata contains functionality to control and monitor Suricata. 16 | package suricata 17 | 18 | import ( 19 | "github.com/google/emitto/source/sensor/suricata/socket" 20 | ) 21 | 22 | // Socket represents a Suricata unix socket connection. 23 | type Socket interface { 24 | // Connect to socket service. 25 | Connect() error 26 | // Send command over socket connection. 27 | Send(*socket.Command) (*socket.Response, error) 28 | // Close socket connection. 29 | Close() error 30 | } 31 | 32 | // Controller controls Suricata via its Unix socket service. 33 | type Controller struct { 34 | sock Socket 35 | } 36 | 37 | // NewController creates a new Controller. 38 | func NewController(sockAddr string) *Controller { 39 | return &Controller{socket.New(sockAddr)} 40 | } 41 | 42 | // ReloadRules issues a command to Suricata to reload the rules engine with updated rules. 43 | func (c *Controller) ReloadRules() error { 44 | if err := c.sock.Connect(); err != nil { 45 | return err 46 | } 47 | defer c.sock.Close() 48 | 49 | _, err := c.sock.Send(&socket.Command{ 50 | Name: socket.ReloadRules, 51 | }) 52 | if err != nil { 53 | return err 54 | } 55 | 56 | return nil 57 | } 58 | -------------------------------------------------------------------------------- /source/sensor/suricata/suricata_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package suricata 16 | 17 | import ( 18 | "net" 19 | "testing" 20 | 21 | "github.com/google/emitto/source/sensor/suricata/socket" 22 | ) 23 | 24 | type fakeSocket struct { 25 | conn net.Conn 26 | 27 | sendResponse *socket.Response 28 | } 29 | 30 | func (s *fakeSocket) Connect() error { return nil } 31 | func (s *fakeSocket) Close() error { return nil } 32 | 33 | func (s *fakeSocket) Send(*socket.Command) (*socket.Response, error) { 34 | return s.sendResponse, nil 35 | } 36 | 37 | func TestReloadRules(t *testing.T) { 38 | fs := new(fakeSocket) 39 | fs.sendResponse = &socket.Response{ 40 | Return: "OK", 41 | } 42 | 43 | ctrl := &Controller{fs} 44 | if err := ctrl.ReloadRules(); err != nil { 45 | t.Fatal(err) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /source/server/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["main.go"], 6 | importpath = "github.com/google/emitto/source/server", 7 | visibility = ["//visibility:private"], 8 | deps = [ 9 | "//source/filestore:go_default_library", 10 | "//source/server/fleetspeak:go_default_library", 11 | "//source/server/proto:go_default_library", 12 | "//source/server/service:go_default_library", 13 | "//source/server/store:go_default_library", 14 | "@com_github_golang_glog//:go_default_library", 15 | "@com_github_google_fleetspeak//fleetspeak/src/server/grpcservice/proto/fleetspeak_grpcservice:go_default_library", 16 | "@com_google_cloud_go//storage:go_default_library", 17 | "@org_golang_google_grpc//:go_default_library", 18 | ], 19 | ) 20 | 21 | go_binary( 22 | name = "server", 23 | embed = [":go_default_library"], 24 | visibility = ["//visibility:public"], 25 | ) 26 | 27 | load("@io_bazel_rules_docker//go:image.bzl", "go_image") 28 | 29 | go_image( 30 | name = "server_image_base", 31 | embed = [":go_default_library"], 32 | ) 33 | 34 | load("@io_bazel_rules_docker//container:container.bzl", "container_image") 35 | 36 | container_image( 37 | name = "server_image", 38 | base = ":server_image_base", 39 | ports = ["4444"], 40 | stamp = True, 41 | ) 42 | -------------------------------------------------------------------------------- /source/server/client/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["client.go"], 6 | importpath = "github.com/google/emitto/source/server/client", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//source/server/proto:go_default_library", 10 | "@com_github_golang_glog//:go_default_library", 11 | "@org_golang_google_grpc//:go_default_library", 12 | ], 13 | ) 14 | 15 | go_test( 16 | name = "go_default_test", 17 | srcs = ["client_test.go"], 18 | embed = [":go_default_library"], 19 | deps = [ 20 | "//source/server/proto:go_default_library", 21 | "@com_github_golang_protobuf//proto:go_default_library", 22 | "@com_github_google_go_cmp//cmp:go_default_library", 23 | "@org_golang_google_grpc//:go_default_library", 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /source/server/client/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package client contains Emitto client functionality. 16 | package client 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "fmt" 22 | "io" 23 | "regexp" 24 | "strconv" 25 | 26 | "google.golang.org/grpc" 27 | 28 | log "github.com/golang/glog" 29 | pb "github.com/google/emitto/source/server/proto" 30 | ) 31 | 32 | var suricataSIDRE = regexp.MustCompile(`sid:(\d+);`) 33 | 34 | // Client represents a Emitto client. 35 | type Client struct { 36 | conn *grpc.ClientConn 37 | emitto pb.EmittoClient 38 | } 39 | 40 | // New returns new Client. 41 | func New(addr string) (*Client, error) { 42 | conn, err := grpc.Dial(addr, grpc.WithDefaultCallOptions()) 43 | if err != nil { 44 | return nil, fmt.Errorf("failed to connect to Emitto (%s): %v", addr, err) 45 | } 46 | return &Client{ 47 | conn: conn, 48 | emitto: pb.NewEmittoClient(conn), 49 | }, nil 50 | } 51 | 52 | // Close Emitto client connection. 53 | func (c *Client) Close() error { 54 | return c.conn.Close() 55 | } 56 | 57 | // DeployRules deploys the rules to the provided location. 58 | func (c *Client) DeployRules(ctx context.Context, loc *pb.Location) ([]*pb.DeployRulesResponse, error) { 59 | stream, err := c.emitto.DeployRules(ctx, &pb.DeployRulesRequest{Location: loc}) 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | var responses []*pb.DeployRulesResponse 65 | for { 66 | resp, err := stream.Recv() 67 | if err == io.EOF { 68 | break 69 | } 70 | if err != nil { 71 | log.Errorf("deployment failure: %v", err) 72 | return responses, err 73 | } 74 | log.Infof("deployment response: %+v", resp) 75 | responses = append(responses, resp) 76 | } 77 | return responses, nil 78 | } 79 | 80 | // getSID extracts the SID from a Suricata rule and casts it to an int64. 81 | func getSID(rule string) (int64, error) { 82 | matches := suricataSIDRE.FindStringSubmatch(rule) 83 | if len(matches) != 2 { 84 | return 0, errors.New("unable to properly locate rule SID") 85 | } 86 | sid, err := strconv.ParseInt(matches[1], 10, 64) 87 | if err != nil { 88 | return 0, fmt.Errorf("failed to cast SID: %v", err) 89 | } 90 | return sid, nil 91 | } 92 | -------------------------------------------------------------------------------- /source/server/client/client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package client 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "io" 21 | "testing" 22 | 23 | "github.com/golang/protobuf/proto" 24 | "github.com/google/go-cmp/cmp" 25 | "google.golang.org/grpc" 26 | 27 | pb "github.com/google/emitto/source/server/proto" 28 | ) 29 | 30 | func TestGetSid(t *testing.T) { 31 | rule := `alert http any any -> any any (msg:"Test"; content:"Test"; nocase; classtype:policy-violation; sid:1234567890; rev:1;)` 32 | want := int64(1234567890) 33 | got, err := getSID(rule) 34 | if err != nil { 35 | t.Errorf("getSID() unexpected failure: %v", err) 36 | } 37 | if diff := cmp.Diff(want, got); diff != "" { 38 | t.Errorf("getSID() expectation mismatch (-want +got):\n%s", diff) 39 | } 40 | } 41 | 42 | func TestDeployRules(t *testing.T) { 43 | responses := []*pb.DeployRulesResponse{ 44 | {ClientId: "id1"}, 45 | {ClientId: "id2"}, 46 | } 47 | c := Client{emitto: &fakeEmittoClient{stream: &fakeEmittoDeployRulesClient{responses: responses}}} 48 | got, err := c.DeployRules(context.Background(), &pb.Location{}) 49 | if err != nil { 50 | t.Errorf("DeployRules() retuned unexpected error: %v", err) 51 | } 52 | if len(responses) != len(got) { 53 | t.Errorf("DeployRules() expected: %v responses, got: %v", len(responses), len(got)) 54 | } 55 | for i, r := range responses { 56 | if diff := cmp.Diff(r, got[i], cmp.Comparer(proto.Equal)); diff != "" { 57 | t.Errorf("DeployRules() expectation mismatch (-want +got):\n%s", diff) 58 | } 59 | } 60 | 61 | wantErr := fmt.Errorf("error") 62 | c = Client{emitto: &fakeEmittoClient{stream: &fakeEmittoDeployRulesClient{err: wantErr}}} 63 | _, err = c.DeployRules(context.Background(), &pb.Location{}) 64 | if diff := cmp.Diff(wantErr.Error(), err.Error()); diff != "" { 65 | t.Errorf("DeployRules() expectation mismatch (-want +got):\n%s", diff) 66 | } 67 | } 68 | 69 | // fakeEmittoClient fakes emitto client, field of Client struct object. 70 | // This does not fake the implementation of DeployRules method of an actual 71 | // Client struct, rather it only fakes the stream to the service. 72 | type fakeEmittoClient struct { 73 | pb.EmittoClient 74 | stream *fakeEmittoDeployRulesClient 75 | } 76 | 77 | func (c *fakeEmittoClient) DeployRules(ctx context.Context, in *pb.DeployRulesRequest, opts ...grpc.CallOption) (pb.Emitto_DeployRulesClient, error) { 78 | return c.stream, nil 79 | } 80 | 81 | type fakeEmittoDeployRulesClient struct { 82 | grpc.ClientStream 83 | responses []*pb.DeployRulesResponse 84 | err error 85 | } 86 | 87 | func (s *fakeEmittoDeployRulesClient) Recv() (*pb.DeployRulesResponse, error) { 88 | if s.err != nil { 89 | return nil, s.err 90 | } 91 | if len(s.responses) == 0 { 92 | return nil, io.EOF 93 | } 94 | r := s.responses[0] 95 | s.responses = s.responses[1:] 96 | return r, nil 97 | } 98 | -------------------------------------------------------------------------------- /source/server/fleetspeak/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["fleetspeak.go"], 6 | importpath = "github.com/google/emitto/source/server/fleetspeak", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//source/sensor/proto:go_default_library", 10 | "@com_github_golang_glog//:go_default_library", 11 | "@com_github_golang_protobuf//ptypes:go_default_library_gen", 12 | "@com_github_google_fleetspeak//fleetspeak/src/common:go_default_library", 13 | "@com_github_google_fleetspeak//fleetspeak/src/common/proto/fleetspeak:go_default_library", 14 | "@com_github_google_fleetspeak//fleetspeak/src/server/proto/fleetspeak_server:go_default_library", 15 | "@org_golang_google_grpc//:go_default_library", 16 | "@org_golang_google_grpc//credentials:go_default_library", 17 | ], 18 | ) 19 | -------------------------------------------------------------------------------- /source/server/fleetspeak/fleetspeak.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package fleetspeak provides administrative functionality for Fleetspeak. 16 | package fleetspeak 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "fmt" 22 | "math/rand" 23 | "sort" 24 | "strings" 25 | "time" 26 | 27 | "github.com/golang/protobuf/ptypes" 28 | "github.com/google/fleetspeak/fleetspeak/src/common" 29 | "google.golang.org/grpc" 30 | "google.golang.org/grpc/credentials" 31 | 32 | log "github.com/golang/glog" 33 | spb "github.com/google/emitto/source/sensor/proto" 34 | fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak" 35 | fsspb "github.com/google/fleetspeak/fleetspeak/src/server/proto/fleetspeak_server" 36 | ) 37 | 38 | const ( 39 | // Service name for Fleetspeak messages. 40 | serviceName = "Emitto" 41 | dateFmt = "15:04:05.000 2006.01.02" 42 | ) 43 | 44 | // Client represents an Fleetspeak admin client. 45 | type Client struct { 46 | admin fsspb.AdminClient 47 | conn *grpc.ClientConn 48 | } 49 | 50 | // New creates a new Client. 51 | func New(addr, certFile string) (*Client, error) { 52 | var client *Client 53 | if certFile != "" { 54 | cred, err := credentials.NewClientTLSFromFile(certFile, "") 55 | if err != nil { 56 | return nil, fmt.Errorf("failed to retrieve certificate from (%q): %v", certFile, err) 57 | } 58 | conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(cred)) 59 | if err != nil { 60 | return nil, fmt.Errorf("unable to connect to Fleetspeak admin interface [%s] with credentials (%q): %v", addr, certFile, err) 61 | } 62 | client = &Client{ 63 | admin: fsspb.NewAdminClient(conn), 64 | conn: conn, 65 | } 66 | } else { 67 | conn, err := grpc.Dial(addr, grpc.WithInsecure()) 68 | if err != nil { 69 | return nil, fmt.Errorf("unable to connect to Fleetspeak admin interface [%s]: %v", addr, err) 70 | } 71 | client = &Client{ 72 | admin: fsspb.NewAdminClient(conn), 73 | conn: conn, 74 | } 75 | } 76 | return client, nil 77 | } 78 | 79 | // Close terminates the Fleetspeak admin connection. 80 | func (c *Client) Close() error { 81 | return c.conn.Close() 82 | } 83 | 84 | // InsertMessage inserts a message into the Fleetspeak system to be delivered to a sensor, where 85 | // the sensor is identified by the id. 86 | func (c *Client) InsertMessage(ctx context.Context, req *spb.SensorRequest, id []byte) error { 87 | data, err := ptypes.MarshalAny(req) 88 | if err != nil { 89 | return fmt.Errorf("failed to marshal operation message (%+v): %v", req, err) 90 | } 91 | mid := make([]byte, 16) 92 | rand.Read(mid) 93 | m := &fspb.Message{ 94 | SourceMessageId: mid, 95 | Source: &fspb.Address{ 96 | ServiceName: serviceName, 97 | }, 98 | Destination: &fspb.Address{ 99 | ServiceName: serviceName, 100 | ClientId: id, 101 | }, 102 | Data: data, 103 | Background: true, 104 | } 105 | if _, err := c.admin.InsertMessage(ctx, m); err != nil { 106 | return fmt.Errorf("failed to insert message for client (%X): %v", id, err) 107 | } 108 | log.Infof("Sent message (%X) to Fleetspeak", m.GetSourceMessageId()) 109 | return nil 110 | } 111 | 112 | // ListClients returns a list of clients from Fleetspeak. 113 | func (c *Client) ListClients(ctx context.Context) ([]*fsspb.Client, error) { 114 | res, err := c.admin.ListClients(ctx, &fsspb.ListClientsRequest{}, grpc.MaxCallRecvMsgSize(1024*1024*1024)) 115 | if err != nil { 116 | return nil, fmt.Errorf("failed to list clients: %v", err) 117 | } 118 | if len(res.Clients) == 0 { 119 | return nil, errors.New("no clients found") 120 | } 121 | sort.Sort(byContactTime(res.Clients)) 122 | return res.Clients, nil 123 | } 124 | 125 | // ParseClients returns human-readable client details. 126 | func ParseClients(clients []*fsspb.Client) []string { 127 | var res []string 128 | for _, c := range clients { 129 | id, err := common.BytesToClientID(c.ClientId) 130 | if err != nil { 131 | log.Errorf("Ignoring invalid client (id=%v), %v", c.ClientId, err) 132 | continue 133 | } 134 | var ls []string 135 | for _, l := range c.Labels { 136 | ls = append(ls, l.ServiceName+":"+l.Label) 137 | } 138 | ts, err := ptypes.Timestamp(c.LastContactTime) 139 | if err != nil { 140 | log.Errorf("Unable to parse last contact time for client (id=%v): %v", id, err) 141 | } 142 | tag := "" 143 | if c.Blacklisted { 144 | tag = " *blacklisted*" 145 | } 146 | res = append(res, fmt.Sprintf("%v %v [%v]%s\n", id, ts.Format(dateFmt), strings.Join(ls, ","), tag)) 147 | } 148 | return res 149 | } 150 | 151 | // byContactTime adapts []*fspb.Client for use by sort.Sort. 152 | type byContactTime []*fsspb.Client 153 | 154 | func (b byContactTime) Len() int { return len(b) } 155 | func (b byContactTime) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 156 | func (b byContactTime) Less(i, j int) bool { return contactTime(b[i]).Before(contactTime(b[j])) } 157 | 158 | func contactTime(c *fsspb.Client) time.Time { 159 | return time.Unix(c.LastContactTime.Seconds, int64(c.LastContactTime.Nanos)) 160 | } 161 | -------------------------------------------------------------------------------- /source/server/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package main initializes and starts the Emitto service. 16 | package main 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | "net" 23 | 24 | "cloud.google.com/go/storage" 25 | "github.com/google/emitto/source/filestore" 26 | "github.com/google/emitto/source/server/fleetspeak" 27 | "github.com/google/emitto/source/server/service" 28 | "github.com/google/emitto/source/server/store" 29 | "google.golang.org/grpc" 30 | 31 | log "github.com/golang/glog" 32 | pb "github.com/google/emitto/source/server/proto" 33 | fspb "github.com/google/fleetspeak/fleetspeak/src/server/grpcservice/proto/fleetspeak_grpcservice" 34 | ) 35 | 36 | var ( 37 | // Server flags. 38 | port = flag.Int("port", 4444, "Emitto server port") 39 | fsAdminAddr = flag.String("admin_addr", "", "Fleetspeak admin server") 40 | memoryStorage = flag.Bool("memory_storage", false, "Use memory store and filestore") 41 | 42 | // Google Cloud Project flags. 43 | projectID = flag.String("project_id", "", "Google Cloud project ID") 44 | storageBucket = flag.String("storage_bucket", "", "Google Cloud Storage bucket for storing rule files") 45 | credFile = flag.String("cred_file", "", "Path of the JSON application credential file.") 46 | 47 | // Fleetspeak flags. 48 | certFile = flag.String("cert_file", "", "Path of the Fleetspeak certificate file") 49 | ) 50 | 51 | func main() { 52 | ctx := context.Background() 53 | 54 | a, err := fleetspeak.New(*fsAdminAddr, *certFile) 55 | if err != nil { 56 | log.Fatalf("unable to connect to the Fleetspeak admin server: %v", err) 57 | } 58 | defer a.Close() 59 | 60 | fs, closeFStore := mustGetFileStore(ctx) 61 | defer closeFStore() 62 | 63 | s, closeStore := mustGetStore(ctx) 64 | defer closeStore() 65 | 66 | server := grpc.NewServer() 67 | svc := service.New(s, fs, a) 68 | pb.RegisterEmittoServer(server, svc) 69 | fspb.RegisterProcessorServer(server, svc) 70 | 71 | l, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) 72 | if err != nil { 73 | log.Exitf("server failed to listen: %v", err) 74 | } 75 | defer l.Close() 76 | server.Serve(l) 77 | } 78 | 79 | func mustGetFileStore(ctx context.Context) (filestore.FileStore, func() error) { 80 | if *memoryStorage { 81 | return filestore.NewMemoryFileStore(), func() error { return nil } 82 | } 83 | c, err := filestore.NewGCSClient(ctx, *credFile, []string{storage.ScopeFullControl}) 84 | if err != nil { 85 | log.Exitf("failed to create Google Cloud Storage client: %v", err) 86 | } 87 | return filestore.NewGCSFileStore(*storageBucket, c), c.Close 88 | } 89 | 90 | func mustGetStore(ctx context.Context) (store.Store, func() error) { 91 | if *memoryStorage { 92 | return store.NewMemoryStore(), func() error { return nil } 93 | } 94 | c, err := store.NewGCDClient(ctx, *projectID, *credFile) 95 | if err != nil { 96 | log.Exitf("failed to create Google Cloud Datastore client: %v", err) 97 | } 98 | return store.NewDataStore(c), c.Close 99 | } 100 | -------------------------------------------------------------------------------- /source/server/proto/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 3 | 4 | proto_library( 5 | name = "emitto_service_proto", 6 | srcs = ["service.proto"], 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "@com_google_protobuf//:empty_proto", 10 | "@com_google_protobuf//:field_mask_proto", 11 | "@go_googleapis//google/rpc:status_proto", 12 | ], 13 | ) 14 | 15 | go_proto_library( 16 | name = "emitto_service_go_proto", 17 | compilers = ["@io_bazel_rules_go//proto:go_grpc"], 18 | importpath = "github.com/google/emitto/source/server/proto", 19 | proto = ":emitto_service_proto", 20 | visibility = ["//visibility:public"], 21 | deps = ["@go_googleapis//google/rpc:status_go_proto"], 22 | ) 23 | 24 | go_library( 25 | name = "go_default_library", 26 | embed = [":emitto_service_go_proto"], 27 | importpath = "github.com/google/emitto/source/server/proto", 28 | visibility = ["//visibility:public"], 29 | ) 30 | -------------------------------------------------------------------------------- /source/server/proto/service.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package emitto.service; 18 | 19 | import "google/protobuf/empty.proto"; 20 | import "google/rpc/status.proto"; 21 | import "google/protobuf/field_mask.proto"; 22 | 23 | service Emitto { 24 | // Deploys rules to the specified location. 25 | rpc DeployRules(DeployRulesRequest) returns (stream DeployRulesResponse) {} 26 | // Adds a new Rule. 27 | rpc AddRule(AddRuleRequest) returns (google.protobuf.Empty) {} 28 | // Modifies an existing Rule. 29 | rpc ModifyRule(ModifyRuleRequest) returns (google.protobuf.Empty) {} 30 | // Deletes an existing Rule. 31 | rpc DeleteRule(DeleteRuleRequest) returns (google.protobuf.Empty) {} 32 | // Lists Rules. 33 | rpc ListRules(ListRulesRequest) returns (ListRulesResponse) {} 34 | // Adds a new Location. 35 | rpc AddLocation(AddLocationRequest) returns (google.protobuf.Empty) {} 36 | // Modifies an existing Location. 37 | rpc ModifyLocation(ModifyLocationRequest) returns (google.protobuf.Empty) {} 38 | // Deletes an existing Location. 39 | rpc DeleteLocation(DeleteLocationRequest) returns (google.protobuf.Empty) {} 40 | // Lists all Locations. 41 | rpc ListLocations(ListLocationsRequest) returns (ListLocationsResponse) {} 42 | } 43 | 44 | // Location defines an arbirary organization of sensors, segmented into a least 45 | // one zone. 46 | message Location { 47 | // The unique name of the location, e.g. "company1". 48 | string name = 1; 49 | // The list of zones or "segments" to organize sensors, e.g. {"dmz", "prod"}. 50 | repeated string zones = 2; 51 | } 52 | 53 | // Rule is an IDS rule, e.g. Snort or Suricata. 54 | message Rule { 55 | // The unique rule ID. 56 | int64 id = 1; 57 | // The rule itself. 58 | string body = 2; 59 | // Select in which organization and zone the rule is enabled, e.g. 60 | // "google:dmz". 61 | repeated string location_zones = 3; 62 | } 63 | 64 | // Deploy rules to the sensors in a specific location. 65 | message DeployRulesRequest { 66 | Location location = 1; 67 | } 68 | 69 | // Contains sensor client information for a deployment request. 70 | message DeployRulesResponse { 71 | // ID of the client. 72 | string client_id = 1; 73 | 74 | // Fleetspeak message insertion status. 75 | google.rpc.Status status = 3; 76 | } 77 | 78 | // Add a rule. 79 | message AddRuleRequest { 80 | Rule rule = 1; 81 | } 82 | 83 | // Modify a rule. 84 | message ModifyRuleRequest { 85 | // Rule to modify. 86 | Rule rule = 1; 87 | 88 | // Fields to be modified. Required. 89 | google.protobuf.FieldMask field_mask = 2; 90 | } 91 | 92 | // Delete a rule by Rule ID. 93 | message DeleteRuleRequest { 94 | int64 rule_id = 1; 95 | } 96 | 97 | // Lists Rules by ID. 98 | message ListRulesRequest { 99 | repeated int64 rule_ids = 1; 100 | } 101 | 102 | // Contains the listed Rules. 103 | message ListRulesResponse { 104 | repeated Rule rules = 1; 105 | } 106 | 107 | // Add a Location. 108 | message AddLocationRequest { 109 | Location location = 1; 110 | } 111 | 112 | // Modify a Location by Location Name. 113 | message ModifyLocationRequest { 114 | // Location to be modified. 115 | Location location = 1; 116 | 117 | // Fields to be modified. Required. 118 | google.protobuf.FieldMask field_mask = 2; 119 | } 120 | 121 | // Delete a Location by Location Name. 122 | message DeleteLocationRequest { 123 | string location_name = 1; 124 | } 125 | 126 | // Lists all Locations. 127 | message ListLocationsRequest {} 128 | 129 | // Contains all the listed Locations. 130 | message ListLocationsResponse { 131 | repeated Location locations = 1; 132 | } 133 | -------------------------------------------------------------------------------- /source/server/service/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "service.go", 7 | "service_helpers.go", 8 | ], 9 | importpath = "github.com/google/emitto/source/server/service", 10 | visibility = ["//visibility:public"], 11 | deps = [ 12 | "//source/filestore:go_default_library", 13 | "//source/resources:go_default_library", 14 | "//source/sensor/proto:go_default_library", 15 | "//source/server/fleetspeak:go_default_library", 16 | "//source/server/proto:go_default_library", 17 | "//source/server/store:go_default_library", 18 | "@com_github_golang_glog//:go_default_library", 19 | "@com_github_golang_protobuf//ptypes:go_default_library_gen", 20 | "@com_github_google_fleetspeak//fleetspeak/src/common/proto/fleetspeak:go_default_library", 21 | "@com_github_google_fleetspeak//fleetspeak/src/server/proto/fleetspeak_server:go_default_library", 22 | "@com_github_google_uuid//:go_default_library", 23 | "@io_bazel_rules_go//proto/wkt:empty_go_proto", 24 | "@io_bazel_rules_go//proto/wkt:field_mask_go_proto", 25 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 26 | "@org_golang_google_grpc//codes:go_default_library", 27 | "@org_golang_google_grpc//status:go_default_library", 28 | ], 29 | ) 30 | 31 | go_test( 32 | name = "go_default_test", 33 | srcs = [ 34 | "service_helpers_test.go", 35 | "service_test.go", 36 | ], 37 | embed = [":go_default_library"], 38 | deps = [ 39 | "//source/filestore:go_default_library", 40 | "//source/resources:go_default_library", 41 | "//source/server/fleetspeak:go_default_library", 42 | "//source/server/proto:go_default_library", 43 | "//source/server/store:go_default_library", 44 | "@com_github_golang_glog//:go_default_library", 45 | "@com_github_golang_protobuf//proto:go_default_library", 46 | "@com_github_google_fleetspeak//fleetspeak/src/common/proto/fleetspeak:go_default_library", 47 | "@com_github_google_fleetspeak//fleetspeak/src/server/proto/fleetspeak_server:go_default_library", 48 | "@com_github_google_go_cmp//cmp:go_default_library", 49 | "@com_github_google_go_cmp//cmp/cmpopts:go_default_library", 50 | "@io_bazel_rules_go//proto/wkt:field_mask_go_proto", 51 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 52 | "@org_golang_google_grpc//:go_default_library", 53 | "@org_golang_google_grpc//codes:go_default_library", 54 | "@org_golang_google_grpc//status:go_default_library", 55 | ], 56 | ) 57 | -------------------------------------------------------------------------------- /source/server/service/service.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package service provides the implementation of the Emitto service. 16 | package service 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "time" 22 | 23 | "github.com/golang/protobuf/ptypes" 24 | "github.com/google/emitto/source/filestore" 25 | "github.com/google/emitto/source/resources" 26 | "github.com/google/emitto/source/server/fleetspeak" 27 | "github.com/google/emitto/source/server/store" 28 | "github.com/google/uuid" 29 | "google.golang.org/grpc/codes" 30 | "google.golang.org/grpc/status" 31 | 32 | log "github.com/golang/glog" 33 | emptypb "github.com/golang/protobuf/ptypes/empty" 34 | tspb "github.com/golang/protobuf/ptypes/timestamp" 35 | spb "github.com/google/emitto/source/sensor/proto" 36 | svpb "github.com/google/emitto/source/server/proto" 37 | fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak" 38 | fsspb "github.com/google/fleetspeak/fleetspeak/src/server/proto/fleetspeak_server" 39 | ) 40 | 41 | // FleetspeakAdminClient represents a Fleetspeak admin client. 42 | type FleetspeakAdminClient interface { 43 | // Insert a SensorRequest message for delivery. 44 | InsertMessage(ctx context.Context, req *spb.SensorRequest, id []byte) error 45 | // List all clients. 46 | ListClients(ctx context.Context) ([]*fsspb.Client, error) 47 | // Close the RPC connection. 48 | Close() error 49 | } 50 | 51 | // Service contains handlers for Emitto server functions. 52 | type Service struct { 53 | store store.Store 54 | fileStore filestore.FileStore 55 | fleetspeak FleetspeakAdminClient 56 | } 57 | 58 | // New returns a new emitto Service. 59 | func New(store store.Store, filestore filestore.FileStore, fs FleetspeakAdminClient) *Service { 60 | return &Service{store, filestore, fs} 61 | } 62 | 63 | // DeployRules generates a rule file and deploys it to the sensors in the provided location. 64 | func (s *Service) DeployRules(req *svpb.DeployRulesRequest, stream svpb.Emitto_DeployRulesServer) error { 65 | ctx := stream.Context() 66 | // Get clients. 67 | clients, err := s.fleetspeak.ListClients(ctx) 68 | if err != nil { 69 | return err 70 | } 71 | log.V(1).Infof("DeployRules() listed clients:\n%s", fleetspeak.ParseClients(clients)) 72 | ids := getClientIDsByLocation(clients, req.GetLocation()) 73 | if len(ids) == 0 { 74 | return status.Errorf(codes.FailedPrecondition, "no clients for location: %v", req.GetLocation()) 75 | } 76 | // Create rule file. 77 | rules, err := s.store.ListRules(ctx, nil) 78 | if err != nil { 79 | return status.Errorf(codes.Internal, "failed to list rules: %v", err) 80 | } 81 | 82 | if rules = filterRulesByLocation(rules, req.GetLocation()); len(rules) == 0 { 83 | return status.Errorf(codes.FailedPrecondition, "no rules found for %q", req.GetLocation()) 84 | } 85 | path := ruleFilepath(req.GetLocation().GetName()) 86 | if err := s.fileStore.AddRuleFile(ctx, path, resources.MakeRuleFile(rules)); err != nil { 87 | return err 88 | } 89 | 90 | for _, id := range ids { 91 | resp := &svpb.DeployRulesResponse{ 92 | ClientId: fmt.Sprintf("%X", id), 93 | Status: status.New(codes.OK, "OK").Proto(), // Default; will be reset for errors. 94 | } 95 | rid := uuid.New().String() 96 | // Log sensor request. This is a precondition to sending the request. 97 | m := &resources.SensorRequest{ 98 | ID: rid, 99 | Time: timeNow().Format(time.RFC1123Z), 100 | ClientID: fmt.Sprintf("%X", id), 101 | Type: resources.DeployRules, 102 | } 103 | if err := s.store.AddSensorRequest(ctx, m); err != nil { 104 | resp.Status = status.New(codes.FailedPrecondition, fmt.Sprintf("failed to add sensor message (%+v): %v", m, err)).Proto() 105 | if err := stream.Send(resp); err != nil { 106 | return err 107 | } 108 | continue 109 | } 110 | // Send sensor request to client. 111 | r := &spb.SensorRequest{ 112 | Id: rid, 113 | Time: &tspb.Timestamp{Seconds: time.Now().Unix()}, 114 | Type: &spb.SensorRequest_DeployRules{&spb.DeployRules{RuleFile: path}}, 115 | } 116 | if err := s.fleetspeak.InsertMessage(ctx, r, id); err != nil { 117 | // Clean up Store entry - do not fail hard on this. 118 | if err := s.store.DeleteSensorRequest(ctx, rid); err != nil { 119 | log.Errorf("Failed to remove sensor request (%+v): %v", r, err) 120 | } 121 | resp.Status = status.New(codes.Internal, fmt.Sprintf("failed to insert message: %v", err)).Proto() 122 | } 123 | if err := stream.Send(resp); err != nil { 124 | return err 125 | } 126 | } 127 | return nil 128 | } 129 | 130 | // AddRule adds the provided Rule. 131 | func (s *Service) AddRule(ctx context.Context, req *svpb.AddRuleRequest) (*emptypb.Empty, error) { 132 | if err := s.store.AddRule(ctx, resources.ProtoToRule(req.GetRule())); err != nil { 133 | return &emptypb.Empty{}, status.Errorf(codes.Internal, "failed to add rule: %v", err) 134 | } 135 | return &emptypb.Empty{}, nil 136 | } 137 | 138 | // ModifyRule modifies an existing Rule with the provided field mask. 139 | func (s *Service) ModifyRule(ctx context.Context, req *svpb.ModifyRuleRequest) (*emptypb.Empty, error) { 140 | r := resources.ProtoToRule(req.GetRule()) 141 | if err := ValidateUpdateMask(*r, req.GetFieldMask()); err != nil { 142 | return &emptypb.Empty{}, status.Errorf(codes.InvalidArgument, 143 | "failed to validate modifications for rule (%+v) and mask (%+v): %v", r, req.GetFieldMask(), err) 144 | } 145 | if err := s.store.ModifyRule(ctx, r); err != nil { 146 | return &emptypb.Empty{}, status.Errorf(codes.Internal, "failed to modify rule (%+v): %v", r, err) 147 | } 148 | return &emptypb.Empty{}, nil 149 | } 150 | 151 | // DeleteRule deletes an existing Rule by Rule ID. 152 | func (s *Service) DeleteRule(ctx context.Context, req *svpb.DeleteRuleRequest) (*emptypb.Empty, error) { 153 | if err := s.store.DeleteRule(ctx, req.GetRuleId()); err != nil { 154 | return &emptypb.Empty{}, status.Errorf(codes.Internal, "failed to delete rule (id=%d): %v", req.GetRuleId(), err) 155 | } 156 | return &emptypb.Empty{}, nil 157 | } 158 | 159 | // ListRules returns Rules for the provided Rule IDs. 160 | func (s *Service) ListRules(ctx context.Context, req *svpb.ListRulesRequest) (*svpb.ListRulesResponse, error) { 161 | rules, err := s.store.ListRules(ctx, req.GetRuleIds()) 162 | if err != nil { 163 | return nil, status.Errorf(codes.Internal, "failed to list rules (%+v): %v", req.GetRuleIds(), err) 164 | } 165 | resp := &svpb.ListRulesResponse{} 166 | for _, r := range rules { 167 | resp.Rules = append(resp.Rules, resources.RuleToProto(r)) 168 | } 169 | return resp, nil 170 | } 171 | 172 | // AddLocation adds the provided Location. 173 | func (s *Service) AddLocation(ctx context.Context, req *svpb.AddLocationRequest) (*emptypb.Empty, error) { 174 | if err := s.store.AddLocation(ctx, resources.ProtoToLocation(req.GetLocation())); err != nil { 175 | return &emptypb.Empty{}, status.Errorf(codes.Internal, "failed to add location (%+v): %v", req.GetLocation(), err) 176 | } 177 | return &emptypb.Empty{}, nil 178 | } 179 | 180 | // ModifyLocation updates the specified location. 181 | func (s *Service) ModifyLocation(ctx context.Context, req *svpb.ModifyLocationRequest) (*emptypb.Empty, error) { 182 | l := resources.ProtoToLocation(req.GetLocation()) 183 | if err := ValidateUpdateMask(*l, req.GetFieldMask()); err != nil { 184 | return &emptypb.Empty{}, status.Errorf(codes.InvalidArgument, "failed to modify location (%+v): %v", l, err) 185 | } 186 | if err := s.store.ModifyLocation(ctx, l); err != nil { 187 | return &emptypb.Empty{}, status.Errorf(codes.Internal, "failed to modify location: %v", err) 188 | } 189 | return &emptypb.Empty{}, nil 190 | } 191 | 192 | // DeleteLocation deletes an existing Location by Location Name. 193 | func (s *Service) DeleteLocation(ctx context.Context, req *svpb.DeleteLocationRequest) (*emptypb.Empty, error) { 194 | if err := s.store.DeleteLocation(ctx, req.GetLocationName()); err != nil { 195 | return &emptypb.Empty{}, status.Errorf(codes.Internal, "failed to add location (%+v): %v", req.GetLocationName(), err) 196 | } 197 | return &emptypb.Empty{}, nil 198 | } 199 | 200 | // ListLocations returns all Locations. 201 | func (s *Service) ListLocations(ctx context.Context, req *svpb.ListLocationsRequest) (*svpb.ListLocationsResponse, error) { 202 | locs, err := s.store.ListLocations(ctx) 203 | if err != nil { 204 | return nil, status.Errorf(codes.Internal, "failed to list locations: %v", err) 205 | } 206 | resp := &svpb.ListLocationsResponse{} 207 | for _, l := range locs { 208 | resp.Locations = append(resp.Locations, resources.LocationToProto(l)) 209 | } 210 | return resp, nil 211 | } 212 | 213 | // Process receives Fleetspeak messages and stores the enclosed SensorResponse. 214 | func (s *Service) Process(ctx context.Context, m *fspb.Message) (*fspb.EmptyMessage, error) { 215 | var msg spb.SensorMessage 216 | if err := ptypes.UnmarshalAny(m.Data, &msg); err != nil { 217 | log.Errorf("Failed to unmarshal sensor message (%s)", m.Data.String()) 218 | return &fspb.EmptyMessage{}, nil 219 | } 220 | log.Infof("Received sensor message (%X) from Fleetspeak", m.GetSourceMessageId()) 221 | switch t := msg.Type.(type) { 222 | case *spb.SensorMessage_Response: 223 | req := resources.ProtoToSensorRequest(&msg) 224 | if err := s.store.ModifySensorRequest(ctx, req); err != nil { 225 | log.Errorf("Failed to update sensor request (%+v)", req) 226 | } 227 | case *spb.SensorMessage_Alert: 228 | if err := s.store.AddSensorMessage(ctx, resources.ProtoToSensorMessage(&msg)); err != nil { 229 | log.Errorf("Failed to store sensor alert (%+v)", msg.GetAlert()) 230 | } 231 | case *spb.SensorMessage_Heartbeat: 232 | if err := s.store.AddSensorMessage(ctx, resources.ProtoToSensorMessage(&msg)); err != nil { 233 | log.Errorf("Failed to store sensor heartbeat (%+v)", msg.GetHeartbeat()) 234 | } 235 | default: 236 | log.Errorf("Unknown sensor message type (%T)", t) 237 | } 238 | return &fspb.EmptyMessage{}, nil 239 | } 240 | -------------------------------------------------------------------------------- /source/server/service/service_helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package service 16 | 17 | import ( 18 | "fmt" 19 | "path/filepath" 20 | "strings" 21 | "time" 22 | 23 | "github.com/google/emitto/source/resources" 24 | 25 | spb "github.com/google/emitto/source/server/proto" 26 | fsspb "github.com/google/fleetspeak/fleetspeak/src/server/proto/fleetspeak_server" 27 | mpb "google.golang.org/genproto/protobuf/field_mask" 28 | ) 29 | 30 | var timeNow = time.Now // Stubbed out for testing. 31 | 32 | func ruleFilepath(location string) string { 33 | return filepath.Join(location, fmt.Sprintf("%s/%d", timeNow().Format("2006/01/02"), timeNow().Unix())) 34 | } 35 | 36 | func filterRulesByLocation(rules []*resources.Rule, loc *spb.Location) []*resources.Rule { 37 | matches := make(map[int64]*resources.Rule) 38 | for _, r := range rules { 39 | for _, lz := range r.LocZones { 40 | // fmt.Printf("LOCZONE: %+v\n", lz) 41 | // Filter by location name first. 42 | if strings.HasPrefix(lz, loc.GetName()+":") { 43 | for _, z := range loc.GetZones() { 44 | // Filter by zone name. 45 | if strings.HasSuffix(lz, ":"+z) { 46 | matches[r.ID] = r 47 | } 48 | } 49 | } 50 | } 51 | } 52 | res := make([]*resources.Rule, 0, len(matches)) 53 | for _, m := range matches { 54 | res = append(res, m) 55 | } 56 | return res 57 | } 58 | 59 | func getClientIDsByLocation(clients []*fsspb.Client, loc *spb.Location) [][]byte { 60 | // Filter by location name. 61 | var filtered []*fsspb.Client 62 | for _, c := range clients { 63 | labels := c.GetLabels() 64 | for _, l := range labels { 65 | if l.GetLabel() == (resources.LocationNamePrefix + loc.GetName()) { 66 | filtered = append(filtered, c) 67 | break 68 | } 69 | } 70 | } 71 | // Filter by location zones. 72 | var ids [][]byte 73 | for _, c := range filtered { 74 | labels := c.GetLabels() 75 | var found bool 76 | for _, l := range labels { 77 | for _, z := range loc.GetZones() { 78 | if l.GetLabel() == (resources.LocationZonePrefix + z) { 79 | ids = append(ids, c.GetClientId()) 80 | found = true 81 | break 82 | } 83 | } 84 | if found { 85 | break 86 | } 87 | } 88 | } 89 | return ids 90 | } 91 | 92 | // ValidateUpdateMask confirms whether the specified update field mask is valid for the 93 | // respective object. If not, the invalid fields are returned in the error. 94 | func ValidateUpdateMask(obj interface{}, mask *mpb.FieldMask) error { 95 | m, err := resources.MutationsMapping(obj) 96 | if err != nil { 97 | return err 98 | } 99 | var invalid []string 100 | for _, p := range mask.GetPaths() { 101 | if !m[p] { 102 | invalid = append(invalid, p) 103 | } 104 | } 105 | if len(invalid) > 0 { 106 | return fmt.Errorf("the following fields are not mutable: %v", invalid) 107 | } 108 | return nil 109 | } 110 | -------------------------------------------------------------------------------- /source/server/service/service_helpers_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package service 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/google/emitto/source/resources" 21 | "github.com/google/go-cmp/cmp" 22 | "github.com/google/go-cmp/cmp/cmpopts" 23 | 24 | tspb "github.com/golang/protobuf/ptypes/timestamp" 25 | spb "github.com/google/emitto/source/server/proto" 26 | fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak" 27 | fsspb "github.com/google/fleetspeak/fleetspeak/src/server/proto/fleetspeak_server" 28 | mpb "google.golang.org/genproto/protobuf/field_mask" 29 | ) 30 | 31 | var ( 32 | testClients = []*fsspb.Client{ 33 | { 34 | ClientId: []byte("client_a"), 35 | Labels: []*fspb.Label{{Label: "alphabet-location-name-a"}, {Label: "alphabet-location-zone-dmz"}}, 36 | LastContactTime: &tspb.Timestamp{Seconds: 1111111111}, 37 | }, 38 | { 39 | ClientId: []byte("client_b"), 40 | Labels: []*fspb.Label{{Label: "alphabet-location-name-a"}, {Label: "alphabet-location-zone-dmz"}}, 41 | LastContactTime: &tspb.Timestamp{Seconds: 2222222222}, 42 | }, 43 | { 44 | ClientId: []byte("client_c"), 45 | Labels: []*fspb.Label{{Label: "alphabet-location-name-a"}, {Label: "alphabet-location-zone-corp"}}, 46 | LastContactTime: &tspb.Timestamp{Seconds: 3333333333}, 47 | }, 48 | { 49 | ClientId: []byte("client_d"), 50 | Labels: []*fspb.Label{{Label: "alphabet-location-name-b"}, {Label: "alphabet-location-zone-dmz"}}, 51 | LastContactTime: &tspb.Timestamp{Seconds: 4444444444}, 52 | }, 53 | { 54 | ClientId: []byte("client_e"), 55 | Labels: []*fspb.Label{{Label: "alphabet-location-name-c"}, {Label: "alphabet-location-zone-corp"}}, 56 | LastContactTime: &tspb.Timestamp{Seconds: 5555555555}, 57 | }, 58 | { 59 | ClientId: []byte("client_f"), 60 | Labels: []*fspb.Label{{Label: "alphabet-location-name-a"}, {Label: "alphabet-location-zone-prod"}}, 61 | LastContactTime: &tspb.Timestamp{Seconds: 6666666666}, 62 | }, 63 | } 64 | testRules = []*resources.Rule{ 65 | { 66 | ID: 1111, 67 | Body: "test", 68 | LocZones: []string{"a:dmz", "b:corp"}, 69 | }, 70 | { 71 | ID: 2222, 72 | Body: "test", 73 | LocZones: []string{"a:corp", "b:dmz"}, 74 | }, 75 | { 76 | ID: 3333, 77 | Body: "test", 78 | LocZones: []string{"c:prod"}, 79 | }, 80 | { 81 | ID: 4444, 82 | Body: "test", 83 | LocZones: []string{"b:dmz", "b:corp"}, 84 | }, 85 | { 86 | ID: 5555, 87 | Body: "test", 88 | LocZones: []string{"google:dmz"}, 89 | }, 90 | } 91 | testLocations = []*resources.Location{ 92 | { 93 | Name: "a", 94 | Zones: []string{"a"}, 95 | }, 96 | { 97 | Name: "b", 98 | Zones: []string{"a", "b"}, 99 | }, 100 | { 101 | Name: "c", 102 | Zones: []string{"a", "b", "c"}, 103 | }, 104 | } 105 | ) 106 | 107 | func TestGetClientIDsByLocation(t *testing.T) { 108 | tests := []struct { 109 | desc string 110 | loc *spb.Location 111 | want []string 112 | }{ 113 | { 114 | desc: "1 zone", 115 | loc: &spb.Location{Name: "a", Zones: []string{"dmz"}}, 116 | want: []string{"client_a", "client_b"}, 117 | }, 118 | { 119 | desc: "2 zones", 120 | loc: &spb.Location{Name: "a", Zones: []string{"dmz", "corp"}}, 121 | want: []string{"client_a", "client_b", "client_c"}, 122 | }, 123 | { 124 | desc: "3 zones", 125 | loc: &spb.Location{Name: "a", Zones: []string{"dmz", "corp", "prod"}}, 126 | want: []string{"client_a", "client_b", "client_c", "client_f"}, 127 | }, 128 | { 129 | desc: "non-existent location", 130 | loc: &spb.Location{Name: "d", Zones: []string{"corp"}}, 131 | want: nil, 132 | }, 133 | } 134 | for _, tt := range tests { 135 | t.Run(tt.desc, func(t *testing.T) { 136 | var got []string 137 | for _, id := range getClientIDsByLocation(testClients, tt.loc) { 138 | got = append(got, string(id)) 139 | } 140 | if diff := cmp.Diff(tt.want, got); diff != "" { 141 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 142 | } 143 | }) 144 | } 145 | } 146 | 147 | func TestFilterRulesByLocation(t *testing.T) { 148 | tests := []struct { 149 | desc string 150 | loc *spb.Location 151 | want []*resources.Rule 152 | }{ 153 | { 154 | desc: "1 rule", 155 | loc: &spb.Location{Name: "a", Zones: []string{"dmz"}}, 156 | want: []*resources.Rule{{ID: 1111, Body: "test", LocZones: []string{"a:dmz", "b:corp"}}}, 157 | }, 158 | { 159 | desc: "3 testRules", 160 | loc: &spb.Location{Name: "b", Zones: []string{"dmz", "corp"}}, 161 | want: []*resources.Rule{ 162 | { 163 | ID: 1111, 164 | Body: "test", 165 | LocZones: []string{"a:dmz", "b:corp"}, 166 | }, 167 | { 168 | ID: 2222, 169 | Body: "test", 170 | LocZones: []string{"a:corp", "b:dmz"}, 171 | }, 172 | { 173 | ID: 4444, 174 | Body: "test", 175 | LocZones: []string{"b:dmz", "b:corp"}, 176 | }, 177 | }, 178 | }, 179 | { 180 | desc: "unknown loczone", 181 | loc: &spb.Location{Name: "goog", Zones: []string{"dmz"}}, 182 | want: []*resources.Rule{}, 183 | }, 184 | } 185 | for _, tt := range tests { 186 | t.Run(tt.desc, func(t *testing.T) { 187 | got := filterRulesByLocation(testRules, tt.loc) 188 | if diff := cmp.Diff(tt.want, got, cmpopts.SortSlices(func(a, b *resources.Rule) bool { return a.ID < b.ID })); diff != "" { 189 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 190 | } 191 | }) 192 | } 193 | } 194 | 195 | func TestValidateUpdateMask(t *testing.T) { 196 | tests := []struct { 197 | desc string 198 | obj interface{} 199 | fieldMask *mpb.FieldMask 200 | wantErr bool 201 | }{ 202 | { 203 | desc: "valid Rule and mask paths", 204 | obj: resources.Rule{}, 205 | fieldMask: &mpb.FieldMask{ 206 | Paths: []string{"body", "loc_zones"}, 207 | }, 208 | }, 209 | { 210 | desc: "valid Rule but invalid mask path", 211 | obj: resources.Rule{}, 212 | fieldMask: &mpb.FieldMask{ 213 | Paths: []string{"id"}, 214 | }, 215 | wantErr: true, 216 | }, 217 | { 218 | desc: "invalid object", 219 | obj: resources.LocationSelector{}, 220 | fieldMask: &mpb.FieldMask{ 221 | Paths: []string{"foo"}, 222 | }, 223 | wantErr: true, 224 | }, 225 | { 226 | desc: "valid Location and valid mask paths", 227 | obj: resources.Location{}, 228 | fieldMask: &mpb.FieldMask{ 229 | Paths: []string{"zones"}, 230 | }, 231 | }, 232 | { 233 | desc: "valid Location but invalid mask path", 234 | obj: resources.Rule{}, 235 | fieldMask: &mpb.FieldMask{ 236 | Paths: []string{"name"}, 237 | }, 238 | wantErr: true, 239 | }, 240 | } 241 | for _, tt := range tests { 242 | t.Run(tt.desc, func(t *testing.T) { 243 | if err := ValidateUpdateMask(tt.obj, tt.fieldMask); (err != nil) != tt.wantErr { 244 | t.Errorf("got err=%v, wantErr=%t", err, tt.wantErr) 245 | } 246 | }) 247 | } 248 | } 249 | -------------------------------------------------------------------------------- /source/server/service/service_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package service 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "io" 21 | "net" 22 | "testing" 23 | "time" 24 | 25 | "github.com/golang/protobuf/proto" 26 | "github.com/google/emitto/source/filestore" 27 | "github.com/google/emitto/source/resources" 28 | "github.com/google/emitto/source/server/fleetspeak" 29 | "github.com/google/emitto/source/server/store" 30 | "github.com/google/go-cmp/cmp" 31 | "google.golang.org/grpc" 32 | "google.golang.org/grpc/codes" 33 | "google.golang.org/grpc/status" 34 | 35 | log "github.com/golang/glog" 36 | spb "github.com/google/emitto/source/server/proto" 37 | fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak" 38 | fsspb "github.com/google/fleetspeak/fleetspeak/src/server/proto/fleetspeak_server" 39 | mpb "google.golang.org/genproto/protobuf/field_mask" 40 | ) 41 | 42 | func TestDeployRules(t *testing.T) { 43 | ctx := context.Background() 44 | timeNow = func() time.Time { 45 | return time.Date(2000, 1, 1, 0, 0, 0, 0, time.FixedZone("UTC", 0)) 46 | } 47 | 48 | tests := []struct { 49 | desc string 50 | req *spb.DeployRulesRequest 51 | fsServer *fakeFSAdminServer 52 | want []*spb.DeployRulesResponse 53 | wantErr bool 54 | }{ 55 | { 56 | desc: "successful deployment", 57 | req: &spb.DeployRulesRequest{Location: &spb.Location{Name: "a", Zones: []string{"dmz"}}}, 58 | fsServer: &fakeFSAdminServer{ 59 | listClients: func(*fsspb.ListClientsRequest) (*fsspb.ListClientsResponse, error) { 60 | return &fsspb.ListClientsResponse{Clients: testClients}, nil 61 | }, 62 | insertMessage: func(*fspb.Message) (*fspb.EmptyMessage, error) { 63 | return &fspb.EmptyMessage{}, nil 64 | }, 65 | }, 66 | want: []*spb.DeployRulesResponse{ 67 | { 68 | ClientId: "636C69656E745F61", // "client_a" 69 | Status: status.New(codes.OK, "OK").Proto(), 70 | }, 71 | { 72 | ClientId: "636C69656E745F62", // "client_b" 73 | Status: status.New(codes.OK, "OK").Proto(), 74 | }, 75 | }, 76 | }, 77 | { 78 | desc: "list client error", 79 | req: &spb.DeployRulesRequest{Location: &spb.Location{Name: "a", Zones: []string{"dmz"}}}, 80 | fsServer: &fakeFSAdminServer{ 81 | listClients: func(*fsspb.ListClientsRequest) (*fsspb.ListClientsResponse, error) { 82 | return nil, errors.New("error") 83 | }, 84 | }, 85 | wantErr: true, 86 | }, 87 | { 88 | desc: "insert message error", 89 | req: &spb.DeployRulesRequest{Location: &spb.Location{Name: "a", Zones: []string{"dmz"}}}, 90 | fsServer: &fakeFSAdminServer{ 91 | listClients: func(*fsspb.ListClientsRequest) (*fsspb.ListClientsResponse, error) { 92 | return &fsspb.ListClientsResponse{Clients: testClients}, nil 93 | }, 94 | insertMessage: func(*fspb.Message) (*fspb.EmptyMessage, error) { 95 | return nil, errors.New("error") 96 | }, 97 | }, 98 | want: []*spb.DeployRulesResponse{ 99 | { 100 | ClientId: "636C69656E745F61", // "client_a" 101 | Status: status.New(codes.Internal, `failed to insert message: failed to insert message for client (636C69656E745F61): rpc error: code = Unknown desc = error`).Proto(), 102 | }, 103 | { 104 | ClientId: "636C69656E745F62", // "client_b" 105 | Status: status.New(codes.Internal, `failed to insert message: failed to insert message for client (636C69656E745F62): rpc error: code = Unknown desc = error`).Proto(), 106 | }, 107 | }, 108 | }, 109 | { 110 | desc: "no rules for location", 111 | req: &spb.DeployRulesRequest{Location: &spb.Location{Name: "unknown", Zones: []string{"dmz"}}}, 112 | fsServer: &fakeFSAdminServer{ 113 | listClients: func(*fsspb.ListClientsRequest) (*fsspb.ListClientsResponse, error) { 114 | return &fsspb.ListClientsResponse{Clients: testClients}, nil 115 | }, 116 | insertMessage: func(*fspb.Message) (*fspb.EmptyMessage, error) { 117 | return nil, nil 118 | }, 119 | }, 120 | wantErr: true, 121 | }, 122 | } 123 | for _, tt := range tests { 124 | // Set up storage. 125 | ds := store.NewMemoryStore() 126 | for _, r := range testRules { 127 | if err := ds.AddRule(ctx, r); err != nil { 128 | t.Fatal(err) 129 | } 130 | } 131 | fs := filestore.NewMemoryFileStore() 132 | // Set up servers and test clients. 133 | fc, stopFs := initFSAdminServerAndClient(t, tt.fsServer) 134 | defer fc.Close() 135 | defer stopFs() 136 | s := &Service{ 137 | store: ds, 138 | fileStore: fs, 139 | fleetspeak: fc, 140 | } 141 | c, stopServer := initServerAndClient(t, s) 142 | defer stopServer() 143 | t.Run(tt.desc, func(t *testing.T) { 144 | stream, err := c.DeployRules(ctx, tt.req) 145 | if err != nil { 146 | t.Fatalf("failed to deploy rules: %v", err) 147 | } 148 | var got []*spb.DeployRulesResponse 149 | for { 150 | r, err := stream.Recv() 151 | if err == io.EOF { 152 | break 153 | } 154 | if (err != nil) != tt.wantErr { 155 | t.Errorf("got err=%v, wantErr=%t", err, tt.wantErr) 156 | } 157 | if err != nil { 158 | return 159 | } 160 | got = append(got, r) 161 | } 162 | if diff := cmp.Diff(tt.want, got, cmp.Comparer(proto.Equal)); diff != "" { 163 | t.Errorf("expectation mismatch (want -> got):\n%s", diff) 164 | } 165 | }) 166 | } 167 | } 168 | 169 | func TestModifyRule(t *testing.T) { 170 | ctx := context.Background() 171 | timeNow := func() time.Time { 172 | return time.Date(2000, 1, 1, 0, 0, 0, 0, time.FixedZone("UTC", 0)) 173 | } 174 | store.TimeNow = timeNow 175 | 176 | tests := []struct { 177 | desc string 178 | rule *spb.Rule 179 | mask *mpb.FieldMask 180 | want *resources.Rule 181 | wantErr bool 182 | }{ 183 | { 184 | desc: "successfully modified", 185 | rule: &spb.Rule{ 186 | Id: 1111, 187 | Body: "updated", 188 | LocationZones: []string{"a:dmz", "b:corp"}, 189 | }, 190 | mask: &mpb.FieldMask{Paths: []string{"body"}}, 191 | want: &resources.Rule{ 192 | ID: 1111, 193 | Body: "updated", 194 | LocZones: []string{"a:dmz", "b:corp"}, 195 | LastModified: timeNow().Format(time.RFC1123Z), 196 | }, 197 | }, 198 | { 199 | desc: "invalid mask path", 200 | rule: &spb.Rule{ 201 | Id: 1111, 202 | Body: "updated", 203 | LocationZones: []string{"a:dmz", "b:corp"}, 204 | }, 205 | mask: &mpb.FieldMask{Paths: []string{"id"}}, 206 | wantErr: true, 207 | }, 208 | } 209 | // Set up storage. 210 | ds := store.NewMemoryStore() 211 | for _, r := range testRules { 212 | if err := ds.AddRule(ctx, r); err != nil { 213 | t.Fatal(err) 214 | } 215 | } 216 | for _, tt := range tests { 217 | // Set up servers and test clients. 218 | s := &Service{store: ds} 219 | c, stopServer := initServerAndClient(t, s) 220 | defer stopServer() 221 | t.Run(tt.desc, func(t *testing.T) { 222 | _, err := c.ModifyRule(ctx, &spb.ModifyRuleRequest{Rule: tt.rule, FieldMask: tt.mask}) 223 | if (err != nil) != tt.wantErr { 224 | t.Errorf("got err=%v, wantErr=%t", err, tt.wantErr) 225 | } 226 | if err != nil { 227 | return 228 | } 229 | got, err := s.store.ListRules(ctx, []int64{tt.want.ID}) 230 | if err != nil { 231 | t.Error(err) 232 | } 233 | if l := len(got); l != 1 { 234 | t.Errorf("expected 1 rule, got %d", l) 235 | } 236 | if diff := cmp.Diff(tt.want, got[0]); diff != "" { 237 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 238 | } 239 | }) 240 | } 241 | } 242 | 243 | func TestModifyLocation(t *testing.T) { 244 | ctx := context.Background() 245 | tn := func() time.Time { 246 | return time.Date(2000, 1, 1, 0, 0, 0, 0, time.FixedZone("UTC", 0)) 247 | } 248 | timeNow = tn 249 | store.TimeNow = tn 250 | 251 | tests := []struct { 252 | desc string 253 | location *spb.Location 254 | mask *mpb.FieldMask 255 | want *resources.Location 256 | wantErr bool 257 | }{ 258 | { 259 | desc: "successfully modified", 260 | location: &spb.Location{ 261 | Name: "a", 262 | Zones: []string{"updated"}, 263 | }, 264 | mask: &mpb.FieldMask{Paths: []string{"zones"}}, 265 | want: &resources.Location{ 266 | Name: "a", 267 | Zones: []string{"updated"}, 268 | LastModified: timeNow().Format(time.RFC1123Z), 269 | }, 270 | }, 271 | { 272 | desc: "invalid mask path", 273 | location: &spb.Location{ 274 | Name: "updated", 275 | Zones: []string{"updated"}, 276 | }, 277 | mask: &mpb.FieldMask{Paths: []string{"name", "zones"}}, 278 | wantErr: true, 279 | }, 280 | } 281 | // Set up storage. 282 | ds := store.NewMemoryStore() 283 | for _, l := range testLocations { 284 | if err := ds.AddLocation(ctx, l); err != nil { 285 | t.Fatal(err) 286 | } 287 | } 288 | for _, tt := range tests { 289 | // Set up servers and testClients. 290 | s := &Service{store: ds} 291 | c, stopServer := initServerAndClient(t, s) 292 | defer stopServer() 293 | t.Run(tt.desc, func(t *testing.T) { 294 | _, err := c.ModifyLocation(ctx, &spb.ModifyLocationRequest{Location: tt.location, FieldMask: tt.mask}) 295 | if (err != nil) != tt.wantErr { 296 | t.Errorf("got err=%v, wantErr=%t", err, tt.wantErr) 297 | } 298 | if err != nil { 299 | return 300 | } 301 | got, err := s.store.GetLocation(ctx, tt.want.Name) 302 | if err != nil { 303 | t.Error(err) 304 | } 305 | if diff := cmp.Diff(tt.want, got); diff != "" { 306 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 307 | } 308 | }) 309 | } 310 | } 311 | 312 | func initServerAndClient(t *testing.T, s *Service) (spb.EmittoClient, func()) { 313 | l, err := net.Listen("tcp", "localhost:") 314 | if err != nil { 315 | log.Fatal(err) 316 | } 317 | 318 | srv := grpc.NewServer() 319 | spb.RegisterEmittoServer(srv, s) 320 | go srv.Serve(l) 321 | 322 | conn, err := grpc.Dial(l.Addr().String(), grpc.WithInsecure()) 323 | if err != nil { 324 | t.Fatal(err) 325 | } 326 | return spb.NewEmittoClient(conn), srv.Stop 327 | } 328 | 329 | func initFSAdminServerAndClient(t *testing.T, f *fakeFSAdminServer) (*fleetspeak.Client, func()) { 330 | l, err := net.Listen("tcp", "localhost:") 331 | if err != nil { 332 | log.Fatal(err) 333 | } 334 | 335 | srv := grpc.NewServer() 336 | fsspb.RegisterAdminServer(srv, f) 337 | go srv.Serve(l) 338 | 339 | fs, err := fleetspeak.New(l.Addr().String(), "") 340 | if err != nil { 341 | t.Fatal(err) 342 | } 343 | 344 | return fs, srv.Stop 345 | } 346 | 347 | type fakeFSAdminServer struct { 348 | fsspb.AdminServer 349 | 350 | insertMessage func(*fspb.Message) (*fspb.EmptyMessage, error) 351 | listClients func(*fsspb.ListClientsRequest) (*fsspb.ListClientsResponse, error) 352 | } 353 | 354 | func (s *fakeFSAdminServer) InsertMessage(_ context.Context, m *fspb.Message) (*fspb.EmptyMessage, error) { 355 | return s.insertMessage(m) 356 | } 357 | 358 | func (s *fakeFSAdminServer) ListClients(_ context.Context, req *fsspb.ListClientsRequest) (*fsspb.ListClientsResponse, error) { 359 | return s.listClients(req) 360 | } 361 | -------------------------------------------------------------------------------- /source/server/store/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "conversions.go", 7 | "datastore.go", 8 | "memory.go", 9 | "store.go", 10 | "store_test_suite.go", 11 | ], 12 | importpath = "github.com/google/emitto/source/server/store", 13 | visibility = ["//visibility:public"], 14 | deps = [ 15 | "//source/resources:go_default_library", 16 | "@com_github_fatih_camelcase//:go_default_library", 17 | "@com_github_google_go_cmp//cmp:go_default_library", 18 | "@com_google_cloud_go//datastore:go_default_library", 19 | "@org_golang_google_api//option:go_default_library", 20 | ], 21 | ) 22 | 23 | go_test( 24 | name = "go_default_test", 25 | srcs = [ 26 | "conversions_test.go", 27 | "datastore_test.go", 28 | "memory_test.go", 29 | ], 30 | embed = [":go_default_library"], 31 | deps = [ 32 | "//source/resources:go_default_library", 33 | "@com_github_google_go_cmp//cmp:go_default_library", 34 | "@com_google_cloud_go//datastore:go_default_library", 35 | "@org_golang_google_api//option:go_default_library", 36 | "@org_golang_google_grpc//:go_default_library", 37 | ], 38 | ) 39 | -------------------------------------------------------------------------------- /source/server/store/conversions.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package store 16 | 17 | import ( 18 | "fmt" 19 | "reflect" 20 | "strings" 21 | "time" 22 | 23 | "github.com/fatih/camelcase" 24 | "github.com/google/emitto/source/resources" 25 | ) 26 | 27 | // TimeNow is stubbed for testing. 28 | var TimeNow = time.Now 29 | 30 | // MutateRule applies mutable, non-empty field mutations from the src to dst Rule. 31 | func MutateRule(src, dst *resources.Rule) error { 32 | m, err := resources.MutationsMapping(resources.Rule{}) 33 | if err != nil { 34 | return err 35 | } 36 | return mutateFields(reflect.ValueOf(*src), dst, m) 37 | } 38 | 39 | // MutateLocation applies mutable, non-empty field mutations from the src to dst Location. 40 | func MutateLocation(src, dst *resources.Location) error { 41 | m, err := resources.MutationsMapping(resources.Location{}) 42 | if err != nil { 43 | return err 44 | } 45 | return mutateFields(reflect.ValueOf(*src), dst, m) 46 | } 47 | 48 | // MutateSensorRequest applies mutable, non-empty field mutations from the src to dst SensorRequest. 49 | func MutateSensorRequest(src, dst *resources.SensorRequest) error { 50 | m, err := resources.MutationsMapping(resources.SensorRequest{}) 51 | if err != nil { 52 | return err 53 | } 54 | return mutateFields(reflect.ValueOf(*src), dst, m) 55 | } 56 | 57 | func mutateFields(src reflect.Value, dst interface{}, fields map[string]bool) error { 58 | for i := 0; i < src.NumField(); i++ { 59 | n := src.Type().Field(i).Name 60 | if fields[strings.ToLower(strings.Join(camelcase.Split(n), "_"))] { 61 | f := src.Field(i) 62 | if f.IsValid() && !isZero(f) { 63 | var d reflect.Value 64 | switch t := dst.(type) { 65 | case *resources.Rule: 66 | d = reflect.ValueOf(t).Elem() 67 | case *resources.Location: 68 | d = reflect.ValueOf(t).Elem() 69 | case *resources.SensorRequest: 70 | d = reflect.ValueOf(t).Elem() 71 | default: 72 | return fmt.Errorf("invalid mutable type: %T", t) 73 | } 74 | d.FieldByName(n).Set(f) 75 | } 76 | } 77 | } 78 | return nil 79 | } 80 | 81 | func isZero(v reflect.Value) bool { 82 | switch v.Kind() { 83 | case reflect.Slice: // Currently supported types with default values of nil. 84 | return v.IsNil() 85 | } 86 | // All other types. 87 | return v.Interface() == reflect.Zero(v.Type()).Interface() 88 | } 89 | -------------------------------------------------------------------------------- /source/server/store/conversions_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package store 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/google/emitto/source/resources" 21 | "github.com/google/go-cmp/cmp" 22 | ) 23 | 24 | func TestMutateRule(t *testing.T) { 25 | for _, tt := range []struct { 26 | desc string 27 | src, dst, want *resources.Rule 28 | overrideObj bool 29 | }{ 30 | { 31 | desc: "Body is mutated", 32 | src: &resources.Rule{ 33 | Body: "new_body", 34 | }, 35 | dst: &resources.Rule{ 36 | ID: 2222, 37 | Body: "old_body", 38 | }, 39 | want: &resources.Rule{ 40 | ID: 2222, 41 | Body: "new_body", 42 | }, 43 | }, 44 | { 45 | desc: "ID is not mutated", 46 | src: &resources.Rule{ 47 | ID: 1111, 48 | }, 49 | dst: &resources.Rule{ 50 | ID: 2222, 51 | }, 52 | want: &resources.Rule{ 53 | ID: 2222, 54 | }, 55 | }, 56 | { 57 | desc: "Body and LocZones are mutated", 58 | src: &resources.Rule{ 59 | ID: 2222, 60 | Body: "new_body", 61 | LocZones: []string{"new_zone_1", "new_zone_2"}, 62 | }, 63 | dst: &resources.Rule{ 64 | ID: 2222, 65 | Body: "old_body", 66 | LocZones: []string{"old_zone_1", "old_zone_2"}, 67 | }, 68 | want: &resources.Rule{ 69 | ID: 2222, 70 | Body: "new_body", 71 | LocZones: []string{"new_zone_1", "new_zone_2"}, 72 | }, 73 | }, 74 | } { 75 | if err := MutateRule(tt.src, tt.dst); err != nil { 76 | t.Errorf("unexpected error: %v", err) 77 | } 78 | if diff := cmp.Diff(tt.want, tt.dst); diff != "" { 79 | t.Errorf("%s: expectation mismatch (-want +got):\n%s", tt.desc, diff) 80 | } 81 | } 82 | } 83 | 84 | func TestMutateLocation(t *testing.T) { 85 | for _, tt := range []struct { 86 | desc string 87 | src, dst, want *resources.Location 88 | }{ 89 | { 90 | desc: "Zones are mutated", 91 | src: &resources.Location{ 92 | Zones: []string{"new_zone_1", "new_zone_2"}, 93 | }, 94 | dst: &resources.Location{ 95 | Zones: []string{"old_zone_1", "old_zone_2"}, 96 | }, 97 | want: &resources.Location{ 98 | Zones: []string{"new_zone_1", "new_zone_2"}, 99 | }, 100 | }, 101 | { 102 | desc: "Name is not mutated", 103 | src: &resources.Location{ 104 | Name: "new_name", 105 | }, 106 | dst: &resources.Location{ 107 | Name: "old_name", 108 | }, 109 | want: &resources.Location{ 110 | Name: "old_name", 111 | }, 112 | }, 113 | } { 114 | if err := MutateLocation(tt.src, tt.dst); err != nil { 115 | t.Errorf("unexpected error: %v", err) 116 | } 117 | if diff := cmp.Diff(tt.want, tt.dst); diff != "" { 118 | t.Errorf("%s: expectation mismatch (-want +got):\n%s", tt.desc, diff) 119 | } 120 | } 121 | } 122 | 123 | func TestMutateSensorRequest(t *testing.T) { 124 | for _, tt := range []struct { 125 | desc string 126 | src, dst, want *resources.SensorRequest 127 | }{ 128 | { 129 | desc: "Status is mutated", 130 | src: &resources.SensorRequest{ 131 | Status: "new", 132 | }, 133 | dst: &resources.SensorRequest{ 134 | Status: "old", 135 | }, 136 | want: &resources.SensorRequest{ 137 | Status: "new", 138 | }, 139 | }, 140 | { 141 | desc: "preserve non mutable fields", 142 | src: &resources.SensorRequest{ 143 | ID: "new_id", 144 | Time: "new_time", 145 | ClientID: "new_client_id", 146 | Type: "new_type", 147 | }, 148 | dst: &resources.SensorRequest{ 149 | ID: "old_id", 150 | Time: "old_time", 151 | ClientID: "old_client_id", 152 | Type: "old_type", 153 | }, 154 | want: &resources.SensorRequest{ 155 | ID: "old_id", 156 | Time: "old_time", 157 | ClientID: "old_client_id", 158 | Type: "old_type", 159 | }, 160 | }, 161 | } { 162 | if err := MutateSensorRequest(tt.src, tt.dst); err != nil { 163 | t.Errorf("unexpected error: %v", err) 164 | } 165 | if diff := cmp.Diff(tt.want, tt.dst); diff != "" { 166 | t.Errorf("%s: expectation mismatch (-want +got):\n%s", tt.desc, diff) 167 | } 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /source/server/store/datastore.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package store 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | "cloud.google.com/go/datastore" 23 | "github.com/google/emitto/source/resources" 24 | "google.golang.org/api/option" 25 | ) 26 | 27 | const ( 28 | datastoreAddr = "dns:///datastore.googleapis.com:443" 29 | locationKind = "Location" 30 | ruleKind = "Rule" 31 | sensorRequestKind = "SensorRequest" 32 | sensorMessageKind = "SensorMessage" 33 | ) 34 | 35 | // DataStore represents a Google Cloud Datastore implementation of a Store. 36 | type DataStore struct { 37 | client *datastore.Client 38 | } 39 | 40 | // NewDataStore returns a new DataStore. 41 | func NewDataStore(client *datastore.Client) *DataStore { 42 | return &DataStore{client} 43 | } 44 | 45 | // NewGCDClient initializes a new Google Cloud Datastore Client. 46 | // Follow these instructions to set up application credentials: 47 | // https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually. 48 | func NewGCDClient(ctx context.Context, projectID, credFile string) (*datastore.Client, error) { 49 | c, err := datastore.NewClient(ctx, projectID, option.WithEndpoint(datastoreAddr), option.WithCredentialsFile(credFile)) 50 | if err != nil { 51 | return nil, fmt.Errorf("GCD client creation failed: %v", err) 52 | } 53 | return c, nil 54 | } 55 | 56 | // Close Datastore client connection. 57 | func (s *DataStore) Close() error { 58 | return s.client.Close() 59 | } 60 | 61 | func locationKey(name string) *datastore.Key { 62 | return &datastore.Key{ 63 | Kind: locationKind, 64 | Name: name, 65 | } 66 | } 67 | 68 | // locationExists returns true if there is a location with the given name. 69 | func (s *DataStore) locationExists(ctx context.Context, name string) (bool, error) { 70 | query := datastore.NewQuery(locationKind).Filter("__key__ =", locationKey(name)).KeysOnly() 71 | c, err := s.client.Count(ctx, query) 72 | if err != nil { 73 | return false, err 74 | } 75 | return c == 1, nil 76 | } 77 | 78 | // AddLocation adds the given location. 79 | func (s *DataStore) AddLocation(ctx context.Context, l *resources.Location) error { 80 | ok, err := s.locationExists(ctx, l.Name) 81 | if err != nil { 82 | return err 83 | } 84 | if ok { 85 | return fmt.Errorf("location %q already exists", l.Name) 86 | } 87 | l.LastModified = TimeNow().Format(time.RFC1123Z) 88 | _, err = s.client.Put(ctx, locationKey(l.Name), l) 89 | return err 90 | } 91 | 92 | // ModifyLocation modifies an existing location with the provided location. 93 | func (s *DataStore) ModifyLocation(ctx context.Context, l *resources.Location) error { 94 | ok, err := s.locationExists(ctx, l.Name) 95 | if err != nil { 96 | return err 97 | } 98 | if !ok { 99 | return fmt.Errorf("location %q does not exist", l.Name) 100 | } 101 | loc, err := s.GetLocation(ctx, l.Name) 102 | if err != nil { 103 | return fmt.Errorf("unable to get location %q: %v", l.Name, err) 104 | } 105 | if err := MutateLocation(l, loc); err != nil { 106 | return fmt.Errorf("unable to mutate location src=%+v dst=%+v: %v", l, loc, err) 107 | } 108 | loc.LastModified = TimeNow().Format(time.RFC1123Z) 109 | _, err = s.client.Put(ctx, locationKey(l.Name), loc) 110 | return err 111 | } 112 | 113 | // DeleteLocation deletes the given location. 114 | func (s *DataStore) DeleteLocation(ctx context.Context, name string) error { 115 | ok, err := s.locationExists(ctx, name) 116 | if err != nil { 117 | return err 118 | } 119 | if !ok { 120 | return fmt.Errorf("location %q does not exist", name) 121 | } 122 | return s.client.Delete(ctx, locationKey(name)) 123 | } 124 | 125 | // GetLocation gets the location with the given name. 126 | func (s *DataStore) GetLocation(ctx context.Context, name string) (*resources.Location, error) { 127 | query := datastore.NewQuery(locationKind).Filter("__key__ =", locationKey(name)) 128 | l := new(resources.Location) 129 | if _, err := s.client.Run(ctx, query).Next(l); err != nil { 130 | return nil, err 131 | } 132 | return l, nil 133 | } 134 | 135 | // ListLocations list all locations, ordered by name. 136 | func (s *DataStore) ListLocations(ctx context.Context) ([]*resources.Location, error) { 137 | var all []*resources.Location 138 | query := datastore.NewQuery(locationKind).Order("Name") 139 | if _, err := s.client.GetAll(ctx, query, &all); err != nil { 140 | return nil, err 141 | } 142 | return all, nil 143 | } 144 | 145 | func ruleKey(ruleID int64) *datastore.Key { 146 | return &datastore.Key{ 147 | Kind: ruleKind, 148 | ID: ruleID, 149 | } 150 | } 151 | 152 | // ruleExists returns trues if there is a rule with the given rule ID. 153 | func (s *DataStore) ruleExists(ctx context.Context, id int64) (bool, error) { 154 | query := datastore.NewQuery(ruleKind).Filter("__key__ =", ruleKey(id)).KeysOnly() 155 | c, err := s.client.Count(ctx, query) 156 | if err != nil { 157 | return false, err 158 | } 159 | return c == 1, nil 160 | } 161 | 162 | // AddRule adds the given rule. 163 | func (s *DataStore) AddRule(ctx context.Context, r *resources.Rule) error { 164 | switch ok, err := s.ruleExists(ctx, r.ID); { 165 | case err != nil: 166 | return err 167 | case ok: 168 | return fmt.Errorf("rule %d already exists", r.ID) 169 | default: 170 | r.LastModified = TimeNow().Format(time.RFC1123Z) 171 | _, err = s.client.Put(ctx, ruleKey(r.ID), r) 172 | return err 173 | } 174 | } 175 | 176 | // ModifyRule modifies an existing rule with the provided rule. 177 | func (s *DataStore) ModifyRule(ctx context.Context, r *resources.Rule) error { 178 | ok, err := s.ruleExists(ctx, r.ID) 179 | if err != nil { 180 | return err 181 | } 182 | if !ok { 183 | return fmt.Errorf("rule %d does not exist", r.ID) 184 | } 185 | rules, err := s.ListRules(ctx, []int64{r.ID}) 186 | if err != nil { 187 | return fmt.Errorf("unable to get rule %d: %v", r.ID, err) 188 | } 189 | if l := len(rules); l > 1 { 190 | return fmt.Errorf("expected 1 rule (ID=%d), got %d", r.ID, l) 191 | } 192 | rule := rules[0] 193 | if err := MutateRule(r, rule); err != nil { 194 | return fmt.Errorf("unable to mutate rule src=%+v dst=%+v: %v", r, rule, err) 195 | } 196 | rule.LastModified = TimeNow().Format(time.RFC1123Z) 197 | _, err = s.client.Put(ctx, ruleKey(r.ID), rule) 198 | return err 199 | } 200 | 201 | // DeleteRule deletes the given rule. 202 | func (s *DataStore) DeleteRule(ctx context.Context, id int64) error { 203 | switch ok, err := s.ruleExists(ctx, id); { 204 | case err != nil: 205 | return err 206 | case !ok: 207 | return fmt.Errorf("rule %d does not exist", id) 208 | default: 209 | return s.client.Delete(ctx, ruleKey(id)) 210 | } 211 | } 212 | 213 | // ListRules lists the rules with the given rule IDs, following the same order. 214 | // If `ids` is nil, lists all rules, ordered by ID. 215 | func (s *DataStore) ListRules(ctx context.Context, ids []int64) ([]*resources.Rule, error) { 216 | var ( 217 | all []*resources.Rule 218 | err error 219 | ) 220 | if l := len(ids); l > 0 { 221 | all = make([]*resources.Rule, l) 222 | keys := make([]*datastore.Key, l) 223 | for j, id := range ids { 224 | keys[j] = ruleKey(id) 225 | } 226 | err = s.client.GetMulti(ctx, keys, all) 227 | } else { 228 | query := datastore.NewQuery(ruleKind).Order("ID") 229 | if _, err := s.client.GetAll(ctx, query, &all); err != nil { 230 | return nil, err 231 | } 232 | } 233 | if err != nil { 234 | return nil, err 235 | } 236 | return all, nil 237 | } 238 | 239 | func sensorRequestKey(id string) *datastore.Key { 240 | return &datastore.Key{ 241 | Kind: sensorRequestKind, 242 | Name: id, 243 | } 244 | } 245 | 246 | // sensorRequestExists returns true if there is a sensor request with the given ID. 247 | func (s *DataStore) sensorRequestExists(ctx context.Context, id string) (bool, error) { 248 | query := datastore.NewQuery(sensorRequestKind).Filter("__key__ =", sensorRequestKey(id)).KeysOnly() 249 | c, err := s.client.Count(ctx, query) 250 | if err != nil { 251 | return false, err 252 | } 253 | return c == 1, nil 254 | } 255 | 256 | // AddSensorRequest adds the given sensor request. 257 | func (s *DataStore) AddSensorRequest(ctx context.Context, r *resources.SensorRequest) error { 258 | switch ok, err := s.sensorRequestExists(ctx, r.ID); { 259 | case err != nil: 260 | return err 261 | case ok: 262 | return fmt.Errorf("sensor request %q already exists", r.ID) 263 | default: 264 | r.LastModified = TimeNow().Format(time.RFC1123Z) 265 | _, err = s.client.Put(ctx, sensorRequestKey(r.ID), r) 266 | return err 267 | } 268 | } 269 | 270 | // ModifySensorRequest modifies an existing sensor request with the provided sensor request. 271 | func (s *DataStore) ModifySensorRequest(ctx context.Context, r *resources.SensorRequest) error { 272 | ok, err := s.sensorRequestExists(ctx, r.ID) 273 | if err != nil { 274 | return err 275 | } 276 | if !ok { 277 | return fmt.Errorf("sensor request %q does not exist", r.ID) 278 | } 279 | req, err := s.GetSensorRequest(ctx, r.ID) 280 | if err != nil { 281 | return fmt.Errorf("unable to get sensor request %q: %v", r.ID, err) 282 | } 283 | if err := MutateSensorRequest(r, req); err != nil { 284 | return fmt.Errorf("unable to mutate sensor request src=%+v dst=%+v: %v", r, req, err) 285 | } 286 | req.LastModified = TimeNow().Format(time.RFC1123Z) 287 | _, err = s.client.Put(ctx, sensorRequestKey(r.ID), req) 288 | return err 289 | } 290 | 291 | // DeleteSensorRequest removes the given sensor request. 292 | func (s *DataStore) DeleteSensorRequest(ctx context.Context, id string) error { 293 | switch ok, err := s.sensorRequestExists(ctx, id); { 294 | case err != nil: 295 | return err 296 | case !ok: 297 | return fmt.Errorf("sensor request %q does not exist", id) 298 | default: 299 | return s.client.Delete(ctx, sensorRequestKey(id)) 300 | } 301 | } 302 | 303 | // GetSensorRequest gets the sensor request with the given ID. 304 | func (s *DataStore) GetSensorRequest(ctx context.Context, id string) (*resources.SensorRequest, error) { 305 | query := datastore.NewQuery(sensorRequestKind).Filter("__key__ =", sensorRequestKey(id)) 306 | l := new(resources.SensorRequest) 307 | if _, err := s.client.Run(ctx, query).Next(l); err != nil { 308 | return nil, err 309 | } 310 | return l, nil 311 | } 312 | 313 | func sensorMessageKey(id string) *datastore.Key { 314 | return &datastore.Key{ 315 | Kind: sensorMessageKind, 316 | Name: id, 317 | } 318 | } 319 | 320 | // sensorMessageExists returns true if there is a sensor request with the given ID. 321 | func (s *DataStore) sensorMessageExists(ctx context.Context, id string) (bool, error) { 322 | query := datastore.NewQuery(sensorMessageKind).Filter("__key__ =", sensorMessageKey(id)).KeysOnly() 323 | c, err := s.client.Count(ctx, query) 324 | if err != nil { 325 | return false, err 326 | } 327 | return c == 1, nil 328 | } 329 | 330 | // AddSensorMessage adds the given sensor message. 331 | func (s *DataStore) AddSensorMessage(ctx context.Context, m *resources.SensorMessage) error { 332 | switch ok, err := s.sensorMessageExists(ctx, m.ID); { 333 | case err != nil: 334 | return err 335 | case ok: 336 | return fmt.Errorf("sensor request %q already exists", m.ID) 337 | default: 338 | _, err = s.client.Put(ctx, sensorMessageKey(m.ID), m) 339 | return err 340 | } 341 | } 342 | -------------------------------------------------------------------------------- /source/server/store/datastore_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package store 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "errors" 21 | "fmt" 22 | "io/ioutil" 23 | "net/http" 24 | "testing" 25 | 26 | "cloud.google.com/go/datastore" 27 | "google.golang.org/api/option" 28 | "google.golang.org/grpc" 29 | ) 30 | 31 | // This test relies on the Google Cloud Datastore emulator with the following configuration 32 | // constants: 33 | const ( 34 | testHost = "localhost:9999" 35 | testProject = "test-project-name" 36 | ) 37 | 38 | func TestDataStore(t *testing.T) { 39 | RunTestSuite(t, func() (Store, error) { 40 | // Reset emulator before each test. 41 | if err := resetEmulator(); err != nil { 42 | return nil, err 43 | } 44 | c, err := datastore.NewClient(context.Background(), testProject, option.WithEndpoint(testHost), option.WithoutAuthentication(), option.WithGRPCDialOption(grpc.WithInsecure())) 45 | if err != nil { 46 | return nil, fmt.Errorf("GCD client creation failed: %v", err) 47 | } 48 | return &DataStore{c}, nil 49 | }) 50 | } 51 | 52 | func resetEmulator() error { 53 | resp, err := http.Post(fmt.Sprintf("http://%s/reset", testHost), "", bytes.NewBuffer([]byte{})) 54 | if err != nil { 55 | return err 56 | } 57 | defer resp.Body.Close() 58 | body, err := ioutil.ReadAll(resp.Body) 59 | if err != nil { 60 | return err 61 | } 62 | if string(body) != "Resetting...\n" { 63 | return errors.New("emulator failed to reset") 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /source/server/store/memory.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package store 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "sort" 21 | "sync" 22 | "time" 23 | 24 | "github.com/google/emitto/source/resources" 25 | ) 26 | 27 | // MemoryStore represents a memory Store implementation. 28 | type MemoryStore struct { 29 | m sync.Mutex 30 | locations map[string]resources.Location 31 | rules map[int64]resources.Rule 32 | sensorRequests map[string]resources.SensorRequest 33 | sensorMessages map[string]resources.SensorMessage 34 | } 35 | 36 | // NewMemoryStore returns a MemoryStore. 37 | func NewMemoryStore() *MemoryStore { 38 | return &MemoryStore{ 39 | locations: make(map[string]resources.Location), 40 | rules: make(map[int64]resources.Rule), 41 | sensorRequests: make(map[string]resources.SensorRequest), 42 | sensorMessages: make(map[string]resources.SensorMessage), 43 | } 44 | } 45 | 46 | // AddLocation adds a Location to the store. 47 | func (s *MemoryStore) AddLocation(ctx context.Context, l *resources.Location) error { 48 | s.m.Lock() 49 | defer s.m.Unlock() 50 | 51 | cp := *l 52 | if _, ok := s.locations[cp.Name]; ok { 53 | return fmt.Errorf("location %q already exists", cp.Name) 54 | } 55 | cp.LastModified = TimeNow().Format(time.RFC1123Z) 56 | s.locations[cp.Name] = cp 57 | return nil 58 | } 59 | 60 | // ModifyLocation modifies an existing location with the provided location. 61 | func (s *MemoryStore) ModifyLocation(ctx context.Context, l *resources.Location) error { 62 | s.m.Lock() 63 | defer s.m.Unlock() 64 | 65 | cp := *l 66 | loc, ok := s.locations[cp.Name] 67 | if !ok { 68 | return fmt.Errorf("location %q does not exist", cp.Name) 69 | } 70 | if err := MutateLocation(&cp, &loc); err != nil { 71 | return fmt.Errorf("unable to mutate location src=%+v dst=%+v: %v", cp, loc, err) 72 | } 73 | loc.LastModified = TimeNow().Format(time.RFC1123Z) 74 | s.locations[cp.Name] = loc 75 | return nil 76 | } 77 | 78 | // DeleteLocation deletes an existing Location. 79 | func (s *MemoryStore) DeleteLocation(ctx context.Context, name string) error { 80 | s.m.Lock() 81 | defer s.m.Unlock() 82 | 83 | if _, ok := s.locations[name]; !ok { 84 | return fmt.Errorf("location %q does not exist", name) 85 | } 86 | delete(s.locations, name) 87 | return nil 88 | } 89 | 90 | // GetLocation returns the Location with the given name. 91 | func (s *MemoryStore) GetLocation(ctx context.Context, name string) (*resources.Location, error) { 92 | s.m.Lock() 93 | defer s.m.Unlock() 94 | 95 | r, ok := s.locations[name] 96 | if !ok { 97 | return nil, fmt.Errorf("location %q does not exist", name) 98 | } 99 | return &r, nil 100 | } 101 | 102 | // ListLocations returns all the locations, sorted by name. 103 | func (s *MemoryStore) ListLocations(ctx context.Context) ([]*resources.Location, error) { 104 | s.m.Lock() 105 | defer s.m.Unlock() 106 | 107 | locations := make([]*resources.Location, 0, len(s.locations)) 108 | for l := range s.locations { 109 | loc := s.locations[l] 110 | locations = append(locations, &loc) 111 | } 112 | sort.Slice(locations, func(i, j int) bool { 113 | return locations[i].Name < locations[j].Name 114 | }) 115 | return locations, nil 116 | } 117 | 118 | // AddRule adds a Rule to the store. 119 | func (s *MemoryStore) AddRule(ctx context.Context, r *resources.Rule) error { 120 | s.m.Lock() 121 | defer s.m.Unlock() 122 | 123 | cp := *r 124 | if _, ok := s.rules[cp.ID]; ok { 125 | return fmt.Errorf("rule %d already exists", cp.ID) 126 | } 127 | cp.LastModified = TimeNow().Format(time.RFC1123Z) 128 | s.rules[cp.ID] = cp 129 | return nil 130 | } 131 | 132 | // ModifyRule modifies an existing rule with the provided rule. 133 | func (s *MemoryStore) ModifyRule(ctx context.Context, r *resources.Rule) error { 134 | s.m.Lock() 135 | defer s.m.Unlock() 136 | 137 | cp := *r 138 | rule, ok := s.rules[cp.ID] 139 | if !ok { 140 | return fmt.Errorf("rule %d does not exist", cp.ID) 141 | } 142 | if err := MutateRule(&cp, &rule); err != nil { 143 | return fmt.Errorf("unable to mutate rule src=%+v dst=%+v: %v", cp, rule, err) 144 | } 145 | rule.LastModified = TimeNow().Format(time.RFC1123Z) 146 | s.rules[cp.ID] = rule 147 | return nil 148 | } 149 | 150 | // DeleteRule deletes an existing Rule. 151 | func (s *MemoryStore) DeleteRule(ctx context.Context, id int64) error { 152 | s.m.Lock() 153 | defer s.m.Unlock() 154 | 155 | if _, ok := s.rules[id]; !ok { 156 | return fmt.Errorf("rule %d does not exist", id) 157 | } 158 | delete(s.rules, id) 159 | return nil 160 | } 161 | 162 | // ListRules returns Rules from a list of rule IDs. All rules are returned if ids is nil. 163 | func (s *MemoryStore) ListRules(ctx context.Context, ids []int64) ([]*resources.Rule, error) { 164 | s.m.Lock() 165 | defer s.m.Unlock() 166 | 167 | if len(ids) == 0 { 168 | var rules []*resources.Rule 169 | for r := range s.rules { 170 | rule := s.rules[r] 171 | rules = append(rules, &rule) 172 | } 173 | return rules, nil 174 | } 175 | 176 | rules := make([]*resources.Rule, 0, len(ids)) 177 | for _, id := range ids { 178 | r, ok := s.rules[id] 179 | if !ok { 180 | return nil, fmt.Errorf("rule %d does not exist", id) 181 | } 182 | rules = append(rules, &r) 183 | } 184 | return rules, nil 185 | } 186 | 187 | // AddSensorRequest adds a sensor request. 188 | func (s *MemoryStore) AddSensorRequest(ctx context.Context, r *resources.SensorRequest) error { 189 | s.m.Lock() 190 | defer s.m.Unlock() 191 | 192 | cp := *r 193 | if _, ok := s.sensorRequests[cp.ID]; ok { 194 | return fmt.Errorf("sensor request %q already exists", cp.ID) 195 | } 196 | cp.LastModified = TimeNow().Format(time.RFC1123Z) 197 | s.sensorRequests[cp.ID] = cp 198 | return nil 199 | } 200 | 201 | // ModifySensorRequest modifies an existing sensor request with the provided sensor request. 202 | func (s *MemoryStore) ModifySensorRequest(ctx context.Context, r *resources.SensorRequest) error { 203 | s.m.Lock() 204 | defer s.m.Unlock() 205 | 206 | cp := *r 207 | req, ok := s.sensorRequests[cp.ID] 208 | if !ok { 209 | return fmt.Errorf("sensor request %q does not exist", cp.ID) 210 | } 211 | if err := MutateSensorRequest(&cp, &req); err != nil { 212 | return fmt.Errorf("unable to mutate sensor request src=%+v dst=%+v: %v", cp, req, err) 213 | } 214 | req.LastModified = TimeNow().Format(time.RFC1123Z) 215 | s.sensorRequests[cp.ID] = req 216 | return nil 217 | } 218 | 219 | // DeleteSensorRequest deletes an existing sensor request. 220 | func (s *MemoryStore) DeleteSensorRequest(ctx context.Context, id string) error { 221 | s.m.Lock() 222 | defer s.m.Unlock() 223 | 224 | if _, ok := s.sensorRequests[id]; !ok { 225 | return fmt.Errorf("sensor request %q does not exist", id) 226 | } 227 | delete(s.sensorRequests, id) 228 | return nil 229 | } 230 | 231 | // GetSensorRequest returns the sensor request with the given ID. 232 | func (s *MemoryStore) GetSensorRequest(ctx context.Context, id string) (*resources.SensorRequest, error) { 233 | s.m.Lock() 234 | defer s.m.Unlock() 235 | 236 | r, ok := s.sensorRequests[id] 237 | if !ok { 238 | return nil, fmt.Errorf("sensor request %q does not exist", id) 239 | } 240 | return &r, nil 241 | } 242 | 243 | // AddSensorMessage adds a sensor message. 244 | func (s *MemoryStore) AddSensorMessage(ctx context.Context, m *resources.SensorMessage) error { 245 | s.m.Lock() 246 | defer s.m.Unlock() 247 | 248 | cp := *m 249 | if _, ok := s.sensorMessages[cp.ID]; ok { 250 | return fmt.Errorf("sensor message %q already exists", cp.ID) 251 | } 252 | s.sensorMessages[cp.ID] = cp 253 | return nil 254 | } 255 | -------------------------------------------------------------------------------- /source/server/store/memory_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package store 16 | 17 | import "testing" 18 | 19 | func TestMemoryStore(t *testing.T) { 20 | RunTestSuite(t, func() (Store, error) { return NewMemoryStore(), nil }) 21 | } 22 | -------------------------------------------------------------------------------- /source/server/store/store.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package store contains functionality to store objects. 16 | package store 17 | 18 | import ( 19 | "context" 20 | 21 | "github.com/google/emitto/source/resources" 22 | ) 23 | 24 | // Store represents object storage. 25 | type Store interface { 26 | // AddLocation adds a new Location. 27 | AddLocation(ctx context.Context, l *resources.Location) error 28 | // ModifyLocation modifies an existing Location. 29 | ModifyLocation(ctx context.Context, l *resources.Location) error 30 | // DeleteLocation removes an existing Loction by name. 31 | DeleteLocation(ctx context.Context, name string) error 32 | // GetLocation retrieves a Location by name. 33 | GetLocation(ctx context.Context, name string) (*resources.Location, error) 34 | // ListLocations lists all stored Locations. 35 | ListLocations(ctx context.Context) ([]*resources.Location, error) 36 | 37 | // AddRule adds a new Rule. 38 | AddRule(ctx context.Context, r *resources.Rule) error 39 | // ModifyRule modifies an existing Rule. 40 | ModifyRule(ctx context.Context, r *resources.Rule) error 41 | // DeleteRule removes an existing Rule by ID. 42 | DeleteRule(ctx context.Context, id int64) error 43 | // ListRules lists stored Rules by ID. 44 | ListRules(ctx context.Context, ids []int64) ([]*resources.Rule, error) 45 | 46 | // AddSensorRequest adds a new SensorRequest. 47 | AddSensorRequest(ctx context.Context, r *resources.SensorRequest) error 48 | // ModifySensorRequest updates an existing SensorRequest. 49 | ModifySensorRequest(ctx context.Context, r *resources.SensorRequest) error 50 | // DeleteSensorRequest removes an existing SensorRequest by request ID. 51 | DeleteSensorRequest(ctx context.Context, id string) error 52 | // GetSensorRequest retrieves a SensorRequest by ID. 53 | GetSensorRequest(ctx context.Context, id string) (*resources.SensorRequest, error) 54 | 55 | // AddSensorMessage adds a new SensorMessage. 56 | AddSensorMessage(ctx context.Context, r *resources.SensorMessage) error 57 | } 58 | -------------------------------------------------------------------------------- /source/server/store/store_test_suite.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package store 16 | 17 | import ( 18 | "context" 19 | "log" 20 | "reflect" 21 | "strings" 22 | "testing" 23 | "time" 24 | 25 | "github.com/google/emitto/source/resources" 26 | "github.com/google/go-cmp/cmp" 27 | ) 28 | 29 | // suite contains store tests and the underlying Store implementation. 30 | type suite struct { 31 | builder func() (Store, error) 32 | } 33 | 34 | // RunTestSuite runs all generic store tests. 35 | // 36 | // The tests use the provided builder to instantiate a Store. 37 | // The builder is expected to always return a valid Store. 38 | func RunTestSuite(t *testing.T, builder func() (Store, error)) { 39 | s := &suite{builder} 40 | s.Run(t) 41 | } 42 | 43 | // Run runs Test* methods of the suite as subtests. 44 | func (s *suite) Run(t *testing.T) { 45 | sv := reflect.ValueOf(s) 46 | st := reflect.TypeOf(s) 47 | for i := 0; i < sv.NumMethod(); i++ { 48 | n := st.Method(i).Name 49 | if strings.HasPrefix(n, "Test") { 50 | mv := sv.MethodByName(n) 51 | mt := mv.Type() 52 | if mt.NumIn() != 1 || !reflect.TypeOf(t).AssignableTo(mt.In(0)) { 53 | log.Fatalf("Method %q of the test suite must have 1 argument of type *testing.T", n) 54 | } 55 | if mt.NumOut() != 0 { 56 | log.Fatalf("Method %q of the test suite must have no return value", n) 57 | } 58 | m := mv.Interface().(func(t *testing.T)) 59 | t.Run(n, m) 60 | } 61 | } 62 | } 63 | 64 | var ( 65 | location1 = &resources.Location{ 66 | Name: "test", 67 | Zones: []string{"unstable", "canary", "prod"}, 68 | } 69 | location2 = &resources.Location{ 70 | Name: "zzz_test", 71 | Zones: []string{"canary", "prod"}, 72 | } 73 | 74 | rule1 = &resources.Rule{ 75 | ID: 1111, 76 | Body: `sid:111 foo:bar`, 77 | } 78 | rule2 = &resources.Rule{ 79 | ID: 2222, 80 | Body: `sid:222 foo:bar`, 81 | } 82 | rule3 = &resources.Rule{ 83 | ID: 3333, 84 | Body: `sid:333 foo:bar`, 85 | } 86 | 87 | sensorRequest1 = &resources.SensorRequest{ 88 | ID: "req1", 89 | ClientID: "dest1", 90 | Type: resources.DeployRules, 91 | Status: "OK", 92 | } 93 | 94 | sensorMessage1 = &resources.SensorMessage{ 95 | ID: "req1", 96 | ClientID: "dest1", 97 | Type: resources.Alert, 98 | Status: "ERROR", 99 | } 100 | ) 101 | 102 | func (s *suite) TestAddLocation(t *testing.T) { 103 | st, err := s.builder() 104 | if err != nil { 105 | t.Fatal(err) 106 | } 107 | ctx := context.Background() 108 | 109 | if err := st.AddLocation(ctx, location1); err != nil { 110 | t.Error(err) 111 | } 112 | if err := st.AddLocation(ctx, location1); err == nil { 113 | t.Error("location.add: adding a duplicate location should have raised an error") 114 | } 115 | } 116 | 117 | func (s *suite) TestModifyLocation(t *testing.T) { 118 | st, err := s.builder() 119 | if err != nil { 120 | t.Fatal(err) 121 | } 122 | ctx := context.Background() 123 | 124 | TimeNow = func() time.Time { 125 | return time.Date(2000, 1, 1, 0, 0, 0, 0, time.FixedZone("UTC", 0)) 126 | } 127 | 128 | if err := st.AddLocation(ctx, location1); err != nil { 129 | t.Error(err) 130 | } 131 | cp := *location1 132 | cp.Zones = append(cp.Zones, "another_zone") 133 | if err := st.ModifyLocation(ctx, &cp); err != nil { 134 | t.Error(err) 135 | } 136 | got, err := st.GetLocation(ctx, cp.Name) 137 | if err != nil { 138 | t.Error(err) 139 | } 140 | cp.LastModified = TimeNow().Format(time.RFC1123Z) 141 | if diff := cmp.Diff(&cp, got); diff != "" { 142 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 143 | } 144 | } 145 | 146 | func (s *suite) TestDeleteLocation(t *testing.T) { 147 | st, err := s.builder() 148 | if err != nil { 149 | t.Fatal(err) 150 | } 151 | ctx := context.Background() 152 | 153 | if err := st.AddLocation(ctx, location1); err != nil { 154 | t.Error(err) 155 | } 156 | if err := st.DeleteLocation(ctx, location1.Name); err != nil { 157 | t.Error(err) 158 | } 159 | if _, err := st.GetLocation(ctx, location1.Name); err == nil { 160 | t.Error("GetLocation on a deleted location should have failed") 161 | } 162 | } 163 | 164 | func (s *suite) TestActionsOnNonExistingLocation(t *testing.T) { 165 | st, err := s.builder() 166 | if err != nil { 167 | t.Fatal(err) 168 | } 169 | ctx := context.Background() 170 | 171 | if err := st.DeleteLocation(ctx, "does not exist"); err == nil { 172 | t.Error("DeleteLocation on a non-existing location should have failed") 173 | } 174 | if err := st.ModifyLocation(ctx, location1); err == nil { 175 | t.Error("ModifyLocation on a non-existing location should have failed") 176 | } 177 | } 178 | 179 | func (s *suite) TestListLocations(t *testing.T) { 180 | st, err := s.builder() 181 | if err != nil { 182 | t.Fatal(err) 183 | } 184 | ctx := context.Background() 185 | 186 | TimeNow = func() time.Time { 187 | return time.Date(2000, 1, 1, 0, 0, 0, 0, time.FixedZone("UTC", 0)) 188 | } 189 | 190 | if err := st.AddLocation(ctx, location1); err != nil { 191 | t.Error(err) 192 | } 193 | if err := st.AddLocation(ctx, location2); err != nil { 194 | t.Error(err) 195 | } 196 | got, err := st.ListLocations(ctx) 197 | if err != nil { 198 | t.Error(err) 199 | } 200 | 201 | location1.LastModified = TimeNow().Format(time.RFC1123Z) 202 | location2.LastModified = TimeNow().Format(time.RFC1123Z) 203 | want := []*resources.Location{location1, location2} 204 | if diff := cmp.Diff(want, got); diff != "" { 205 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 206 | } 207 | } 208 | 209 | func (s *suite) TestAddRule(t *testing.T) { 210 | st, err := s.builder() 211 | if err != nil { 212 | t.Fatal(err) 213 | } 214 | ctx := context.Background() 215 | 216 | if err := st.AddRule(ctx, rule1); err != nil { 217 | t.Error(err) 218 | } 219 | if err := st.AddRule(ctx, rule1); err == nil { 220 | t.Error("adding a duplicate rule should have raised an error") 221 | } 222 | } 223 | 224 | func (s *suite) TestDeleteRule(t *testing.T) { 225 | st, err := s.builder() 226 | if err != nil { 227 | t.Fatal(err) 228 | } 229 | ctx := context.Background() 230 | 231 | if err := st.AddRule(ctx, rule1); err != nil { 232 | t.Error(err) 233 | } 234 | if err := st.DeleteRule(ctx, rule1.ID); err != nil { 235 | t.Error(err) 236 | } 237 | if _, err := st.ListRules(ctx, []int64{rule1.ID}); err == nil { 238 | t.Error("ListRules on a deleted rule should have failed") 239 | } 240 | } 241 | 242 | func (s *suite) TestActionsOnNonExistingRule(t *testing.T) { 243 | st, err := s.builder() 244 | if err != nil { 245 | t.Fatal(err) 246 | } 247 | ctx := context.Background() 248 | 249 | if err := st.DeleteRule(ctx, -1); err == nil { 250 | t.Error("DeleteRule on a non-existing rule should have failed") 251 | } 252 | if err := st.ModifyRule(ctx, rule1); err == nil { 253 | t.Error("ModifyRule on a non-existing rule should have failed") 254 | } 255 | } 256 | 257 | func (s *suite) TestModifyRule(t *testing.T) { 258 | st, err := s.builder() 259 | if err != nil { 260 | t.Fatal(err) 261 | } 262 | ctx := context.Background() 263 | 264 | TimeNow = func() time.Time { 265 | return time.Date(2000, 1, 1, 0, 0, 0, 0, time.FixedZone("UTC", 0)) 266 | } 267 | 268 | if err := st.AddRule(ctx, rule1); err != nil { 269 | t.Error(err) 270 | } 271 | cp := *rule1 272 | cp.Body += " some_suffix:111" 273 | if err := st.ModifyRule(ctx, &cp); err != nil { 274 | t.Error(err) 275 | } 276 | got, err := st.ListRules(ctx, []int64{cp.ID}) 277 | if err != nil { 278 | t.Error(err) 279 | } 280 | cp.LastModified = TimeNow().Format(time.RFC1123Z) 281 | if diff := cmp.Diff([]*resources.Rule{&cp}, got); diff != "" { 282 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 283 | } 284 | } 285 | 286 | func (s *suite) TestListAllRules(t *testing.T) { 287 | st, err := s.builder() 288 | if err != nil { 289 | t.Fatal(err) 290 | } 291 | ctx := context.Background() 292 | 293 | if err := st.AddRule(ctx, rule1); err != nil { 294 | t.Error(err) 295 | } 296 | if err := st.AddRule(ctx, rule2); err != nil { 297 | t.Error(err) 298 | } 299 | if err := st.AddRule(ctx, rule3); err != nil { 300 | t.Error(err) 301 | } 302 | got, err := st.ListRules(ctx, nil) 303 | if err != nil { 304 | t.Error(err) 305 | } 306 | if l := len(got); l != 3 { 307 | t.Errorf("expected 3 rules, got %d", l) 308 | } 309 | } 310 | 311 | func (s *suite) TestAddSensorRequest(t *testing.T) { 312 | st, err := s.builder() 313 | if err != nil { 314 | t.Fatal(err) 315 | } 316 | ctx := context.Background() 317 | 318 | if err := st.AddSensorRequest(ctx, sensorRequest1); err != nil { 319 | t.Error(err) 320 | } 321 | if err := st.AddSensorRequest(ctx, sensorRequest1); err == nil { 322 | t.Error("adding a duplicate sensor request should have raised an error") 323 | } 324 | } 325 | 326 | func (s *suite) TestModifySensorRequest(t *testing.T) { 327 | st, err := s.builder() 328 | if err != nil { 329 | t.Fatal(err) 330 | } 331 | ctx := context.Background() 332 | 333 | TimeNow = func() time.Time { 334 | return time.Date(2000, 1, 1, 0, 0, 0, 0, time.FixedZone("UTC", 0)) 335 | } 336 | 337 | if err := st.AddSensorRequest(ctx, sensorRequest1); err != nil { 338 | t.Error(err) 339 | } 340 | cp := *sensorRequest1 341 | cp.Status = "NEW" 342 | if err := st.ModifySensorRequest(ctx, &cp); err != nil { 343 | t.Error(err) 344 | } 345 | got, err := st.GetSensorRequest(ctx, cp.ID) 346 | if err != nil { 347 | t.Error(err) 348 | } 349 | cp.LastModified = TimeNow().Format(time.RFC1123Z) 350 | if diff := cmp.Diff(&cp, got); diff != "" { 351 | t.Errorf("expectation mismatch (-want +got):\n%s", diff) 352 | } 353 | } 354 | 355 | func (s *suite) TestDeleteSensorRequest(t *testing.T) { 356 | st, err := s.builder() 357 | if err != nil { 358 | t.Fatal(err) 359 | } 360 | ctx := context.Background() 361 | 362 | if err := st.AddSensorRequest(ctx, sensorRequest1); err != nil { 363 | t.Error(err) 364 | } 365 | if err := st.DeleteSensorRequest(ctx, sensorRequest1.ID); err != nil { 366 | t.Error(err) 367 | } 368 | if _, err := st.GetSensorRequest(ctx, sensorRequest1.ID); err == nil { 369 | t.Error("GetSensorRequest on a deleted sensor request should have failed") 370 | } 371 | } 372 | 373 | func (s *suite) TestAddSensorMessage(t *testing.T) { 374 | st, err := s.builder() 375 | if err != nil { 376 | t.Fatal(err) 377 | } 378 | ctx := context.Background() 379 | 380 | if err := st.AddSensorMessage(ctx, sensorMessage1); err != nil { 381 | t.Error(err) 382 | } 383 | if err := st.AddSensorMessage(ctx, sensorMessage1); err == nil { 384 | t.Error("adding a duplicate sensor message should have raised an error") 385 | } 386 | } 387 | --------------------------------------------------------------------------------