├── .github ├── CODEOWNERS ├── release.sh └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── Contributing.md ├── Dockerfile ├── LICENSE ├── Makefile ├── Readme.md ├── commands ├── Readme.md ├── clean.go ├── commands.go ├── delete-database.go ├── formatting.go ├── import.go ├── import_test.go ├── reporting.go ├── show-beacons-proxy.go ├── show-beacons-sni.go ├── show-beacons.go ├── show-bl-hostname.go ├── show-bl-ip.go ├── show-databases.go ├── show-dns-fqdn-ips.go ├── show-explodedDns.go ├── show-ip-dns-fqdns.go ├── show-long-connections.go ├── show-open-connections.go ├── show-strobes.go ├── show-user-agents.go ├── test-config.go ├── update-check.go └── update_test.go ├── config ├── config.go ├── config_test.go ├── running.go ├── static.go ├── static_test.go ├── tables.go └── testing.go ├── database ├── db.go ├── meta.go └── writer.go ├── docker-compose.yml ├── docs ├── Docker Usage.md ├── Manual Installation.md ├── Mongo Configuration.md ├── RITA Gittiquette.md ├── Releases.md ├── Rolling Datasets.md ├── System Requirements.md └── Upgrading.md ├── etc ├── bash_completion.d │ └── rita ├── rita.yaml └── rita_docker.yaml ├── go.mod ├── go.sum ├── install.sh ├── parser ├── conn.go ├── dns.go ├── files │ ├── indexing.go │ ├── reading.go │ └── repository.go ├── filter.go ├── filter_test.go ├── fsimporter.go ├── http.go ├── open_conn.go ├── parsetypes │ ├── conn.go │ ├── dns.go │ ├── http.go │ ├── open_conn.go │ ├── parsetypes.go │ ├── parsetypes_test.go │ └── ssl.go ├── repository.go └── ssl.go ├── pkg ├── beacon │ ├── Readme.md │ ├── analyzer.go │ ├── dissector.go │ ├── mongodb.go │ ├── mongodb_test.go │ ├── repository.go │ ├── results.go │ ├── siphon.go │ ├── sorter.go │ └── summarizer.go ├── beaconproxy │ ├── Readme.md │ ├── analyzer.go │ ├── dissector.go │ ├── mongodb.go │ ├── repository.go │ ├── results.go │ ├── siphon.go │ ├── sorter.go │ └── summarizer.go ├── beaconsni │ ├── Readme.md │ ├── analyzer.go │ ├── dissector.go │ ├── mongodb.go │ ├── repository.go │ ├── results.go │ ├── siphon.go │ ├── sorter.go │ └── summarizer.go ├── blacklist │ ├── Readme.md │ ├── analyzer.go │ ├── mongodb.go │ ├── repository.go │ ├── results.go │ └── service.go ├── certificate │ ├── Readme.md │ ├── analyzer.go │ ├── mongodb.go │ ├── mongodb_test.go │ └── repository.go ├── data │ ├── fqdn.go │ ├── ip.go │ ├── ip_test.go │ ├── sets.go │ └── zeek.go ├── explodeddns │ ├── Readme.md │ ├── analyzer.go │ ├── mongodb.go │ ├── mongodb_test.go │ ├── repository.go │ └── results.go ├── host │ ├── Readme.md │ ├── analyzer.go │ ├── mongodb.go │ ├── mongodb_test.go │ └── repository.go ├── hostname │ ├── Readme.md │ ├── analyzer.go │ ├── mongodb.go │ ├── mongodb_test.go │ ├── repository.go │ └── results.go ├── remover │ ├── analyzer.go │ ├── mongodb.go │ ├── repository.go │ └── writer.go ├── sniconn │ ├── Readme.md │ ├── analyzer.go │ ├── mongodb.go │ └── repository.go ├── uconn │ ├── Readme.md │ ├── analyzer.go │ ├── mongodb.go │ ├── mongodb_test.go │ ├── repository.go │ ├── results.go │ └── summarizer.go ├── uconnproxy │ ├── Readme.md │ ├── analyzer.go │ ├── mongodb.go │ ├── mongodb_test.go │ └── repository.go └── useragent │ ├── Readme.md │ ├── analyzer.go │ ├── mongodb.go │ ├── mongodb_test.go │ ├── repository.go │ ├── results.go │ └── summarizer.go ├── reporting ├── report-beacons.go ├── report-beaconsproxy.go ├── report-beaconssni.go ├── report-bl-dest-ips.go ├── report-bl-hostnames.go ├── report-bl-source-ips.go ├── report-explodedDns.go ├── report-long-connections.go ├── report-strobes.go ├── report-useragents.go ├── report.go └── templates │ ├── csstempl.go │ ├── svgtempl.go │ └── templates.go ├── resources ├── logging.go ├── resources.go └── testing.go ├── rita-logo.png ├── rita.go ├── static-tests.sh ├── test.Dockerfile ├── tests ├── helpers.sh └── install.bats └── util ├── ip.go ├── ip_test.go ├── util.go └── util_test.go /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @activecm/lumberghs -------------------------------------------------------------------------------- /.github/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # based on: https://github.com/skx/github-action-publish-binaries/blob/master/upload-script 3 | # 4 | # Upload binary artifacts when a new release is made. 5 | # 6 | 7 | set -e 8 | 9 | # Ensure that the GITHUB_TOKEN secret is included 10 | if [[ -z "$GITHUB_TOKEN" ]]; then 11 | echo "Set the GITHUB_TOKEN env variable." 12 | exit 1 13 | fi 14 | 15 | # Ensure that the file path is present 16 | if [[ -z "$1" ]]; then 17 | echo "Missing file (pattern) to upload." 18 | exit 1 19 | fi 20 | 21 | # Only upload to non-draft releases 22 | IS_DRAFT=$(jq --raw-output '.release.draft' $GITHUB_EVENT_PATH) 23 | if [ "$IS_DRAFT" = true ]; then 24 | echo "This is a draft, so nothing to do!" 25 | exit 0 26 | fi 27 | 28 | # Run the build-script 29 | make docker-build 30 | docker container create --name rita quay.io/activecm/rita-legacy:latest 31 | docker container cp rita:/rita ./rita 32 | 33 | # Prepare the headers 34 | AUTH_HEADER="Authorization: token ${GITHUB_TOKEN}" 35 | 36 | # Build the Upload URL from the various pieces 37 | RELEASE_ID=$(jq --raw-output '.release.id' $GITHUB_EVENT_PATH) 38 | 39 | # For each matching file 40 | for file in $*; do 41 | echo "Processing file ${file}" 42 | 43 | FILENAME=$(basename ${file}) 44 | UPLOAD_URL="https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets?name=${FILENAME}" 45 | echo "$UPLOAD_URL" 46 | 47 | # Upload the file 48 | curl \ 49 | -sSL \ 50 | -XPOST \ 51 | -H "${AUTH_HEADER}" \ 52 | --upload-file "${file}" \ 53 | --header "Content-Type:application/octet-stream" \ 54 | "${UPLOAD_URL}" 55 | done 56 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Upload files to new release 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | #- unpublished 8 | - created 9 | - edited 10 | #- deleted 11 | - prereleased 12 | - released 13 | 14 | jobs: 15 | upload: 16 | name: Upload Artifacts 17 | runs-on: ubuntu-20.04 18 | steps: 19 | - uses: actions/checkout@v1 20 | - run: .github/release.sh rita install.sh 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Rita tests 2 | 3 | on: 4 | pull_request: 5 | # Run tests except if only markdown files are changed 6 | paths: 7 | - '**' 8 | - '!*.md' 9 | 10 | jobs: 11 | test: 12 | name: static and unit tests 13 | runs-on: ubuntu-20.04 14 | steps: 15 | - uses: actions/checkout@v1 16 | - run: make docker-test 17 | 18 | # integration: 19 | # name: integration tests 20 | # runs-on: ubuntu-16.04 21 | # steps: 22 | # - uses: actions/checkout@v1 23 | # - run: | 24 | # make docker-build-test 25 | # docker container create --name rita --entrypoint /bin/sleep quay.io/activecm/rita-legacy:test 5m 26 | # docker container start rita 27 | # docker container exec rita tar cf vendor.tar vendor 28 | # docker container cp rita:/go/src/github.com/activecm/rita-legacy/rita ./rita 29 | # docker container cp rita:/go/src/github.com/activecm/rita-legacy/vendor.tar ./vendor.tar 30 | # docker container stop rita 31 | # tar xf vendor.tar 32 | # - run: make integration-test 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /pprof 2 | *.pprof 3 | *.swp 4 | *\# 5 | *\~ 6 | *.swp 7 | *.swo 8 | *.exe 9 | /.vscode/ 10 | /rita 11 | /vendor/ 12 | /rita-html-report/ 13 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | disable-all: true 3 | enable: 4 | #- depguard # useful for Sirupsen and old mgo 5 | # Suggests code refactoring 6 | #- govet 7 | - staticcheck # aka megacheck, supercedes govet 8 | #- goconst 9 | - ineffassign 10 | #- maligned 11 | #- prealloc 12 | #- gosimple 13 | # Finds unused and dead code 14 | - unused 15 | # - structcheck 16 | - varcheck 17 | - deadcode 18 | # - unparam 19 | # Style changes 20 | - golint 21 | # - goimports # supercedes gofmt 22 | - misspell 23 | #- lll 24 | 25 | issues: 26 | exclude: 27 | # https://github.com/golangci/golangci-lint/issues/446 28 | - composites 29 | -------------------------------------------------------------------------------- /Contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing to RITA 2 | --- 3 | ## Want to help? We would love that! 4 | Here are some ways to get involved, ranging in 5 | difficulty from easiest to hardest 6 | 7 | ## Bug Hunting 8 | Run the software and tell us when it breaks. We are happy to receive bug 9 | reports 10 | 11 | Just be sure to do the following: 12 | * Check if the bug is already accounted for on the 13 | [Github issue tracker](https://github.com/activecm/rita-legacy/issues) 14 | * If an issue already exists, add the relevant info in a comment 15 | * If not, create an issue and include the relevant info 16 | * Give very specific descriptions of how to reproduce the bug 17 | * Include the output of `rita --version` 18 | * Include a description of your hardware (e.g. CPU, RAM, filesystems) 19 | * Tell us about the size of the test and the physical resources available 20 | 21 | ## Contributing Code 22 | There are several ways to contribute code to the RITA project. 23 | Before diving in, follow the [Manual Installation Instructions](docs/Manual%20Installation.md) 24 | 25 | * Work on bug fixes: 26 | * Find an issue you would like to work on in the Github tracker, especially [unassigned issues marked "good first issue"](https://github.com/activecm/rita-legacy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22+no%3Aassignee) 27 | * Leave a comment letting us know you would like to work on it 28 | * Add new features: 29 | * If you would like to become involved in the development effort, open a new issue or continue a discussion on an existing issue 30 | 31 | ### Running Static Tests 32 | * You must have a RITA [development environment](https://github.com/activecm/rita-legacy/blob/master/docs/Manual%20Installation.md#installing-golang) set up and [golangci-lint](https://github.com/golangci/golangci-lint#install) installed to run the tests. 33 | * Check the [Makefile](https://github.com/activecm/rita-legacy/blob/master/Makefile) for all options. Currently you can run `make test`, `make static-test`, and `make unit-test`. There is also `make integration-test` and docker variants that will require you install docker as well. 34 | 35 | ### Reviewing Automated Test Results 36 | Automated tests are run against each pull request. Build results may be viewed [here](https://github.com/activecm/rita-legacy/actions). 37 | 38 | ### Gittiquette Summary 39 | * In order to contribute to RITA, you must [fork it](https://github.com/activecm/rita-legacy/fork). 40 | * Once you have a forked repo you will need to clone it to a very specific path which corresponds to _the original repo location_. This is due to the way packages are imported in Go programs. 41 | * `git clone [your forked repo git url]` 42 | * Add `https://github.com/activecm/rita-legacy` as a new remote so you can pull new changes. 43 | * `git remote add upstream https://github.com/activecm/rita-legacy` 44 | * Split a branch off of master . 45 | * `git checkout -b [your new feature]` 46 | * When your work is finished, pull the latest changes from the upstream master and rebase your changes on it. 47 | * `git checkout master; git pull -r upstream master` 48 | * `git checkout [your new feature]; git rebase master` 49 | * Push your commits to your repo and submit a pull request on Github. 50 | 51 | Further info can be found in the [Gittiquette doc](docs/RITA%20Gittiquette.md) under the guidelines and contributors sections. 52 | 53 | ### Common Issues 54 | * Building Rita using `go build` or `go install` yields a RITA version of `UNDEFINED` 55 | * Use `make` or `make install` instead 56 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17-alpine as rita-builder 2 | 3 | RUN apk add --no-cache git make ca-certificates wget build-base 4 | 5 | WORKDIR /go/src/github.com/activecm/rita-legacy 6 | 7 | # cache dependencies 8 | COPY go.mod go.sum ./ 9 | RUN go mod download 10 | 11 | # copy the rest of the code 12 | COPY . ./ 13 | 14 | # Change ARGs with --build-arg to target other architectures 15 | # Produce a self-contained statically linked binary 16 | ARG CGO_ENABLED=0 17 | # Set the build target architecture and OS 18 | ARG GOARCH=amd64 19 | ARG GOOS=linux 20 | # Passing arguments in to make result in them being set as 21 | # environment variables for the call to go build 22 | RUN make CGO_ENABLED=$CGO_ENABLED GOARCH=$GOARCH GOOS=$GOOS 23 | 24 | FROM scratch 25 | 26 | WORKDIR / 27 | COPY --from=rita-builder /go/src/github.com/activecm/rita-legacy/etc/rita_docker.yaml /etc/rita/config.yaml 28 | COPY --from=rita-builder /go/src/github.com/activecm/rita-legacy/rita /rita 29 | 30 | ENTRYPOINT ["/rita"] 31 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION := $(shell git describe --abbrev=0 --tags) 2 | EXACT_VERSION := $(shell git describe --always --long --dirty --tags) 3 | PREFIX ?= /usr/local 4 | 5 | LDFLAGS := -ldflags='-X github.com/activecm/rita-legacy/config.Version=${VERSION} -X github.com/activecm/rita-legacy/config.ExactVersion=${EXACT_VERSION}' 6 | TESTFLAGS := -p=1 -v 7 | # go source files 8 | SRC := $(shell find . -path ./vendor -prune -o -type f -name '*.go' -print) 9 | 10 | # Allow a variable to be initialized and cached on first use. Subsequent uses will 11 | # use the cached value instead of evaluating the variable's declaration again. 12 | # Use like this: VAR = $(call cache,VAR) 13 | # https://www.cmcrossroads.com/article/makefile-optimization-eval-and-macro-caching 14 | cache = $(if $(cached-$1),,$(eval cached-$1 := 1)$(eval cache-$1 := $($1)))$(cache-$1) 15 | 16 | # force rita to be rebuilt even if it's up to date 17 | .PHONY: rita 18 | rita: $(SRC) 19 | @# remove any existing vendor directory from dep 20 | @rm -rf vendor 21 | go build -o rita ${LDFLAGS} 22 | 23 | .PHONY: install 24 | install: rita 25 | mv rita $(PREFIX)/bin/ 26 | mkdir -p $(PREFIX)/etc/bash_completion.d/ $(PREFIX)/etc/rita/ 27 | sudo cp etc/bash_completion.d/rita $(PREFIX)/etc/bash_completion.d/rita 28 | sudo cp etc/rita.yaml $(PREFIX)/etc/rita/config.yaml 29 | 30 | .PHONY: docker-check 31 | # Use this recipe if you want to fail if docker is missing 32 | docker-check: 33 | @if ! docker ps > /dev/null; then echo "Ensure docker is installed and accessible from the current user context"; return 1; fi 34 | 35 | .PHONY: integration-test 36 | integration-test: docker-check 37 | # docker run should only get executed once on initialization using the cache trick 38 | integration-test: MONGO_EXE = $(shell docker run --rm -d mongo:4.2) 39 | integration-test: MONGO_ID = $(call cache,MONGO_EXE) 40 | integration-test: MONGO_IP = $(shell docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(MONGO_ID)) 41 | integration-test: 42 | @echo Waiting for Mongo to respond to connection attempts 43 | @until nc -z $(MONGO_IP) 27017; do sleep 1; done; true 44 | @echo Running tests 45 | @bash -c "trap 'docker stop $(MONGO_ID) > /dev/null' EXIT; go test $(TESTFLAGS) -tags=integration $(LDFLAGS) ./... -args mongodb://$(MONGO_IP):27017" 46 | 47 | 48 | .PHONY: test 49 | test: static-test unit-test 50 | @echo Ran tests on rita $(EXACT_VERSION) 51 | 52 | .PHONY: static-test 53 | static-test: 54 | golangci-lint run ./... 55 | 56 | .PHONY: unit-test 57 | unit-test: 58 | go test -race -cover $(shell go list ./... | grep -v /vendor/) 59 | 60 | 61 | # The following targets all use docker 62 | 63 | .PHONY: docker-build 64 | docker-build: 65 | docker build -t quay.io/activecm/rita-legacy:latest -f Dockerfile . 66 | 67 | .PHONY: docker-build-test 68 | docker-build-test: 69 | docker build -t quay.io/activecm/rita-legacy:test -f test.Dockerfile . 70 | 71 | # Runs all tests inside docker container 72 | .PHONY: docker-test 73 | docker-test: docker-build-test 74 | docker run --rm quay.io/activecm/rita-legacy:test make test 75 | 76 | .PHONY: docker-unit-test 77 | docker-unit-test: docker-build-test 78 | docker run --rm quay.io/activecm/rita-legacy:test make unit-test 79 | 80 | .PHONY: docker-static-test 81 | docker-static-test: docker-build-test 82 | docker run --rm quay.io/activecm/rita-legacy:test make static-test 83 | 84 | # .PHONY: docker-integration-test 85 | # docker-integration-test: docker-build-test 86 | # docker run --rm quay.io/activecm/rita-legacy:test make integration-test 87 | -------------------------------------------------------------------------------- /commands/Readme.md: -------------------------------------------------------------------------------- 1 | # RITA Commands 2 | 3 | ## A note on philosophy 4 | 5 | As RITA is written in Go and targeted towards Unix based systems, we believe that RITA too should follow the Unix Philosophy. 6 | 7 | As such RITA commands should implement this philosophy by default. 8 | 9 | For example, the human readable output from `show-beacons` or `show-blacklisted` is easy to read, but the default action is to print an unformatted comma separated list. This "ugly" format is much easier to parse and process with tools such as `sed`, `awk`, and `cut`. 10 | 11 | These tools are great at processing the results that come out of RITA, and we believe that RITA should do its best to support them. 12 | 13 | ## How to create a new command 14 | 15 | Adding commands to RITA is a straight-forward process. 16 | 17 | 1. Create a new command file in the commands directory 18 | 1. Create an `init` function in the newly created file that declares your command and boostraps it 19 | 1. Create a function that executes the business logic of your command 20 | 1. Profit. 21 | 22 | ## Example Command 23 | 24 | ```go 25 | //init functions run import 26 | func init() { 27 | // command to do something 28 | command := cli.Command{ 29 | Flags: []cli.Flags{ 30 | cli.IntFlag{ 31 | Name: "test, t", 32 | Usage: "set test flag", 33 | Value: 29, 34 | }, 35 | // There are also a few pre-defined flags for you to use in commands.go 36 | allFlag, 37 | }, 38 | Name: "name-of-command", 39 | Usage: "how to use the command", 40 | Action: nameOfCmdFunc, 41 | } 42 | 43 | // command to do something else 44 | otherCommand := cli.Command{ 45 | Flags: []cli.Flags{ 46 | cli.IntFlag{ 47 | Name: "test, t", 48 | Usage: "set test flag", 49 | Value: 29, 50 | }, 51 | ConfigFlag, 52 | }, 53 | Name: "name-of-other-command", 54 | Usage: "how to use the other command", 55 | Action: nameOfOtherCmdFunc, 56 | } 57 | 58 | // Bootstrap the commands (IMPORTANT) 59 | bootstrapCommands(command, otherCommand) 60 | } 61 | 62 | // It is very important that we use a function of this type (for compatibility with cli) 63 | func nameOfCmdFunc(c *cli.Context) error { 64 | // do stuff 65 | return nil 66 | } 67 | 68 | // another command function 69 | func nameOfOtherCmdFunc(c *cli.Context) error { 70 | // do stuff 71 | return nil 72 | } 73 | ``` 74 | -------------------------------------------------------------------------------- /commands/formatting.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | // helper functions for formatting floats and integers 8 | func f(f float64) string { 9 | return strconv.FormatFloat(f, 'g', 6, 64) 10 | } 11 | func i(i int64) string { 12 | return strconv.FormatInt(i, 10) 13 | } 14 | -------------------------------------------------------------------------------- /commands/reporting.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/reporting" 5 | "github.com/activecm/rita-legacy/resources" 6 | "github.com/urfave/cli" 7 | ) 8 | 9 | func init() { 10 | command := cli.Command{ 11 | 12 | Name: "html-report", 13 | Usage: "Create an html report for an analyzed database", 14 | UsageText: "rita html-report [command-options] [database]\n\n" + 15 | "If no database is specified, a report will be created for every database.", 16 | Flags: []cli.Flag{ 17 | ConfigFlag, 18 | netNamesFlag, 19 | noBrowserFlag, 20 | }, 21 | Action: func(c *cli.Context) error { 22 | res := resources.InitResources(getConfigFilePath(c)) 23 | databaseName := c.Args().Get(0) 24 | var databases []string 25 | if databaseName != "" { 26 | databases = append(databases, databaseName) 27 | } else { 28 | databases = res.MetaDB.GetAnalyzedDatabases() 29 | } 30 | err := reporting.PrintHTML(databases, c.Bool("network-names"), c.Bool("no-browser"), res) 31 | if err != nil { 32 | return cli.NewExitError(err.Error(), -1) 33 | } 34 | return nil 35 | }, 36 | } 37 | bootstrapCommands(command) 38 | } 39 | -------------------------------------------------------------------------------- /commands/show-databases.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/activecm/rita-legacy/resources" 7 | "github.com/urfave/cli" 8 | ) 9 | 10 | func init() { 11 | 12 | databases := cli.Command{ 13 | Name: "list", 14 | Aliases: []string{"show-databases"}, 15 | Usage: "Print the databases currently stored", 16 | Flags: []cli.Flag{ 17 | ConfigFlag, 18 | }, 19 | Action: func(c *cli.Context) error { 20 | res := resources.InitResources(getConfigFilePath(c)) 21 | 22 | if res != nil { 23 | for _, name := range res.MetaDB.GetDatabases() { 24 | fmt.Println(name) 25 | } 26 | } else { 27 | fmt.Println("\t[-] Cannot display databases due to outdated metadatabase entries.") 28 | } 29 | 30 | return nil 31 | }, 32 | } 33 | 34 | bootstrapCommands(databases) 35 | } 36 | -------------------------------------------------------------------------------- /commands/show-dns-fqdn-ips.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "github.com/activecm/rita-legacy/pkg/data" 9 | "github.com/activecm/rita-legacy/pkg/hostname" 10 | "github.com/activecm/rita-legacy/resources" 11 | "github.com/olekukonko/tablewriter" 12 | "github.com/urfave/cli" 13 | ) 14 | 15 | func init() { 16 | command := cli.Command{ 17 | Name: "show-dns-fqdn-ips", 18 | Usage: "Print IPs associated with FQDN via DNS", 19 | ArgsUsage: " ", 20 | Flags: []cli.Flag{ 21 | ConfigFlag, 22 | humanFlag, 23 | delimFlag, 24 | netNamesFlag, 25 | }, 26 | Action: showFqdnIps, 27 | } 28 | 29 | bootstrapCommands(command) 30 | } 31 | 32 | func showFqdnIps(c *cli.Context) error { 33 | db, fqdn := c.Args().Get(0), c.Args().Get(1) 34 | if db == "" || fqdn == "" { 35 | return cli.NewExitError("Specify a database and FQDN", -1) 36 | } 37 | res := resources.InitResources(getConfigFilePath(c)) 38 | res.DB.SelectDB(db) 39 | 40 | ipResults, err := hostname.IPResults(res, fqdn) 41 | 42 | if err != nil { 43 | res.Log.Error(err) 44 | return cli.NewExitError(err, -1) 45 | } 46 | 47 | if !(len(ipResults) > 0) { 48 | return cli.NewExitError("No results were found for "+db, -1) 49 | } 50 | 51 | showNetNames := c.Bool("network-names") 52 | 53 | if c.Bool("human-readable") { 54 | err := showFqdnIpsHuman(ipResults, showNetNames) 55 | if err != nil { 56 | return cli.NewExitError(err.Error(), -1) 57 | } 58 | return nil 59 | } 60 | 61 | err = showFqdnIpsDelim(ipResults, c.String("delimiter"), showNetNames) 62 | if err != nil { 63 | return cli.NewExitError(err.Error(), -1) 64 | } 65 | 66 | return nil 67 | } 68 | 69 | func showFqdnIpsHuman(data []data.UniqueIP, showNetNames bool) error { 70 | table := tablewriter.NewWriter(os.Stdout) 71 | var headerFields []string 72 | if showNetNames { 73 | headerFields = []string{ 74 | "Resolved IP", "Network", 75 | } 76 | } else { 77 | headerFields = []string{ 78 | "Resolved IP", 79 | } 80 | } 81 | 82 | table.SetHeader(headerFields) 83 | 84 | for _, d := range data { 85 | var row []string 86 | if showNetNames { 87 | row = []string{ 88 | d.IP, d.NetworkName, 89 | } 90 | } else { 91 | row = []string{ 92 | d.IP, 93 | } 94 | } 95 | table.Append(row) 96 | } 97 | table.Render() 98 | return nil 99 | } 100 | 101 | func showFqdnIpsDelim(data []data.UniqueIP, delim string, showNetNames bool) error { 102 | var headerFields []string 103 | if showNetNames { 104 | headerFields = []string{ 105 | "Resolved IP", "Network", 106 | } 107 | } else { 108 | headerFields = []string{ 109 | "Resolved IP", 110 | } 111 | } 112 | 113 | // Print the headers and analytic values, separated by a delimiter 114 | fmt.Println(strings.Join(headerFields, delim)) 115 | for _, d := range data { 116 | var row []string 117 | if showNetNames { 118 | row = []string{ 119 | d.IP, d.NetworkName, 120 | } 121 | } else { 122 | row = []string{ 123 | d.IP, 124 | } 125 | } 126 | 127 | fmt.Println(strings.Join(row, delim)) 128 | } 129 | return nil 130 | } 131 | -------------------------------------------------------------------------------- /commands/show-explodedDns.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/activecm/rita-legacy/pkg/explodeddns" 10 | "github.com/activecm/rita-legacy/resources" 11 | "github.com/olekukonko/tablewriter" 12 | "github.com/urfave/cli" 13 | ) 14 | 15 | func init() { 16 | command := cli.Command{ 17 | 18 | Name: "show-exploded-dns", 19 | Usage: "Print dns analysis. Exposes covert dns channels", 20 | ArgsUsage: "", 21 | Flags: []cli.Flag{ 22 | ConfigFlag, 23 | humanFlag, 24 | limitFlag, 25 | noLimitFlag, 26 | delimFlag, 27 | }, 28 | Action: func(c *cli.Context) error { 29 | db := c.Args().Get(0) 30 | if db == "" { 31 | return cli.NewExitError("Specify a database", -1) 32 | } 33 | 34 | res := resources.InitResources(getConfigFilePath(c)) 35 | res.DB.SelectDB(db) 36 | 37 | data, err := explodeddns.Results(res, c.Int("limit"), c.Bool("no-limit")) 38 | 39 | if err != nil { 40 | res.Log.Error(err) 41 | return cli.NewExitError(err, -1) 42 | } 43 | 44 | if len(data) == 0 { 45 | return cli.NewExitError("No results were found for "+db, -1) 46 | } 47 | 48 | if c.Bool("human-readable") { 49 | err := showDNSResultsHuman(data) 50 | if err != nil { 51 | return cli.NewExitError(err.Error(), -1) 52 | } 53 | return nil 54 | } 55 | err = showDNSResults(data, c.String("delimiter")) 56 | if err != nil { 57 | return cli.NewExitError(err.Error(), -1) 58 | } 59 | return nil 60 | }, 61 | } 62 | bootstrapCommands(command) 63 | } 64 | 65 | // splitSubN splits s every n characters 66 | func splitSubN(s string, n int) []string { 67 | sub := "" 68 | subs := []string{} 69 | 70 | runes := bytes.Runes([]byte(s)) 71 | l := len(runes) 72 | for i, r := range runes { 73 | sub = sub + string(r) 74 | if (i+1)%n == 0 { 75 | subs = append(subs, sub) 76 | sub = "" 77 | } else if (i + 1) == l { 78 | subs = append(subs, sub) 79 | } 80 | } 81 | 82 | return subs 83 | } 84 | 85 | func showDNSResults(dnsResults []explodeddns.Result, delim string) error { 86 | headers := []string{"Domain", "Unique Subdomains", "Times Looked Up"} 87 | 88 | // Print the headers and analytic values, separated by a delimiter 89 | fmt.Println(strings.Join(headers, delim)) 90 | for _, result := range dnsResults { 91 | fmt.Println( 92 | strings.Join( 93 | []string{result.Domain, i(result.SubdomainCount), i(result.Visited)}, 94 | delim, 95 | ), 96 | ) 97 | } 98 | return nil 99 | } 100 | 101 | func showDNSResultsHuman(dnsResults []explodeddns.Result) error { 102 | const DOMAINRECLEN = 80 103 | table := tablewriter.NewWriter(os.Stdout) 104 | table.SetAutoWrapText(true) 105 | table.SetRowSeparator("-") 106 | table.SetRowLine(true) 107 | table.SetHeader([]string{"Domain", "Unique Subdomains", "Times Looked Up"}) 108 | for _, result := range dnsResults { 109 | domain := result.Domain 110 | if len(domain) > DOMAINRECLEN { 111 | // Reformat the result.Domain value adding a newline every DOMAINRECLEN chars for wrapping 112 | subs := splitSubN(result.Domain, DOMAINRECLEN) 113 | domain = strings.Join(subs, "\n") 114 | } 115 | table.Append([]string{ 116 | domain, i(result.SubdomainCount), i(result.Visited), 117 | }) 118 | } 119 | table.Render() 120 | return nil 121 | } 122 | -------------------------------------------------------------------------------- /commands/show-ip-dns-fqdns.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/activecm/rita-legacy/pkg/hostname" 8 | "github.com/activecm/rita-legacy/resources" 9 | "github.com/olekukonko/tablewriter" 10 | "github.com/urfave/cli" 11 | ) 12 | 13 | func init() { 14 | command := cli.Command{ 15 | Name: "show-ip-dns-fqdns", 16 | Usage: "Print FQDNs associated with IP Address via DNS", 17 | ArgsUsage: " ", 18 | Flags: []cli.Flag{ 19 | ConfigFlag, 20 | humanFlag, 21 | delimFlag, 22 | }, 23 | Action: showIPFqdns, 24 | } 25 | 26 | bootstrapCommands(command) 27 | } 28 | 29 | func showIPFqdns(c *cli.Context) error { 30 | db, ip := c.Args().Get(0), c.Args().Get(1) 31 | if db == "" || ip == "" { 32 | return cli.NewExitError("Specify a database and IP address", -1) 33 | } 34 | 35 | res := resources.InitResources(getConfigFilePath(c)) 36 | res.DB.SelectDB(db) 37 | 38 | fqdnResults, err := hostname.FQDNResults(res, ip) 39 | 40 | if err != nil { 41 | res.Log.Error(err) 42 | return cli.NewExitError(err, -1) 43 | } 44 | 45 | if !(len(fqdnResults) > 0) { 46 | return cli.NewExitError("No results were found for "+db, -1) 47 | } 48 | 49 | if c.Bool("human-readable") { 50 | err := showIPFqdnsHuman(fqdnResults) 51 | if err != nil { 52 | return cli.NewExitError(err.Error(), -1) 53 | } 54 | return nil 55 | } 56 | 57 | err = showIPFqdnsRaw(fqdnResults) 58 | if err != nil { 59 | return cli.NewExitError(err.Error(), -1) 60 | } 61 | 62 | return nil 63 | } 64 | 65 | func showIPFqdnsHuman(data []*hostname.FQDNResult) error { 66 | table := tablewriter.NewWriter(os.Stdout) 67 | headerFields := []string{ 68 | "Queried FQDN", 69 | } 70 | 71 | table.SetHeader(headerFields) 72 | 73 | for _, d := range data { 74 | row := []string{ 75 | d.Hostname, 76 | } 77 | table.Append(row) 78 | } 79 | table.Render() 80 | return nil 81 | } 82 | 83 | func showIPFqdnsRaw(data []*hostname.FQDNResult) error { 84 | fmt.Println("Queried FQDN") 85 | for _, d := range data { 86 | fmt.Println(d.Hostname) 87 | } 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /commands/show-strobes.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "github.com/activecm/rita-legacy/pkg/beacon" 9 | "github.com/activecm/rita-legacy/resources" 10 | "github.com/olekukonko/tablewriter" 11 | "github.com/urfave/cli" 12 | ) 13 | 14 | func init() { 15 | command := cli.Command{ 16 | 17 | Name: "show-strobes", 18 | Usage: "Print strobe information", 19 | ArgsUsage: "", 20 | Flags: []cli.Flag{ 21 | ConfigFlag, 22 | humanFlag, 23 | cli.BoolFlag{ 24 | Name: "connection-count, l", 25 | Usage: "Sort the strobes by largest connection count.", 26 | }, 27 | limitFlag, 28 | noLimitFlag, 29 | delimFlag, 30 | netNamesFlag, 31 | }, 32 | Action: func(c *cli.Context) error { 33 | db := c.Args().Get(0) 34 | if db == "" { 35 | return cli.NewExitError("Specify a database", -1) 36 | } 37 | 38 | res := resources.InitResources(getConfigFilePath(c)) 39 | res.DB.SelectDB(db) 40 | 41 | sortDirection := -1 42 | if !c.Bool("connection-count") { 43 | sortDirection = 1 44 | } 45 | 46 | data, err := beacon.StrobeResults(res, sortDirection, c.Int("limit"), c.Bool("no-limit")) 47 | 48 | if err != nil { 49 | res.Log.Error(err) 50 | return cli.NewExitError(err, -1) 51 | } 52 | 53 | if len(data) == 0 { 54 | return cli.NewExitError("No results were found for "+db, -1) 55 | } 56 | 57 | if c.Bool("human-readable") { 58 | err := showStrobesHuman(data, c.Bool("network-names")) 59 | if err != nil { 60 | return cli.NewExitError(err.Error(), -1) 61 | } 62 | return nil 63 | } 64 | err = showStrobes(data, c.String("delimiter"), c.Bool("network-names")) 65 | if err != nil { 66 | return cli.NewExitError(err.Error(), -1) 67 | } 68 | return nil 69 | }, 70 | } 71 | bootstrapCommands(command) 72 | } 73 | 74 | func showStrobes(strobes []beacon.StrobeResult, delim string, showNetNames bool) error { 75 | var headerFields []string 76 | if showNetNames { 77 | headerFields = []string{"Source Network", "Destination Network", "Source", "Destination", "Connection Count"} 78 | } else { 79 | headerFields = []string{"Source", "Destination", "Connection Count"} 80 | } 81 | 82 | // Print the headers and analytic values, separated by a delimiter 83 | fmt.Println(strings.Join(headerFields, delim)) 84 | for _, strobe := range strobes { 85 | var row []string 86 | if showNetNames { 87 | row = []string{ 88 | strobe.SrcNetworkName, 89 | strobe.DstNetworkName, 90 | strobe.SrcIP, 91 | strobe.DstIP, 92 | i(strobe.ConnectionCount), 93 | } 94 | } else { 95 | row = []string{ 96 | strobe.SrcIP, 97 | strobe.DstIP, 98 | i(strobe.ConnectionCount), 99 | } 100 | } 101 | fmt.Println(strings.Join(row, delim)) 102 | } 103 | return nil 104 | } 105 | 106 | func showStrobesHuman(strobes []beacon.StrobeResult, showNetNames bool) error { 107 | table := tablewriter.NewWriter(os.Stdout) 108 | table.SetColWidth(100) 109 | 110 | var headerFields []string 111 | if showNetNames { 112 | headerFields = []string{"Source Network", "Destination Network", "Source", "Destination", "Connection Count"} 113 | } else { 114 | headerFields = []string{"Source", "Destination", "Connection Count"} 115 | } 116 | table.SetHeader(headerFields) 117 | 118 | for _, strobe := range strobes { 119 | var row []string 120 | if showNetNames { 121 | row = []string{ 122 | strobe.SrcNetworkName, 123 | strobe.DstNetworkName, 124 | strobe.SrcIP, 125 | strobe.DstIP, 126 | i(strobe.ConnectionCount), 127 | } 128 | } else { 129 | row = []string{ 130 | strobe.SrcIP, 131 | strobe.DstIP, 132 | i(strobe.ConnectionCount), 133 | } 134 | } 135 | table.Append(row) 136 | } 137 | table.Render() 138 | return nil 139 | } 140 | -------------------------------------------------------------------------------- /commands/show-user-agents.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "github.com/activecm/rita-legacy/pkg/useragent" 9 | "github.com/activecm/rita-legacy/resources" 10 | "github.com/olekukonko/tablewriter" 11 | "github.com/urfave/cli" 12 | ) 13 | 14 | func init() { 15 | command := cli.Command{ 16 | 17 | Name: "show-useragents", 18 | Usage: "Print user agent information", 19 | ArgsUsage: "", 20 | Flags: []cli.Flag{ 21 | ConfigFlag, 22 | humanFlag, 23 | cli.BoolFlag{ 24 | Name: "least-used, l", 25 | Usage: "Sort the user agents from least used to most used.", 26 | }, 27 | limitFlag, 28 | noLimitFlag, 29 | delimFlag, 30 | }, 31 | Action: func(c *cli.Context) error { 32 | db := c.Args().Get(0) 33 | if db == "" { 34 | return cli.NewExitError("Specify a database", -1) 35 | } 36 | 37 | res := resources.InitResources(getConfigFilePath(c)) 38 | res.DB.SelectDB(db) 39 | 40 | sortDirection := 1 41 | if !c.Bool("least-used") { 42 | sortDirection = -1 43 | } 44 | 45 | data, err := useragent.Results(res, sortDirection, c.Int("limit"), c.Bool("no-limit")) 46 | 47 | if err != nil { 48 | res.Log.Error(err) 49 | return cli.NewExitError(err, -1) 50 | } 51 | 52 | if len(data) == 0 { 53 | return cli.NewExitError("No results were found for "+db, -1) 54 | } 55 | 56 | if c.Bool("human-readable") { 57 | err := showAgentsHuman(data) 58 | if err != nil { 59 | return cli.NewExitError(err.Error(), -1) 60 | } 61 | return nil 62 | } 63 | err = showAgents(data, c.String("delimiter")) 64 | if err != nil { 65 | return cli.NewExitError(err.Error(), -1) 66 | } 67 | return nil 68 | }, 69 | } 70 | bootstrapCommands(command) 71 | } 72 | 73 | func showAgents(agents []useragent.Result, delim string) error { 74 | headers := []string{"User Agent", "Times Used"} 75 | 76 | // Print the headers and analytic values, separated by a delimiter 77 | fmt.Println(strings.Join(headers, delim)) 78 | for _, agent := range agents { 79 | fmt.Println( 80 | strings.Join( 81 | []string{agent.UserAgent, i(agent.TimesUsed)}, 82 | delim, 83 | ), 84 | ) 85 | } 86 | return nil 87 | } 88 | 89 | func showAgentsHuman(agents []useragent.Result) error { 90 | table := tablewriter.NewWriter(os.Stdout) 91 | table.SetColWidth(100) 92 | table.SetHeader([]string{"User Agent", "Times Used"}) 93 | for _, agent := range agents { 94 | table.Append([]string{agent.UserAgent, i(agent.TimesUsed)}) 95 | } 96 | table.Render() 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /commands/test-config.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/activecm/rita-legacy/config" 8 | "github.com/activecm/rita-legacy/resources" 9 | 10 | "github.com/urfave/cli" 11 | yaml "gopkg.in/yaml.v2" 12 | ) 13 | 14 | func init() { 15 | command := cli.Command{ 16 | Flags: []cli.Flag{ConfigFlag}, 17 | Name: "test-config", 18 | Usage: "Check the configuration file for validity", 19 | Before: SetConfigFilePath, 20 | Action: testConfiguration, 21 | } 22 | 23 | allCommands = append(allCommands, command) 24 | } 25 | 26 | // testConfiguration prints out the result of parsing the config file 27 | func testConfiguration(c *cli.Context) error { 28 | // First, print out the config as it was parsed 29 | conf, err := config.LoadConfig(getConfigFilePath(c)) 30 | if err != nil { 31 | fmt.Fprintf(os.Stdout, "Failed to config: %s\n", err.Error()) 32 | os.Exit(-1) 33 | } 34 | 35 | staticConfig, err := yaml.Marshal(conf.S) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | tableConfig, err := yaml.Marshal(conf.T) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | fmt.Fprintf(os.Stdout, "\n%s\n", string(staticConfig)) 46 | fmt.Fprintf(os.Stdout, "\n%s\n", string(tableConfig)) 47 | 48 | // Then test initializing external resources like db connection and file handles 49 | resources.InitResources(getConfigFilePath(c)) 50 | 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /commands/update-check.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/activecm/rita-legacy/config" 10 | "github.com/activecm/rita-legacy/resources" 11 | "github.com/blang/semver" 12 | "github.com/google/go-github/github" 13 | log "github.com/sirupsen/logrus" 14 | "github.com/urfave/cli" 15 | ) 16 | 17 | // Strings used for informing the user of a new version. 18 | var informFmtStr = "\nTheres a new %s version of RITA %s available at:\nhttps://github.com/activecm/rita-legacy/releases\n" 19 | var versions = []string{"Major", "Minor", "Patch"} 20 | 21 | // GetVersionPrinter prints the version info 22 | func GetVersionPrinter() func(*cli.Context) { 23 | return func(c *cli.Context) { 24 | fmt.Printf("%s version %s\n", c.App.Name, c.App.Version) 25 | fmt.Println(updateCheck(getConfigFilePath(c))) 26 | } 27 | } 28 | 29 | // UpdateCheck Performs a check for the new version of RITA against the git repository and 30 | // returns a string indicating the new version if available 31 | func updateCheck(configFile string) string { 32 | res := resources.InitResources(configFile) 33 | delta := res.Config.S.UserConfig.UpdateCheckFrequency 34 | var newVersion semver.Version 35 | var err error 36 | var timestamp time.Time 37 | 38 | if delta <= 0 { 39 | return "" 40 | } 41 | 42 | //Check Logs for Versioning 43 | m := res.MetaDB 44 | timestamp, newVersion = m.LastCheck() 45 | 46 | days := time.Since(timestamp).Hours() / 24 47 | 48 | if days > float64(delta) { 49 | newVersion, err = getRemoteVersion() 50 | 51 | if err != nil { 52 | return "" 53 | } 54 | 55 | //Log checked version. 56 | res.Log.WithFields(log.Fields{ 57 | "Message": "Checking versions...", 58 | "LastUpdateCheck": time.Now(), 59 | "NewestVersion": fmt.Sprint(newVersion), 60 | }).Info("Checking for new version") 61 | 62 | } 63 | 64 | configVersion, err := semver.ParseTolerant(config.Version) 65 | if err != nil { 66 | return "" 67 | } 68 | 69 | if newVersion.GT(configVersion) { 70 | return informUser(configVersion, newVersion) 71 | } 72 | 73 | return "" 74 | } 75 | 76 | // Returns the first index where v1 is greater than v2 77 | func versionDiffIndex(v1 semver.Version, v2 semver.Version) int { 78 | 79 | if v1.Major > v2.Major { 80 | return 0 81 | } 82 | if v1.Minor > v2.Minor { 83 | return 1 84 | } 85 | 86 | return 2 87 | } 88 | 89 | func getRemoteVersion() (semver.Version, error) { 90 | client := github.NewClient(nil) 91 | refs, _, err := client.Git.GetRefs(context.Background(), "activecm", "rita", "refs/tags/v") 92 | 93 | if err == nil { 94 | s := strings.TrimPrefix(*refs[len(refs)-1].Ref, "refs/tags/") 95 | return semver.ParseTolerant(s) 96 | } 97 | return semver.Version{}, err 98 | } 99 | 100 | // Assembles a notice for the user informing them of an upgrade. 101 | // The return value is printed regardless so, "" is returned on errror. 102 | // func informUser( verStr string, index int ) string { 103 | func informUser(local semver.Version, remote semver.Version) string { 104 | return fmt.Sprintf(informFmtStr, 105 | versions[versionDiffIndex(remote, local)], 106 | fmt.Sprint(remote)) 107 | } 108 | -------------------------------------------------------------------------------- /commands/update_test.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/blang/semver" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestVersionCompare(t *testing.T) { 12 | 13 | Base, _ := semver.Parse("1.1.1") 14 | Major, _ := semver.Parse("2.3.4") 15 | Minor, _ := semver.Parse("1.2.3") 16 | Patch, _ := semver.Parse("1.1.9") 17 | 18 | assert.Equal(t, 0, versionDiffIndex(Major, Base), 19 | "Should return new Major") 20 | assert.Equal(t, 1, versionDiffIndex(Minor, Base), 21 | "Should return new Minor") 22 | assert.Equal(t, 2, versionDiffIndex(Patch, Base), 23 | "Should return new Patch") 24 | 25 | } 26 | 27 | func TestInformUser(t *testing.T) { 28 | 29 | Base, _ := semver.Parse("1.1.1") 30 | Major, _ := semver.Parse("2.3.4") 31 | assert.Equal(t, 32 | fmt.Sprintf(informFmtStr, "Major", "2.3.4"), 33 | informUser(Base, Major), 34 | "Should be identical strings") 35 | } 36 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | 7 | "github.com/creasty/defaults" 8 | ) 9 | 10 | // Version is filled at compile time with the git version of RITA 11 | // Version is filled by "git describe --abbrev=0 --tags" 12 | var Version = "undefined" 13 | 14 | // ExactVersion is filled at compile time with the git version of RITA 15 | // ExactVersion is filled by "git describe --always --long --dirty --tags" 16 | var ExactVersion = "undefined" 17 | 18 | type ( 19 | //Config holds the configuration for the running system 20 | Config struct { 21 | R RunningCfg 22 | S StaticCfg 23 | T TableCfg 24 | } 25 | ) 26 | 27 | // defaultConfigPath specifies the path of RITA's static config file 28 | const defaultConfigPath = "/etc/rita/config.yaml" 29 | 30 | // LoadConfig initializes a Config struct with values read 31 | // from a config file. It takes a string for the path to the file. 32 | // If the string is empty it uses the default path. 33 | func LoadConfig(customConfigPath string) (*Config, error) { 34 | // Use the default path unless a custom path is given 35 | configPath := defaultConfigPath 36 | if customConfigPath != "" { 37 | configPath = customConfigPath 38 | } 39 | 40 | config := &Config{} 41 | 42 | // Initialize table config to the default values 43 | if err := defaults.Set(&config.T); err != nil { 44 | return nil, err 45 | } 46 | 47 | // Initialize static config to the default values 48 | if err := defaults.Set(&config.S); err != nil { 49 | return nil, err 50 | } 51 | 52 | // Read the contents from the config file 53 | contents, err := readStaticConfigFile(configPath) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | // Deserialize the yaml file contents into the static config 59 | if err := parseStaticConfig(contents, &config.S); err != nil { 60 | return nil, err 61 | } 62 | 63 | // Use the static config to initialize the running config 64 | if err := initRunningConfig(&config.S, &config.R); err != nil { 65 | return nil, err 66 | } 67 | 68 | return config, nil 69 | } 70 | 71 | // expandConfig expands environment variables in config strings 72 | func expandConfig(reflected reflect.Value) { 73 | for i := 0; i < reflected.NumField(); i++ { 74 | f := reflected.Field(i) 75 | // process sub configs 76 | if f.Kind() == reflect.Struct { 77 | expandConfig(f) 78 | } else if f.Kind() == reflect.String { 79 | f.SetString(os.ExpandEnv(f.String())) 80 | } else if f.Kind() == reflect.Slice && f.Type().Elem().Kind() == reflect.String { 81 | strs := f.Interface().([]string) 82 | for i, str := range strs { 83 | strs[i] = os.ExpandEnv(str) 84 | } 85 | f.Set(reflect.ValueOf(strs)) 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | type TestStruct struct { 12 | InertString string 13 | ExpandString string 14 | ExpandStringSlice []string 15 | Inner TestStructInner 16 | } 17 | 18 | type TestStructInner struct { 19 | InertString string 20 | ExpandString string 21 | ExpandStringSlice []string 22 | } 23 | 24 | func TestExpandConfig(t *testing.T) { 25 | inert := "DO_NOT_CHANGE" 26 | outerEnvVarName := "_OUTER_ENV_VAR" 27 | outerEnvVarValue := "OUTER_VALUE" 28 | innerEnvVarName := "_INNER_ENV_VAR" 29 | innerEnvVarValue := "INNER_VALUE" 30 | test := TestStruct{ 31 | InertString: inert, 32 | ExpandString: "$" + outerEnvVarName, 33 | ExpandStringSlice: []string{"$" + outerEnvVarName, inert}, 34 | } 35 | innerStruct := TestStructInner{ 36 | InertString: inert, 37 | ExpandString: "$" + innerEnvVarName, 38 | ExpandStringSlice: []string{"$" + innerEnvVarName, inert}, 39 | } 40 | test.Inner = innerStruct 41 | 42 | os.Setenv(outerEnvVarName, outerEnvVarValue) 43 | os.Setenv(innerEnvVarName, innerEnvVarValue) 44 | assert.Equal(t, outerEnvVarValue, os.ExpandEnv("$"+outerEnvVarName)) 45 | assert.Equal(t, innerEnvVarValue, os.ExpandEnv("$"+innerEnvVarName)) 46 | expandConfig(reflect.ValueOf(&test).Elem()) 47 | 48 | assert.Equal(t, inert, test.InertString) 49 | assert.Equal(t, outerEnvVarValue, test.ExpandString) 50 | assert.Equal(t, innerEnvVarValue, test.Inner.ExpandString) 51 | os.Unsetenv(outerEnvVarName) 52 | os.Unsetenv(innerEnvVarName) 53 | } 54 | -------------------------------------------------------------------------------- /config/running.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "io/ioutil" 8 | 9 | "github.com/activecm/mgosec" 10 | "github.com/blang/semver" 11 | ) 12 | 13 | type ( 14 | //RunningCfg holds configuration options that are parsed at run time 15 | RunningCfg struct { 16 | MongoDB MongoDBRunningCfg 17 | Version semver.Version 18 | } 19 | 20 | //MongoDBRunningCfg holds parsed information for connecting to MongoDB 21 | MongoDBRunningCfg struct { 22 | AuthMechanismParsed mgosec.AuthMechanism 23 | TLS struct { 24 | TLSConfig *tls.Config 25 | } 26 | } 27 | ) 28 | 29 | // initRunningConfig uses data in the static config initialize 30 | // the passed in running config 31 | func initRunningConfig(static *StaticCfg, running *RunningCfg) error { 32 | var err error 33 | 34 | //parse the tls configuration 35 | if static.MongoDB.TLS.Enabled { 36 | tlsConf := &tls.Config{} 37 | if !static.MongoDB.TLS.VerifyCertificate { 38 | tlsConf.InsecureSkipVerify = true 39 | } 40 | if len(static.MongoDB.TLS.CAFile) > 0 { 41 | pem, err2 := ioutil.ReadFile(static.MongoDB.TLS.CAFile) 42 | err = err2 43 | if err != nil { 44 | fmt.Println("[!] Could not read MongoDB CA file") 45 | } else { 46 | tlsConf.RootCAs = x509.NewCertPool() 47 | tlsConf.RootCAs.AppendCertsFromPEM(pem) 48 | } 49 | } 50 | running.MongoDB.TLS.TLSConfig = tlsConf 51 | } 52 | 53 | //parse out the mongo authentication mechanism 54 | authMechanism, err := mgosec.ParseAuthMechanism( 55 | static.MongoDB.AuthMechanism, 56 | ) 57 | if err != nil { 58 | authMechanism = mgosec.None 59 | fmt.Println("[!] Could not parse MongoDB authentication mechanism") 60 | } 61 | running.MongoDB.AuthMechanismParsed = authMechanism 62 | 63 | running.Version, err = semver.ParseTolerant(static.Version) 64 | if err != nil { 65 | fmt.Println("\t[!] Version error: please ensure that you cloned the git repo and are using make to build.") 66 | fmt.Println("\t[!] See the following resources for further information:") 67 | fmt.Println("\t[>] https://github.com/activecm/rita-legacy/blob/master/Contributing.md#common-issues") 68 | fmt.Println("\t[>] https://github.com/activecm/rita-legacy/blob/master/docs/Manual%20Installation.md") 69 | } 70 | return err 71 | } 72 | -------------------------------------------------------------------------------- /config/tables.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type ( 4 | //TableCfg is the container for other table config sections 5 | TableCfg struct { 6 | Log LogTableCfg 7 | DNS DNSTableCfg 8 | Structure StructureTableCfg 9 | Beacon BeaconTableCfg 10 | BeaconSNI BeaconSNITableCfg 11 | BeaconProxy BeaconProxyTableCfg 12 | UserAgent UserAgentTableCfg 13 | Cert CertificateTableCfg 14 | Meta MetaTableCfg 15 | } 16 | 17 | //LogTableCfg contains the configuration for logging 18 | LogTableCfg struct { 19 | RitaLogTable string `default:"logs"` 20 | } 21 | 22 | //StructureTableCfg contains the names of the base level collections 23 | StructureTableCfg struct { 24 | ConnTable string `default:"conn"` 25 | DNSTable string `default:"dns"` 26 | HostTable string `default:"host"` 27 | HTTPTable string `default:"http"` 28 | OpenConnTable string `default:"openconn"` 29 | SSLTable string `default:"ssl"` 30 | UniqueConnTable string `default:"uconn"` 31 | UniqueConnProxyTable string `default:"uconnProxy"` 32 | SNIConnTable string `default:"SNIconn"` 33 | } 34 | 35 | //DNSTableCfg is used to control the dns analysis module 36 | DNSTableCfg struct { 37 | ExplodedDNSTable string `default:"explodedDns"` 38 | HostnamesTable string `default:"hostnames"` 39 | } 40 | 41 | //BeaconTableCfg is used to control the beaconing analysis module 42 | BeaconTableCfg struct { 43 | BeaconTable string `default:"beacon"` 44 | } 45 | 46 | //BeaconSNITableCfg is used to control the SNI beaconing analysis module 47 | BeaconSNITableCfg struct { 48 | BeaconSNITable string `default:"beaconSNI"` 49 | } 50 | 51 | //BeaconProxyTableCfg is used to control the beaconing analysis module 52 | BeaconProxyTableCfg struct { 53 | BeaconProxyTable string `default:"beaconProxy"` 54 | } 55 | 56 | //UserAgentTableCfg is used to control the useragent analysis module 57 | UserAgentTableCfg struct { 58 | UserAgentTable string `default:"useragent"` 59 | } 60 | 61 | //CertificateTableCfg is used to control the useragent analysis module 62 | CertificateTableCfg struct { 63 | CertificateTable string `default:"cert"` 64 | } 65 | 66 | //MetaTableCfg contains the meta db collection names 67 | MetaTableCfg struct { 68 | FilesTable string `default:"files"` 69 | DatabasesTable string `default:"databases"` 70 | } 71 | ) 72 | -------------------------------------------------------------------------------- /config/testing.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/creasty/defaults" 5 | ) 6 | 7 | const testConfig = ` 8 | MongoDB: 9 | ConnectionString: null 10 | AuthenticationMechanism: null 11 | SocketTimeout: 2 12 | TLS: 13 | Enable: false 14 | VerifyCertificate: false 15 | CAFile: null 16 | MetaDB: RITA-TEST-MetaDatabase 17 | LogConfig: 18 | LogLevel: 3 19 | RitaLogPath: null 20 | LogToFile: false 21 | LogToDB: true 22 | UserConfig: 23 | UpdateCheckFrequency: 14 24 | BlackListed: 25 | myIP.ms: false 26 | MalwareDomains.com: false 27 | MalwareDomainList.com: false 28 | CustomIPBlacklists: [] 29 | CustomHostnameBlacklists: [] 30 | DNS: 31 | Enabled: true 32 | Beacon: 33 | Enabled: true 34 | DefaultConnectionThresh: 23 35 | TimestampScoreWeight: 0.25 36 | DatasizeScoreWeight: 0.25 37 | DurationScoreWeight: 0.25 38 | HistogramScoreWeight: 0.25 39 | DurationMinHoursSeen: 6 40 | DurationConsistencyIdealHoursSeen: 12 41 | HistogramBimodalBucketSize: 0.05 42 | HistogramBimodalOutlierRemoval: 1 43 | HistogramBimodalMinHoursSeen: 11 44 | BeaconSNI: 45 | Enabled: true 46 | DefaultConnectionThresh: 23 47 | TimestampScoreWeight: 0.25 48 | DatasizeScoreWeight: 0.25 49 | DurationScoreWeight: 0.25 50 | HistogramScoreWeight: 0.25 51 | DurationMinHoursSeen: 6 52 | DurationConsistencyIdealHoursSeen: 12 53 | HistogramBimodalBucketSize: 0.05 54 | HistogramBimodalOutlierRemoval: 1 55 | HistogramBimodalMinHoursSeen: 11 56 | BeaconProxy: 57 | Enabled: true 58 | DefaultConnectionThresh: 23 59 | TimestampScoreWeight: 0.333 60 | DurationScoreWeight: 0.333 61 | HistogramScoreWeight: 0.333 62 | DurationMinHoursSeen: 6 63 | DurationConsistencyIdealHoursSeen: 12 64 | HistogramBimodalBucketSize: 0.05 65 | HistogramBimodalOutlierRemoval: 1 66 | HistogramBimodalMinHoursSeen: 11 67 | Strobe: 68 | ConnectionLimit: 250000 69 | Filtering: 70 | AlwaysInclude: ["8.8.8.8/32"] 71 | NeverInclude: ["8.8.4.4/32"] 72 | InternalSubnets: ["10.0.0.0/8","172.16.0.0/12","192.168.0.0/16"] 73 | HTTPProxyServers: ["1.1.1.1", "1.1.1.2/32", "1.2.0.0/16"] 74 | AlwaysIncludeDomain: ["bad.com", "google.com", "*.myotherdomain.com"] 75 | NeverIncludeDomain: ["good.com", "google.com", "*.mydomain.com"] 76 | FilterExternalToInternal: false 77 | ` 78 | 79 | // LoadTestingConfig loads the hard coded testing config 80 | func LoadTestingConfig(mongoURI string) (*Config, error) { 81 | config := &Config{} 82 | 83 | // Initialize table config to the default values 84 | if err := defaults.Set(&config.T); err != nil { 85 | return nil, err 86 | } 87 | 88 | // Initialize static config to the default values 89 | if err := defaults.Set(&config.S); err != nil { 90 | return nil, err 91 | } 92 | 93 | config.S.MongoDB.ConnectionString = mongoURI 94 | 95 | // Deserialize the yaml file contents into the static config 96 | if err := parseStaticConfig([]byte(testConfig), &config.S); err != nil { 97 | return nil, err 98 | } 99 | 100 | config.S.Version = "v0.0.0+testing" 101 | config.S.ExactVersion = "v0.0.0+testing" 102 | 103 | // Use the static config to initialize the running config 104 | if err := initRunningConfig(&config.S, &config.R); err != nil { 105 | return nil, err 106 | } 107 | 108 | return config, nil 109 | } 110 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | db: 5 | image: mongo:4.2 6 | volumes: 7 | - db:/data/db/ 8 | 9 | rita: 10 | image: quay.io/activecm/rita-legacy:${VERSION:-latest} 11 | build: . 12 | depends_on: 13 | - db 14 | volumes: 15 | - ${CONFIG:?You must provide a path to your CONFIG}:/etc/rita/config.yaml:ro 16 | - ${LOGS:?You must provide a path to your LOGS}:/logs:ro 17 | 18 | volumes: 19 | db: 20 | -------------------------------------------------------------------------------- /docs/Docker Usage.md: -------------------------------------------------------------------------------- 1 | # Docker Usage 2 | 3 | You can run RITA using Docker! You have several options depending on your specific needs. 4 | * [Running RITA with Docker Compose](#running-rita-with-docker-compose) - This is the simplest option and requires the least setup. You will have to provide your own Zeek logs. 5 | * [Running RITA with Docker Using External Mongo](#running-rita-with-docker-using-external-mongo) - This option is useful if you do not want to use Docker Compose or you have an external Mongo server you wish to use. 6 | * [Using Docker to Build RITA](#using-docker-to-build-rita) - You can use Docker to build a standalone RITA binary that runs on any Linux 64-bit CPU. This is useful if you want a portable binary but don't want to use Docker to actually run RITA. 7 | 8 | ## Obtaining the RITA Docker Image 9 | 10 | The easiest way is to pull down the pre-built image. 11 | 12 | ``` 13 | sudo docker pull quay.io/activecm/rita-legacy 14 | ``` 15 | 16 | You can also build the image from source. 17 | 18 | ``` 19 | sudo docker build -t quay.io/activecm/rita-legacy . 20 | ``` 21 | 22 | ## Running RITA with Docker Compose 23 | 24 | You will need a config file where you have [put in your `InternalSubnets`](../Readme.md#configuration-file). 25 | You will also need the path to your Zeek log files. 26 | 27 | ``` 28 | export CONFIG=/path/to/your/rita/config.yaml 29 | export LOGS=/path/to/your/zeek/logs 30 | sudo -E docker compose run --rm rita import /logs your-dataset 31 | ``` 32 | 33 | Note: If you'd like to use a specific version of RITA than the default `latest` you can do so using the `VERSION` variable. 34 | 35 | ``` 36 | export VERSION=v4.3.0 37 | sudo -E docker compose run --rm rita --version 38 | ``` 39 | 40 | ## Running RITA with Docker Using External Mongo 41 | 42 | If you don't need/want the convenience of Docker Compose running the Mongo server for you, you can also use RITA without it. You will need to modify RITA's config file to point to your external Mongo server and invoke RITA like this: 43 | 44 | ``` 45 | sudo docker run -it --rm \ 46 | -v /path/to/your/zeek/logs:/logs:ro \ 47 | -v /path/to/your/rita/config.yaml:/etc/rita/config.yaml:ro \ 48 | quay.io/activecm/rita-legacy import /logs your-dataset 49 | ``` 50 | 51 | ## Using Docker to Build RITA 52 | 53 | You can use Docker to build a statically linked RITA binary for you. This binary should be portable between Linux 64-bit systems. Once you've obtained the RITA docker image (see the "Obtaining the RITA Docker Image" section above) you can run the following commands to copy the binary to your host system. 54 | 55 | ``` 56 | sudo docker create --name rita quay.io/activecm/rita-legacy 57 | sudo docker cp rita:/rita ./rita 58 | sudo docker rm rita 59 | ``` 60 | 61 | Note that you will have to manually install the `config.yaml` files into `/etc/rita/` as well as create any directories referenced inside the `config.yaml` file. 62 | -------------------------------------------------------------------------------- /docs/Manual Installation.md: -------------------------------------------------------------------------------- 1 | 2 | ### Installation 3 | 4 | This guide walks through installing several components. 5 | 6 | * [Zeek](https://www.zeek.org) 7 | * [MongoDB](https://www.mongodb.com) 8 | * [RITA](https://github.com/activecm/rita-legacy/) 9 | 10 | #### Zeek 11 | 12 | Installing Zeek is recommended. RITA needs Zeek logs as input so if you already have Zeek or its logs you can skip installing Zeek. 13 | 14 | 1. Follow the directions at https://zeek.org/get-zeek/. 15 | 1. Use the [quick start guide](https://docs.zeek.org/en/current/quickstart/index.html) to configure. 16 | 17 | #### MongoDB 18 | 19 | RITA requires Mongo for storing and processing data. The current supported version is 4.2, but anything >= 4.0.0 may work. 20 | 21 | 1. Follow the MongoDB installation guide at https://docs.mongodb.com/v4.2/installation/ 22 | * Alternatively, this is a direct link to the [download page](https://www.mongodb.com/try/download/community). Be sure to choose version 4.2 23 | 1. Ensure MongoDB is running before running RITA. 24 | 25 | #### RITA 26 | 27 | You have a few options for installing RITA. 28 | 1. The main install script. You can disable Zeek and Mongo from being installed with the `--disable-zeek` and `--disable-mongo` flags. 29 | 1. A prebuilt binary is available for download on [RITA's release page](https://github.com/activecm/rita-legacy/releases). In this case you will need to download the config file from the same release and create some directories manually, as described below in the "Configuring the system" section. 30 | 1. Compile RITA manually from source. See below. 31 | 32 | ##### Installing Golang 33 | 34 | In order to compile RITA manually you will need to install [Golang](https://golang.org/doc/install) (v1.13 or greater). 35 | 36 | ##### Building RITA 37 | 38 | At this point you can build RITA from source code. 39 | 40 | 1. ```git clone https://github.com/activecm/rita-legacy.git``` 41 | 1. ```cd rita``` 42 | 1. ```make``` (Note that you will need to have `make` installed. You can use your system's package manager to install it.) 43 | 44 | This will yield a `rita` binary in the current directory. You can use `make install` to install the binary to `/usr/local/bin/rita` or `PREFIX=/ make install` to install to a different location (`/bin/rita` in this case). 45 | 46 | ##### Configuring the system 47 | 48 | RITA requires a few directories to be created for it to function correctly. 49 | 50 | 1. ```sudo mkdir /etc/rita && sudo chmod 755 /etc/rita``` 51 | 1. ```sudo mkdir -p /var/lib/rita/logs && sudo chmod -R 755 /var/lib/rita``` 52 | 1. ```sudo chmod 777 /var/lib/rita/logs``` 53 | 54 | Copy the config file from your local RITA source code. 55 | * ```sudo cp etc/rita.yaml /etc/rita/config.yaml && sudo chmod 666 /etc/rita/config.yaml``` 56 | 57 | At this point, you can modify the config file as needed and test using the ```rita test-config``` command. There will be empty quotes or 0's assigned to empty fields. [RITA's readme](../Readme.md#configuration-file) has more information on changing the configuration. 58 | -------------------------------------------------------------------------------- /docs/Releases.md: -------------------------------------------------------------------------------- 1 | # Releases 2 | 3 | Steps for creating a RITA release. 4 | 5 | - Update the `install.sh` script so that the `_RITA_VERSION` variable reflects the desired version tag 6 | - Create a branch with this change and go through the pull request process 7 | - Ensure that all tests pass on this branch 8 | - Note: after merging this pull request, the master install script will break until the release files are built and uploaded. 9 | 10 | - Go to the [releases](https://github.com/activecm/rita-legacy/releases) page 11 | - Click `Draft a new release` or pick the already existing draft 12 | - Select the new `[version]` tag 13 | - Fill out the title and description with recent changes 14 | - If the config file changed, give a thorough description of the needed changes 15 | - Publish the release 16 | 17 | - Keep refreshing the release page until the `rita` binary and `install.sh` script are automatically added. You can keep an eye on the progress on the [actions page](https://github.com/activecm/rita-legacy/actions) 18 | -------------------------------------------------------------------------------- /docs/Rolling Datasets.md: -------------------------------------------------------------------------------- 1 | # Rolling Datsets 2 | 3 | Please see the readme section on [rolling datasets](../Readme.md#rolling-datasets) for the simplest and most common use case. The following section covers the various options you can customize and more complicated use cases. 4 | 5 | Each rolling dataset has a total number of chunks it can hold before it rotates data out. For instance, if the dataset currently contains 24 chunks of data and is set to hold a max of 24 chunks then the next chunk to be imported will automatically remove the first chunk before brining the new data in. This will result in a database that still contains 24 chunks. If each chunk contains an hour of data your dataset will have 24 hours of data in it. You can specify the number of chunks manually with `--numchunks` when creating a rolling database but if this is omitted RITA will use the `Rolling: DefaultChunks` value from the config file. 6 | 7 | Likewise, when importing a new chunk you can specify a chunk number that you wish to replace in a dataset with `--chunk`. If you leave this off RITA will auto-increment the chunk for you. The chunk must be 0 (inclusive) through the total number of chunks (exclusive). This must be between 0 (inclusive) and the total number of chunks (exclusive). You will get an error if you try to use a chunk number greater or equal to the total number of chunks. 8 | 9 | All files and folders that you give RITA to import will be imported into a single chunk. This could be 1 hour, 2 hours, 10 hours, 24 hours, or more. RITA doesn't care how much data is in each chunk so even though it's normal for each chunk to represent the same amount of time, each chunk could have a different number of hours of logs. This means that you can run RITA on a regular interval without worrying if systems were offline for a little while or the data was delayed. You might get a little more or less data than you intended but as time passes and new data is added it will slowly correct itself. 10 | 11 | **Example:** If you wanted to have a dataset with a week's worth of data you could run the following rita command once per day. 12 | ``` 13 | rita import --rolling --numchunks 7 /opt/bro/logs/current week-dataset 14 | ``` 15 | This would import a day's worth of data into each chunk and you'd get a week's in total. After the first 7 days were imported, the dataset would rotate out old data to keep the most recent 7 days' worth of data. Note that you'd have to make sure new logs were being added to in `/opt/bro/logs/current` in this example. 16 | 17 | **Example:** If you wanted to have a dataset with 48 hours of data you could run the following rita command every hour. 18 | ``` 19 | rita import --rolling --numchunks 48 /opt/bro/logs/current 48-hour-dataset 20 | ``` 21 | -------------------------------------------------------------------------------- /docs/System Requirements.md: -------------------------------------------------------------------------------- 1 | # System Requirements 2 | 3 | * Operating System - The preferred platform is 64-bit Ubuntu 18.04 LTS. The system should be patched and up to date using apt-get. 4 | * The automated installer will also support Security Onion and CentOS 7. You can install on other operating systems using [docker](Docker%20Usage.md) or our [manual installation](Manual%20Installation.md). 5 | 6 | If RITA is used on a separate system from Zeek our recommended specs are: 7 | * Processor - Two or more cores. RITA uses parallel processing and benefits from more CPU cores. 8 | * Memory - 16GB. Larger datasets may require more memory. 9 | * Storage - RITA's datasets are significantly smaller than the Zeek logs so storage requirements are minimal compared to retaining the Zeek log files. 10 | 11 | 12 | ## Zeek 13 | The following requirements apply to the Zeek system. 14 | 15 | * Processor - Two cores plus an additional core for every 100 Mb of traffic being captured. (three cores minimum). This should be dedicated hardware, as resource congestion with other VMs can cause packets to be dropped or missed. 16 | * Memory - 16GB minimum. 64GB if monitoring 100Mb or more of network traffic. 128GB if monitoring 1Gb or more of network traffic. 17 | * Storage - 300GB minimum. 1TB or more is recommended to reduce log maintenance. 18 | * Network - In order to capture traffic with Zeek, you will need at least 2 network interface cards (NICs). One will be for management of the system and the other will be the dedicated capture port. Intel NICs perform well and are recommended. 19 | -------------------------------------------------------------------------------- /docs/Upgrading.md: -------------------------------------------------------------------------------- 1 | # Upgrading RITA 2 | 3 | > :exclamation: **IMPORTANT** :exclamation: 4 | > If you are upgrading from a version prior to v2 you will need to modify your config file to include values for `Filtering: InternalSubnets`. 5 | 6 | ## Upgrading Between Major Versions 7 | 8 | If you are upgrading across major versions (e.g. v1.x.x to v2.x.x, or v2.x.x to v3.x.x), you will need to delete your existing datasets and re-import them. Major version bumps typically bring massive performance gains as well as new features at the cost of removing compatibility for older datasets. 9 | 10 | You will likely need to update your config file as well. See the [Updating RITA's Config File](#updating-ritas-config-file) section below. 11 | 12 | ## Upgrading Between Minor or Patch Versions 13 | 14 | If you are upgrading within the same major version (e.g. v2.0.0 to v2.0.1, or v3.0.0 to v3.1.0) all you need to do is download the newest RITA binary and replace the one on your system. Alternatively, you can download and run the `install.sh` file from the [releases page](https://github.com/activecm/rita-legacy/releases) that corresponds with the version of RITA you wish to install. 15 | 16 | You may not need to update your config file at all as RITA includes sane default settings for any new config value. This means that if your config file is missing a value, RITA will still have a default to use. However, if you need to customize any of these new values you'll have to update to the newer config file. 17 | 18 | ## Updating RITA's Config File 19 | 20 | In some cases you may also need to update your config file to a newer version. You can always find the latest config file in [`etc/rita.yml`](https://github.com/activecm/rita-legacy/blob/master/etc/rita.yaml). If you use the `install.sh` script, the correct version of the config file will be downloaded for you to `/etc/rita/config.yaml.new`. 21 | 22 | To update the config file, transfer over any values you customized in your existing config to the equivalent section of the new config. Then save a backup of your existing `/etc/rita/config.yaml` before you replace it with the new version. 23 | 24 | Here are other useful tips for comparing differences between configs: 25 | * Check the release notes for each of the versions of RITA for details on config file changes. 26 | * Run `diff /etc/rita/config.yaml /etc/rita/config.yaml.new` to see a summary of both your customizations and any changes to the new config. 27 | * Use `rita test-config` to see the config values RITA is using while it runs. This includes any default values set when your config file doesn't specify them. You can also specify a custom config file to further compare the differences like this: `rita test-config --config /etc/rita/config.yaml.new`. -------------------------------------------------------------------------------- /etc/bash_completion.d/rita: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This file comes from: 4 | # https://github.com/urfave/cli/blob/v1.20.0/autocomplete/bash_autocomplete 5 | # Update to a newer version as needed. 6 | 7 | : ${PROG:=$(basename ${BASH_SOURCE})} 8 | 9 | _cli_bash_autocomplete() { 10 | local cur opts base 11 | COMPREPLY=() 12 | cur="${COMP_WORDS[COMP_CWORD]}" 13 | opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion ) 14 | COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) 15 | return 0 16 | } 17 | 18 | complete -F _cli_bash_autocomplete $PROG 19 | 20 | unset PROG 21 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/activecm/rita-legacy 2 | 3 | go 1.22.0 4 | 5 | require ( 6 | github.com/activecm/mgorus v0.1.1 7 | github.com/activecm/mgosec v0.1.1 8 | github.com/activecm/rita-bl v0.0.0-20220823191806-f014db21453d 9 | github.com/blang/semver v3.5.1+incompatible 10 | github.com/creasty/defaults v1.7.0 11 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 12 | github.com/google/go-github v17.0.0+incompatible 13 | github.com/google/uuid v1.6.0 14 | github.com/json-iterator/go v1.1.12 15 | github.com/olekukonko/tablewriter v0.0.5 16 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 17 | github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 18 | github.com/sirupsen/logrus v1.9.3 19 | github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 20 | github.com/stretchr/testify v1.9.0 21 | github.com/urfave/cli v1.22.15 22 | github.com/vbauerster/mpb v3.4.0+incompatible 23 | gopkg.in/yaml.v2 v2.4.0 24 | ) 25 | 26 | require ( 27 | github.com/VividCortex/ewma v1.2.0 // indirect 28 | github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect 29 | github.com/davecgh/go-spew v1.1.1 // indirect 30 | github.com/golang/protobuf v1.1.0 // indirect 31 | github.com/google/go-querystring v1.1.0 // indirect 32 | github.com/google/safebrowsing v0.0.0-20171128203709-fe6951d7ef01 // indirect 33 | github.com/kr/text v0.2.0 // indirect 34 | github.com/mattn/go-isatty v0.0.20 // indirect 35 | github.com/mattn/go-runewidth v0.0.9 // indirect 36 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect 37 | github.com/modern-go/reflect2 v1.0.2 // indirect 38 | github.com/pmezard/go-difflib v1.0.0 // indirect 39 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 40 | golang.org/x/crypto v0.24.0 // indirect 41 | golang.org/x/net v0.21.0 // indirect 42 | golang.org/x/sys v0.21.0 // indirect 43 | golang.org/x/term v0.21.0 // indirect 44 | golang.org/x/text v0.16.0 // indirect 45 | gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect 46 | gopkg.in/yaml.v3 v3.0.1 // indirect 47 | ) 48 | -------------------------------------------------------------------------------- /parser/dns.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "net" 5 | 6 | "github.com/activecm/rita-legacy/parser/parsetypes" 7 | "github.com/activecm/rita-legacy/pkg/data" 8 | "github.com/activecm/rita-legacy/pkg/hostname" 9 | 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | func parseDNSEntry(parseDNS *parsetypes.DNS, filter filter, retVals ParseResults, logger *log.Logger) { 14 | // get source destination pair 15 | src := parseDNS.Source 16 | dst := parseDNS.Destination 17 | 18 | // parse addresses into binary format 19 | srcIP := net.ParseIP(src) 20 | dstIP := net.ParseIP(dst) 21 | 22 | // verify that both addresses were able to be parsed successfully 23 | if (srcIP == nil) || (dstIP == nil) { 24 | logger.WithFields(log.Fields{ 25 | "uid": parseDNS.UID, 26 | "src": parseDNS.Source, 27 | "dst": parseDNS.Destination, 28 | }).Error("Unable to parse valid ip address pair from dns log entry, skipping entry.") 29 | return 30 | } 31 | 32 | // get domain 33 | domain := parseDNS.Query 34 | 35 | // Run domain through filter to filter out certain domains and 36 | // filter out traffic which is external -> external or external -> internal (if specified in the config file) 37 | ignore := (filter.filterDomain(domain) || filter.filterDNSPair(srcIP, dstIP)) 38 | 39 | // If domain is not subject to filtering, process 40 | if ignore { 41 | return 42 | } 43 | 44 | srcUniqIP := data.NewUniqueIP(srcIP, parseDNS.AgentUUID, parseDNS.AgentHostname) 45 | 46 | updateExplodedDNSbyDNS(domain, retVals) 47 | updateHostnamesByDNS(srcUniqIP, domain, parseDNS, retVals) 48 | } 49 | 50 | func updateExplodedDNSbyDNS(domain string, retVals ParseResults) { 51 | 52 | retVals.ExplodedDNSLock.Lock() 53 | defer retVals.ExplodedDNSLock.Unlock() 54 | 55 | retVals.ExplodedDNSMap[domain]++ 56 | } 57 | 58 | func updateHostnamesByDNS(srcUniqIP data.UniqueIP, domain string, parseDNS *parsetypes.DNS, retVals ParseResults) { 59 | 60 | retVals.HostnameLock.Lock() 61 | defer retVals.HostnameLock.Unlock() 62 | 63 | if _, ok := retVals.HostnameMap[domain]; !ok { 64 | retVals.HostnameMap[domain] = &hostname.Input{ 65 | Host: domain, 66 | ClientIPs: make(data.UniqueIPSet), 67 | ResolvedIPs: make(data.UniqueIPSet), 68 | } 69 | } 70 | 71 | // ///// UNION SOURCE HOST INTO HOSTNAME CLIENT SET ///// 72 | retVals.HostnameMap[domain].ClientIPs.Insert(srcUniqIP) 73 | 74 | // ///// UNION HOST ANSWERS INTO HOSTNAME RESOLVED HOST SET ///// 75 | if parseDNS.QTypeName == "A" { 76 | for _, answer := range parseDNS.Answers { 77 | answerIP := net.ParseIP(answer) 78 | // Check if answer is an IP address and store it if it is 79 | if answerIP != nil { 80 | answerUniqIP := data.NewUniqueIP(answerIP, parseDNS.AgentUUID, parseDNS.AgentHostname) 81 | retVals.HostnameMap[domain].ResolvedIPs.Insert(answerUniqIP) 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /parser/files/repository.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "time" 5 | 6 | pt "github.com/activecm/rita-legacy/parser/parsetypes" 7 | "github.com/globalsign/mgo/bson" 8 | ) 9 | 10 | // BroHeader contains the parse information contained within the comment lines 11 | // of Zeek files 12 | type BroHeader struct { 13 | Names []string // Names of fields 14 | Types []string // Types of fields 15 | Separator string // Field separator 16 | SetSep string // Set separator 17 | Empty string // Empty field tag 18 | Unset string // Unset field tag 19 | ObjType string // Object type (comes from #path) 20 | } 21 | 22 | // ZeekHeaderIndexMap maps the indexes of the fields in the ZeekHeader to the respective 23 | // indexes in the parsetype.BroData structs 24 | type ZeekHeaderIndexMap struct { 25 | NthLogFieldExistsInParseType []bool 26 | NthLogFieldParseTypeOffset []int 27 | } 28 | 29 | // IndexedFile ties a file to a target collection and database 30 | type IndexedFile struct { 31 | ID bson.ObjectId `bson:"_id,omitempty"` 32 | Path string `bson:"filepath"` 33 | Length int64 `bson:"length"` 34 | ModTime time.Time `bson:"modified"` 35 | Hash string `bson:"hash"` 36 | TargetCollection string `bson:"collection"` 37 | TargetDatabase string `bson:"database"` 38 | CID int `bson:"cid"` 39 | ParseTime time.Time `bson:"time_complete"` 40 | header *BroHeader 41 | broDataFactory func() pt.BroData 42 | fieldMap ZeekHeaderIndexMap 43 | json bool 44 | } 45 | 46 | //The following functions are for interacting with the private data in 47 | //IndexedFile as if it were public. The fields are private so they don't get 48 | //marshalled into MongoDB 49 | 50 | // IsJSON returns whether the file is a json file 51 | func (i *IndexedFile) IsJSON() bool { 52 | return i.json 53 | } 54 | 55 | // SetJSON sets the json flag 56 | func (i *IndexedFile) SetJSON() { 57 | i.json = true 58 | } 59 | 60 | // SetHeader sets the broHeader on the indexed file 61 | func (i *IndexedFile) SetHeader(header *BroHeader) { 62 | i.header = header 63 | } 64 | 65 | // GetHeader retrieves the broHeader on the indexed file 66 | func (i *IndexedFile) GetHeader() *BroHeader { 67 | return i.header 68 | } 69 | 70 | // SetBroDataFactory sets the function which makes broData corresponding 71 | // with this type of Zeek file 72 | func (i *IndexedFile) SetBroDataFactory(broDataFactory func() pt.BroData) { 73 | i.broDataFactory = broDataFactory 74 | } 75 | 76 | // GetBroDataFactory retrieves the function which makes broData corresponding 77 | // with this type of Zeek file 78 | func (i *IndexedFile) GetBroDataFactory() func() pt.BroData { 79 | return i.broDataFactory 80 | } 81 | 82 | // SetFieldMap sets the map which maps the indexes of Zeek fields in the log header to the indexes 83 | // in their respective broData structs 84 | func (i *IndexedFile) SetFieldMap(fieldMap ZeekHeaderIndexMap) { 85 | i.fieldMap = fieldMap 86 | } 87 | 88 | // GetFieldMap retrieves the map which maps the indexes of Zeek fields in the log header to the indexes 89 | // in their respective broData structs 90 | func (i *IndexedFile) GetFieldMap() ZeekHeaderIndexMap { 91 | return i.fieldMap 92 | } 93 | -------------------------------------------------------------------------------- /parser/parsetypes/parsetypes_test.go: -------------------------------------------------------------------------------- 1 | package parsetypes 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestNewBroDataFactory(t *testing.T) { 10 | 11 | testCasesIn := []string{"conn", "http", "dns", "httpa", "http_a", "http_eth0", "httpasdf12345=-ASDF?", "open_conn", "ASDF"} 12 | testCasesOut := []BroData{&Conn{}, &HTTP{}, &DNS{}, &HTTP{}, &HTTP{}, &HTTP{}, &HTTP{}, &OpenConn{}, nil} 13 | for i := range testCasesIn { 14 | factory := NewBroDataFactory(testCasesIn[i]) 15 | if factory == nil { 16 | require.Nil(t, testCasesOut[i]) 17 | } else { 18 | require.Equal(t, testCasesOut[i], factory()) 19 | } 20 | } 21 | } 22 | 23 | func TestConvertTimestamp(t *testing.T) { 24 | testCases := []struct { 25 | input interface{} 26 | expected int64 27 | }{ 28 | {1517336042.090842, 1517336042}, 29 | {1517336042, 1517336042}, 30 | {"2018-01-30T18:14:02Z", 1517336042}, 31 | {0, 0}, 32 | {"", 0}, 33 | {nil, 0}, 34 | } 35 | 36 | for _, testCase := range testCases { 37 | actual := convertTimestamp(testCase.input) 38 | require.Equal(t, testCase.expected, actual, "input: %v", testCase.input) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /parser/repository.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/activecm/rita-legacy/pkg/certificate" 7 | "github.com/activecm/rita-legacy/pkg/data" 8 | "github.com/activecm/rita-legacy/pkg/host" 9 | "github.com/activecm/rita-legacy/pkg/hostname" 10 | "github.com/activecm/rita-legacy/pkg/sniconn" 11 | "github.com/activecm/rita-legacy/pkg/uconn" 12 | "github.com/activecm/rita-legacy/pkg/uconnproxy" 13 | "github.com/activecm/rita-legacy/pkg/useragent" 14 | ) 15 | 16 | // ParseResults contains the data which the analysis packages 17 | // expect from the parser as well as locks for safely 18 | // accessing the data from multipel goroutines. 19 | type ParseResults struct { 20 | UniqueConnMap map[string]*uconn.Input 21 | UniqueConnLock *sync.Mutex 22 | ProxyUniqueConnMap map[string]*uconnproxy.Input 23 | ProxyUniqueConnLock *sync.Mutex 24 | HostMap map[string]*host.Input 25 | HostLock *sync.Mutex 26 | HostnameMap map[string]*hostname.Input 27 | HostnameLock *sync.Mutex 28 | UseragentMap map[string]*useragent.Input 29 | UseragentLock *sync.Mutex 30 | CertificateMap map[string]*certificate.Input 31 | CertificateLock *sync.Mutex 32 | ExplodedDNSMap map[string]int 33 | ExplodedDNSLock *sync.Mutex 34 | TLSConnMap map[string]*sniconn.TLSInput 35 | TLSConnLock *sync.Mutex 36 | HTTPConnMap map[string]*sniconn.HTTPInput 37 | HTTPConnLock *sync.Mutex 38 | ZeekUIDMap map[string]*data.ZeekUIDRecord 39 | ZeekUIDLock *sync.Mutex 40 | } 41 | 42 | // newParseResults instantiates a ParseResults struct 43 | func newParseResults() ParseResults { 44 | return ParseResults{ 45 | UniqueConnMap: make(map[string]*uconn.Input), 46 | UniqueConnLock: new(sync.Mutex), 47 | ProxyUniqueConnMap: make(map[string]*uconnproxy.Input), 48 | ProxyUniqueConnLock: new(sync.Mutex), 49 | HostMap: make(map[string]*host.Input), 50 | HostLock: new(sync.Mutex), 51 | HostnameMap: make(map[string]*hostname.Input), 52 | HostnameLock: new(sync.Mutex), 53 | UseragentMap: make(map[string]*useragent.Input), 54 | UseragentLock: new(sync.Mutex), 55 | CertificateMap: make(map[string]*certificate.Input), 56 | CertificateLock: new(sync.Mutex), 57 | ExplodedDNSMap: make(map[string]int), 58 | ExplodedDNSLock: new(sync.Mutex), 59 | TLSConnMap: make(map[string]*sniconn.TLSInput), 60 | TLSConnLock: new(sync.Mutex), 61 | HTTPConnMap: make(map[string]*sniconn.HTTPInput), 62 | HTTPConnLock: new(sync.Mutex), 63 | ZeekUIDMap: make(map[string]*data.ZeekUIDRecord), 64 | ZeekUIDLock: new(sync.Mutex), 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/beacon/mongodb_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package beacon 5 | 6 | import ( 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | 11 | "github.com/activecm/rita-legacy/pkg/data" 12 | "github.com/activecm/rita-legacy/pkg/uconn" 13 | "github.com/activecm/rita-legacy/resources" 14 | "github.com/activecm/rita-legacy/util" 15 | "github.com/globalsign/mgo/dbtest" 16 | ) 17 | 18 | // Server holds the dbtest DBServer 19 | var Server dbtest.DBServer 20 | 21 | // Set the test database 22 | var testTargetDB = "tmp_test_db" 23 | 24 | var testRepo Repository 25 | 26 | var testHost = map[string]*uconn.Input{ 27 | "test": { 28 | Hosts: data.UniqueIPPair{ 29 | UniqueSrcIP: data.UniqueSrcIP{ 30 | SrcIP: "127.0.0.1", 31 | SrcNetworkUUID: util.UnknownPrivateNetworkUUID, 32 | SrcNetworkName: util.UnknownPrivateNetworkName, 33 | }, 34 | UniqueDstIP: data.UniqueDstIP{ 35 | DstIP: "127.0.0.1", 36 | DstNetworkUUID: util.UnknownPrivateNetworkUUID, 37 | DstNetworkName: util.UnknownPrivateNetworkName, 38 | }, 39 | }, 40 | ConnectionCount: 12, 41 | IsLocalSrc: true, 42 | IsLocalDst: true, 43 | TotalBytes: 123, 44 | TsList: []int64{1234567, 1234567}, 45 | OrigBytesList: []int64{12, 12}, 46 | TotalDuration: 123.0, 47 | MaxDuration: 12, 48 | }, 49 | } 50 | 51 | func TestUpsert(t *testing.T) { 52 | testRepo.Upsert(testHost, 1234560, 1234570) 53 | } 54 | 55 | // TestMain wraps all tests with the needed initialized mock DB and fixtures 56 | func TestMain(m *testing.M) { 57 | // Store temporary databases files in a temporary directory 58 | tempDir, _ := ioutil.TempDir("", "testing") 59 | Server.SetPath(tempDir) 60 | 61 | // Set the main session variable to the temporary MongoDB instance 62 | res := resources.InitTestResources() 63 | 64 | testRepo = NewMongoRepository(res.DB, res.Config, res.Log) 65 | 66 | // Run the test suite 67 | retCode := m.Run() 68 | 69 | // Shut down the temporary server and removes data on disk. 70 | Server.Stop() 71 | 72 | // call with result of m.Run() 73 | os.Exit(retCode) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/beacon/repository.go: -------------------------------------------------------------------------------- 1 | package beacon 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | "github.com/activecm/rita-legacy/pkg/host" 6 | "github.com/activecm/rita-legacy/pkg/uconn" 7 | ) 8 | 9 | // Repository for beacon collection 10 | type Repository interface { 11 | CreateIndexes() error 12 | Upsert(uconnMap map[string]*uconn.Input, hostMap map[string]*host.Input, minTimestamp, maxTimestamp int64) 13 | } 14 | 15 | // TSData ... 16 | type TSData struct { 17 | Score float64 `bson:"score"` 18 | Range int64 `bson:"range"` 19 | Mode int64 `bson:"mode"` 20 | ModeCount int64 `bson:"mode_count"` 21 | Skew float64 `bson:"skew"` 22 | Dispersion int64 `bson:"dispersion"` 23 | } 24 | 25 | // DSData ... 26 | type DSData struct { 27 | Score float64 `bson:"score"` 28 | Skew float64 `bson:"skew"` 29 | Dispersion int64 `bson:"dispersion"` 30 | Range int64 `bson:"range"` 31 | Mode int64 `bson:"mode"` 32 | ModeCount int64 `bson:"mode_count"` 33 | } 34 | 35 | // Result represents a beacon between two hosts. Contains information 36 | // on connection delta times and the amount of data transferred 37 | type Result struct { 38 | data.UniqueIPPair `bson:",inline"` 39 | Connections int64 `bson:"connection_count"` 40 | AvgBytes float64 `bson:"avg_bytes"` 41 | TotalBytes int64 `bson:"total_bytes"` 42 | Ts TSData `bson:"ts"` 43 | Ds DSData `bson:"ds"` 44 | DurScore float64 `bson:"duration_score"` 45 | HistScore float64 `bson:"hist_score"` 46 | Score float64 `bson:"score"` 47 | } 48 | 49 | // StrobeResult represents a unique connection with a large amount 50 | // of connections between the hosts 51 | type StrobeResult struct { 52 | data.UniqueIPPair `bson:",inline"` 53 | ConnectionCount int64 `bson:"connection_count"` 54 | } 55 | -------------------------------------------------------------------------------- /pkg/beacon/results.go: -------------------------------------------------------------------------------- 1 | package beacon 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/resources" 5 | "github.com/globalsign/mgo/bson" 6 | ) 7 | 8 | // Results finds beacons in the database greater than a given cutoffScore 9 | func Results(res *resources.Resources, cutoffScore float64) ([]Result, error) { 10 | ssn := res.DB.Session.Copy() 11 | defer ssn.Close() 12 | 13 | var beacons []Result 14 | 15 | beaconQuery := bson.M{"score": bson.M{"$gt": cutoffScore}} 16 | 17 | err := ssn.DB(res.DB.GetSelectedDB()).C(res.Config.T.Beacon.BeaconTable).Find(beaconQuery).Sort("-score").All(&beacons) 18 | 19 | return beacons, err 20 | } 21 | 22 | // StrobeResults finds strobes (beacons with an immense number of connections) in the database. 23 | // The results will be sorted by connection count ordered by sortDir (-1 or 1). 24 | // limit and noLimit control how many results are returned. 25 | func StrobeResults(res *resources.Resources, sortDir, limit int, noLimit bool) ([]StrobeResult, error) { 26 | ssn := res.DB.Session.Copy() 27 | defer ssn.Close() 28 | 29 | var strobes []StrobeResult 30 | 31 | strobeQuery := []bson.M{ 32 | {"$match": bson.M{"strobe": true}}, 33 | {"$unwind": "$dat"}, 34 | {"$project": bson.M{ 35 | "src": 1, 36 | "src_network_uuid": 1, 37 | "src_network_name": 1, 38 | "dst": 1, 39 | "dst_network_uuid": 1, 40 | "dst_network_name": 1, 41 | "conns": "$dat.count", 42 | }}, 43 | {"$group": bson.M{ 44 | "_id": "$_id", 45 | "src": bson.M{"$first": "$src"}, 46 | "src_network_uuid": bson.M{"$first": "$src_network_uuid"}, 47 | "src_network_name": bson.M{"$first": "$src_network_name"}, 48 | "dst": bson.M{"$first": "$dst"}, 49 | "dst_network_uuid": bson.M{"$first": "$dst_network_uuid"}, 50 | "dst_network_name": bson.M{"$first": "$dst_network_name"}, 51 | "connection_count": bson.M{"$sum": "$conns"}, 52 | }}, 53 | {"$sort": bson.M{"connection_count": sortDir}}, 54 | } 55 | 56 | if !noLimit { 57 | strobeQuery = append(strobeQuery, bson.M{"$limit": limit}) 58 | } 59 | 60 | err := ssn.DB(res.DB.GetSelectedDB()).C(res.Config.T.Structure.UniqueConnTable).Pipe(strobeQuery).AllowDiskUse().All(&strobes) 61 | 62 | return strobes, err 63 | 64 | } 65 | -------------------------------------------------------------------------------- /pkg/beacon/sorter.go: -------------------------------------------------------------------------------- 1 | package beacon 2 | 3 | import ( 4 | "sort" 5 | "sync" 6 | 7 | "github.com/activecm/rita-legacy/config" 8 | "github.com/activecm/rita-legacy/database" 9 | "github.com/activecm/rita-legacy/pkg/uconn" 10 | "github.com/activecm/rita-legacy/util" 11 | ) 12 | 13 | type ( 14 | //sorter handles sorting the timestamp deltas and data sizes 15 | //of pairs of hosts in order to prepare the data for quantile based 16 | //statistical analysis 17 | sorter struct { 18 | db *database.DB // provides access to MongoDB 19 | conf *config.Config // contains details needed to access MongoDB 20 | sortedCallback func(*uconn.Input) // called on each sorted result 21 | closedCallback func() // called when .close() is called and no more calls to sortedCallback will be made 22 | sortChannel chan *uconn.Input // holds unsorted data 23 | sortWg sync.WaitGroup // wait for analysis to finish 24 | } 25 | ) 26 | 27 | // newSorter creates a new sorter which sorts unique connection data 28 | // for use in quantile based statistics 29 | func newSorter(db *database.DB, conf *config.Config, sortedCallback func(*uconn.Input), closedCallback func()) *sorter { 30 | return &sorter{ 31 | db: db, 32 | conf: conf, 33 | sortedCallback: sortedCallback, 34 | closedCallback: closedCallback, 35 | sortChannel: make(chan *uconn.Input), 36 | } 37 | } 38 | 39 | // collect gathers a chunk of data to be sorted 40 | func (s *sorter) collect(data *uconn.Input) { 41 | s.sortChannel <- data 42 | } 43 | 44 | // close waits for the sorter to finish 45 | func (s *sorter) close() { 46 | close(s.sortChannel) 47 | s.sortWg.Wait() 48 | s.closedCallback() 49 | } 50 | 51 | // start kicks off a new sorter thread 52 | func (s *sorter) start() { 53 | s.sortWg.Add(1) 54 | go func() { 55 | 56 | for data := range s.sortChannel { 57 | if (data.TsList) != nil { 58 | //sort the size and timestamps to compute quantiles in the analyzer 59 | sort.Sort(util.SortableInt64(data.TsList)) 60 | sort.Sort(util.SortableInt64(data.OrigBytesList)) 61 | } 62 | s.sortedCallback(data) 63 | } 64 | s.sortWg.Done() 65 | }() 66 | } 67 | -------------------------------------------------------------------------------- /pkg/beaconproxy/repository.go: -------------------------------------------------------------------------------- 1 | package beaconproxy 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | "github.com/activecm/rita-legacy/pkg/host" 6 | "github.com/activecm/rita-legacy/pkg/uconnproxy" 7 | "github.com/globalsign/mgo/bson" 8 | ) 9 | 10 | type ( 11 | 12 | // Repository for host collection 13 | Repository interface { 14 | CreateIndexes() error 15 | Upsert(uconnProxyMap map[string]*uconnproxy.Input, hostMap map[string]*host.Input, minTimestamp, maxTimestamp int64) 16 | } 17 | 18 | //TSData ... 19 | TSData struct { 20 | Score float64 `bson:"score"` 21 | Range int64 `bson:"range"` 22 | Mode int64 `bson:"mode"` 23 | ModeCount int64 `bson:"mode_count"` 24 | Skew float64 `bson:"skew"` 25 | Dispersion int64 `bson:"dispersion"` 26 | } 27 | 28 | //Result represents a beacon proxy between a source IP and 29 | // an fqdn. 30 | Result struct { 31 | FQDN string `bson:"fqdn"` 32 | SrcIP string `bson:"src"` 33 | SrcNetworkName string `bson:"src_network_name"` 34 | SrcNetworkUUID bson.Binary `bson:"src_network_uuid"` 35 | Connections int64 `bson:"connection_count"` 36 | Ts TSData `bson:"ts"` 37 | DurScore float64 `bson:"duration_score"` 38 | HistScore float64 `bson:"hist_score"` 39 | Score float64 `bson:"score"` 40 | Proxy data.UniqueIP `bson:"proxy"` 41 | } 42 | 43 | //StrobeResult represents a unique connection with a large amount 44 | //of connections between the hosts 45 | StrobeResult struct { 46 | data.UniqueSrcFQDNPair `bson:",inline"` 47 | ConnectionCount int64 `bson:"connection_count"` 48 | } 49 | ) 50 | -------------------------------------------------------------------------------- /pkg/beaconproxy/results.go: -------------------------------------------------------------------------------- 1 | package beaconproxy 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/resources" 5 | "github.com/globalsign/mgo/bson" 6 | ) 7 | 8 | // Results finds beacons FQDN in the database greater than a given cutoffScore 9 | func Results(res *resources.Resources, cutoffScore float64) ([]Result, error) { 10 | ssn := res.DB.Session.Copy() 11 | defer ssn.Close() 12 | 13 | var beaconsProxy []Result 14 | 15 | BeaconProxyQuery := bson.M{"score": bson.M{"$gt": cutoffScore}} 16 | 17 | err := ssn.DB(res.DB.GetSelectedDB()).C(res.Config.T.BeaconProxy.BeaconProxyTable).Find(BeaconProxyQuery).Sort("-score").All(&beaconsProxy) 18 | 19 | return beaconsProxy, err 20 | } 21 | -------------------------------------------------------------------------------- /pkg/beaconproxy/sorter.go: -------------------------------------------------------------------------------- 1 | package beaconproxy 2 | 3 | import ( 4 | "sort" 5 | "sync" 6 | 7 | "github.com/activecm/rita-legacy/config" 8 | "github.com/activecm/rita-legacy/database" 9 | "github.com/activecm/rita-legacy/pkg/uconnproxy" 10 | "github.com/activecm/rita-legacy/util" 11 | ) 12 | 13 | type ( 14 | sorter struct { 15 | db *database.DB // provides access to MongoDB 16 | conf *config.Config // contains details needed to access MongoDB 17 | sortedCallback func(*uconnproxy.Input) // called on each analyzed result 18 | closedCallback func() // called when .close() is called and no more calls to analyzedCallback will be made 19 | sortChannel chan *uconnproxy.Input // holds unanalyzed data 20 | sortWg sync.WaitGroup // wait for analysis to finish 21 | } 22 | ) 23 | 24 | // newsorter creates a new collector for gathering data 25 | func newSorter(db *database.DB, conf *config.Config, sortedCallback func(*uconnproxy.Input), closedCallback func()) *sorter { 26 | return &sorter{ 27 | db: db, 28 | conf: conf, 29 | sortedCallback: sortedCallback, 30 | closedCallback: closedCallback, 31 | sortChannel: make(chan *uconnproxy.Input), 32 | } 33 | } 34 | 35 | // collect sends a chunk of data to be analyzed 36 | func (s *sorter) collect(entry *uconnproxy.Input) { 37 | s.sortChannel <- entry 38 | } 39 | 40 | // close waits for the collector to finish 41 | func (s *sorter) close() { 42 | close(s.sortChannel) 43 | s.sortWg.Wait() 44 | s.closedCallback() 45 | } 46 | 47 | // start kicks off a new analysis thread 48 | func (s *sorter) start() { 49 | s.sortWg.Add(1) 50 | go func() { 51 | 52 | for entry := range s.sortChannel { 53 | 54 | if (entry.TsList) != nil { 55 | //sort the timestamp lists to compute quantiles in the analyzer 56 | sort.Sort(util.SortableInt64(entry.TsList)) 57 | sort.Sort(util.SortableInt64(entry.TsListFull)) 58 | } 59 | 60 | s.sortedCallback(entry) 61 | 62 | } 63 | s.sortWg.Done() 64 | }() 65 | } 66 | -------------------------------------------------------------------------------- /pkg/beaconsni/repository.go: -------------------------------------------------------------------------------- 1 | package beaconsni 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | "github.com/activecm/rita-legacy/pkg/host" 6 | "github.com/activecm/rita-legacy/pkg/sniconn" 7 | ) 8 | 9 | // Repository for beaconsni collection 10 | type Repository interface { 11 | CreateIndexes() error 12 | Upsert(tlsMap map[string]*sniconn.TLSInput, httpMap map[string]*sniconn.HTTPInput, hostMap map[string]*host.Input, minTimestamp, maxTimestamp int64) 13 | } 14 | 15 | type dissectorResults struct { 16 | Hosts data.UniqueSrcFQDNPair 17 | RespondingIPs []data.UniqueIP 18 | ConnectionCount int64 19 | TotalBytes int64 20 | TsList []int64 21 | TsListFull []int64 22 | OrigBytesList []int64 23 | } 24 | 25 | // Result represents an SNI beacon between a source IP and 26 | // an SNI. An SNI can be comprised of one or more destination IPs. 27 | // Contains information on connection delta times and the amount of data transferred 28 | type Result struct { 29 | data.UniqueSrcFQDNPair `bson:",inline"` 30 | Connections int64 `bson:"connection_count"` 31 | AvgBytes float64 `bson:"avg_bytes"` 32 | TotalBytes int64 `bson:"total_bytes"` 33 | Ts TSData `bson:"ts"` 34 | Ds DSData `bson:"ds"` 35 | DurScore float64 `bson:"duration_score"` 36 | HistScore float64 `bson:"hist_score"` 37 | Score float64 `bson:"score"` 38 | // ResolvedIPs []data.UniqueIP // Requires lookup on SNIconn collection 39 | } 40 | 41 | // TSData ... 42 | type TSData struct { 43 | Score float64 `bson:"score"` 44 | Range int64 `bson:"range"` 45 | Mode int64 `bson:"mode"` 46 | ModeCount int64 `bson:"mode_count"` 47 | Skew float64 `bson:"skew"` 48 | Dispersion int64 `bson:"dispersion"` 49 | Duration float64 `bson:"duration"` 50 | } 51 | 52 | // DSData ... 53 | type DSData struct { 54 | Score float64 `bson:"score"` 55 | Skew float64 `bson:"skew"` 56 | Dispersion int64 `bson:"dispersion"` 57 | Range int64 `bson:"range"` 58 | Mode int64 `bson:"mode"` 59 | ModeCount int64 `bson:"mode_count"` 60 | } 61 | -------------------------------------------------------------------------------- /pkg/beaconsni/results.go: -------------------------------------------------------------------------------- 1 | package beaconsni 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/resources" 5 | "github.com/globalsign/mgo/bson" 6 | ) 7 | 8 | // Results finds SNI beacons in the database greater than a given cutoffScore 9 | func Results(res *resources.Resources, cutoffScore float64) ([]Result, error) { 10 | ssn := res.DB.Session.Copy() 11 | defer ssn.Close() 12 | 13 | var beaconsSNI []Result 14 | 15 | beaconSNIQuery := bson.M{"score": bson.M{"$gt": cutoffScore}} 16 | 17 | err := ssn.DB(res.DB.GetSelectedDB()).C(res.Config.T.BeaconSNI.BeaconSNITable).Find(beaconSNIQuery).Sort("-score").All(&beaconsSNI) 18 | if err != nil { 19 | return beaconsSNI, err 20 | } 21 | 22 | return beaconsSNI, err 23 | } 24 | -------------------------------------------------------------------------------- /pkg/beaconsni/sorter.go: -------------------------------------------------------------------------------- 1 | package beaconsni 2 | 3 | import ( 4 | "sort" 5 | "sync" 6 | 7 | "github.com/activecm/rita-legacy/config" 8 | "github.com/activecm/rita-legacy/database" 9 | "github.com/activecm/rita-legacy/util" 10 | ) 11 | 12 | type ( 13 | //sorter handles sorting the timestamp deltas and data sizes 14 | //of pairs of hosts in order to prepare the data for quantile based 15 | //statistical analysis 16 | sorter struct { 17 | db *database.DB // provides access to MongoDB 18 | conf *config.Config // contains details needed to access MongoDB 19 | sortedCallback func(dissectorResults) // called on each sorted result 20 | closedCallback func() // called when .close() is called and no more calls to sortedCallback will be made 21 | sortChannel chan *dissectorResults // holds unsorted data 22 | sortWg sync.WaitGroup // wait for analysis to finish 23 | } 24 | ) 25 | 26 | // newSorter creates a new sorter which sorts SNI connection data 27 | // for use in quantile based statistics 28 | func newSorter(db *database.DB, conf *config.Config, sortedCallback func(dissectorResults), closedCallback func()) *sorter { 29 | return &sorter{ 30 | db: db, 31 | conf: conf, 32 | sortedCallback: sortedCallback, 33 | closedCallback: closedCallback, 34 | sortChannel: make(chan *dissectorResults), 35 | } 36 | } 37 | 38 | // collect gathers a chunk of data to be sorted 39 | func (s *sorter) collect(data *dissectorResults) { 40 | s.sortChannel <- data 41 | } 42 | 43 | // close waits for the sorter to finish 44 | func (s *sorter) close() { 45 | close(s.sortChannel) 46 | s.sortWg.Wait() 47 | s.closedCallback() 48 | } 49 | 50 | // start kicks off a new sorter thread 51 | func (s *sorter) start() { 52 | s.sortWg.Add(1) 53 | go func() { 54 | 55 | for data := range s.sortChannel { 56 | if (data.TsList) != nil { 57 | //sort the size and timestamps to compute quantiles in the analyzer 58 | sort.Sort(util.SortableInt64(data.TsList)) 59 | sort.Sort(util.SortableInt64(data.TsListFull)) 60 | sort.Sort(util.SortableInt64(data.OrigBytesList)) 61 | } 62 | s.sortedCallback(*data) 63 | } 64 | s.sortWg.Done() 65 | }() 66 | } 67 | -------------------------------------------------------------------------------- /pkg/blacklist/Readme.md: -------------------------------------------------------------------------------- 1 | ## Threat Intelligence Package 2 | 3 | *Documented on June 7, 2022* 4 | 5 | --- 6 | 7 | This packages summarizes the connections made between internal hosts and external hosts which appear on threat intelligence lists. 8 | 9 | ## Package Outputs 10 | 11 | ### Peer Connection Summary 12 | Inputs: 13 | - MongoDB `host` collection: 14 | - Field: `ip` 15 | - Type: string 16 | - Field: `network_uuid` 17 | - Type: UUID 18 | - Field: `network_name` 19 | - Type: string 20 | - Field: `blacklisted` 21 | - Type: bool 22 | - MongoDB `uconn` collection: 23 | - Field: `src` 24 | - Type: string 25 | - Field: `src_network_uuid` 26 | - Type: UUID 27 | - Field: `src_network_name` 28 | - Type: string 29 | - Field: `dst` 30 | - Type: string 31 | - Field: `dst_network_uuid` 32 | - Type: UUID 33 | - Field: `dst_network_name` 34 | - Type: string 35 | - Array Field: `dat` 36 | - Field: `count` 37 | - Type: int 38 | - Field: `tbytes` 39 | - Type: int 40 | - Field: `open_bytes` 41 | - Type: int 42 | - Field: `open_connection_count` 43 | - Type: int 44 | 45 | Outputs: 46 | - MongoDB `host` collection: 47 | - Array Field: `dat` 48 | - Field: `bl` 49 | - Field: `ip` 50 | - Type: string 51 | - Field: `network_uuid` 52 | - Type: UUID 53 | - Field: `network_name` 54 | - Type: string 55 | - Field: `bl_conn_count` 56 | - Type: int 57 | - Field: `bl_total_bytes` 58 | - Type: int 59 | - Field: `bl_in_count` 60 | - Type: int 61 | - Field: `bl_out_count` 62 | - Type: int 63 | - Field: `cid` 64 | - Type: int 65 | 66 | After the `host` package and `uconn` packages run, the threat intelligence package creates summaries for the hosts which were the connection peers of other hosts which were marked as unsafe. 67 | 68 | Unsafe hosts are first gathered using the `host` collection. Then, the connection peers of these hosts are then queried using the `uconn` collection. The connection counts and bytes of each of these `uconn` entries are derived from their respective `dat` subdocuments. Finally, the results are stored in new `dat` subdocuments in the peers' `host` entries. 69 | 70 | `bl` stores the unsafe host this host contacted. `bl_conn_count` tracks how many times the hosts connected. `bl_total_bytes` tracks how many bytes were sent back and forth between the hosts. `bl_in_count` and `bl_out_count` are either absent or set to 1 in each `dat` subdocument. 71 | 72 | The current chunk ID is recorded in this subdocument in order to track when the entry was created. 73 | 74 | There should always be one `dat` subdocument per unsafe host this host contacted. Multiple subdocuments with the same `bl` field should not exist. -------------------------------------------------------------------------------- /pkg/blacklist/mongodb.go: -------------------------------------------------------------------------------- 1 | package blacklist 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/activecm/rita-legacy/config" 7 | "github.com/activecm/rita-legacy/database" 8 | "github.com/activecm/rita-legacy/pkg/data" 9 | "github.com/activecm/rita-legacy/util" 10 | "github.com/vbauerster/mpb" 11 | "github.com/vbauerster/mpb/decor" 12 | 13 | "github.com/globalsign/mgo" 14 | "github.com/globalsign/mgo/bson" 15 | 16 | log "github.com/sirupsen/logrus" 17 | ) 18 | 19 | type repo struct { 20 | database *database.DB 21 | config *config.Config 22 | log *log.Logger 23 | } 24 | 25 | // NewMongoRepository bundles the given resources for updating MongoDB with threat intel data 26 | func NewMongoRepository(db *database.DB, conf *config.Config, logger *log.Logger) Repository { 27 | return &repo{ 28 | database: db, 29 | config: conf, 30 | log: logger, 31 | } 32 | } 33 | 34 | // CreateIndexes sets up the indices needed to find hosts which contacted unsafe hosts 35 | func (r *repo) CreateIndexes() error { 36 | session := r.database.Session.Copy() 37 | defer session.Close() 38 | 39 | coll := session.DB(r.database.GetSelectedDB()).C(r.config.T.Structure.HostTable) 40 | 41 | // create hosts collection 42 | // Desired indexes 43 | indexes := []mgo.Index{ 44 | {Key: []string{"dat.bl.ip", "dat.bl.network_uuid"}}, 45 | } 46 | 47 | for _, index := range indexes { 48 | err := coll.EnsureIndex(index) 49 | if err != nil { 50 | return err 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | // Upsert creates threat intel records in the host collection for the hosts which 57 | // contacted hosts which have been marked unsafe 58 | func (r *repo) Upsert() { 59 | 60 | // Create the workers 61 | writerWorker := database.NewBulkWriter(r.database, r.config, r.log, true, "bl_updater") 62 | 63 | analyzerWorker := newAnalyzer( 64 | r.config.S.Rolling.CurrentChunk, 65 | r.database, 66 | r.config, 67 | r.log, 68 | writerWorker.Collect, 69 | writerWorker.Close, 70 | ) 71 | 72 | // kick off the threaded goroutines 73 | for i := 0; i < util.Max(1, runtime.NumCPU()/2); i++ { 74 | analyzerWorker.start() 75 | writerWorker.Start() 76 | } 77 | 78 | // ensure the worker closing cascade fires when we exit this method 79 | defer analyzerWorker.close() 80 | 81 | // grab all of the unsafe hosts we have ever seen 82 | // NOTE: we cannot use the (hostMap map[string]*host.Input) 83 | // since we are creating peer statistic summaries for the entire 84 | // observation period not just this import session 85 | session := r.database.Session.Copy() 86 | defer session.Close() 87 | 88 | unsafeHostsQuery := session.DB(r.database.GetSelectedDB()).C(r.config.T.Structure.HostTable).Find(bson.M{"blacklisted": true}) 89 | 90 | numUnsafeHosts, err := unsafeHostsQuery.Count() 91 | if err != nil { 92 | r.log.WithFields(log.Fields{ 93 | "Module": "bl_updater", 94 | }).Error(err) 95 | } 96 | if numUnsafeHosts == 0 { 97 | // fmt.Println("\t[!] No blacklisted hosts to update") 98 | return 99 | } 100 | 101 | // add a progress bar for troubleshooting 102 | p := mpb.New(mpb.WithWidth(20)) 103 | bar := p.AddBar(int64(numUnsafeHosts), 104 | mpb.PrependDecorators( 105 | decor.Name("\t[-] Updating blacklisted peers:", decor.WC{W: 30, C: decor.DidentRight}), 106 | decor.CountersNoUnit(" %d / %d ", decor.WCSyncWidth), 107 | ), 108 | mpb.AppendDecorators(decor.Percentage()), 109 | ) 110 | 111 | var unsafeHost data.UniqueIP 112 | unsafeHostIter := unsafeHostsQuery.Iter() 113 | for unsafeHostIter.Next(&unsafeHost) { 114 | analyzerWorker.collect(unsafeHost) 115 | bar.IncrBy(1) 116 | } 117 | if err := unsafeHostIter.Close(); err != nil { 118 | r.log.WithFields(log.Fields{ 119 | "Module": "bl_updater", 120 | }).Error(err) 121 | } 122 | 123 | p.Wait() 124 | 125 | } 126 | -------------------------------------------------------------------------------- /pkg/blacklist/repository.go: -------------------------------------------------------------------------------- 1 | package blacklist 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | ) 6 | 7 | // Repository for blacklist results in host collection 8 | type Repository interface { 9 | CreateIndexes() error 10 | Upsert() 11 | } 12 | 13 | // connectionPeer records how many connections were made to/ from a given host and how many bytes were sent/ received 14 | type connectionPeer struct { 15 | Host data.UniqueIP `bson:"_id"` 16 | Connections int `bson:"bl_conn_count"` 17 | TotalBytes int `bson:"bl_total_bytes"` 18 | } 19 | 20 | // IPResult represtes a blacklisted IP and summary data 21 | // about the connections involving that IP 22 | type IPResult struct { 23 | Host data.UniqueIP `bson:",inline"` 24 | Connections int `bson:"conn_count"` 25 | UniqueConnections int `bson:"uconn_count"` 26 | TotalBytes int `bson:"total_bytes"` 27 | Peers []data.UniqueIP `bson:"peers"` 28 | } 29 | 30 | // HostnameResult represents a blacklisted hostname and summary 31 | // data about the connections made to that hostname 32 | type HostnameResult struct { 33 | Host string `bson:"host"` 34 | Connections int `bson:"conn_count"` 35 | UniqueConnections int `bson:"uconn_count"` 36 | TotalBytes int `bson:"total_bytes"` 37 | ConnectedHosts []data.UniqueIP `bson:"sources,omitempty"` 38 | } 39 | -------------------------------------------------------------------------------- /pkg/certificate/analyzer.go: -------------------------------------------------------------------------------- 1 | package certificate 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/activecm/rita-legacy/config" 7 | "github.com/activecm/rita-legacy/database" 8 | "github.com/globalsign/mgo/bson" 9 | ) 10 | 11 | type ( 12 | //analyzer is a structure for invalid certificate analysis 13 | analyzer struct { 14 | chunk int // current chunk (0 if not on rolling analysis) 15 | db *database.DB // provides access to MongoDB 16 | conf *config.Config // contains details needed to access MongoDB 17 | analyzedCallback func(database.BulkChanges) // called on each analyzed result 18 | closedCallback func() // called when .close() is called and no more calls to analyzedCallback will be made 19 | analysisChannel chan *Input // holds unanalyzed data 20 | analysisWg sync.WaitGroup // wait for analysis to finish 21 | } 22 | ) 23 | 24 | // newAnalyzer creates a new analyzer for recording connections that were made 25 | // with invalid certificates 26 | func newAnalyzer(chunk int, db *database.DB, conf *config.Config, analyzedCallback func(database.BulkChanges), closedCallback func()) *analyzer { 27 | return &analyzer{ 28 | chunk: chunk, 29 | db: db, 30 | conf: conf, 31 | analyzedCallback: analyzedCallback, 32 | closedCallback: closedCallback, 33 | analysisChannel: make(chan *Input), 34 | } 35 | } 36 | 37 | // collect gathers invalid certificate connection records for analysis 38 | func (a *analyzer) collect(datum *Input) { 39 | a.analysisChannel <- datum 40 | } 41 | 42 | // close waits for the analyzer to finish 43 | func (a *analyzer) close() { 44 | close(a.analysisChannel) 45 | a.analysisWg.Wait() 46 | a.closedCallback() 47 | } 48 | 49 | // start kicks off a new analysis thread 50 | func (a *analyzer) start() { 51 | a.analysisWg.Add(1) 52 | go func() { 53 | ssn := a.db.Session.Copy() 54 | defer ssn.Close() 55 | 56 | for datum := range a.analysisChannel { 57 | // cap the list to an arbitrary amount (hopefully smaller than the 16 MB document size cap) 58 | // anything approaching this limit will cause performance issues in software that depends on rita 59 | // anything tuncated over this limit won't be visible as an IP connecting to an invalid cert 60 | origIPs := datum.OrigIps.Items() 61 | if len(origIPs) > 200003 { 62 | origIPs = origIPs[:200003] 63 | } 64 | 65 | tuples := datum.Tuples.Items() 66 | if len(tuples) > 20 { 67 | tuples = tuples[:20] 68 | } 69 | 70 | invalidCerts := datum.InvalidCerts.Items() 71 | if len(invalidCerts) > 10 { 72 | invalidCerts = invalidCerts[:10] 73 | } 74 | 75 | // create certificateQuery 76 | certificateQuery := bson.M{ 77 | "$push": bson.M{ 78 | "dat": bson.M{ 79 | "seen": datum.Seen, 80 | "orig_ips": origIPs, 81 | "tuples": tuples, 82 | "icodes": invalidCerts, 83 | "cid": a.chunk, 84 | }, 85 | }, 86 | "$set": bson.M{ 87 | "cid": a.chunk, 88 | "network_name": datum.Host.NetworkName, 89 | }, 90 | } 91 | 92 | // set to writer channel 93 | a.analyzedCallback(database.BulkChanges{ 94 | a.conf.T.Cert.CertificateTable: []database.BulkChange{{ 95 | Selector: datum.Host.BSONKey(), 96 | Update: certificateQuery, 97 | Upsert: true, 98 | }}, 99 | }) 100 | } 101 | 102 | a.analysisWg.Done() 103 | }() 104 | } 105 | -------------------------------------------------------------------------------- /pkg/certificate/mongodb.go: -------------------------------------------------------------------------------- 1 | package certificate 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/activecm/rita-legacy/config" 7 | "github.com/activecm/rita-legacy/database" 8 | "github.com/activecm/rita-legacy/util" 9 | "github.com/globalsign/mgo" 10 | "github.com/vbauerster/mpb" 11 | "github.com/vbauerster/mpb/decor" 12 | 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | type repo struct { 17 | database *database.DB 18 | config *config.Config 19 | log *log.Logger 20 | } 21 | 22 | // NewMongoRepository bundles the given resources for updating MongoDB with invalid certificate data 23 | func NewMongoRepository(db *database.DB, conf *config.Config, logger *log.Logger) Repository { 24 | return &repo{ 25 | database: db, 26 | config: conf, 27 | log: logger, 28 | } 29 | } 30 | 31 | // CreateIndexes creates indexes for the certificate collection 32 | func (r *repo) CreateIndexes() error { 33 | session := r.database.Session.Copy() 34 | defer session.Close() 35 | 36 | // set collection name 37 | collectionName := r.config.T.Cert.CertificateTable 38 | 39 | // check if collection already exists 40 | names, _ := session.DB(r.database.GetSelectedDB()).CollectionNames() 41 | 42 | // if collection exists, we don't need to do anything else 43 | for _, name := range names { 44 | if name == collectionName { 45 | return nil 46 | } 47 | } 48 | 49 | indexes := []mgo.Index{ 50 | {Key: []string{"ip", "network_uuid"}, Unique: true}, 51 | {Key: []string{"dat.seen"}}, 52 | } 53 | 54 | // create collection 55 | err := r.database.CreateCollection(collectionName, indexes) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | return nil 61 | } 62 | 63 | // Upsert records the given certificate data in MongoDB 64 | func (r *repo) Upsert(certMap map[string]*Input) { 65 | // Create the workers 66 | writerWorker := database.NewBulkWriter(r.database, r.config, r.log, true, "certificate") 67 | 68 | analyzerWorker := newAnalyzer( 69 | r.config.S.Rolling.CurrentChunk, 70 | r.database, 71 | r.config, 72 | writerWorker.Collect, 73 | writerWorker.Close, 74 | ) 75 | 76 | // kick off the threaded goroutines 77 | for i := 0; i < util.Max(1, runtime.NumCPU()/2); i++ { 78 | analyzerWorker.start() 79 | writerWorker.Start() 80 | } 81 | 82 | // progress bar for troubleshooting 83 | p := mpb.New(mpb.WithWidth(20)) 84 | bar := p.AddBar(int64(len(certMap)), 85 | mpb.PrependDecorators( 86 | decor.Name("\t[-] Invalid Cert Analysis:", decor.WC{W: 30, C: decor.DidentRight}), 87 | decor.CountersNoUnit(" %d / %d ", decor.WCSyncWidth), 88 | ), 89 | mpb.AppendDecorators(decor.Percentage()), 90 | ) 91 | 92 | // loop over map entries 93 | for _, value := range certMap { 94 | analyzerWorker.collect(value) 95 | bar.IncrBy(1) 96 | } 97 | 98 | p.Wait() 99 | 100 | // start the closing cascade (this will also close the other channels) 101 | analyzerWorker.close() 102 | } 103 | -------------------------------------------------------------------------------- /pkg/certificate/mongodb_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package certificate 5 | 6 | import ( 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | 11 | "github.com/activecm/rita-legacy/pkg/data" 12 | "github.com/activecm/rita-legacy/resources" 13 | "github.com/activecm/rita-legacy/util" 14 | "github.com/globalsign/mgo/dbtest" 15 | ) 16 | 17 | // Server holds the dbtest DBServer 18 | var Server dbtest.DBServer 19 | 20 | // Set the test database 21 | var testTargetDB = "tmp_test_db" 22 | 23 | var testRepo Repository 24 | 25 | var testCertificate = map[string]*Input{ 26 | "Debian APT-HTTP/1.3 (1.2.24)": { 27 | Host: data.UniqueIP{ 28 | IP: "1.2.3.4", 29 | NetworkUUID: util.PublicNetworkUUID, 30 | NetworkName: util.PublicNetworkName, 31 | }, 32 | InvalidCerts: []string{"I'm an invalid cert!", "me too!"}, 33 | Seen: 123, 34 | }, 35 | } 36 | 37 | func init() { 38 | testCertificate["Debian APT-HTTP/1.3 (1.2.24)"].OrigIps.Insert( 39 | data.UniqueIP{ 40 | IP: "5.6.7.8", 41 | NetworkUUID: util.PublicNetworkUUID, 42 | NetworkName: util.PublicNetworkName, 43 | }, 44 | ) 45 | testCertificate["Debian APT-HTTP/1.3 (1.2.24)"].OrigIps.Insert( 46 | data.UniqueIP{ 47 | IP: "9.10.11.12", 48 | NetworkUUID: util.PublicNetworkUUID, 49 | NetworkName: util.PublicNetworkName, 50 | }, 51 | ) 52 | } 53 | 54 | func TestUpsert(t *testing.T) { 55 | testRepo.Upsert(testCertificate) 56 | 57 | } 58 | 59 | // TestMain wraps all tests with the needed initialized mock DB and fixtures 60 | func TestMain(m *testing.M) { 61 | // Store temporary databases files in a temporary directory 62 | tempDir, _ := ioutil.TempDir("", "testing") 63 | Server.SetPath(tempDir) 64 | 65 | // Set the main session variable to the temporary MongoDB instance 66 | res := resources.InitTestResources() 67 | 68 | testRepo = NewMongoRepository(res.DB, res.Config, res.Log) 69 | 70 | // Run the test suite 71 | retCode := m.Run() 72 | 73 | // Shut down the temporary server and removes data on disk. 74 | Server.Stop() 75 | 76 | // call with result of m.Run() 77 | os.Exit(retCode) 78 | } 79 | -------------------------------------------------------------------------------- /pkg/certificate/repository.go: -------------------------------------------------------------------------------- 1 | package certificate 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | ) 6 | 7 | // Repository for uconn collection 8 | type Repository interface { 9 | CreateIndexes() error 10 | Upsert(useragentMap map[string]*Input) 11 | } 12 | 13 | // Input .... 14 | type Input struct { 15 | Host data.UniqueIP 16 | Seen int64 17 | OrigIps data.UniqueIPSet 18 | InvalidCerts data.StringSet 19 | Tuples data.StringSet 20 | } 21 | 22 | // AnalysisView (for reporting) 23 | type AnalysisView struct { 24 | UserAgent string `bson:"user_agent"` 25 | TimesUsed int64 `bson:"seen"` 26 | } 27 | -------------------------------------------------------------------------------- /pkg/data/fqdn.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/globalsign/mgo/bson" 7 | ) 8 | 9 | //UniqueSrcFQDNPair is used to make a tuple of 10 | // Src IP/UUID/Name and an FQDN to which the Src IP 11 | // was attempting to communicate 12 | type UniqueSrcFQDNPair struct { 13 | UniqueSrcIP `bson:",inline"` 14 | FQDN string `bson:"fqdn"` 15 | } 16 | 17 | //NewUniqueSrcFQDNPair binds a pair of UniqueIPs and an FQDN 18 | func NewUniqueSrcFQDNPair(source UniqueIP, fqdn string) UniqueSrcFQDNPair { 19 | return UniqueSrcFQDNPair{ 20 | UniqueSrcIP: UniqueSrcIP{ 21 | SrcIP: source.IP, 22 | SrcNetworkUUID: source.NetworkUUID, 23 | SrcNetworkName: source.NetworkName, 24 | }, 25 | FQDN: fqdn, 26 | } 27 | } 28 | 29 | //MapKey generates a string which may be used to index a Unique SrcIP / FQDN pair. Concatenates IPs and UUIDs. 30 | func (p UniqueSrcFQDNPair) MapKey() string { 31 | var builder strings.Builder 32 | 33 | srcUUIDLen := 1 + len(p.SrcNetworkUUID.Data) 34 | 35 | builder.Grow(len(p.SrcIP) + srcUUIDLen + len(p.FQDN)) 36 | builder.WriteString(p.SrcIP) 37 | builder.WriteByte(p.SrcNetworkUUID.Kind) 38 | builder.Write(p.SrcNetworkUUID.Data) 39 | 40 | builder.WriteString(p.FQDN) 41 | 42 | return builder.String() 43 | } 44 | 45 | //BSONKey generates a BSON map which may be used to index a given a unique 46 | // src-fqdn pair. Includes IP and Network UUID. 47 | func (p UniqueSrcFQDNPair) BSONKey() bson.M { 48 | key := bson.M{ 49 | "src": p.SrcIP, 50 | "src_network_uuid": p.SrcNetworkUUID, 51 | "fqdn": p.FQDN, 52 | } 53 | return key 54 | } 55 | -------------------------------------------------------------------------------- /pkg/data/ip_test.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/activecm/rita-legacy/util" 8 | "github.com/globalsign/mgo/bson" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestNewUniqueIP(t *testing.T) { 13 | ip := NewUniqueIP(net.ParseIP("192.168.1.1"), "ff0d0776-0cdc-4a10-b793-522bcd48a560", "test") 14 | assert.Equal(t, "192.168.1.1", ip.IP, "ip correctly assigned on private ip with valid data") 15 | assert.Equal(t, bson.BinaryUUID, ip.NetworkUUID.Kind, "uuid kind set for private ip with valid data") 16 | assert.Equal(t, []byte{ 17 | 0xff, 0x0d, 0x07, 0x76, 18 | 0x0c, 0xdc, 0x4a, 0x10, 19 | 0xb7, 0x93, 0x52, 0x2b, 20 | 0xcd, 0x48, 0xa5, 0x60, 21 | }, ip.NetworkUUID.Data, "uuid binary correctly parsed for private ip with valid data") 22 | assert.Equal(t, "test", ip.NetworkName, "net name set for private ip with valid data") 23 | 24 | ip = NewUniqueIP(net.ParseIP("192.168.1.1"), "", "") 25 | assert.Equal(t, "192.168.1.1", ip.IP, "ip correctly assigned on private ip with no network data") 26 | assert.Equal(t, util.UnknownPrivateNetworkUUID.Kind, ip.NetworkUUID.Kind, "uuid kind set for private ip with no network data") 27 | assert.Equal(t, util.UnknownPrivateNetworkUUID.Data, ip.NetworkUUID.Data, "uuid binary set to flag value for private ip with no network data") 28 | assert.Equal(t, util.UnknownPrivateNetworkName, ip.NetworkName, "net name set to flag value for private ip with no network data") 29 | 30 | ip = NewUniqueIP(net.ParseIP("192.168.1.1"), "invalid-uuid-here", "test") 31 | assert.Equal(t, "192.168.1.1", ip.IP, "ip correctly assigned on private ip with invalid network data") 32 | assert.Equal(t, util.UnknownPrivateNetworkUUID.Kind, ip.NetworkUUID.Kind, "uuid kind set for private ip with invalid network data") 33 | assert.Equal(t, util.UnknownPrivateNetworkUUID.Data, ip.NetworkUUID.Data, "uuid binary set to flag value for private ip with invalid network data") 34 | assert.Equal(t, util.UnknownPrivateNetworkName, ip.NetworkName, "net name set to flag value for private ip with invalid network data") 35 | 36 | ip = NewUniqueIP(net.ParseIP("8.8.8.8"), "", "") 37 | assert.Equal(t, "8.8.8.8", ip.IP, "ip correctly assigned on public ip with no network data") 38 | assert.Equal(t, util.PublicNetworkUUID.Kind, ip.NetworkUUID.Kind, "uuid kind set for public ip with no network data") 39 | assert.Equal(t, util.PublicNetworkUUID.Data, ip.NetworkUUID.Data, "uuid binary set to flag value for public ip with no network data") 40 | assert.Equal(t, util.PublicNetworkName, ip.NetworkName, "net name set to flag value for public ip with no network data") 41 | 42 | ip = NewUniqueIP(net.ParseIP("8.8.8.8"), "invalid-uuid-here", "test") 43 | assert.Equal(t, "8.8.8.8", ip.IP, "ip correctly assigned on public ip with invalid network data") 44 | assert.Equal(t, util.PublicNetworkUUID.Kind, ip.NetworkUUID.Kind, "uuid kind set for public ip with invalid network data") 45 | assert.Equal(t, util.PublicNetworkUUID.Data, ip.NetworkUUID.Data, "uuid binary set to flag value for public ip with invalid network data") 46 | assert.Equal(t, util.PublicNetworkName, ip.NetworkName, "net name set to flag value for public ip with invalid network data") 47 | 48 | ip = NewUniqueIP(net.ParseIP("8.8.8.8"), "ff0d0776-0cdc-4a10-b793-522bcd48a560", "test") 49 | assert.Equal(t, "8.8.8.8", ip.IP, "ip correctly assigned on public ip with valid network data") 50 | assert.Equal(t, util.PublicNetworkUUID.Kind, ip.NetworkUUID.Kind, "uuid kind set for public ip with valid network data") 51 | assert.Equal(t, util.PublicNetworkUUID.Data, ip.NetworkUUID.Data, "uuid binary set to flag value for public ip with valid network data") 52 | assert.Equal(t, util.PublicNetworkName, ip.NetworkName, "net name set to flag value for public ip with valid network data") 53 | } 54 | -------------------------------------------------------------------------------- /pkg/data/sets.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | type StringSet map[string]struct{} 4 | 5 | //Items returns the strings in the set as a slice. 6 | func (s StringSet) Items() []string { 7 | retVal := make([]string, 0, len(s)) 8 | for str := range s { 9 | retVal = append(retVal, str) 10 | } 11 | return retVal 12 | } 13 | 14 | //Insert adds a string to the set 15 | func (s StringSet) Insert(str string) { 16 | s[str] = struct{}{} 17 | } 18 | 19 | //Contains checks if a given string is in the set 20 | func (s StringSet) Contains(str string) bool { 21 | _, ok := s[str] 22 | return ok 23 | } 24 | 25 | type IntSet map[int]struct{} 26 | 27 | //Items returns the integers in the set as a slice. 28 | func (s IntSet) Items() []int { 29 | retVal := make([]int, 0, len(s)) 30 | for intVal := range s { 31 | retVal = append(retVal, intVal) 32 | } 33 | return retVal 34 | } 35 | 36 | //Insert adds a integer to the set 37 | func (s IntSet) Insert(intVal int) { 38 | s[intVal] = struct{}{} 39 | } 40 | 41 | //Contains checks if a given integer is in the set 42 | func (s IntSet) Contains(intVal int) bool { 43 | _, ok := s[intVal] 44 | return ok 45 | } 46 | 47 | type Int64Set map[int64]struct{} 48 | 49 | //Items returns the integers in the set as a slice. 50 | func (s Int64Set) Items() []int64 { 51 | retVal := make([]int64, 0, len(s)) 52 | for intVal := range s { 53 | retVal = append(retVal, intVal) 54 | } 55 | return retVal 56 | } 57 | 58 | //Insert adds a integer to the set 59 | func (s Int64Set) Insert(intVal int64) { 60 | s[intVal] = struct{}{} 61 | } 62 | 63 | //Contains checks if a given integer is in the set 64 | func (s Int64Set) Contains(intVal int64) bool { 65 | _, ok := s[intVal] 66 | return ok 67 | } 68 | -------------------------------------------------------------------------------- /pkg/data/zeek.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | type ZeekUIDRecord struct { 4 | Conn struct { 5 | OrigBytes int64 6 | RespBytes int64 7 | Duration float64 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /pkg/explodeddns/mongodb.go: -------------------------------------------------------------------------------- 1 | package explodeddns 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/activecm/rita-legacy/config" 7 | "github.com/activecm/rita-legacy/database" 8 | "github.com/activecm/rita-legacy/util" 9 | "github.com/globalsign/mgo" 10 | "github.com/vbauerster/mpb" 11 | "github.com/vbauerster/mpb/decor" 12 | 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | type repo struct { 17 | database *database.DB 18 | config *config.Config 19 | log *log.Logger 20 | } 21 | 22 | // NewMongoRepository bundles the given resources for updating MongoDB with exploded DNS data 23 | func NewMongoRepository(db *database.DB, conf *config.Config, logger *log.Logger) Repository { 24 | return &repo{ 25 | database: db, 26 | config: conf, 27 | log: logger, 28 | } 29 | } 30 | 31 | // CreateIndexes creates indexes for the explodedDns collection 32 | func (r *repo) CreateIndexes() error { 33 | session := r.database.Session.Copy() 34 | defer session.Close() 35 | 36 | // set collection name 37 | collectionName := r.config.T.DNS.ExplodedDNSTable 38 | 39 | // check if collection already exists 40 | names, _ := session.DB(r.database.GetSelectedDB()).CollectionNames() 41 | 42 | // if collection exists, we don't need to do anything else 43 | for _, name := range names { 44 | if name == collectionName { 45 | return nil 46 | } 47 | } 48 | 49 | // set desired indexes 50 | indexes := []mgo.Index{ 51 | {Key: []string{"domain"}, Unique: true}, 52 | // {Key: []string{"visited"}}, 53 | {Key: []string{"subdomain_count"}}, 54 | } 55 | 56 | // create collection 57 | err := r.database.CreateCollection(collectionName, indexes) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | return nil 63 | } 64 | 65 | // Upsert records the given dns query count data in MongoDB 66 | func (r *repo) Upsert(domainMap map[string]int) { 67 | 68 | //Create the workers 69 | writerWorker := database.NewBulkWriter(r.database, r.config, r.log, true, "exploded_dns") 70 | 71 | analyzerWorker := newAnalyzer( 72 | r.config.S.Rolling.CurrentChunk, 73 | r.database, 74 | r.config, 75 | writerWorker.Collect, 76 | writerWorker.Close, 77 | ) 78 | 79 | //kick off the threaded goroutines 80 | for i := 0; i < util.Max(1, runtime.NumCPU()/2); i++ { 81 | analyzerWorker.start() 82 | writerWorker.Start() 83 | } 84 | 85 | // progress bar for troubleshooting 86 | p := mpb.New(mpb.WithWidth(20)) 87 | bar := p.AddBar(int64(len(domainMap)), 88 | mpb.PrependDecorators( 89 | decor.Name("\t[-] Exploded DNS Analysis:", decor.WC{W: 30, C: decor.DidentRight}), 90 | decor.CountersNoUnit(" %d / %d ", decor.WCSyncWidth), 91 | ), 92 | mpb.AppendDecorators(decor.Percentage()), 93 | ) 94 | 95 | // loop over map entries 96 | for entry, count := range domainMap { 97 | //Mongo Index key is limited to a size of 1024 https://docs.mongodb.com/v3.4/reference/limits/#index-limitations 98 | // so if the key is too large, we should cut it back, this is rough but 99 | // works. Figured 800 allows some wiggle room, while also not being too large 100 | if len(entry) > 1024 { 101 | entry = entry[:800] 102 | } 103 | analyzerWorker.collect(domain{entry, count}) 104 | bar.IncrBy(1) 105 | } 106 | 107 | p.Wait() 108 | 109 | // start the closing cascade (this will also close the other channels) 110 | analyzerWorker.close() 111 | } 112 | -------------------------------------------------------------------------------- /pkg/explodeddns/mongodb_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package explodeddns 5 | 6 | import ( 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | 11 | "github.com/activecm/rita-legacy/resources" 12 | "github.com/globalsign/mgo/dbtest" 13 | ) 14 | 15 | // Server holds the dbtest DBServer 16 | var Server dbtest.DBServer 17 | 18 | // Set the test database 19 | var testTargetDB = "tmp_test_db" 20 | 21 | var testRepo Repository 22 | 23 | var testExplodedDNS = map[string]int{ 24 | "a.b.activecountermeasures.com": 123, 25 | "x.a.b.activecountermeasures.com": 38, 26 | "activecountermeasures.com": 1, 27 | "google.com": 912, 28 | } 29 | 30 | func TestUpdateDomains(t *testing.T) { 31 | testRepo.Upsert(testExplodedDNS) 32 | // if err != nil { 33 | // t.Errorf("Error creating explodedDNS upserts") 34 | // } 35 | } 36 | 37 | // TestMain wraps all tests with the needed initialized mock DB and fixtures 38 | func TestMain(m *testing.M) { 39 | // Store temporary databases files in a temporary directory 40 | tempDir, _ := ioutil.TempDir("", "testing") 41 | Server.SetPath(tempDir) 42 | 43 | // Set the main session variable to the temporary MongoDB instance 44 | res := resources.InitTestResources() 45 | 46 | testRepo = NewMongoRepository(res) 47 | 48 | // Run the test suite 49 | retCode := m.Run() 50 | 51 | // Shut down the temporary server and removes data on disk. 52 | Server.Stop() 53 | 54 | // call with result of m.Run() 55 | os.Exit(retCode) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/explodeddns/repository.go: -------------------------------------------------------------------------------- 1 | package explodeddns 2 | 3 | // Repository for explodedDNS collection 4 | type Repository interface { 5 | CreateIndexes() error 6 | // Upsert(explodedDNS *parsetypes.ExplodedDNS) error 7 | Upsert(domainMap map[string]int) 8 | } 9 | 10 | // domain .... 11 | type domain struct { 12 | name string 13 | count int 14 | } 15 | 16 | // dns .... 17 | type dns struct { 18 | Domain string `bson:"domain"` 19 | SubdomainCount int64 `bson:"subdomain_count"` 20 | CID int `bson:"cid"` 21 | } 22 | 23 | // Result represents a hostname, how many subdomains were found 24 | // for that hostname, and how many times that hostname and its subdomains 25 | // were looked up. 26 | type Result struct { 27 | Domain string `bson:"domain"` 28 | SubdomainCount int64 `bson:"subdomain_count"` 29 | Visited int64 `bson:"visited"` 30 | } 31 | -------------------------------------------------------------------------------- /pkg/explodeddns/results.go: -------------------------------------------------------------------------------- 1 | package explodeddns 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/resources" 5 | "github.com/globalsign/mgo/bson" 6 | ) 7 | 8 | // Results returns hostnames and their subdomain/ lookup statistics from the database. 9 | // limit and noLimit control how many results are returned. 10 | func Results(res *resources.Resources, limit int, noLimit bool) ([]Result, error) { 11 | ssn := res.DB.Session.Copy() 12 | defer ssn.Close() 13 | 14 | var explodedDNSResults []Result 15 | 16 | explodedDNSQuery := []bson.M{ 17 | bson.M{"$unwind": "$dat"}, 18 | bson.M{"$project": bson.M{"domain": 1, "subdomain_count": 1, "visited": "$dat.visited"}}, 19 | bson.M{"$group": bson.M{ 20 | "_id": "$domain", 21 | "visited": bson.M{"$sum": "$visited"}, 22 | "subdomain_count": bson.M{"$first": "$subdomain_count"}, 23 | }}, 24 | bson.M{"$project": bson.M{ 25 | "_id": 0, 26 | "domain": "$_id", 27 | "visited": 1, 28 | "subdomain_count": 1, 29 | }}, 30 | bson.M{"$sort": bson.M{"visited": -1}}, 31 | bson.M{"$sort": bson.M{"subdomain_count": -1}}, 32 | } 33 | 34 | if !noLimit { 35 | explodedDNSQuery = append(explodedDNSQuery, bson.M{"$limit": limit}) 36 | } 37 | 38 | err := ssn.DB(res.DB.GetSelectedDB()).C(res.Config.T.DNS.ExplodedDNSTable).Pipe(explodedDNSQuery).AllowDiskUse().All(&explodedDNSResults) 39 | 40 | return explodedDNSResults, err 41 | 42 | } 43 | -------------------------------------------------------------------------------- /pkg/host/mongodb.go: -------------------------------------------------------------------------------- 1 | package host 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/activecm/rita-legacy/config" 7 | "github.com/activecm/rita-legacy/database" 8 | "github.com/activecm/rita-legacy/util" 9 | 10 | "github.com/globalsign/mgo" 11 | "github.com/vbauerster/mpb" 12 | "github.com/vbauerster/mpb/decor" 13 | 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | type repo struct { 18 | database *database.DB 19 | config *config.Config 20 | log *log.Logger 21 | } 22 | 23 | // NewMongoRepository bundles the given resources for updating MongoDB with host data 24 | func NewMongoRepository(db *database.DB, conf *config.Config, logger *log.Logger) Repository { 25 | return &repo{ 26 | database: db, 27 | config: conf, 28 | log: logger, 29 | } 30 | } 31 | 32 | // CreateIndexes creates indexes for the host collection 33 | func (r *repo) CreateIndexes() error { 34 | session := r.database.Session.Copy() 35 | defer session.Close() 36 | 37 | coll := session.DB(r.database.GetSelectedDB()).C(r.config.T.Structure.HostTable) 38 | 39 | // create hosts collection 40 | // Desired indexes 41 | indexes := []mgo.Index{ 42 | {Key: []string{"ip", "network_uuid"}, Unique: true}, 43 | {Key: []string{"local"}}, 44 | {Key: []string{"ipv4_binary"}}, 45 | {Key: []string{"dat.mdip.ip", "dat.mdip.network_uuid"}}, 46 | {Key: []string{"dat.mbdst.ip", "dat.mbdst.network_uuid"}}, 47 | {Key: []string{"dat.mbproxy"}}, 48 | } 49 | 50 | for _, index := range indexes { 51 | err := coll.EnsureIndex(index) 52 | if err != nil { 53 | return err 54 | } 55 | } 56 | return nil 57 | } 58 | 59 | // Upsert records the given host data in MongoDB 60 | func (r *repo) Upsert(hostMap map[string]*Input) { 61 | 62 | // 1st Phase: Analysis 63 | 64 | // Create the workers 65 | writerWorker := database.NewBulkWriter(r.database, r.config, r.log, true, "host") 66 | 67 | analyzerWorker := newAnalyzer( 68 | r.config.S.Rolling.CurrentChunk, 69 | r.config, 70 | r.database, 71 | r.log, 72 | writerWorker.Collect, 73 | writerWorker.Close, 74 | ) 75 | 76 | // kick off the threaded goroutines 77 | for i := 0; i < util.Max(1, runtime.NumCPU()/2); i++ { 78 | analyzerWorker.start() 79 | writerWorker.Start() 80 | } 81 | 82 | // progress bar for troubleshooting 83 | p := mpb.New(mpb.WithWidth(20)) 84 | bar := p.AddBar(int64(len(hostMap)), 85 | mpb.PrependDecorators( 86 | decor.Name("\t[-] Host Analysis:", decor.WC{W: 30, C: decor.DidentRight}), 87 | decor.CountersNoUnit(" %d / %d ", decor.WCSyncWidth), 88 | ), 89 | mpb.AppendDecorators(decor.Percentage()), 90 | ) 91 | 92 | // loop over map entries 93 | for _, entry := range hostMap { 94 | analyzerWorker.collect(entry) 95 | bar.IncrBy(1) 96 | } 97 | p.Wait() 98 | 99 | // start the closing cascade (this will also close the other channels) 100 | analyzerWorker.close() 101 | } 102 | -------------------------------------------------------------------------------- /pkg/host/mongodb_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package host 5 | 6 | import ( 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | 11 | "github.com/activecm/rita-legacy/pkg/data" 12 | "github.com/activecm/rita-legacy/resources" 13 | "github.com/activecm/rita-legacy/util" 14 | "github.com/globalsign/mgo/dbtest" 15 | ) 16 | 17 | // Server holds the dbtest DBServer 18 | var Server dbtest.DBServer 19 | 20 | // Set the test database 21 | var testTargetDB = "tmp_test_db" 22 | 23 | var testRepo Repository 24 | 25 | var testHost = map[string]*Input{ 26 | "test": &Input{ 27 | Host: data.UniqueIP{ 28 | IP: "1.2.3.4", 29 | NetworkUUID: util.PublicNetworkUUID, 30 | NetworkName: util.PublicNetworkName, 31 | }, 32 | ConnectionCount: 12, 33 | TotalBytes: 123, 34 | TotalDuration: 123.0, 35 | MaxDuration: 12, 36 | }, 37 | } 38 | 39 | func TestUpsert(t *testing.T) { 40 | testRepo.Upsert(testHost) 41 | } 42 | 43 | // TestMain wraps all tests with the needed initialized mock DB and fixtures 44 | func TestMain(m *testing.M) { 45 | // Store temporary databases files in a temporary directory 46 | tempDir, _ := ioutil.TempDir("", "testing") 47 | Server.SetPath(tempDir) 48 | 49 | // Set the main session variable to the temporary MongoDB instance 50 | res := resources.InitTestResources() 51 | 52 | testRepo = NewMongoRepository(res.DB, res.Config, res.Log) 53 | 54 | // Run the test suite 55 | retCode := m.Run() 56 | 57 | // Shut down the temporary server and removes data on disk. 58 | Server.Stop() 59 | 60 | // call with result of m.Run() 61 | os.Exit(retCode) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/host/repository.go: -------------------------------------------------------------------------------- 1 | package host 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | ) 6 | 7 | // Repository for host collection 8 | type Repository interface { 9 | CreateIndexes() error 10 | Upsert(uconnMap map[string]*Input) 11 | } 12 | 13 | // Input ... 14 | type Input struct { 15 | Host data.UniqueIP 16 | IsLocal bool 17 | CountSrc int 18 | CountDst int 19 | ConnectionCount int64 20 | TotalBytes int64 21 | MaxDuration float64 22 | TotalDuration float64 23 | UntrustedAppConnCount int64 24 | MaxTS int64 25 | MinTS int64 26 | IP4 bool 27 | IP4Bin int64 28 | } 29 | -------------------------------------------------------------------------------- /pkg/hostname/Readme.md: -------------------------------------------------------------------------------- 1 | ## Hostname Package 2 | 3 | *Documented on May 24, 2022* 4 | 5 | --- 6 | 7 | This package records the hostnames/ fully qualified domain names (FQDN) seen in the current set of network logs under consideration. DNS query logs are used to gather FQDNs and the IP addresses they resolve to. 8 | 9 | This package records the following: 10 | - Fully qualified domain names 11 | - Whether the FQDN appears on any threat intel lists 12 | - The IP addresses the FQDN was seen resolving to 13 | - The IP addresses which made DNS queries for the FQDN 14 | 15 | ## Package Outputs 16 | 17 | ### Fully Qualified Domain Name 18 | 19 | Inputs: 20 | - `ParseResults.HostnameMap` created by `FSImporter` 21 | - Field: `Host` 22 | - Type: string 23 | 24 | Outputs: 25 | - MongoDB `hostname` collection: 26 | - Field: `host` 27 | - Type: string 28 | 29 | The `host` field of each document in the `hostnames` collection records the FQDN of a host that was queried for in the DNS logs currently under consideration. 30 | 31 | ### Chunk ID 32 | Inputs: 33 | - `Config.S.Rolling.CurrentChunk` 34 | - Type: int 35 | 36 | Outputs: 37 | - MongoDB `host` collection: 38 | - Field: `cid` 39 | - Type: int 40 | 41 | The `cid` field records the chunk ID of the import session in which this host document was last updated. This field is used to support rolling imports. 42 | 43 | ### Threat Intel Designation 44 | Inputs: 45 | - MongoDB `hostname` collection in the `rita-bl` database 46 | - Field: `index` 47 | - Type: string 48 | - `ParseResults.HostnameMap` created by `FSImporter` 49 | - Field: `Host` 50 | - Type: string 51 | 52 | Outputs: 53 | - MongoDB `hostname` collection: 54 | - Field: `blacklisted` 55 | - Type: bool 56 | 57 | This field marks whether the FQDN has appeared on any threat intelligence lists managed by `rita-bl`. These lists are registered in the RITA configuration file. 58 | 59 | ### Query Originator and Resolved IP Addresses 60 | - `ParseResults.HostnameMap` created by `FSImporter` 61 | - Field: `ClientIPs` 62 | - Type: data.UniqueIPSet 63 | - Field: `ResolvedIPs` 64 | - Type: data.UniqueIPSet 65 | 66 | Outputs: 67 | - MongoDB `hostname` collection: 68 | - Array Field: `dat` 69 | - Array Field: `ips` 70 | - Field: `ip` 71 | - Type: string 72 | - Field: `network_uuid` 73 | - Type: UUID 74 | - Field: `network_name` 75 | - Type: string 76 | - Array Field: `src_ips` 77 | - Field: `ip` 78 | - Type: string 79 | - Field: `network_uuid` 80 | - Type: UUID 81 | - Field: `network_name` 82 | - Type: string 83 | - Field: `cid` 84 | - Type: int 85 | 86 | The set of IP addresses which queried for a given FQDN is stored in `dat.src_ips` as an array of Unique IP addresses. Similarly, the set of IP addresses which the FQDN was seen to resolve to are stored in `dat.ips` as an array of Unique IP addresses. 87 | 88 | In order to gather all of the query originator IP addresses or resolved IP addresses for an FQDN across chunked imports, the `src_ips` or `ips` arrays from each of the `dat` documents must be unioned together. -------------------------------------------------------------------------------- /pkg/hostname/mongodb.go: -------------------------------------------------------------------------------- 1 | package hostname 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/activecm/rita-legacy/config" 7 | "github.com/activecm/rita-legacy/database" 8 | "github.com/activecm/rita-legacy/util" 9 | "github.com/globalsign/mgo" 10 | log "github.com/sirupsen/logrus" 11 | "github.com/vbauerster/mpb" 12 | "github.com/vbauerster/mpb/decor" 13 | ) 14 | 15 | type repo struct { 16 | database *database.DB 17 | config *config.Config 18 | log *log.Logger 19 | } 20 | 21 | // NewMongoRepository bundles the given resources for updating MongoDB with hostname data 22 | func NewMongoRepository(db *database.DB, conf *config.Config, logger *log.Logger) Repository { 23 | return &repo{ 24 | database: db, 25 | config: conf, 26 | log: logger, 27 | } 28 | } 29 | 30 | // CreateIndexes creates indexes for the hostname collection 31 | func (r *repo) CreateIndexes() error { 32 | session := r.database.Session.Copy() 33 | defer session.Close() 34 | 35 | // set collection name 36 | collectionName := r.config.T.DNS.HostnamesTable 37 | 38 | // check if collection already exists 39 | names, _ := session.DB(r.database.GetSelectedDB()).CollectionNames() 40 | 41 | // if collection exists, we don't need to do anything else 42 | for _, name := range names { 43 | if name == collectionName { 44 | return nil 45 | } 46 | } 47 | 48 | // set desired indexes 49 | indexes := []mgo.Index{ 50 | {Key: []string{"host"}, Unique: true}, 51 | {Key: []string{"dat.ips.ip", "dat.ips.network_uuid"}}, 52 | } 53 | 54 | // create collection 55 | err := r.database.CreateCollection(collectionName, indexes) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | return nil 61 | } 62 | 63 | // Upsert records the given hostname data in MongoDB 64 | func (r *repo) Upsert(hostnameMap map[string]*Input) { 65 | 66 | // Create the workers 67 | writerWorker := database.NewBulkWriter(r.database, r.config, r.log, true, "hostname") 68 | 69 | analyzerWorker := newAnalyzer( 70 | r.config.S.Rolling.CurrentChunk, 71 | r.database, 72 | r.config, 73 | r.log, 74 | writerWorker.Collect, 75 | writerWorker.Close, 76 | ) 77 | 78 | // kick off the threaded goroutines 79 | for i := 0; i < util.Max(1, runtime.NumCPU()/2); i++ { 80 | analyzerWorker.start() 81 | writerWorker.Start() 82 | } 83 | 84 | // progress bar for troubleshooting 85 | p := mpb.New(mpb.WithWidth(20)) 86 | bar := p.AddBar(int64(len(hostnameMap)), 87 | mpb.PrependDecorators( 88 | decor.Name("\t[-] Hostname Analysis:", decor.WC{W: 30, C: decor.DidentRight}), 89 | decor.CountersNoUnit(" %d / %d ", decor.WCSyncWidth), 90 | ), 91 | mpb.AppendDecorators(decor.Percentage()), 92 | ) 93 | 94 | // loop over map entries 95 | for _, entry := range hostnameMap { 96 | //Mongo Index key is limited to a size of 1024 https://docs.mongodb.com/v3.4/reference/limits/#index-limitations 97 | // so if the key is too large, we should cut it back, this is rough but 98 | // works. Figured 800 allows some wiggle room, while also not being too large 99 | if len(entry.Host) > 1024 { 100 | entry.Host = entry.Host[:800] 101 | } 102 | analyzerWorker.collect(entry) 103 | bar.IncrBy(1) 104 | } 105 | 106 | p.Wait() 107 | 108 | // start the closing cascade (this will also close the other channels) 109 | analyzerWorker.close() 110 | 111 | } 112 | -------------------------------------------------------------------------------- /pkg/hostname/mongodb_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package hostname 5 | 6 | import ( 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | 11 | "github.com/activecm/rita-legacy/pkg/data" 12 | "github.com/activecm/rita-legacy/resources" 13 | "github.com/activecm/rita-legacy/util" 14 | "github.com/globalsign/mgo/dbtest" 15 | ) 16 | 17 | // Server holds the dbtest DBServer 18 | var Server dbtest.DBServer 19 | 20 | // Set the test database 21 | var testTargetDB = "tmp_test_db" 22 | 23 | var testRepo Repository 24 | 25 | func ipFactory(ip string) data.UniqueIP { 26 | return data.UniqueIP{ 27 | IP: ip, 28 | NetworkUUID: util.UnknownPrivateNetworkUUID, 29 | NetworkName: util.UnknownPrivateNetworkName, 30 | } 31 | } 32 | 33 | var testHostname = map[string]*Input{ 34 | "a.b.activecountermeasures.com": &Input{}, 35 | "x.a.b.activecountermeasures.com": &Input{}, 36 | "activecountermeasures.com": &Input{}, 37 | "google.com": &Input{}, 38 | } 39 | 40 | func init() { 41 | testHostname["a.b.activecountermeasures.com"].ClientIPs.Insert(ipFactory("192.168.1.1")) 42 | testHostname["a.b.activecountermeasures.com"].ResolvedIPs.Insert(ipFactory("127.0.0.1")) 43 | testHostname["a.b.activecountermeasures.com"].ResolvedIPs.Insert(ipFactory("127.0.0.2")) 44 | 45 | testHostname["x.a.b.activecountermeasures.com"].ClientIPs.Insert(ipFactory("192.168.1.1")) 46 | testHostname["x.a.b.activecountermeasures.com"].ResolvedIPs.Insert(ipFactory("127.0.0.1")) 47 | testHostname["x.a.b.activecountermeasures.com"].ResolvedIPs.Insert(ipFactory("127.0.0.2")) 48 | 49 | testHostname["activecountermeasures.com"].ClientIPs.Insert(ipFactory("192.168.1.1")) 50 | 51 | testHostname["google.com"].ClientIPs.Insert(ipFactory("192.168.1.1")) 52 | testHostname["google.com"].ClientIPs.Insert(ipFactory("192.168.1.2")) 53 | 54 | testHostname["google.com"].ResolvedIPs.Insert(ipFactory("127.0.0.1")) 55 | testHostname["google.com"].ResolvedIPs.Insert(ipFactory("127.0.0.2")) 56 | testHostname["google.com"].ResolvedIPs.Insert(ipFactory("0.0.0.0")) 57 | 58 | } 59 | 60 | func TestUpsert(t *testing.T) { 61 | testRepo.Upsert(testHostname) 62 | } 63 | 64 | // TestMain wraps all tests with the needed initialized mock DB and fixtures 65 | func TestMain(m *testing.M) { 66 | // Store temporary databases files in a temporary directory 67 | tempDir, _ := ioutil.TempDir("", "testing") 68 | Server.SetPath(tempDir) 69 | 70 | // Set the main session variable to the temporary MongoDB instance 71 | res := resources.InitTestResources() 72 | 73 | testRepo = NewMongoRepository(res.DB, res.Config, res.Log) 74 | 75 | // Run the test suite 76 | retCode := m.Run() 77 | 78 | // Shut down the temporary server and removes data on disk. 79 | Server.Stop() 80 | 81 | // call with result of m.Run() 82 | os.Exit(retCode) 83 | } 84 | -------------------------------------------------------------------------------- /pkg/hostname/repository.go: -------------------------------------------------------------------------------- 1 | package hostname 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | ) 6 | 7 | type ( 8 | // Repository for hostnames collection 9 | Repository interface { 10 | CreateIndexes() error 11 | Upsert(domainMap map[string]*Input) 12 | } 13 | 14 | //Input .... 15 | Input struct { 16 | Host string //A hostname 17 | ResolvedIPs data.UniqueIPSet //Set of resolved UniqueIPs associated with a given hostname 18 | ClientIPs data.UniqueIPSet //Set of DNS Client UniqueIPs which issued queries for a given hostname 19 | } 20 | 21 | // FQDN Results for show-ip-dns-fqdns 22 | FQDNResult struct { 23 | Hostname string `bson:"_id"` 24 | } 25 | ) 26 | -------------------------------------------------------------------------------- /pkg/hostname/results.go: -------------------------------------------------------------------------------- 1 | package hostname 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | "github.com/activecm/rita-legacy/resources" 6 | "github.com/globalsign/mgo/bson" 7 | ) 8 | 9 | // IPResults returns the IP addresses the hostname was seen resolving to in the dataset 10 | func IPResults(res *resources.Resources, hostname string) ([]data.UniqueIP, error) { 11 | ssn := res.DB.Session.Copy() 12 | defer ssn.Close() 13 | 14 | ipsForHostnameQuery := []bson.M{ 15 | {"$match": bson.M{ 16 | "host": hostname, 17 | }}, 18 | {"$project": bson.M{ 19 | "ips": "$dat.ips", 20 | }}, 21 | {"$unwind": "$ips"}, 22 | {"$unwind": "$ips"}, 23 | {"$group": bson.M{ 24 | "_id": bson.M{ 25 | "ip": "$ips.ip", 26 | "network_uuid": "$ips.network_uuid", 27 | }, 28 | "network_name": bson.M{"$last": "$ips.network_name"}, 29 | }}, 30 | {"$project": bson.M{ 31 | "_id": 0, 32 | "ip": "$_id.ip", 33 | "network_uuid": "$_id.network_uuid", 34 | "network_name": "$network_name", 35 | }}, 36 | {"$sort": bson.M{ 37 | "ip": 1, 38 | }}, 39 | } 40 | 41 | var ipResults []data.UniqueIP 42 | err := ssn.DB(res.DB.GetSelectedDB()).C(res.Config.T.DNS.HostnamesTable).Pipe(ipsForHostnameQuery).AllowDiskUse().All(&ipResults) 43 | return ipResults, err 44 | } 45 | 46 | // FQDNResults returns the FQDNs the IP address was seen resolving to in the dataset 47 | func FQDNResults(res *resources.Resources, hostIP string) ([]*FQDNResult, error) { 48 | ssn := res.DB.Session.Copy() 49 | defer ssn.Close() 50 | 51 | fqdnsForHostnameQuery := []bson.M{ 52 | {"$match": bson.M{ 53 | "dat.ips.ip": hostIP, 54 | }}, 55 | {"$group": bson.M{ 56 | "_id": "$host", 57 | }}, 58 | } 59 | 60 | var fqdnResults []*FQDNResult 61 | err := ssn.DB(res.DB.GetSelectedDB()).C(res.Config.T.DNS.HostnamesTable).Pipe(fqdnsForHostnameQuery).AllowDiskUse().All(&fqdnResults) 62 | return fqdnResults, err 63 | } 64 | -------------------------------------------------------------------------------- /pkg/remover/analyzer.go: -------------------------------------------------------------------------------- 1 | package remover 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | "sync" 7 | 8 | "github.com/activecm/rita-legacy/config" 9 | "github.com/activecm/rita-legacy/database" 10 | "github.com/globalsign/mgo/bson" 11 | ) 12 | 13 | type ( 14 | //analyzer : structure for exploded dns analysis 15 | analyzer struct { 16 | chunk int //current chunk (0 if not on rolling analysis) 17 | chunkStr string //current chunk (0 if not on rolling analysis) 18 | db *database.DB // provides access to MongoDB 19 | conf *config.Config // contains details needed to access MongoDB 20 | analyzedCallback func(update) // called on each analyzed result 21 | closedCallback func() // called when .close() is called and no more calls to analyzedCallback will be made 22 | analysisChannel chan string // holds unanalyzed data 23 | analysisWg sync.WaitGroup // wait for analysis to finish 24 | } 25 | ) 26 | 27 | // newAnalyzer creates a new collector for parsing hostnames 28 | func newAnalyzer(chunk int, db *database.DB, conf *config.Config, analyzedCallback func(update), closedCallback func()) *analyzer { 29 | return &analyzer{ 30 | chunk: chunk, 31 | chunkStr: strconv.Itoa(chunk), 32 | db: db, 33 | conf: conf, 34 | analyzedCallback: analyzedCallback, 35 | closedCallback: closedCallback, 36 | analysisChannel: make(chan string), 37 | } 38 | } 39 | 40 | // collect sends a group of domains to be analyzed 41 | func (a *analyzer) collect(data string) { 42 | a.analysisChannel <- data 43 | } 44 | 45 | // close waits for the collector to finish 46 | func (a *analyzer) close() { 47 | close(a.analysisChannel) 48 | a.analysisWg.Wait() 49 | a.closedCallback() 50 | } 51 | 52 | // start kicks off a new analysis thread 53 | func (a *analyzer) start() { 54 | a.analysisWg.Add(1) 55 | go func() { 56 | 57 | for data := range a.analysisChannel { 58 | 59 | a.reduceDNSSubCount(data) 60 | 61 | } 62 | 63 | a.analysisWg.Done() 64 | }() 65 | } 66 | 67 | func (a *analyzer) reduceDNSSubCount(name string) { 68 | 69 | // split name on periods 70 | split := strings.Split(name, ".") 71 | 72 | // we will not count the very last item, because it will be either all or 73 | // a part of the tlds. This means that something like ".co.uk" will still 74 | // not be fully excluded, but it will greatly reduce the complexity for the 75 | // most common tlds 76 | max := len(split) - 1 77 | 78 | for i := 1; i <= max; i++ { 79 | // parse domain which will be the part we are on until the end of the string 80 | entry := strings.Join(split[max-i:], ".") 81 | 82 | // in some of these strings, the empty space will get counted as a domain, 83 | // this was an issue in the old version of exploded dns and caused inaccuracies 84 | if (entry == "") || (entry == "in-addr.arpa") { 85 | break 86 | } 87 | 88 | // set up writer output 89 | var output update 90 | 91 | output.query = bson.M{ 92 | "$inc": bson.M{ 93 | "subdomain_count": -1, 94 | }, 95 | } 96 | 97 | // create selector for output 98 | output.selector = bson.M{"domain": entry} 99 | 100 | output.collection = a.conf.T.DNS.ExplodedDNSTable 101 | 102 | // set to writer channel 103 | a.analyzedCallback(output) 104 | 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /pkg/remover/repository.go: -------------------------------------------------------------------------------- 1 | package remover 2 | 3 | // Repository .... 4 | type Repository interface { 5 | Remove(int) error 6 | } 7 | 8 | //update .... 9 | type update struct { 10 | selector interface{} 11 | query interface{} 12 | collection string 13 | } 14 | 15 | // //Input .... 16 | // type Input struct { 17 | // name string 18 | // Seen int64 19 | // OrigIps []string 20 | // Requests []string 21 | // } 22 | -------------------------------------------------------------------------------- /pkg/sniconn/repository.go: -------------------------------------------------------------------------------- 1 | package sniconn 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | "github.com/activecm/rita-legacy/pkg/host" 6 | ) 7 | 8 | // Repository for uconn collection 9 | type Repository interface { 10 | CreateIndexes() error 11 | Upsert(tlsMap map[string]*TLSInput, httpMap map[string]*HTTPInput, zeekUIDMap map[string]*data.ZeekUIDRecord, hostMap map[string]*host.Input) 12 | } 13 | 14 | type linkedInput struct { 15 | TLS *TLSInput 16 | TLSZeekRecords []*data.ZeekUIDRecord 17 | 18 | HTTP *HTTPInput 19 | HTTPZeekRecords []*data.ZeekUIDRecord 20 | } 21 | 22 | type TLSInput struct { 23 | Hosts data.UniqueSrcFQDNPair 24 | 25 | IsLocalSrc bool 26 | 27 | ConnectionCount int64 28 | Timestamps []int64 29 | RespondingIPs data.UniqueIPSet 30 | RespondingPorts data.IntSet 31 | 32 | RespondingCertInvalid bool 33 | Subjects data.StringSet 34 | JA3s data.StringSet 35 | JA3Ss data.StringSet 36 | 37 | ZeekUIDs []string 38 | } 39 | 40 | type HTTPInput struct { 41 | Hosts data.UniqueSrcFQDNPair 42 | 43 | IsLocalSrc bool 44 | 45 | ConnectionCount int64 46 | Timestamps []int64 47 | RespondingIPs data.UniqueIPSet 48 | RespondingPorts data.IntSet 49 | 50 | Methods data.StringSet 51 | UserAgents data.StringSet 52 | 53 | ZeekUIDs []string 54 | } 55 | -------------------------------------------------------------------------------- /pkg/uconn/mongodb_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package uconn 5 | 6 | import ( 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | 11 | "github.com/activecm/rita-legacy/pkg/data" 12 | "github.com/activecm/rita-legacy/resources" 13 | "github.com/activecm/rita-legacy/util" 14 | "github.com/globalsign/mgo/dbtest" 15 | ) 16 | 17 | // Server holds the dbtest DBServer 18 | var Server dbtest.DBServer 19 | 20 | // Set the test database 21 | var testTargetDB = "tmp_test_db" 22 | 23 | var testRepo Repository 24 | 25 | var testUconn = map[string]*Input{ 26 | "test": &Input{ 27 | Hosts: data.UniqueIPPair{ 28 | UniqueSrcIP: data.UniqueSrcIP{ 29 | SrcIP: "127.0.0.1", 30 | SrcNetworkUUID: util.UnknownPrivateNetworkUUID, 31 | SrcNetworkName: util.UnknownPrivateNetworkName, 32 | }, 33 | UniqueDstIP: data.UniqueDstIP{ 34 | DstIP: "127.0.0.1", 35 | DstNetworkUUID: util.UnknownPrivateNetworkUUID, 36 | DstNetworkName: util.UnknownPrivateNetworkName, 37 | }, 38 | }, 39 | ConnectionCount: 12, 40 | IsLocalSrc: true, 41 | IsLocalDst: true, 42 | TotalBytes: 123, 43 | TsList: []int64{1234567, 1234567}, 44 | OrigBytesList: []int64{12, 12}, 45 | TotalDuration: 123.0, 46 | MaxDuration: 12, 47 | }, 48 | } 49 | 50 | func TestUpsert(t *testing.T) { 51 | testRepo.Upsert(testUconn) 52 | 53 | } 54 | 55 | // TestMain wraps all tests with the needed initialized mock DB and fixtures 56 | func TestMain(m *testing.M) { 57 | // Store temporary databases files in a temporary directory 58 | tempDir, _ := ioutil.TempDir("", "testing") 59 | Server.SetPath(tempDir) 60 | 61 | // Set the main session variable to the temporary MongoDB instance 62 | res := resources.InitTestResources() 63 | 64 | testRepo = NewMongoRepository(res.DB, res.Config, res.Log) 65 | 66 | // Run the test suite 67 | retCode := m.Run() 68 | 69 | // Shut down the temporary server and removes data on disk. 70 | Server.Stop() 71 | 72 | // call with result of m.Run() 73 | os.Exit(retCode) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/uconn/repository.go: -------------------------------------------------------------------------------- 1 | package uconn 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | "github.com/activecm/rita-legacy/pkg/host" 6 | ) 7 | 8 | // Repository for uconn collection 9 | type Repository interface { 10 | CreateIndexes() error 11 | Upsert(uconnMap map[string]*Input, hostMap map[string]*host.Input) 12 | } 13 | 14 | // Input holds aggregated connection information between two hosts in a dataset 15 | type Input struct { 16 | Hosts data.UniqueIPPair 17 | ConnectionCount int64 18 | IsLocalSrc bool 19 | IsLocalDst bool 20 | TotalBytes int64 21 | MaxDuration float64 22 | TotalDuration float64 23 | TsList []int64 24 | UniqueTsListLength int64 25 | OrigBytesList []int64 26 | Tuples data.StringSet 27 | InvalidCertFlag bool 28 | UPPSFlag bool 29 | ConnStateMap map[string]*ConnState 30 | } 31 | 32 | // LongConnResult represents a pair of hosts that communicated and 33 | // the longest connection between those hosts. 34 | type LongConnResult struct { 35 | data.UniqueIPPair `bson:",inline"` 36 | ConnectionCount int64 `bson:"count"` 37 | TotalBytes int64 `bson:"tbytes"` 38 | TotalDuration float64 `bson:"tdur"` 39 | MaxDuration float64 `bson:"maxdur"` 40 | Tuples []string `bson:"tuples"` 41 | Open bool `bson:"open"` 42 | } 43 | 44 | // OpenConnResult represents a pair of hosts that currently 45 | // have an open connection. It shows the current number of 46 | // bytes that have been transferred, the total duration thus far, 47 | // the port:protocol:service tuple, and the Zeek UID in case 48 | // the user wants to look for that connection in their zeek logs 49 | type OpenConnResult struct { 50 | data.UniqueIPPair `bson:",inline"` 51 | Bytes int `bson:"bytes"` 52 | Duration float64 `bson:"duration"` 53 | Tuple string `bson:"tuple"` 54 | UID string `bson:"uid"` 55 | } 56 | 57 | // ConnState is used to determine if a particular 58 | // connection, keyed by Zeek's UID field, is open 59 | // or closed. If a connection is still open, we 60 | // will write its bytes and duration info out in 61 | // a separate field in mongo. This is needed so 62 | // that we can keep a running score of data from 63 | // open connections without double-counting the information 64 | // when the connection closes. 65 | // Parameters: 66 | // 67 | // Bytes: total bytes for current connection 68 | // Duration: total duration for current connection 69 | // Open: shows if a connection is still open 70 | // OrigBytes: total origin bytes for current connection 71 | // Ts: timestamp of the start of the open connection 72 | // Tuple: destination port:protocol:service of current connection 73 | type ConnState struct { 74 | Bytes int64 `bson:"bytes"` 75 | Duration float64 `bson:"duration"` 76 | Open bool `bson:"open"` 77 | OrigBytes int64 `bson:"orig_bytes"` 78 | Ts int64 `bson:"ts"` 79 | Tuple string `bson:"tuple"` 80 | } 81 | -------------------------------------------------------------------------------- /pkg/uconnproxy/analyzer.go: -------------------------------------------------------------------------------- 1 | package uconnproxy 2 | 3 | import ( 4 | "strconv" 5 | "sync" 6 | 7 | "github.com/activecm/rita-legacy/config" 8 | "github.com/activecm/rita-legacy/database" 9 | "github.com/globalsign/mgo/bson" 10 | ) 11 | 12 | type ( 13 | //analyzer : structure for proxy beacon analysis 14 | analyzer struct { 15 | chunk int //current chunk (0 if not on rolling analysis) 16 | chunkStr string //current chunk (0 if not on rolling analysis) 17 | connLimit int64 // limit for strobe classification 18 | db *database.DB // provides access to MongoDB 19 | conf *config.Config // contains details needed to access MongoDB 20 | analyzedCallback func(database.BulkChanges) // called on each analyzed result 21 | closedCallback func() // called when .close() is called and no more calls to analyzedCallback will be made 22 | analysisChannel chan *Input // holds unanalyzed data 23 | analysisWg sync.WaitGroup // wait for analysis to finish 24 | } 25 | ) 26 | 27 | // newAnalyzer creates a new collector for parsing uconnproxy 28 | func newAnalyzer(chunk int, connLimit int64, db *database.DB, conf *config.Config, analyzedCallback func(database.BulkChanges), closedCallback func()) *analyzer { 29 | return &analyzer{ 30 | chunk: chunk, 31 | chunkStr: strconv.Itoa(chunk), 32 | connLimit: connLimit, 33 | db: db, 34 | conf: conf, 35 | analyzedCallback: analyzedCallback, 36 | closedCallback: closedCallback, 37 | analysisChannel: make(chan *Input), 38 | } 39 | } 40 | 41 | // collect sends a group of uconnproxy data to be analyzed 42 | func (a *analyzer) collect(datum *Input) { 43 | a.analysisChannel <- datum 44 | } 45 | 46 | // close waits for the collector to finish 47 | func (a *analyzer) close() { 48 | close(a.analysisChannel) 49 | a.analysisWg.Wait() 50 | a.closedCallback() 51 | } 52 | 53 | // start kicks off a new analysis thread 54 | func (a *analyzer) start() { 55 | a.analysisWg.Add(1) 56 | go func() { 57 | 58 | for datum := range a.analysisChannel { 59 | 60 | mainUpdate := mainQuery(datum, a.connLimit, a.chunk) 61 | 62 | a.analyzedCallback(database.BulkChanges{ 63 | a.conf.T.Structure.UniqueConnProxyTable: []database.BulkChange{{ 64 | Selector: datum.Hosts.BSONKey(), 65 | Update: mainUpdate, 66 | Upsert: true, 67 | }}, 68 | }) 69 | } 70 | a.analysisWg.Done() 71 | }() 72 | } 73 | 74 | // mainQuery records the bulk of the information about communications between two hosts 75 | // over an HTTP proxy 76 | func mainQuery(datum *Input, strobeLimit int64, chunk int) bson.M { 77 | 78 | // if this connection qualifies to be a strobe with the current number 79 | // of connections in the current datum, don't store ts. 80 | // it will not qualify to be downgraded to a proxy beacon until this chunk is 81 | // outdated and removed. If only importing once - still just a strobe. 82 | ts := datum.TsList 83 | 84 | isStrobe := datum.ConnectionCount >= strobeLimit 85 | if isStrobe { 86 | ts = []int64{} 87 | } 88 | 89 | return bson.M{ 90 | "$set": bson.M{ 91 | "strobeFQDN": isStrobe, 92 | "cid": chunk, 93 | "src_network_name": datum.Hosts.SrcNetworkName, 94 | "proxy": datum.Proxy, 95 | }, 96 | "$push": bson.M{ 97 | "dat": bson.M{ 98 | "$each": []bson.M{{ 99 | "count": datum.ConnectionCount, 100 | "ts": ts, 101 | "cid": chunk, 102 | }}, 103 | }, 104 | }, 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /pkg/uconnproxy/mongodb.go: -------------------------------------------------------------------------------- 1 | package uconnproxy 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/activecm/rita-legacy/config" 7 | "github.com/activecm/rita-legacy/database" 8 | "github.com/activecm/rita-legacy/util" 9 | "github.com/globalsign/mgo" 10 | log "github.com/sirupsen/logrus" 11 | "github.com/vbauerster/mpb" 12 | "github.com/vbauerster/mpb/decor" 13 | ) 14 | 15 | type repo struct { 16 | database *database.DB 17 | config *config.Config 18 | log *log.Logger 19 | } 20 | 21 | // NewMongoRepository bundles the given resources for updating MongoDB with proxy connection data 22 | func NewMongoRepository(db *database.DB, conf *config.Config, logger *log.Logger) Repository { 23 | return &repo{ 24 | database: db, 25 | config: conf, 26 | log: logger, 27 | } 28 | } 29 | 30 | // CreateIndexes creates indexes for the uconnProxy collection 31 | func (r *repo) CreateIndexes() error { 32 | session := r.database.Session.Copy() 33 | defer session.Close() 34 | 35 | // set collection name 36 | collectionName := r.config.T.Structure.UniqueConnProxyTable 37 | 38 | // check if collection already exists 39 | names, _ := session.DB(r.database.GetSelectedDB()).CollectionNames() 40 | 41 | // if collection exists, we don't need to do anything else 42 | for _, name := range names { 43 | if name == collectionName { 44 | return nil 45 | } 46 | } 47 | 48 | indexes := []mgo.Index{ 49 | {Key: []string{"src", "fqdn", "src_network_uuid"}, Unique: true}, 50 | {Key: []string{"fqdn"}}, 51 | {Key: []string{"src", "src_network_uuid"}}, 52 | {Key: []string{"dat.count"}}, 53 | } 54 | 55 | // create collection 56 | err := r.database.CreateCollection(collectionName, indexes) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | return nil 62 | } 63 | 64 | // Upsert records the given proxy connection data in MongoDB 65 | func (r *repo) Upsert(uconnProxyMap map[string]*Input) { 66 | // Create the workers 67 | writerWorker := database.NewBulkWriter(r.database, r.config, r.log, true, "uconnproxy") 68 | 69 | analyzerWorker := newAnalyzer( 70 | r.config.S.Rolling.CurrentChunk, 71 | int64(r.config.S.Strobe.ConnectionLimit), 72 | r.database, 73 | r.config, 74 | writerWorker.Collect, 75 | writerWorker.Close, 76 | ) 77 | 78 | // kick off the threaded goroutines 79 | for i := 0; i < util.Max(1, runtime.NumCPU()/2); i++ { 80 | analyzerWorker.start() 81 | writerWorker.Start() 82 | } 83 | 84 | // progress bar for troubleshooting 85 | p := mpb.New(mpb.WithWidth(20)) 86 | bar := p.AddBar(int64(len(uconnProxyMap)), 87 | mpb.PrependDecorators( 88 | decor.Name("\t[-] Uconn Proxy Analysis:", decor.WC{W: 30, C: decor.DidentRight}), 89 | decor.CountersNoUnit(" %d / %d ", decor.WCSyncWidth), 90 | ), 91 | mpb.AppendDecorators(decor.Percentage()), 92 | ) 93 | 94 | // loop over map entries 95 | for _, entry := range uconnProxyMap { 96 | analyzerWorker.collect(entry) 97 | bar.IncrBy(1) 98 | } 99 | p.Wait() 100 | 101 | // start the closing cascade (this will also close the other channels) 102 | analyzerWorker.close() 103 | } 104 | -------------------------------------------------------------------------------- /pkg/uconnproxy/mongodb_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package uconnproxy 5 | 6 | import ( 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | 11 | "github.com/activecm/rita-legacy/pkg/data" 12 | "github.com/activecm/rita-legacy/resources" 13 | "github.com/activecm/rita-legacy/util" 14 | "github.com/globalsign/mgo/dbtest" 15 | ) 16 | 17 | // Server holds the dbtest DBServer 18 | var Server dbtest.DBServer 19 | 20 | // Set the test database 21 | var testTargetDB = "tmp_test_db" 22 | 23 | var testRepo Repository 24 | 25 | var testUconn = map[string]*Input{ 26 | "test": &Input{ 27 | Hosts: data.UniqueIPPair{ 28 | SrcIP: "127.0.0.1", 29 | SrcNetworkUUID: util.UnknownPrivateNetworkUUID, 30 | SrcNetworkName: util.UnknownPrivateNetworkName, 31 | DstIP: "127.0.0.1", 32 | DstNetworkUUID: util.UnknownPrivateNetworkUUID, 33 | DstNetworkName: util.UnknownPrivateNetworkName, 34 | }, 35 | ConnectionCount: 12, 36 | IsLocalSrc: true, 37 | IsLocalDst: true, 38 | TotalBytes: 123, 39 | TsList: []int64{1234567, 1234567}, 40 | OrigBytesList: []int64{12, 12}, 41 | TotalDuration: 123.0, 42 | MaxDuration: 12, 43 | }, 44 | } 45 | 46 | func TestUpsert(t *testing.T) { 47 | testRepo.Upsert(testUconn) 48 | 49 | } 50 | 51 | // TestMain wraps all tests with the needed initialized mock DB and fixtures 52 | func TestMain(m *testing.M) { 53 | // Store temporary databases files in a temporary directory 54 | tempDir, _ := ioutil.TempDir("", "testing") 55 | Server.SetPath(tempDir) 56 | 57 | // Set the main session variable to the temporary MongoDB instance 58 | res := resources.InitTestResources() 59 | 60 | testRepo = NewMongoRepository(res) 61 | 62 | // Run the test suite 63 | retCode := m.Run() 64 | 65 | // Shut down the temporary server and removes data on disk. 66 | Server.Stop() 67 | 68 | // call with result of m.Run() 69 | os.Exit(retCode) 70 | } 71 | -------------------------------------------------------------------------------- /pkg/uconnproxy/repository.go: -------------------------------------------------------------------------------- 1 | package uconnproxy 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | ) 6 | 7 | // Repository for uconnproxy collection 8 | type Repository interface { 9 | CreateIndexes() error 10 | Upsert(uconnProxyMap map[string]*Input) 11 | } 12 | 13 | // Input structure for sending data 14 | // to the analyzer. Contains a tuple of 15 | // Src IP/UUID/Name and an FQDN to which the Src IP 16 | // was attempting to communicate. 17 | // Contains a list of unique time stamps for the 18 | // connections out from the Src to the FQDN via the 19 | // proxy server and a count of the connections. 20 | type Input struct { 21 | Hosts data.UniqueSrcFQDNPair 22 | TsList []int64 23 | TsListFull []int64 24 | Proxy data.UniqueIP 25 | ConnectionCount int64 26 | } 27 | -------------------------------------------------------------------------------- /pkg/useragent/analyzer.go: -------------------------------------------------------------------------------- 1 | package useragent 2 | 3 | import ( 4 | "strconv" 5 | "sync" 6 | 7 | "github.com/activecm/rita-legacy/config" 8 | "github.com/activecm/rita-legacy/database" 9 | "github.com/globalsign/mgo/bson" 10 | ) 11 | 12 | // rareSignatureOrigIPsCutoff determines the cutoff for marking a particular IP as having used 13 | // rare signature on an HTTP(s) connection. If a particular signature/ user agent is associated 14 | // with less than `rareSignatureOrigIPsCutoff` originating IPs, we mark those IPs as having used 15 | // a rare signature. 16 | const rareSignatureOrigIPsCutoff = 5 17 | 18 | type ( 19 | //analyzer is a structure for useragent analysis 20 | analyzer struct { 21 | chunk int //current chunk (0 if not on rolling analysis) 22 | chunkStr string //current chunk (0 if not on rolling analysis) 23 | db *database.DB // provides access to MongoDB 24 | conf *config.Config // contains details needed to access MongoDB 25 | analyzedCallback func(database.BulkChanges) // called on each analyzed result 26 | closedCallback func() // called when .close() is called and no more calls to analyzedCallback will be made 27 | analysisChannel chan *Input // holds unanalyzed data 28 | analysisWg sync.WaitGroup // wait for analysis to finish 29 | } 30 | ) 31 | 32 | // newAnalyzer creates a new analyzer for recording connections that were made 33 | // with HTTP useragents and TLS JA3 hashes 34 | func newAnalyzer(chunk int, db *database.DB, conf *config.Config, analyzedCallback func(database.BulkChanges), closedCallback func()) *analyzer { 35 | return &analyzer{ 36 | chunk: chunk, 37 | chunkStr: strconv.Itoa(chunk), 38 | db: db, 39 | conf: conf, 40 | analyzedCallback: analyzedCallback, 41 | closedCallback: closedCallback, 42 | analysisChannel: make(chan *Input), 43 | } 44 | } 45 | 46 | // collect gathers connection signature records for analysis 47 | func (a *analyzer) collect(datum *Input) { 48 | a.analysisChannel <- datum 49 | } 50 | 51 | // close waits for the analyzer to finish 52 | func (a *analyzer) close() { 53 | close(a.analysisChannel) 54 | a.analysisWg.Wait() 55 | a.closedCallback() 56 | } 57 | 58 | // start kicks off a new analysis thread 59 | func (a *analyzer) start() { 60 | a.analysisWg.Add(1) 61 | go func() { 62 | ssn := a.db.Session.Copy() 63 | defer ssn.Close() 64 | 65 | for datum := range a.analysisChannel { 66 | useragentsSelector := bson.M{"user_agent": datum.Name} 67 | useragentsQuery := useragentsQuery(datum, a.chunk) 68 | a.analyzedCallback(database.BulkChanges{ 69 | a.conf.T.UserAgent.UserAgentTable: []database.BulkChange{{ 70 | Selector: useragentsSelector, 71 | Update: useragentsQuery, 72 | Upsert: true, 73 | }}, 74 | }) 75 | } 76 | 77 | a.analysisWg.Done() 78 | }() 79 | } 80 | 81 | // useragentsQuery returns a mgo query which inserts the given datum into the useragent collection. The useragent's 82 | // originating IPs and requested FQDNs are capped in order to prevent hitting the MongoDB document size limits. 83 | func useragentsQuery(datum *Input, chunk int) bson.M { 84 | origIPs := datum.OrigIps.Items() 85 | if len(origIPs) > 10 { 86 | origIPs = origIPs[:10] 87 | } 88 | 89 | requests := datum.Requests.Items() 90 | if len(requests) > 10 { 91 | requests = requests[:10] 92 | } 93 | 94 | return bson.M{ 95 | "$push": bson.M{ 96 | "dat": bson.M{ 97 | "seen": datum.Seen, 98 | "orig_ips": origIPs, 99 | "hosts": requests, 100 | "cid": chunk, 101 | }, 102 | }, 103 | "$set": bson.M{"cid": chunk}, 104 | "$setOnInsert": bson.M{"ja3": datum.JA3}, 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /pkg/useragent/mongodb_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package useragent 5 | 6 | import ( 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | 11 | "github.com/activecm/rita-legacy/pkg/data" 12 | "github.com/activecm/rita-legacy/resources" 13 | "github.com/activecm/rita-legacy/util" 14 | "github.com/globalsign/mgo/dbtest" 15 | ) 16 | 17 | // Server holds the dbtest DBServer 18 | var Server dbtest.DBServer 19 | 20 | // Set the test database 21 | var testTargetDB = "tmp_test_db" 22 | 23 | var testRepo Repository 24 | 25 | var testUserAgent = map[string]*Input{ 26 | "Debian APT-HTTP/1.3 (1.2.24)": { 27 | Seen: 123, 28 | }, 29 | } 30 | 31 | func init() { 32 | testUserAgent["Debian APT-HTTP/1.3 (1.2.24)"].OrigIps.Insert( 33 | data.UniqueIP{ 34 | IP: "5.6.7.8", 35 | NetworkUUID: util.PublicNetworkUUID, 36 | NetworkName: util.PublicNetworkName, 37 | }, 38 | ) 39 | testUserAgent["Debian APT-HTTP/1.3 (1.2.24)"].OrigIps.Insert( 40 | data.UniqueIP{ 41 | IP: "9.10.11.12", 42 | NetworkUUID: util.PublicNetworkUUID, 43 | NetworkName: util.PublicNetworkName, 44 | }, 45 | ) 46 | } 47 | 48 | func TestUpsert(t *testing.T) { 49 | testRepo.Upsert(testUserAgent) 50 | 51 | } 52 | 53 | // TestMain wraps all tests with the needed initialized mock DB and fixtures 54 | func TestMain(m *testing.M) { 55 | // Store temporary databases files in a temporary directory 56 | tempDir, _ := ioutil.TempDir("", "testing") 57 | Server.SetPath(tempDir) 58 | 59 | // Set the main session variable to the temporary MongoDB instance 60 | res := resources.InitTestResources() 61 | 62 | testRepo = NewMongoRepository(res.DB, res.Config, res.Log) 63 | 64 | // Run the test suite 65 | retCode := m.Run() 66 | 67 | // Shut down the temporary server and removes data on disk. 68 | Server.Stop() 69 | 70 | // call with result of m.Run() 71 | os.Exit(retCode) 72 | } 73 | -------------------------------------------------------------------------------- /pkg/useragent/repository.go: -------------------------------------------------------------------------------- 1 | package useragent 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/pkg/data" 5 | "github.com/activecm/rita-legacy/pkg/host" 6 | ) 7 | 8 | // Repository for uconn collection 9 | type Repository interface { 10 | CreateIndexes() error 11 | Upsert(useragentMap map[string]*Input, hostMap map[string]*host.Input) 12 | } 13 | 14 | // Input .... 15 | type Input struct { 16 | Name string 17 | Seen int64 18 | OrigIps data.UniqueIPSet 19 | Requests data.StringSet 20 | JA3 bool 21 | } 22 | 23 | // Result represents a user agent and how many times that user agent 24 | // was seen in the dataset 25 | type Result struct { 26 | UserAgent string `bson:"user_agent"` 27 | TimesUsed int64 `bson:"seen"` 28 | } 29 | -------------------------------------------------------------------------------- /pkg/useragent/results.go: -------------------------------------------------------------------------------- 1 | package useragent 2 | 3 | import ( 4 | "github.com/activecm/rita-legacy/resources" 5 | "github.com/globalsign/mgo/bson" 6 | ) 7 | 8 | // Results returns useragents sorted by how many times each useragent was 9 | // seen in the dataset. sortDirection controls where the useragents are 10 | // sorted in descending (sortDirection=-1) or ascending order (sortDirection=1). 11 | // limit and noLimit control how many results are returned. 12 | func Results(res *resources.Resources, sortDirection, limit int, noLimit bool) ([]Result, error) { 13 | ssn := res.DB.Session.Copy() 14 | defer ssn.Close() 15 | 16 | var useragentResults []Result 17 | 18 | useragentQuery := []bson.M{ 19 | {"$project": bson.M{"user_agent": 1, "seen": "$dat.seen"}}, 20 | {"$unwind": "$seen"}, 21 | {"$group": bson.M{ 22 | "_id": "$user_agent", 23 | "seen": bson.M{"$sum": "$seen"}, 24 | }}, 25 | {"$project": bson.M{ 26 | "_id": 0, 27 | "user_agent": "$_id", 28 | "seen": 1, 29 | }}, 30 | {"$sort": bson.M{"seen": sortDirection}}, 31 | } 32 | 33 | if !noLimit { 34 | useragentQuery = append(useragentQuery, bson.M{"$limit": limit}) 35 | } 36 | 37 | err := ssn.DB(res.DB.GetSelectedDB()).C(res.Config.T.UserAgent.UserAgentTable).Pipe(useragentQuery).AllowDiskUse().All(&useragentResults) 38 | 39 | return useragentResults, err 40 | 41 | } 42 | -------------------------------------------------------------------------------- /reporting/report-beacons.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | "os" 7 | 8 | "github.com/activecm/rita-legacy/pkg/beacon" 9 | "github.com/activecm/rita-legacy/reporting/templates" 10 | "github.com/activecm/rita-legacy/resources" 11 | ) 12 | 13 | func printBeacons(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 14 | var w string 15 | f, err := os.Create("beacons.html") 16 | if err != nil { 17 | return err 18 | } 19 | defer f.Close() 20 | 21 | var beaconsTempl string 22 | if showNetNames { 23 | beaconsTempl = templates.BeaconsNetNamesTempl 24 | } else { 25 | beaconsTempl = templates.BeaconsTempl 26 | } 27 | 28 | out, err := template.New("beacon.html").Parse(beaconsTempl) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | data, err := beacon.Results(res, 0) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | if len(data) == 0 { 39 | w = "" 40 | } else { 41 | w, err = getBeaconWriter(data, showNetNames) 42 | if err != nil { 43 | return err 44 | } 45 | } 46 | 47 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 48 | } 49 | 50 | func getBeaconWriter(beacons []beacon.Result, showNetNames bool) (string, error) { 51 | tmpl := "" 52 | 53 | tmpl += "{{printf \"%.3f\" .Score}}" 54 | 55 | if showNetNames { 56 | tmpl += "{{.SrcNetworkName}}{{.DstNetworkName}}{{.SrcIP}}{{.DstIP}}" 57 | } else { 58 | tmpl += "{{.SrcIP}}{{.DstIP}}" 59 | } 60 | tmpl += "{{.Connections}}{{printf \"%.3f\" .AvgBytes}}{{.TotalBytes}}{{printf \"%.3f\" .Ts.Score}}" 61 | tmpl += "{{printf \"%.3f\" .Ds.Score}}{{printf \"%.3f\" .DurScore}}{{printf \"%.3f\" .HistScore}}{{.Ts.Mode}}" 62 | tmpl += "\n" 63 | 64 | out, err := template.New("beacon").Parse(tmpl) 65 | if err != nil { 66 | return "", err 67 | } 68 | 69 | w := new(bytes.Buffer) 70 | 71 | for _, result := range beacons { 72 | err = out.Execute(w, result) 73 | if err != nil { 74 | return "", err 75 | } 76 | } 77 | 78 | return w.String(), nil 79 | } 80 | -------------------------------------------------------------------------------- /reporting/report-beaconsproxy.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | "os" 7 | 8 | "github.com/activecm/rita-legacy/pkg/beaconproxy" 9 | "github.com/activecm/rita-legacy/reporting/templates" 10 | "github.com/activecm/rita-legacy/resources" 11 | ) 12 | 13 | func printBeaconsProxy(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 14 | var w string 15 | f, err := os.Create("beaconsproxy.html") 16 | if err != nil { 17 | return err 18 | } 19 | defer f.Close() 20 | 21 | var beaconsProxyTempl string 22 | if showNetNames { 23 | beaconsProxyTempl = templates.BeaconsProxyNetNamesTempl 24 | } else { 25 | beaconsProxyTempl = templates.BeaconsProxyTempl 26 | } 27 | 28 | out, err := template.New("beaconproxy.html").Parse(beaconsProxyTempl) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | data, err := beaconproxy.Results(res, 0) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | if len(data) == 0 { 39 | w = "" 40 | } else { 41 | w, err = getBeaconProxyWriter(data, showNetNames) 42 | if err != nil { 43 | return err 44 | } 45 | } 46 | 47 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 48 | } 49 | 50 | func getBeaconProxyWriter(beaconsProxy []beaconproxy.Result, showNetNames bool) (string, error) { 51 | tmpl := "" 52 | 53 | tmpl += "{{printf \"%.3f\" .Score}}" 54 | 55 | if showNetNames { 56 | tmpl += "{{.SrcNetworkName}}" 57 | } 58 | 59 | tmpl += "{{.SrcIP}}{{.FQDN}}" 60 | 61 | if showNetNames { 62 | tmpl += "{{.Proxy.NetworkName}}" 63 | } 64 | 65 | tmpl += "{{.Proxy.IP}}" 66 | 67 | tmpl += "{{.Connections}}{{printf \"%.3f\" .Ts.Score}}" 68 | tmpl += "{{printf \"%.3f\" .DurScore}}{{printf \"%.3f\" .HistScore}}{{.Ts.Mode}}" 69 | tmpl += "\n" 70 | 71 | out, err := template.New("beaconproxy").Parse(tmpl) 72 | if err != nil { 73 | return "", err 74 | } 75 | 76 | w := new(bytes.Buffer) 77 | 78 | for _, result := range beaconsProxy { 79 | err = out.Execute(w, result) 80 | if err != nil { 81 | return "", err 82 | } 83 | } 84 | 85 | return w.String(), nil 86 | } 87 | -------------------------------------------------------------------------------- /reporting/report-beaconssni.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | "os" 7 | 8 | "github.com/activecm/rita-legacy/pkg/beaconsni" 9 | "github.com/activecm/rita-legacy/reporting/templates" 10 | "github.com/activecm/rita-legacy/resources" 11 | ) 12 | 13 | func printBeaconsSNI(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 14 | var w string 15 | f, err := os.Create("beaconssni.html") 16 | if err != nil { 17 | return err 18 | } 19 | defer f.Close() 20 | 21 | var beaconsSNITempl string 22 | if showNetNames { 23 | beaconsSNITempl = templates.BeaconsSNINetNamesTempl 24 | } else { 25 | beaconsSNITempl = templates.BeaconsSNITempl 26 | } 27 | 28 | out, err := template.New("beaconssni.html").Parse(beaconsSNITempl) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | data, err := beaconsni.Results(res, 0) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | if len(data) == 0 { 39 | w = "" 40 | } else { 41 | w, err = getBeaconSNIWriter(data, showNetNames) 42 | if err != nil { 43 | return err 44 | } 45 | } 46 | 47 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 48 | } 49 | 50 | func getBeaconSNIWriter(beaconsSNI []beaconsni.Result, showNetNames bool) (string, error) { 51 | tmpl := "" 52 | 53 | tmpl += "{{printf \"%.3f\" .Score}}" 54 | 55 | if showNetNames { 56 | tmpl += "{{.SrcNetworkName}}{{.SrcIP}}{{.FQDN}}" 57 | } else { 58 | tmpl += "{{.SrcIP}}{{.FQDN}}" 59 | } 60 | tmpl += "{{.Connections}}{{printf \"%.3f\" .AvgBytes}}{{.TotalBytes}}{{printf \"%.3f\" .Ts.Score}}" 61 | tmpl += "{{printf \"%.3f\" .Ds.Score}}{{printf \"%.3f\" .DurScore}}{{printf \"%.3f\" .HistScore}}{{.Ts.Mode}}" 62 | tmpl += "\n" 63 | 64 | out, err := template.New("beaconsni").Parse(tmpl) 65 | if err != nil { 66 | return "", err 67 | } 68 | 69 | w := new(bytes.Buffer) 70 | 71 | for _, result := range beaconsSNI { 72 | err = out.Execute(w, result) 73 | if err != nil { 74 | return "", err 75 | } 76 | } 77 | 78 | return w.String(), nil 79 | } 80 | -------------------------------------------------------------------------------- /reporting/report-bl-dest-ips.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "html/template" 5 | "os" 6 | 7 | "github.com/activecm/rita-legacy/pkg/blacklist" 8 | "github.com/activecm/rita-legacy/reporting/templates" 9 | "github.com/activecm/rita-legacy/resources" 10 | ) 11 | 12 | func printBLDestIPs(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 13 | f, err := os.Create("bl-dest-ips.html") 14 | if err != nil { 15 | return err 16 | } 17 | defer f.Close() 18 | 19 | data, err := blacklist.DstIPResults(res, "conn_count", 1000, false) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | var blDestIPTempl string 25 | if showNetNames { 26 | blDestIPTempl = templates.BLDestIPNetNamesTempl 27 | } else { 28 | blDestIPTempl = templates.BLDestIPTempl 29 | } 30 | 31 | out, err := template.New("bl-dest-ips.html").Parse(blDestIPTempl) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | w, err := getBLIPWriter(data, showNetNames) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 42 | } 43 | -------------------------------------------------------------------------------- /reporting/report-bl-hostnames.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | "os" 7 | "sort" 8 | "strings" 9 | 10 | "github.com/activecm/rita-legacy/pkg/blacklist" 11 | "github.com/activecm/rita-legacy/reporting/templates" 12 | "github.com/activecm/rita-legacy/resources" 13 | ) 14 | 15 | func printBLHostnames(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 16 | f, err := os.Create("bl-hostnames.html") 17 | if err != nil { 18 | return err 19 | } 20 | defer f.Close() 21 | 22 | data, err := blacklist.HostnameResults(res, "conn_count", 1000, false) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | out, err := template.New("bl-hostnames.html").Parse(templates.BLHostnameTempl) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | w, err := getBLHostnameWriter(data, showNetNames) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 38 | } 39 | 40 | func getBLHostnameWriter(results []blacklist.HostnameResult, showNetNames bool) (string, error) { 41 | tmpl := "{{.Host}}{{.Connections}}{{.UniqueConnections}}" + 42 | "{{.TotalBytes}}" + 43 | "{{range $idx, $host := .ConnectedHostStrs}}{{if $idx}}, {{end}}{{ $host }}{{end}}" + 44 | "\n" 45 | 46 | out, err := template.New("blhostname").Parse(tmpl) 47 | if err != nil { 48 | return "", err 49 | } 50 | 51 | w := new(bytes.Buffer) 52 | 53 | for _, result := range results { 54 | 55 | //format UniqueIP destinations 56 | var connectedHostStrs []string 57 | for _, connectedUniqIP := range result.ConnectedHosts { 58 | 59 | var connectedIPStr string 60 | if showNetNames { 61 | escapedNetName := strings.ReplaceAll(connectedUniqIP.NetworkName, " ", "_") 62 | escapedNetName = strings.ReplaceAll(escapedNetName, ":", "_") 63 | connectedIPStr = escapedNetName + ":" + connectedUniqIP.IP 64 | } else { 65 | connectedIPStr = connectedUniqIP.IP 66 | } 67 | 68 | connectedHostStrs = append(connectedHostStrs, connectedIPStr) 69 | } 70 | sort.Strings(connectedHostStrs) 71 | 72 | formattedResult := struct { 73 | blacklist.HostnameResult 74 | ConnectedHostStrs []string 75 | }{result, connectedHostStrs} 76 | 77 | err := out.Execute(w, formattedResult) 78 | if err != nil { 79 | return "", err 80 | } 81 | } 82 | return w.String(), nil 83 | } 84 | -------------------------------------------------------------------------------- /reporting/report-bl-source-ips.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | "os" 7 | "sort" 8 | "strings" 9 | 10 | "github.com/activecm/rita-legacy/pkg/blacklist" 11 | "github.com/activecm/rita-legacy/reporting/templates" 12 | "github.com/activecm/rita-legacy/resources" 13 | ) 14 | 15 | func printBLSourceIPs(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 16 | f, err := os.Create("bl-source-ips.html") 17 | if err != nil { 18 | return err 19 | } 20 | defer f.Close() 21 | 22 | data, err := blacklist.SrcIPResults(res, "conn_count", 1000, false) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | var blSourceIPTempl string 28 | if showNetNames { 29 | blSourceIPTempl = templates.BLSourceIPNetNamesTempl 30 | } else { 31 | blSourceIPTempl = templates.BLSourceIPTempl 32 | } 33 | 34 | out, err := template.New("bl-source-ips.html").Parse(blSourceIPTempl) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | w, err := getBLIPWriter(data, showNetNames) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 45 | } 46 | 47 | func getBLIPWriter(results []blacklist.IPResult, showNetNames bool) (string, error) { 48 | var tmpl string 49 | if showNetNames { 50 | tmpl = "{{.Host.IP}}{{.Host.NetworkName}}{{.Connections}}{{.UniqueConnections}}" + 51 | "{{.TotalBytes}}" + 52 | "{{range $idx, $host := .ConnectedHostStrs}}{{if $idx}}, {{end}}{{ $host }}{{end}}" + 53 | "\n" 54 | } else { 55 | tmpl = "{{.Host.IP}}{{.Connections}}{{.UniqueConnections}}" + 56 | "{{.TotalBytes}}" + 57 | "{{range $idx, $host := .ConnectedHostStrs}}{{if $idx}}, {{end}}{{ $host }}{{end}}" + 58 | "\n" 59 | } 60 | 61 | out, err := template.New("blip").Parse(tmpl) 62 | if err != nil { 63 | return "", err 64 | } 65 | 66 | w := new(bytes.Buffer) 67 | 68 | for _, result := range results { 69 | 70 | //format UniqueIP destinations 71 | var connectedHostStrs []string 72 | for _, connectedUniqIP := range result.Peers { 73 | 74 | var connectedIPStr string 75 | if showNetNames { 76 | escapedNetName := strings.ReplaceAll(connectedUniqIP.NetworkName, " ", "_") 77 | escapedNetName = strings.ReplaceAll(escapedNetName, ":", "_") 78 | connectedIPStr = escapedNetName + ":" + connectedUniqIP.IP 79 | } else { 80 | connectedIPStr = connectedUniqIP.IP 81 | } 82 | 83 | connectedHostStrs = append(connectedHostStrs, connectedIPStr) 84 | } 85 | sort.Strings(connectedHostStrs) 86 | 87 | formattedResult := struct { 88 | blacklist.IPResult 89 | ConnectedHostStrs []string 90 | }{result, connectedHostStrs} 91 | 92 | err := out.Execute(w, formattedResult) 93 | if err != nil { 94 | return "", err 95 | } 96 | } 97 | return w.String(), nil 98 | } 99 | -------------------------------------------------------------------------------- /reporting/report-explodedDns.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | "os" 7 | 8 | "github.com/activecm/rita-legacy/pkg/explodeddns" 9 | "github.com/activecm/rita-legacy/reporting/templates" 10 | "github.com/activecm/rita-legacy/resources" 11 | ) 12 | 13 | func printDNS(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 14 | f, err := os.Create("dns.html") 15 | if err != nil { 16 | return err 17 | } 18 | defer f.Close() 19 | 20 | res.DB.SelectDB(db) 21 | 22 | limit := 1000 23 | 24 | data, err := explodeddns.Results(res, limit, false) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | out, err := template.New("dns.html").Parse(templates.DNStempl) 30 | if err != nil { 31 | return err 32 | } 33 | 34 | w, err := getDNSWriter(data) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 40 | } 41 | 42 | func getDNSWriter(results []explodeddns.Result) (string, error) { 43 | tmpl := "{{.SubdomainCount}}{{.Visited}}{{.Domain}}\n" 44 | 45 | out, err := template.New("dns").Parse(tmpl) 46 | if err != nil { 47 | return "", err 48 | } 49 | 50 | w := new(bytes.Buffer) 51 | 52 | for _, result := range results { 53 | err := out.Execute(w, result) 54 | if err != nil { 55 | return "", err 56 | } 57 | } 58 | return w.String(), nil 59 | } 60 | -------------------------------------------------------------------------------- /reporting/report-long-connections.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | "github.com/activecm/rita-legacy/pkg/uconn" 11 | "github.com/activecm/rita-legacy/reporting/templates" 12 | "github.com/activecm/rita-legacy/resources" 13 | "github.com/activecm/rita-legacy/util" 14 | ) 15 | 16 | func printLongConns(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 17 | f, err := os.Create("long-conns.html") 18 | if err != nil { 19 | return err 20 | } 21 | defer f.Close() 22 | 23 | var longConnsTempl string 24 | if showNetNames { 25 | longConnsTempl = templates.LongConnsNetNamesTempl 26 | } else { 27 | longConnsTempl = templates.LongConnsTempl 28 | } 29 | 30 | out, err := template.New("long-conns.html").Parse(longConnsTempl) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | res.DB.SelectDB(db) 36 | 37 | thresh := 60 // 1 minute 38 | data, err := uconn.LongConnResults(res, thresh, 1000, false) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | w, err := getLongConnWriter(data, showNetNames) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 49 | } 50 | 51 | func getLongConnWriter(conns []uconn.LongConnResult, showNetNames bool) (string, error) { 52 | var tmpl string 53 | if showNetNames { 54 | tmpl = "{{.SrcNetworkName}}{{.DstNetworkName}}{{.SrcIP}}{{.DstIP}}{{.TupleStr}}{{.TotalDurationStr}}{{.MaxDurationStr}}{{.ConnectionCount}}{{.TotalBytes}}{{.State}}\n" 55 | } else { 56 | tmpl = "{{.SrcIP}}{{.DstIP}}{{.TupleStr}}{{.TotalDurationStr}}{{.MaxDurationStr}}{{.ConnectionCount}}{{.TotalBytes}}{{.State}}\n" 57 | } 58 | 59 | out, err := template.New("Conn").Parse(tmpl) 60 | if err != nil { 61 | return "", err 62 | } 63 | w := new(bytes.Buffer) 64 | for _, conn := range conns { 65 | state := "closed" 66 | if conn.Open { 67 | state = "open" 68 | } 69 | connTmplData := struct { 70 | uconn.LongConnResult 71 | TupleStr string 72 | TotalDurationStr string 73 | MaxDurationStr string 74 | State string 75 | }{ 76 | LongConnResult: conn, 77 | TupleStr: strings.Join(conn.Tuples, ", "), 78 | TotalDurationStr: util.FormatDuration(time.Duration(int(conn.TotalDuration * float64(time.Second)))), 79 | MaxDurationStr: util.FormatDuration(time.Duration(int(conn.MaxDuration * float64(time.Second)))), 80 | State: state, 81 | } 82 | 83 | err := out.Execute(w, connTmplData) 84 | if err != nil { 85 | return "", err 86 | } 87 | } 88 | return w.String(), nil 89 | } 90 | -------------------------------------------------------------------------------- /reporting/report-strobes.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | "os" 7 | 8 | "github.com/activecm/rita-legacy/pkg/beacon" 9 | "github.com/activecm/rita-legacy/reporting/templates" 10 | "github.com/activecm/rita-legacy/resources" 11 | ) 12 | 13 | func printStrobes(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 14 | f, err := os.Create("strobes.html") 15 | if err != nil { 16 | return err 17 | } 18 | defer f.Close() 19 | 20 | var strobesTempl string 21 | if showNetNames { 22 | strobesTempl = templates.StrobesNetNamesTempl 23 | } else { 24 | strobesTempl = templates.StrobesTempl 25 | } 26 | 27 | out, err := template.New("strobes.html").Parse(strobesTempl) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | data, err := beacon.StrobeResults(res, -1, 1000, false) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | w, err := getStrobesWriter(data, showNetNames) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 43 | } 44 | 45 | func getStrobesWriter(strobes []beacon.StrobeResult, showNetNames bool) (string, error) { 46 | var tmpl string 47 | if showNetNames { 48 | tmpl = "{{.SrcNetworkName}}{{.DstNetworkName}}{{.SrcIP}}{{.DstIP}}{{.ConnectionCount}}\n" 49 | } else { 50 | tmpl = "{{.SrcIP}}{{.DstIP}}{{.ConnectionCount}}\n" 51 | } 52 | 53 | out, err := template.New("Strobes").Parse(tmpl) 54 | if err != nil { 55 | return "", err 56 | } 57 | w := new(bytes.Buffer) 58 | for _, strobe := range strobes { 59 | err := out.Execute(w, strobe) 60 | if err != nil { 61 | return "", err 62 | } 63 | } 64 | return w.String(), nil 65 | } 66 | -------------------------------------------------------------------------------- /reporting/report-useragents.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | "os" 7 | 8 | "github.com/activecm/rita-legacy/pkg/useragent" 9 | "github.com/activecm/rita-legacy/reporting/templates" 10 | "github.com/activecm/rita-legacy/resources" 11 | ) 12 | 13 | func printUserAgents(db string, showNetNames bool, res *resources.Resources, logsGeneratedAt string) error { 14 | f, err := os.Create("useragents.html") 15 | if err != nil { 16 | return err 17 | } 18 | defer f.Close() 19 | out, err := template.New("useragents.html").Parse(templates.UserAgentsTempl) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | data, err := useragent.Results(res, 1, 1000, false) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | w, err := getUserAgentsWriter(data) 30 | if err != nil { 31 | return err 32 | } 33 | 34 | return out.Execute(f, &templates.ReportingInfo{DB: db, Writer: template.HTML(w), LogsGeneratedAt: logsGeneratedAt}) 35 | } 36 | 37 | func getUserAgentsWriter(agents []useragent.Result) (string, error) { 38 | tmpl := "{{.UserAgent}}{{.TimesUsed}}\n" 39 | out, err := template.New("Agents").Parse(tmpl) 40 | if err != nil { 41 | return "", err 42 | } 43 | w := new(bytes.Buffer) 44 | for _, agent := range agents { 45 | err := out.Execute(w, agent) 46 | if err != nil { 47 | return "", err 48 | } 49 | } 50 | return w.String(), nil 51 | } 52 | -------------------------------------------------------------------------------- /reporting/templates/csstempl.go: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | // CSStempl is our css template sheet 4 | var CSStempl = []byte(`p { 5 | margin-bottom: 1.625em; 6 | font-family: 'Lucida Sans', Arial, sans-serif; 7 | } 8 | 9 | p { 10 | font-family: 'Lucida Sans', Arial, sans-serif; 11 | text-indent: 30px; 12 | } 13 | 14 | h1 { 15 | color: #000; 16 | font-family: 'Lato', sans-serif; 17 | font-size: 32px; 18 | font-weight: 300; 19 | line-height: 58px; 20 | margin: 0 0 58px; 21 | text-indent: 30px; 22 | } 23 | 24 | ul { 25 | list-style-type: none; 26 | margin: 0; 27 | padding: 0; 28 | overflow: hidden; 29 | background-color: #000; 30 | font-family: "Arial", Helvetica, sans-serif; 31 | } 32 | 33 | li { 34 | float: left; 35 | border-right: 1px solid #bbb; 36 | } 37 | 38 | li:last-child { 39 | border-right: none; 40 | } 41 | 42 | li a { 43 | display: block; 44 | color: white; 45 | text-align: center; 46 | padding: 14px 16px; 47 | text-decoration: none; 48 | } 49 | 50 | div { 51 | color: #adb7bd; 52 | font-family: 'Lucida Sans', Arial, sans-serif; 53 | font-size: 16px; 54 | line-height: 26px; 55 | margin: 0; 56 | } 57 | 58 | li a:hover { 59 | background-color: #34C6CD; 60 | } 61 | 62 | .vertical-menu { 63 | width: auto; 64 | } 65 | 66 | .vertical-menu a { 67 | background-color: #000; 68 | color: white; 69 | display: block; 70 | padding: 12px; 71 | text-decoration: none; 72 | text-align: center; 73 | vertical-align: middle; 74 | } 75 | 76 | .vertical-menu a:hover { 77 | background-color: #34C6CD; 78 | } 79 | 80 | .active { 81 | background-color: #A66F00; 82 | color: white; 83 | } 84 | 85 | .info { 86 | margin: 10px 0px; 87 | padding:12px; 88 | color: white; 89 | background-color: #333; 90 | } 91 | 92 | .container { 93 | overflow-x: auto; 94 | white-space: nowrap; 95 | } 96 | 97 | table { 98 | border-collapse: collapse; 99 | width: 100%; 100 | } 101 | 102 | th, td { 103 | text-align: left; 104 | padding: 8px; 105 | } 106 | 107 | tr:nth-child(even){ 108 | background-color: #f2f2f2 109 | } 110 | 111 | #github { 112 | height: 1em; 113 | } 114 | 115 | `) 116 | -------------------------------------------------------------------------------- /reporting/templates/svgtempl.go: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | //GithubSVG icon from font awesome 4 | var GithubSVG = []byte(` 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | `) 13 | -------------------------------------------------------------------------------- /resources/logging.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "time" 9 | 10 | log "github.com/sirupsen/logrus" 11 | 12 | "github.com/activecm/rita-legacy/config" 13 | "github.com/rifflock/lfshook" 14 | ) 15 | 16 | // DayFormat stores a correctly formatted timestamp for the day 17 | const DayFormat string = "2006-01-02" 18 | 19 | // initLogger creates the logger for logging to stdout and file 20 | func initLogger(logConfig *config.LogStaticCfg) *log.Logger { 21 | var logs = &log.Logger{} 22 | 23 | logs.Formatter = new(log.TextFormatter) 24 | 25 | logs.Out = ioutil.Discard 26 | logs.Hooks = make(log.LevelHooks) 27 | 28 | switch logConfig.LogLevel { 29 | case 3: 30 | logs.Level = log.DebugLevel 31 | case 2: 32 | logs.Level = log.InfoLevel 33 | case 1: 34 | logs.Level = log.WarnLevel 35 | case 0: 36 | logs.Level = log.ErrorLevel 37 | } 38 | if logConfig.LogToFile { 39 | addFileLogger(logs, logConfig.RitaLogPath) 40 | } 41 | return logs 42 | } 43 | 44 | func addFileLogger(logger *log.Logger, logPath string) { 45 | _, err := os.Stat(logPath) 46 | if err != nil && os.IsNotExist(err) { 47 | err = os.MkdirAll(logPath, 0755) 48 | if err != nil { 49 | fmt.Println("[!] Could not initialize file logger. Check RitaLogDir.") 50 | return 51 | } 52 | } 53 | 54 | time := time.Now().Format(DayFormat) 55 | logFile := time + ".log" 56 | logger.Hooks.Add(lfshook.NewHook(lfshook.PathMap{ 57 | log.DebugLevel: path.Join(logPath, logFile), 58 | log.InfoLevel: path.Join(logPath, logFile), 59 | log.WarnLevel: path.Join(logPath, logFile), 60 | log.ErrorLevel: path.Join(logPath, logFile), 61 | log.FatalLevel: path.Join(logPath, logFile), 62 | log.PanicLevel: path.Join(logPath, logFile), 63 | }, nil)) 64 | } 65 | -------------------------------------------------------------------------------- /resources/resources.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/activecm/mgorus" 8 | "github.com/activecm/rita-legacy/config" 9 | "github.com/activecm/rita-legacy/database" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | type ( 14 | // Resources provides a data structure for passing system Resources 15 | Resources struct { 16 | Config *config.Config 17 | Log *log.Logger 18 | DB *database.DB 19 | MetaDB *database.MetaDB 20 | } 21 | ) 22 | 23 | // InitResources grabs the configuration file and intitializes the configuration data 24 | // returning a *Resources object which has all of the necessary configuration information 25 | func InitResources(userConfig string) *Resources { 26 | conf, err := config.LoadConfig(userConfig) 27 | if err != nil { 28 | fmt.Fprintf(os.Stdout, "Failed to config: %s\n", err.Error()) 29 | os.Exit(-1) 30 | } 31 | 32 | // Fire up the logging system 33 | log := initLogger(&conf.S.Log) 34 | 35 | // Allows code to interact with the database 36 | db, err := database.NewDB(conf, log) 37 | if err != nil { 38 | fmt.Printf("Failed to connect to database: %s\n", err.Error()) 39 | os.Exit(-1) 40 | } 41 | 42 | // Allows code to create and remove tracked databases 43 | metaDB := database.NewMetaDB(conf, db.Session, log) 44 | 45 | //Begin logging to the metadatabase 46 | if conf.S.Log.LogToDB { 47 | log.Hooks.Add( 48 | mgorus.NewHookerFromSession( 49 | db.Session, conf.S.MongoDB.MetaDB, conf.T.Log.RitaLogTable, 50 | ), 51 | ) 52 | } 53 | 54 | //bundle up the system resources 55 | r := &Resources{ 56 | Config: conf, 57 | Log: log, 58 | DB: db, 59 | MetaDB: metaDB, 60 | } 61 | return r 62 | } 63 | -------------------------------------------------------------------------------- /resources/testing.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/activecm/mgorus" 10 | "github.com/activecm/rita-legacy/config" 11 | "github.com/activecm/rita-legacy/database" 12 | ) 13 | 14 | // InitIntegrationTestingResources creates a default testing 15 | // resource bundle for use with integration testing. 16 | // The MongoDB server is contacted via the URI provided 17 | // as by go test -args [MongoDB URI]. 18 | func InitIntegrationTestingResources(t *testing.T) *Resources { 19 | if testing.Short() { 20 | t.Skip() 21 | } 22 | 23 | mongoURI := os.Args[len(os.Args)-1] 24 | 25 | if !strings.Contains(mongoURI, "mongodb://") { 26 | t.Fatal("-args [MongoDB URI] is required to run RITA integration tests with go test") 27 | } 28 | 29 | conf, err := config.LoadTestingConfig(mongoURI) 30 | if err != nil { 31 | t.Fatal(err) 32 | } 33 | 34 | // Fire up the logging system 35 | log := initLogger(&conf.S.Log) 36 | 37 | // Allows code to interact with the database 38 | db, err := database.NewDB(conf, log) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | 43 | // Allows code to create and remove tracked databases 44 | metaDB := database.NewMetaDB(conf, db.Session, log) 45 | 46 | //Begin logging to the metadatabase 47 | if conf.S.Log.LogToDB { 48 | log.Hooks.Add( 49 | mgorus.NewHookerFromSession( 50 | db.Session, conf.S.MongoDB.MetaDB, conf.T.Log.RitaLogTable, 51 | ), 52 | ) 53 | } 54 | 55 | //bundle up the system resources 56 | r := &Resources{ 57 | Config: conf, 58 | Log: log, 59 | DB: db, 60 | MetaDB: metaDB, 61 | } 62 | return r 63 | } 64 | 65 | // InitTestResources creates a default testing 66 | // resource bundle for use with integration testing. 67 | func InitTestResources() *Resources { 68 | 69 | conf, err := config.LoadTestingConfig("mongodb://localhost:27017") 70 | if err != nil { 71 | fmt.Println(err) 72 | return nil 73 | } 74 | 75 | // Fire up the logging system 76 | log := initLogger(&conf.S.Log) 77 | 78 | // Allows code to interact with the database 79 | db, err := database.NewDB(conf, log) 80 | if err != nil { 81 | fmt.Println(err) 82 | return nil 83 | } 84 | 85 | // Allows code to create and remove tracked databases 86 | metaDB := database.NewMetaDB(conf, db.Session, log) 87 | 88 | //Begin logging to the metadatabase 89 | if conf.S.Log.LogToDB { 90 | log.Hooks.Add( 91 | mgorus.NewHookerFromSession( 92 | db.Session, conf.S.MongoDB.MetaDB, conf.T.Log.RitaLogTable, 93 | ), 94 | ) 95 | } 96 | 97 | //bundle up the system resources 98 | r := &Resources{ 99 | Config: conf, 100 | Log: log, 101 | DB: db, 102 | MetaDB: metaDB, 103 | } 104 | return r 105 | } 106 | -------------------------------------------------------------------------------- /rita-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/activecm/rita-legacy/0614d82df24ef4d71cba057dab1cc6b33fb52756/rita-logo.png -------------------------------------------------------------------------------- /rita.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | 7 | "github.com/activecm/rita-legacy/commands" 8 | "github.com/activecm/rita-legacy/config" 9 | "github.com/urfave/cli" 10 | ) 11 | 12 | // Entry point of ac-hunt 13 | func main() { 14 | app := cli.NewApp() 15 | app.Name = "rita" 16 | app.Usage = "Look for evil needles in big haystacks." 17 | app.Flags = []cli.Flag{commands.ConfigFlag} 18 | 19 | cli.VersionPrinter = commands.GetVersionPrinter() 20 | 21 | // Change the version string with updates so that a quick help command will 22 | // let the testers know what version of HT they're on 23 | app.Version = config.Version 24 | app.EnableBashCompletion = true 25 | 26 | // Define commands used with this application 27 | app.Commands = commands.Commands() 28 | app.Before = commands.SetConfigFilePath 29 | 30 | runtime.GOMAXPROCS(runtime.NumCPU()) 31 | app.Run(os.Args) 32 | } 33 | -------------------------------------------------------------------------------- /static-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Running static test - gofmt" 4 | GOFMT_FILES=$(gofmt -l .) 5 | if [ -n "${GOFMT_FILES}" ]; then 6 | printf >&2 "gofmt failed for the following files:\n%s\n\nplease run 'gofmt -w .' on your changes before committing.\n" "${GOFMT_FILES}" 7 | TEST_STATUS=FAIL 8 | fi 9 | 10 | echo "Running static test - golint" 11 | GOLINT_ERRORS=$(golint ./...) 12 | if [ -n "${GOLINT_ERRORS}" ]; then 13 | printf >&2 "golint failed for the following reasons:\n%s\n\nplease run 'golint ./...' on your changes before committing.\n" "${GOLINT_ERRORS}" 14 | TEST_STATUS=FAIL 15 | fi 16 | 17 | echo "Running static test - go tool vet" 18 | GOVET_ERRORS=$(go tool vet $(find . -name '*.go' | grep -v '/vendor/') 2>&1) 19 | if [ -n "${GOVET_ERRORS}" ]; then 20 | printf >&2 "go vet failed for the following reasons:\n%s\n\nplease run \"go tool vet \$(find . -name '*.go' | grep -v '/vendor/')\" on your changes before committing.\n" "${GOVET_ERRORS}" 21 | TEST_STATUS=FAIL 22 | fi 23 | 24 | if [ "$TEST_STATUS" = "FAIL" ]; then 25 | exit 1 26 | fi 27 | -------------------------------------------------------------------------------- /test.Dockerfile: -------------------------------------------------------------------------------- 1 | # use debian instead of alpine because the go race requires glibc 2 | # https://github.com/golang/go/issues/14481 3 | FROM golang:1.17 4 | 5 | RUN apt-get update && apt-get install -y git make ca-certificates wget build-essential 6 | WORKDIR /go 7 | # install testing dependencies 8 | RUN wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ 9 | | sh -s v1.41.1 10 | 11 | WORKDIR /go/src/github.com/activecm/rita-legacy 12 | 13 | # cache dependencies 14 | COPY go.mod go.sum ./ 15 | RUN go mod download 16 | 17 | # copy the rest of the code 18 | COPY . ./ 19 | 20 | CMD ["make", "test"] 21 | -------------------------------------------------------------------------------- /tests/helpers.sh: -------------------------------------------------------------------------------- 1 | 2 | # e.g. Ubuntu, CentOS 3 | OS="$(lsb_release -is)" 4 | # e.g. trusty, xenial, bionic 5 | OS_CODENAME="$(lsb_release -cs)" 6 | 7 | 8 | ensure_readable() { 9 | # Print out any offending path with a readable error msg 10 | [ -r "$1" ] || echo "$1 is not readable" 11 | # Perform the actual test to get the error code 12 | [ -r "$1" ] 13 | } 14 | 15 | ensure_writable() { 16 | # Print out any offending path with a readable error msg 17 | [ -w "$1" ] || echo "$1 is not writable" 18 | # Perform the actual test to get the error code 19 | [ -w "$1" ] 20 | } 21 | 22 | ensure_executable() { 23 | # Print out any offending path with a readable error msg 24 | [ -x "$1" ] || echo "$1 is not executable" 25 | # Perform the actual test to get the error code 26 | [ -x "$1" ] 27 | } 28 | 29 | ensure_exists() { 30 | # Print out any offending path with a readable error msg 31 | [ -e "$1" ] || echo "$1 does not exist" 32 | # Perform the actual test to get the error code 33 | [ -e "$1" ] 34 | } 35 | 36 | ensure_file() { 37 | # Print out any offending path with a readable error msg 38 | [ -f "$1" ] || echo "$1 is not a file" 39 | # Perform the actual test to get the error code 40 | [ -f "$1" ] 41 | } 42 | 43 | ensure_directory() { 44 | # Print out any offending path with a readable error msg 45 | [ -d "$1" ] || echo "$1 is not a directory" 46 | # Perform the actual test to get the error code 47 | [ -d "$1" ] 48 | } -------------------------------------------------------------------------------- /tests/install.bats: -------------------------------------------------------------------------------- 1 | 2 | source $BATS_TEST_DIRNAME/helpers.sh 3 | 4 | # These tests only apply to Ubuntu 16.04 Xenial 5 | # [ $OS = "Ubuntu" ] && [ $OS_CODENAME = "xenial" ] || exit 0 6 | 7 | @test "rita is installed" { 8 | _rita_binary="/usr/local/bin/rita" 9 | _rita_config="/etc/rita/config.yaml" 10 | _rita_logs="/var/lib/rita/logs" 11 | 12 | ensure_file $_rita_binary 13 | ensure_readable $_rita_binary 14 | ensure_executable $_rita_binary 15 | 16 | ensure_file $_rita_config 17 | ensure_readable $_rita_config 18 | ensure_writable $_rita_config 19 | 20 | ensure_directory $_rita_logs 21 | ensure_readable $_rita_logs 22 | ensure_writable $_rita_logs 23 | ensure_executable $_rita_logs 24 | } 25 | 26 | @test "bro is installed" { 27 | _bro_pkg="/opt/bro" 28 | _bro_src="/usr/local/bro" 29 | 30 | if [ -d $_bro_pkg ]; then 31 | _bro_path=$_bro_pkg 32 | elif [ -d $_bro_src ]; then 33 | _bro_path=$_bro_src 34 | else 35 | echo "bro was not installed" 36 | exit 1 37 | fi 38 | 39 | _bro_binary="$_bro_path/bin/bro" 40 | _broctl_binary="$_bro_path/bin/broctl" 41 | _bro_node_cfg="" 42 | 43 | ensure_readable $_bro_binary 44 | ensure_executable $_bro_binary 45 | 46 | ensure_readable $_broctl_binary 47 | ensure_executable $_broctl_binary 48 | } 49 | 50 | @test "bro is configured to start on boot" { 51 | if [ $(sudo crontab -l | grep 'broctl cron' | wc -l) -eq 0 ]; then 52 | echo "broctl crontab entry does not exist" 53 | exit 1 54 | fi 55 | } 56 | 57 | @test "mongo is installed" { 58 | _mongo_binary="/usr/bin/mongod" 59 | 60 | ensure_file $_mongo_binary 61 | ensure_readable $_mongo_binary 62 | ensure_executable $_mongo_binary 63 | } 64 | 65 | @test "mongo is configured to start on boot" { 66 | ensure_exists "/etc/systemd/system/multi-user.target.wants/mongod.service" 67 | } 68 | 69 | -------------------------------------------------------------------------------- /util/ip_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | type ipBoolTestCase struct { 11 | ip string 12 | out bool 13 | msg string 14 | } 15 | 16 | type parseSubnetsTestCase struct { 17 | nets []string 18 | out []*net.IPNet 19 | wantErr bool 20 | msg string 21 | } 22 | 23 | func TestIPIsPublicRoutable(t *testing.T) { 24 | 25 | testCases := []ipBoolTestCase{ 26 | {"10.1.2.3", false, "RFC1918 Class A"}, 27 | {"172.16.1.2", false, "RFC1918 Class B"}, 28 | {"192.168.1.2", false, "RFC1918 Class C"}, 29 | {"fc00:1234::", false, "IPv6 local address"}, 30 | {"127.0.0.5", false, "IPv4 loopback"}, 31 | {"::1", false, "IPv6 loopback"}, 32 | {"169.254.1.2", false, "IPv4 link local"}, 33 | {"fe80:1234::", false, "IPv6 link local"}, 34 | {"224.0.0.1", false, "IPv4 multicast"}, 35 | {"ff12:1234::", false, "IPv6 multicast"}, 36 | {"8.8.8.8", true, "google dns ipv4"}, 37 | {"2001:4860:4860::8888", true, "google dns ipv6"}, 38 | } 39 | 40 | for _, testCase := range testCases { 41 | output := IPIsPubliclyRoutable(net.ParseIP(testCase.ip)) 42 | assert.Equal(t, testCase.out, output, testCase.msg) 43 | } 44 | } 45 | 46 | func TestIsIP(t *testing.T) { 47 | testIP := "1.1.1.1" 48 | notIP := "a.b.c.d" 49 | assert.True(t, IsIP(testIP)) 50 | assert.False(t, IsIP(notIP)) 51 | } 52 | 53 | // Ensures ParseSubnets returns expected net.IPNets and returns 54 | // error when invalid IP address/CIDR network is provided. 55 | func TestParseSubnets(t *testing.T) { 56 | validIPv4Nets := []string{"192.168.0.0/24", "10.0.0.0/16"} 57 | validIPv4NetsOut := parseCIDRs([]string{"192.168.0.0/24", "10.0.0.0/16"}) 58 | validIPv4Hosts := []string{"192.168.0.1", "10.0.123.45"} 59 | validIPv4HostsOut := parseCIDRs([]string{"192.168.0.1/32", "10.0.123.45/32"}) 60 | validIPv6Nets := []string{"2001:db8::/32", "2400:cb00:2048::/64"} 61 | validIPv6NetsOut := parseCIDRs([]string{"2001:db8::/32", "2400:cb00:2048::/64"}) 62 | validIPv6Hosts := []string{"2001:db8::1", "2400:cb00:2048::1"} 63 | validIPv6HostsOut := parseCIDRs([]string{"2001:db8::1/128", "2400:cb00:2048::1/128"}) 64 | invalidNets := []string{"invalidIP", "300.0.0.0/24"} 65 | 66 | testCases := []parseSubnetsTestCase{ 67 | { 68 | nets: validIPv4Nets, 69 | out: validIPv4NetsOut, 70 | wantErr: false, 71 | msg: "Valid IPv4 subnetworks", 72 | }, 73 | { 74 | nets: validIPv4Hosts, 75 | out: validIPv4HostsOut, 76 | wantErr: false, 77 | msg: "Valid IPv4 host IPs", 78 | }, 79 | { 80 | nets: validIPv6Nets, 81 | out: validIPv6NetsOut, 82 | wantErr: false, 83 | msg: "Valid IPv6 subnetworks", 84 | }, 85 | { 86 | nets: validIPv6Hosts, 87 | out: validIPv6HostsOut, 88 | wantErr: false, 89 | msg: "Valid IPv6 host IPs", 90 | }, 91 | { 92 | nets: invalidNets, 93 | out: nil, 94 | wantErr: true, 95 | msg: "Invalid IP and subnetwork (Expecting Error)", 96 | }, 97 | } 98 | 99 | for _, testCase := range testCases { 100 | output, err := ParseSubnets(testCase.nets) 101 | if testCase.wantErr { 102 | assert.Error(t, err, testCase.msg) 103 | } else { 104 | assert.Equal(t, testCase.out, output, testCase.msg) 105 | } 106 | } 107 | } 108 | 109 | func parseCIDRs(cidr []string) []*net.IPNet { 110 | ipNets := make([]*net.IPNet, len(cidr)) 111 | 112 | for i, ip := range cidr { 113 | _, ipNet, _ := net.ParseCIDR(ip) 114 | ipNets[i] = ipNet 115 | } 116 | 117 | return ipNets 118 | } 119 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "os" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | //TimeFormat stores a correctly formatted timestamp 12 | const TimeFormat string = "2006-01-02-T15:04:05-0700" 13 | 14 | //DayFormat stores a correctly formatted timestamp for the day 15 | const DayFormat string = "2006-01-02" 16 | 17 | // Exists returns true if file or directory exists 18 | func Exists(path string) bool { 19 | _, err := os.Stat(path) 20 | if err == nil { 21 | return true 22 | } 23 | if os.IsNotExist(err) { 24 | return false 25 | } 26 | return true 27 | } 28 | 29 | // IsDir returns true if argument is a directory 30 | func IsDir(path string) bool { 31 | file, err := os.Stat(path) 32 | if err != nil { 33 | return false 34 | } 35 | if file.IsDir() { 36 | return true 37 | } 38 | return false 39 | } 40 | 41 | // ByStringLength Functions that, in combination with golang sort, 42 | // allow users to sort a slice/list of strings by string length 43 | // (shortest -> longest) 44 | type ByStringLength []string 45 | 46 | func (s ByStringLength) Len() int { return len(s) } 47 | func (s ByStringLength) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 48 | func (s ByStringLength) Less(i, j int) bool { return len(s[i]) < len(s[j]) } 49 | 50 | // SortableInt64 functions that allow a golang sort of int64s 51 | type SortableInt64 []int64 52 | 53 | func (s SortableInt64) Len() int { return len(s) } 54 | func (s SortableInt64) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 55 | func (s SortableInt64) Less(i, j int) bool { return s[i] < s[j] } 56 | 57 | //Abs returns two's complement 64 bit absolute value 58 | func Abs(a int64) int64 { 59 | mask := a >> 63 60 | a = a ^ mask 61 | return a - mask 62 | } 63 | 64 | //Round returns rounded int64 65 | func Round(f float64) int64 { 66 | return int64(math.Floor(f + .5)) 67 | } 68 | 69 | //Min returns the smaller of two integers 70 | func Min(a int, b int) int { 71 | if a < b { 72 | return a 73 | } 74 | return b 75 | } 76 | 77 | //Max returns the larger of two integers 78 | func Max(a int, b int) int { 79 | if a > b { 80 | return a 81 | } 82 | return b 83 | } 84 | 85 | //MaxUint64 returns the larger of two 64 bit unsigned integers 86 | func MaxUint64(a uint64, b uint64) uint64 { 87 | if a > b { 88 | return a 89 | } 90 | return b 91 | } 92 | 93 | //StringInSlice returns true if the string is an element of the array 94 | func StringInSlice(value string, list []string) bool { 95 | for _, entry := range list { 96 | if entry == value { 97 | return true 98 | } 99 | } 100 | return false 101 | } 102 | 103 | //Int64InSlice returns true if the int64 is an element of the array 104 | func Int64InSlice(value int64, list []int64) bool { 105 | for _, entry := range list { 106 | if entry == value { 107 | return true 108 | } 109 | } 110 | return false 111 | } 112 | 113 | const ( 114 | day = time.Minute * 60 * 24 115 | year = 365 * day 116 | ) 117 | 118 | // FormatDuration properly prints a given time.Duration 119 | // https://gist.github.com/harshavardhana/327e0577c4fed9211f65#gistcomment-2557682 120 | func FormatDuration(d time.Duration) string { 121 | if d < day { 122 | return d.String() 123 | } 124 | 125 | var b strings.Builder 126 | 127 | if d >= year { 128 | years := d / year 129 | fmt.Fprintf(&b, "%dy", years) 130 | d -= years * year 131 | } 132 | 133 | days := d / day 134 | d -= days * day 135 | fmt.Fprintf(&b, "%dd%s", days, d) 136 | 137 | return b.String() 138 | } 139 | -------------------------------------------------------------------------------- /util/util_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "math" 5 | // "os" 6 | // "path" 7 | "sort" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | // func TestFileExists(t *testing.T) { 15 | // filePath := "./.jeinwei8380243unt4u" 16 | // os.Remove(filePath) 17 | // file, err := os.OpenFile(filePath, os.O_RDONLY|os.O_CREATE, 0666) 18 | // assert.Nil(t, err) 19 | // file.Close() 20 | // exists, err := Exists(filePath) 21 | // assert.Nil(t, err) 22 | // assert.True(t, exists) 23 | // os.Remove(filePath) 24 | // exists, err = Exists(filePath) 25 | // assert.Nil(t, err) 26 | // assert.False(t, exists) 27 | 28 | // currBinary, err := os.Executable() 29 | // assert.Nil(t, err) 30 | // badPath := path.Join(currBinary, "non-existent-file") 31 | 32 | // _, err = Exists(badPath) 33 | // assert.NotNil(t, err) 34 | // } 35 | 36 | func TestSortByStringLength(t *testing.T) { 37 | strings := []string{"yy", "z", "aaaa"} 38 | sort.Sort(ByStringLength(strings)) 39 | assert.Equal(t, "z", strings[0]) 40 | assert.Equal(t, "yy", strings[1]) 41 | assert.Equal(t, "aaaa", strings[2]) 42 | } 43 | 44 | func TestSortableInt64(t *testing.T) { 45 | ints := []int64{3434, -1, -20, 0} 46 | sort.Sort(SortableInt64(ints)) 47 | assert.Equal(t, int64(-20), ints[0]) 48 | assert.Equal(t, int64(-1), ints[1]) 49 | assert.Equal(t, int64(0), ints[2]) 50 | assert.Equal(t, int64(3434), ints[3]) 51 | } 52 | 53 | func TestAbs(t *testing.T) { 54 | max := int64(math.MaxInt64) 55 | pos := int64(1) 56 | zero := int64(0) 57 | neg := int64(-1) 58 | min := int64(math.MinInt64) 59 | 60 | assert.Equal(t, max, Abs(max)) 61 | assert.Equal(t, pos, Abs(pos)) 62 | assert.Equal(t, zero, Abs(zero)) 63 | assert.Equal(t, -1*neg, Abs(neg)) 64 | assert.Equal(t, -1*min, Abs(min)) 65 | } 66 | 67 | func TestRound(t *testing.T) { 68 | negDown := -16.6 69 | negDownExp := int64(-17) 70 | negUp := -16.1 71 | negUpExp := int64(-16) 72 | posDown := 16.1 73 | posDownExp := int64(16) 74 | posUp := 16.6 75 | posUpExp := int64(17) 76 | assert.Equal(t, negDownExp, Round(negDown)) 77 | assert.Equal(t, negUpExp, Round(negUp)) 78 | assert.Equal(t, posDownExp, Round(posDown)) 79 | assert.Equal(t, posUpExp, Round(posUp)) 80 | } 81 | 82 | func TestMinMax(t *testing.T) { 83 | large := 100 84 | small := -100 85 | assert.Equal(t, large, Max(large, small)) 86 | assert.Equal(t, large, Max(small, large)) 87 | assert.Equal(t, small, Min(large, small)) 88 | assert.Equal(t, small, Min(small, large)) 89 | } 90 | 91 | func TestStringInSlice(t *testing.T) { 92 | tables := []struct { 93 | val string 94 | list []string 95 | out bool 96 | }{ 97 | {"a", []string{"a", "b", "c", "d"}, true}, 98 | {"abc", []string{"a", "b", "c", "d"}, false}, 99 | {"ethan", []string{"ethan", "melissa"}, true}, 100 | {"-1", []string{}, false}, 101 | {"-1", []string{"-1"}, true}, 102 | {"somethingsomething999", []string{"somethingsomething"}, false}, 103 | } 104 | 105 | for _, test := range tables { 106 | output := StringInSlice(test.val, test.list) 107 | require.Equal(t, test.out, output) 108 | } 109 | 110 | } 111 | --------------------------------------------------------------------------------