├── .gitignore ├── services ├── cuckoo │ ├── service.conf.example │ ├── Dockerfile │ ├── main.go │ ├── processing.go │ └── cuckoo │ │ └── cuckoo.go └── README.md ├── .travis.yml ├── config └── totem-dynamic.conf.example ├── main.go ├── lib ├── service.go ├── amqp.go └── lib.go ├── README.md ├── check └── check.go ├── submit └── submit.go ├── feed └── feed.go ├── CONTRIBUTING.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | Holmes-Totem-Dynamic 2 | config/totem-dynamic.conf 3 | services/**/service.conf 4 | -------------------------------------------------------------------------------- /services/cuckoo/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "HTTPBinding":":8080", 3 | "VerifySSL":true, 4 | "CheckFreeSpace":true, 5 | "CuckooURL":"http://127.0.0.1:8090", 6 | "MaxPending":5, 7 | "MaxAPICalls":10000, 8 | "LogFile":"", 9 | "LogLevel":"debug" 10 | } 11 | -------------------------------------------------------------------------------- /services/README.md: -------------------------------------------------------------------------------- 1 | This folder contains all services that can be used with Totem-Dynamic. 2 | 3 | These services are not compiled together with Totem-Dynamic and need to be build and deployed seperately. After this it is necessary to add their links to the `config/totem-dynamic.conf` file so that TD is able to find them. 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.6.2 5 | 6 | # this fixes go imports 7 | before_install: 8 | - RepoName=`basename $PWD`; SrcDir=`dirname $PWD`; DestDir="`dirname $SrcDir`/HolmesProcessing" 9 | - if [[ "$SrcDir" != "$DestDir" ]]; then mv "$SrcDir" "$DestDir"; cd ../../HolmesProcessing/$RepoName; export TRAVIS_BUILD_DIR=`dirname $TRAVIS_BUILD_DIR`/$RepoName; fi 10 | 11 | install: 12 | - go get github.com/streadway/amqp 13 | 14 | script: go test -v . 15 | -------------------------------------------------------------------------------- /services/cuckoo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine 2 | 3 | # create folder 4 | RUN mkdir -p /go/src/cuckoo 5 | WORKDIR /go/src/cuckoo 6 | 7 | # get go dependencies 8 | RUN apk add --no-cache \ 9 | git \ 10 | && go get github.com/julienschmidt/httprouter \ 11 | && rm -rf /var/cache/apk/* 12 | 13 | # add the files to the container 14 | ADD . /go/src/cuckoo/ 15 | 16 | # build vtsample 17 | RUN go build -o bin 18 | 19 | # add the configuration file (possibly from a storage uri) 20 | CMD "/go/src/cuckoo/bin" 21 | -------------------------------------------------------------------------------- /config/totem-dynamic.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "Amqp" : "amqp://USER:PASSWORD@HOST:PORT", 3 | "QueueSuffix" : "", 4 | "ConsumeQueue" : "totem_dynamic_input", 5 | "ResultsQueue" : "totem_results", 6 | "FailedQueue" : "totem_dynamic_failed", 7 | 8 | "LogFile" : "/leave/empty/for/no/log/or/path/to/file.txt", 9 | "LogLevel" : "info", 10 | "VerifySSL" : true, 11 | 12 | "Services" : { 13 | "virustotal": [], 14 | "cuckoo": [] 15 | }, 16 | 17 | "FeedPrefetchCount": 1, 18 | 19 | "CheckPrefetchCount": 100, 20 | "WaitBetweenRequests": 30, 21 | 22 | "SubmitPrefetchCount": 5 23 | } 24 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | 6 | "github.com/HolmesProcessing/Holmes-Totem-Dynamic/check" 7 | "github.com/HolmesProcessing/Holmes-Totem-Dynamic/feed" 8 | "github.com/HolmesProcessing/Holmes-Totem-Dynamic/lib" 9 | "github.com/HolmesProcessing/Holmes-Totem-Dynamic/submit" 10 | ) 11 | 12 | func main() { 13 | cPath := flag.String("config", "", "Path to the configuration file") 14 | flag.Parse() 15 | 16 | ctx := &lib.Ctx{} 17 | 18 | err := ctx.Init(*cPath) 19 | if err != nil { 20 | panic(err.Error()) 21 | } 22 | 23 | err = feed.Run(ctx, false) 24 | if err != nil { 25 | panic(err.Error()) 26 | } 27 | 28 | err = check.Run(ctx, false) 29 | if err != nil { 30 | panic(err.Error()) 31 | } 32 | 33 | err = submit.Run(ctx, true) 34 | if err != nil { 35 | panic(err.Error()) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /lib/service.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | "net/url" 7 | ) 8 | 9 | type Service struct { 10 | Name string 11 | URL string 12 | Client *http.Client 13 | } 14 | 15 | // json return of status request 16 | type Status struct { 17 | Degraded bool 18 | Error string 19 | FreeSlots int 20 | } 21 | 22 | // json return of feed request 23 | type NewTask struct { 24 | Error string 25 | TaskID string 26 | } 27 | 28 | // json return of check request 29 | type CheckTask struct { 30 | Error string 31 | Done bool 32 | } 33 | 34 | // json return of results request 35 | type TaskResults struct { 36 | Error string 37 | Results interface{} 38 | } 39 | 40 | // Status gets the current status of the service and returns it 41 | // as a Status struct. 42 | func (s *Service) Status() (*Status, error) { 43 | status := &Status{} 44 | _, httpStatus, err := FastGet(s.Client, s.URL+"/status/", status) 45 | if httpStatus != 200 && err == nil { 46 | err = errors.New("Returned non-200 status code") 47 | } 48 | 49 | return status, err 50 | } 51 | 52 | // NewTask sends a new task to the service and returns the result 53 | // as a NewTask struct. 54 | func (s *Service) NewTask(sample string) (*NewTask, error) { 55 | nt := &NewTask{} 56 | _, httpStatus, err := FastGet(s.Client, s.URL+"/feed/?obj="+url.QueryEscape(sample), nt) 57 | if httpStatus != 200 && err == nil { 58 | err = errors.New("Returned non-200 status code") 59 | } 60 | 61 | if nt.Error != "" { 62 | err = errors.New(nt.Error) 63 | } 64 | 65 | return nt, err 66 | } 67 | 68 | // CheckTask gets the current status of a task from the service and 69 | // return the result as a CheckTask struct. 70 | func (s *Service) CheckTask(taskID string) (*CheckTask, error) { 71 | ct := &CheckTask{} 72 | _, httpStatus, err := FastGet(s.Client, s.URL+"/check/?taskid="+url.QueryEscape(taskID), ct) 73 | if httpStatus != 200 && err == nil { 74 | err = errors.New("Returned non-200 status code") 75 | } 76 | 77 | if ct.Error != "" { 78 | err = errors.New(ct.Error) 79 | } 80 | 81 | return ct, err 82 | } 83 | 84 | // TaskResults collects the results for a given task from the service 85 | // and returns them as a TaskResults struct. 86 | func (s *Service) TaskResults(taskID string) (*TaskResults, error) { 87 | tr := &TaskResults{} 88 | _, httpStatus, err := FastGet(s.Client, s.URL+"/results/?taskid="+url.QueryEscape(taskID), tr) 89 | if httpStatus != 200 && err == nil { 90 | err = errors.New("Returned non-200 status code") 91 | } 92 | 93 | if tr.Error != "" { 94 | err = errors.New(tr.Error) 95 | } 96 | 97 | return tr, err 98 | } 99 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Holmes-Totem-Dynamic: An Investigation Planner for managing long running services. 2 | 3 | ## Overview 4 | 5 | Just like Holmes-Totem the "Dynamic" Planner is responsible for turning data into information by performing feature extraction against submitted objects. When tasked, Holmes-Totem-Dynamic schedules the execution of its services which are focused on dynamic and other long or indefinite running analysis tasks. 6 | 7 | 8 | ## Highlights 9 | 10 | ### Inner workings 11 | 12 | Totem-Dynamic is based on [distributed_cuckoo](https://github.com/cynexit/cuckoo_distributed) and uses a similar concept. Just like Totem it accepts incoming tasks via AMQP and sends the results back via AMQP as soon as they are availible. 13 | Internally the application is mostly devided in three parts. `feed` is accepting new tasks from the outside, checks the status of the requested service and - if everything is fine - submits the task to the service or else postpones it until the service status is good again. `check` then periodically checks if the service is done with the task or if it still need time or if an error occured. `submit` then collects the results from the service and sends it out 14 | 15 | ![Diagramm of Totem-Dynamic](https://i.imgur.com/WmZzxzF.png) 16 | 17 | 18 | ### Services 19 | 20 | | Name | Object Type | Description | 21 | | ------------------ | --------------------- | --------------- | 22 | | cuckoo | Binary Executable | Performs a cuckoo analysis on the given binaries 23 | | VirusTotal **PLANNED** | Binary Executable, IPv4/6, Domain | Returns available [VirusTotal](https://www.virustotal.com/) information with a public or private key 24 | 25 | 26 | ## Dependencies 27 | 28 | The only needed dependency is a AMQP library for Go. If you setup your [Go environment](https://golang.org/doc/install) correctly you can simple do a 29 | 30 | go get -u github.com/streadway/amqp 31 | 32 | and you are done. 33 | 34 | 35 | ## Compilation 36 | 37 | After cloning this project you can build it using `go build` or `go install`. This requires a configured and working [Go environment](https://golang.org/doc/install). 38 | 39 | 40 | ## Installation 41 | 42 | You can deploy the binary everywhere you want, per default it is statically linked and does not need any libraries on the machine. 43 | 44 | The only thing you need to keep in mind is that the Docker containers hosting the services need to be running on the same machine. To build them, setup Docker on the machine, decent into the `./services/` folder, choose the services you would like to run and forward Port 8080 from the container. Don't forget to also configure the `service.conf` file in each folder before building the container. 45 | 46 | Next up you need to fill the `config/totem-dynamic.conf.example` with your own values (and services you'd like to run) and rename it to `totem-dynamic.conf`. 47 | 48 | After this simply execute the compiled binary and add tasks to the amqp input queue defined in your configuration file. 49 | -------------------------------------------------------------------------------- /check/check.go: -------------------------------------------------------------------------------- 1 | package check 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "sync" 7 | "time" 8 | 9 | "github.com/HolmesProcessing/Holmes-Totem-Dynamic/lib" 10 | 11 | "github.com/streadway/amqp" 12 | ) 13 | 14 | // local context 15 | type cCtx struct { 16 | *lib.Ctx 17 | 18 | Producer *lib.QueueHandler // the queue read by submit 19 | } 20 | 21 | // elemt of the watch map 22 | type watchElem struct { 23 | Req *lib.InternalRequest 24 | Msg *amqp.Delivery 25 | Service *lib.Service 26 | } 27 | 28 | var ( 29 | watchMap = make(map[string]*watchElem) 30 | watchMapMutex = &sync.Mutex{} 31 | ) 32 | 33 | // Run starts the check module either blocking or non-blocking. 34 | func Run(ctx *lib.Ctx, blocking bool) error { 35 | producer, err := ctx.SetupQueue("totem-dynamic-submit-" + ctx.Config.QueueSuffix) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | c := &cCtx{ 41 | ctx, 42 | producer, 43 | } 44 | 45 | go c.checkLoop() 46 | if blocking { 47 | c.Consume("totem-dynamic-check-"+ctx.Config.QueueSuffix, ctx.Config.CheckPrefetchCount, c.parseMsg) 48 | } else { 49 | go c.Consume("totem-dynamic-check-"+ctx.Config.QueueSuffix, ctx.Config.CheckPrefetchCount, c.parseMsg) 50 | } 51 | 52 | return nil 53 | } 54 | 55 | // parseMsg accepts an *amqp.Delivery and parses the body assuming 56 | // it's a request from feed. On success the parsed struct is 57 | // added to the watchMap. 58 | func (c *cCtx) parseMsg(msg amqp.Delivery) { 59 | req := &lib.InternalRequest{} 60 | err := json.Unmarshal(msg.Body, req) 61 | if c.NackOnError(err, "Could not decode json!", &msg) { 62 | return 63 | } 64 | 65 | // TODO: Validate msg 66 | //if c.NackOnError(m.Validate(), "Could not validate msg", &msg) { 67 | // return 68 | //} 69 | 70 | watchMapMutex.Lock() 71 | watchMap[req.FilePath] = &watchElem{ 72 | Req: req, 73 | Msg: &msg, 74 | Service: &lib.Service{ 75 | Name: req.Service, 76 | URL: req.URL, 77 | Client: c.Client, 78 | }, 79 | } 80 | watchMapMutex.Unlock() 81 | } 82 | 83 | // checkLoop loops over the watch map and checks if the 84 | // task is done or if an error occured and if so sends 85 | // the task to submit or the failed queue. 86 | func (c *cCtx) checkLoop() { 87 | waitDuration := time.Second * time.Duration(c.Config.WaitBetweenRequests) 88 | 89 | for { 90 | time.Sleep(waitDuration) //This is here so an empty list does not result in full load 91 | 92 | for k, v := range watchMap { 93 | time.Sleep(waitDuration) 94 | 95 | // try to get task status 96 | check, err := v.Service.CheckTask(v.Req.TaskID) 97 | if c.NackOnError(err, "Couldn't get status of task!", v.Msg) { 98 | delete(watchMap, k) 99 | continue 100 | } 101 | 102 | // if an error occured, remove from map and nack 103 | if check.Error != "" { 104 | c.NackOnError(errors.New(check.Error), "Checking task returned an error!", v.Msg) 105 | delete(watchMap, k) 106 | continue 107 | } 108 | 109 | // if task is not done continue to next task 110 | if !check.Done { 111 | continue 112 | } 113 | 114 | // task is done, send it to submit 115 | internalReq, err := json.Marshal(v.Req) 116 | if c.NackOnError(err, "Could not create internalRequest!", v.Msg) { 117 | return 118 | } 119 | 120 | c.Producer.Send(internalReq) 121 | if err := v.Msg.Ack(false); err != nil { 122 | c.Warning.Println("Sending ACK failed!", err.Error()) 123 | } 124 | 125 | delete(watchMap, k) 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /lib/amqp.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/streadway/amqp" 7 | ) 8 | 9 | type QueueHandler struct { 10 | Queue string 11 | Channel *amqp.Channel 12 | C *Ctx 13 | } 14 | 15 | type FailedMsg struct { 16 | Queue string 17 | Error string 18 | Desc string 19 | Msg string 20 | } 21 | 22 | // SetupQueue creates a new channel on top of the established 23 | // amqp connection and declares a persistent queue with the 24 | // given name. It then returns a pointer to a QueueHandler. 25 | func (c *Ctx) SetupQueue(queue string) (*QueueHandler, error) { 26 | if queue == "" { 27 | c.Warning.Println("Queue name is empty! A persistent and anonymous queue will be created.") 28 | } 29 | 30 | c.Debug.Println("Creating new queue handler for", queue) 31 | 32 | channel, err := c.AmqpConn.Channel() 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | _, err = channel.QueueDeclare( 38 | queue, // name 39 | true, // durable 40 | false, // delete when unused 41 | false, // exclusive 42 | false, // no-wait 43 | nil, // arguments 44 | ) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | return &QueueHandler{queue, channel, c}, nil 50 | } 51 | 52 | // Consume connects to a queue as a consumer, sets the QoS 53 | // and relays all incoming messages to the supplied function. 54 | func (c *Ctx) Consume(queue string, prefetchCount int, fn func(msg amqp.Delivery)) error { 55 | c.Debug.Println("Starting to consume on", queue) 56 | 57 | handle, err := c.SetupQueue(queue) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | err = handle.Channel.Qos( 63 | prefetchCount, // prefetch count 64 | 0, // prefetch size 65 | false, // global 66 | ) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | msgs, err := handle.Channel.Consume( 72 | handle.Queue, // queue 73 | "", // consumer 74 | false, // auto-ack 75 | false, // exclusive 76 | false, // no-local 77 | false, // no-wait 78 | nil, // args 79 | ) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | forever := make(chan bool) 85 | 86 | go func() { 87 | for m := range msgs { 88 | c.Debug.Println("Received a message on", queue) 89 | fn(m) 90 | } 91 | }() 92 | 93 | c.Info.Println("Consuming", queue, "...") 94 | <-forever 95 | 96 | return nil 97 | } 98 | 99 | // Send is used to send a message to a amqp 100 | // queue. Channel and queue name are taken from 101 | // the QueueHandler struct. 102 | func (q *QueueHandler) Send(msg []byte) error { 103 | err := q.Channel.Publish( 104 | "", // exchange 105 | q.Queue, // routing key 106 | false, // mandatory 107 | false, // immediate 108 | amqp.Publishing{ 109 | DeliveryMode: amqp.Persistent, 110 | ContentType: "text/plain", 111 | Body: msg, 112 | }) 113 | if err != nil { 114 | return err 115 | } 116 | 117 | i := "" 118 | if len(msg) > 700 { 119 | i = string(msg[:700]) + " [...]" 120 | } else { 121 | i = string(msg) 122 | } 123 | 124 | q.C.Info.Println("Dispatched", i) 125 | return nil 126 | } 127 | 128 | // NackOnError accepts an error, error description, and amqp 129 | // message. If the error is not nil a NACK is sent in reply 130 | // to the msg. The msg will be redirected to the failed queue 131 | // so the overseer, ehhm, "something" can handle it. 132 | func (c *Ctx) NackOnError(err error, desc string, msg *amqp.Delivery) bool { 133 | if err != nil { 134 | c.Warning.Println("[NACK]", desc, err.Error()) 135 | 136 | jm, err := json.Marshal(FailedMsg{ 137 | msg.RoutingKey, 138 | err.Error(), 139 | desc, 140 | string(msg.Body), 141 | }) 142 | if err != nil { 143 | c.Warning.Println(err.Error()) 144 | } 145 | 146 | c.Failed.Send(jm) 147 | 148 | err = msg.Nack(false, false) 149 | if err != nil { 150 | c.Warning.Println("Sending NACK failed!", err.Error()) 151 | } 152 | 153 | return true 154 | } 155 | 156 | return false 157 | } 158 | -------------------------------------------------------------------------------- /submit/submit.go: -------------------------------------------------------------------------------- 1 | package submit 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha1" 6 | "crypto/sha256" 7 | "encoding/json" 8 | "fmt" 9 | "io/ioutil" 10 | "os" 11 | "time" 12 | 13 | "github.com/HolmesProcessing/Holmes-Totem-Dynamic/lib" 14 | 15 | "github.com/streadway/amqp" 16 | ) 17 | 18 | type sCtx struct { 19 | *lib.Ctx 20 | 21 | Producer *lib.QueueHandler // the queue read by submit 22 | } 23 | 24 | type Result struct { 25 | Filename string `json:"filename"` 26 | Data string `json:"data"` 27 | MD5 string `json:"md5"` 28 | SHA1 string `json:"sha1"` 29 | SHA256 string `json:"sha256"` 30 | ServiceName string `json:"service_name"` 31 | Tags []string `json:"tags"` 32 | Comment string `json:"comment"` 33 | StartedDateTime time.Time `json:"started_date_time"` 34 | FinishedDateTime time.Time `json:"finished_date_time"` 35 | } 36 | 37 | // Run starts the submit module either blocking or non-blocking. 38 | func Run(ctx *lib.Ctx, blocking bool) error { 39 | producer, err := ctx.SetupQueue(ctx.Config.ResultsQueue) // should be "totem_output" 40 | if err != nil { 41 | return err 42 | } 43 | 44 | c := &sCtx{ 45 | ctx, 46 | producer, 47 | } 48 | 49 | if blocking { 50 | c.Consume("totem-dynamic-submit-"+ctx.Config.QueueSuffix, ctx.Config.SubmitPrefetchCount, c.parseMsg) 51 | } else { 52 | go c.Consume("totem-dynamic-submit-"+ctx.Config.QueueSuffix, ctx.Config.SubmitPrefetchCount, c.parseMsg) 53 | } 54 | 55 | return nil 56 | } 57 | 58 | // parseMsg accepts an *amqp.Delivery and parses the body assuming 59 | // it's a request from crits. On success the parsed struct is 60 | // send to handleSubmit. 61 | func (c *sCtx) parseMsg(msg amqp.Delivery) { 62 | req := &lib.InternalRequest{} 63 | err := json.Unmarshal(msg.Body, req) 64 | if c.NackOnError(err, "Could not decode json!", &msg) { 65 | return 66 | } 67 | 68 | // TODO: Validate msg 69 | //if c.NackOnError(m.Validate(), "Could not validate msg", &msg) { 70 | // return 71 | //} 72 | 73 | go c.submitResults(req, &msg) 74 | } 75 | 76 | func (c *sCtx) submitResults(req *lib.InternalRequest, msg *amqp.Delivery) { 77 | service := &lib.Service{ 78 | Name: req.Service, 79 | URL: req.URL, 80 | Client: c.Client, 81 | } 82 | 83 | serviceResults, err := service.TaskResults(req.TaskID) 84 | if c.NackOnError(err, "Could not get results", msg) { 85 | return 86 | } 87 | 88 | // TODO: check if it is still necessary to handle service results as string 89 | resultsJ, err := json.Marshal(serviceResults.Results) 90 | 91 | // generate the necessary hashes, differentiate between samples and urls 92 | var fileBytes []byte 93 | if req.OriginalRequest.Download { 94 | fileBytes, err = ioutil.ReadFile("/tmp/" + req.FilePath) 95 | if c.NackOnError(err, "Could not read sample file", msg) { 96 | return 97 | } 98 | } else { 99 | fileBytes = []byte("/tmp/" + req.FilePath) 100 | } 101 | 102 | hSHA256 := sha256.New() 103 | hSHA256.Write(fileBytes) 104 | sha256String := fmt.Sprintf("%x", hSHA256.Sum(nil)) 105 | 106 | hSHA1 := sha1.New() 107 | hSHA1.Write(fileBytes) 108 | sha1String := fmt.Sprintf("%x", hSHA1.Sum(nil)) 109 | 110 | hMD5 := md5.New() 111 | hMD5.Write(fileBytes) 112 | md5String := fmt.Sprintf("%x", hMD5.Sum(nil)) 113 | 114 | // build the final result obj 115 | 116 | resultMsg, err := json.Marshal(Result{ 117 | Filename: req.OriginalRequest.Filename, 118 | Data: string(resultsJ), 119 | MD5: md5String, 120 | SHA1: sha1String, 121 | SHA256: sha256String, 122 | ServiceName: req.Service, 123 | Tags: req.OriginalRequest.Tags, 124 | Comment: req.OriginalRequest.Comment, 125 | StartedDateTime: req.Started, 126 | FinishedDateTime: time.Now(), 127 | }) 128 | 129 | if c.NackOnError(err, "Could not marshal final result", msg) { 130 | return 131 | } 132 | 133 | c.Producer.Channel.Publish( 134 | "totem", // exchange 135 | req.Service+".result.static.totem", // routing key 136 | false, // mandatory 137 | false, // immediate 138 | amqp.Publishing{ 139 | DeliveryMode: amqp.Persistent, 140 | ContentType: "text/plain", 141 | Body: resultMsg, 142 | }, 143 | ) 144 | 145 | if err := msg.Ack(false); err != nil { 146 | c.Warning.Println("Sending ACK failed!", err.Error()) 147 | } 148 | 149 | // cleanup time 150 | if err := os.Remove("/tmp/" + req.FilePath); err != nil { 151 | c.Warning.Printf("Could not delete file %s: %s\n", req.FilePath, err.Error()) 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /feed/feed.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "io/ioutil" 7 | "math/rand" 8 | "path/filepath" 9 | "time" 10 | 11 | "github.com/HolmesProcessing/Holmes-Totem-Dynamic/lib" 12 | 13 | "github.com/streadway/amqp" 14 | ) 15 | 16 | // local context 17 | type fCtx struct { 18 | *lib.Ctx 19 | 20 | Producer *lib.QueueHandler // the queue read by check 21 | } 22 | 23 | // Run starts the feed module either blocking or non-blocking. 24 | func Run(ctx *lib.Ctx, blocking bool) error { 25 | producer, err := ctx.SetupQueue("totem-dynamic-check-" + ctx.Config.QueueSuffix) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | c := &fCtx{ 31 | ctx, 32 | producer, 33 | } 34 | 35 | if blocking { 36 | c.Consume(ctx.Config.ConsumeQueue, ctx.Config.FeedPrefetchCount, c.parseMsg) 37 | } else { 38 | go c.Consume(ctx.Config.ConsumeQueue, ctx.Config.FeedPrefetchCount, c.parseMsg) 39 | } 40 | 41 | return nil 42 | } 43 | 44 | // parseMsg accepts an *amqp.Delivery and parses the body assuming 45 | // it's a request from the gateway. On success the parsed struct is 46 | // send to handleFeeding. 47 | func (c *fCtx) parseMsg(msg amqp.Delivery) { 48 | req := &lib.ExternalRequest{} 49 | err := json.Unmarshal(msg.Body, req) 50 | if c.NackOnError(err, "Could not decode json!", &msg) { 51 | return 52 | } 53 | 54 | // TODO: Validate msg 55 | //if c.NackOnError(m.Validate(), "Could not validate msg", &msg) { 56 | // return 57 | //} 58 | 59 | for serviceName, _ := range req.Tasks { 60 | urls, check := c.Config.Services[serviceName] 61 | if !check { 62 | //c.NackOnError(errors.New(serviceName+" not found"), "Service is not existing on this node", msg) 63 | //return 64 | c.Warning.Println("Service is not existing on this node") 65 | continue 66 | } 67 | 68 | if len(urls) == 0 { 69 | c.Warning.Println("Service is existing in config but no URLs are supplied") 70 | continue 71 | } 72 | 73 | service := &lib.Service{ 74 | Name: serviceName, 75 | URL: urls[rand.Intn(len(urls))], 76 | Client: c.Client, 77 | } 78 | 79 | go c.handleFeeding(req, service, &msg) 80 | } 81 | } 82 | 83 | // handleFeeding checks the status of the respective service 84 | // and uploads the new sample if everything is fine. If not 85 | // either an error is send or a waiting timer is actived. 86 | func (c *fCtx) handleFeeding(req *lib.ExternalRequest, service *lib.Service, msg *amqp.Delivery) { 87 | // get the status of the service 88 | status, err := service.Status() 89 | if c.NackOnError(err, "Service is not existing on this node", msg) { 90 | return 91 | } 92 | 93 | // check if the service has free capacity 94 | for status.FreeSlots <= 0 { 95 | c.Debug.Println("Slowdown: No free slots") 96 | time.Sleep(time.Second * 30) 97 | 98 | status, err = service.Status() 99 | if c.NackOnError(err, "Service is not existing on this node", msg) { 100 | return 101 | } 102 | } 103 | 104 | // differentiate between downloadable samples and URLs 105 | sample := "" 106 | if req.Download { 107 | // we need to download the sample to /tmp 108 | resp, err := c.Client.Get(req.PrimaryURI) 109 | if c.NackOnError(err, "Downloading the file failed from "+req.PrimaryURI+" failed", msg) { 110 | return 111 | } 112 | defer lib.SafeResponseClose(resp) 113 | 114 | // return if file does not exist 115 | if resp.StatusCode != 200 { 116 | c.NackOnError(errors.New(resp.Status), req.PrimaryURI+" returned non-200 status code", msg) 117 | return 118 | } 119 | 120 | fileBytes, err := ioutil.ReadAll(resp.Body) 121 | if c.NackOnError(err, "couldn't read downloaded file!", msg) { 122 | return 123 | } 124 | 125 | tmpFile, err := ioutil.TempFile("/tmp/", "totem-dynamic") 126 | if c.NackOnError(err, "couldn't create file in /tmp", msg) { 127 | return 128 | } 129 | 130 | err = ioutil.WriteFile(tmpFile.Name(), fileBytes, 0644) 131 | if c.NackOnError(err, "couldn't create file in /tmp", msg) { 132 | return 133 | } 134 | 135 | sample = filepath.Base(tmpFile.Name()) 136 | } else { 137 | // we do not need to download the sample 138 | // the filename "is the sample data" 139 | sample = req.Filename 140 | } 141 | 142 | // create new task 143 | resp, err := service.NewTask(sample) 144 | if c.NackOnError(err, "Feeding sample to service failed", msg) { 145 | return 146 | } 147 | 148 | internalReq, err := json.Marshal(lib.InternalRequest{ 149 | Service: service.Name, 150 | URL: service.URL, 151 | TaskID: resp.TaskID, 152 | FilePath: sample, 153 | Started: time.Now(), 154 | OriginalRequest: req, 155 | }) 156 | if c.NackOnError(err, "Could not create internalRequest!", msg) { 157 | return 158 | } 159 | 160 | // send to check 161 | c.Producer.Send(internalReq) 162 | if err := msg.Ack(false); err != nil { 163 | c.Warning.Println("Sending ACK failed!", err.Error()) 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Holmes-Totem-Dynamic: A Holmes Processing Investigation Planner for Time Intensive Large-scale Analysis 2 | 3 | Contributions are always welcome and appreciated! If you would like to contribute please read the [official document](http://holmes-processing.readthedocs.io/en/latest/) and use the information in this CONTRIBUTING.md file as a guide. If you have questions or are unsure about something you would like to implement, please open a new issue. We would be happy to discuss the idea with you. 4 | 5 | ## Services 6 | New Services are simple to create and are always much appreciated. When implementing a Service you will need to provide: 7 | 8 | 1. A RESTful interface for Totem-Dynamic to interact with. 9 | 2. Add service information to Totem-Dynamic's configuration file. 10 | 11 | Additionally, please keep in mind that Totem-Dynamic is optimized for executing long running tasks. For example, this Planner is perfect for Dynamic analysis and querying 3rd party services that take a long time to execute. When needing to rapidly perform quick tasks, such as static analysis, please consider making a [Totem Service](https://github.com/HolmesProcessing/Holmes-Totem) instead. 12 | 13 | ### Core Components 14 | #### RESTful Endpoints 15 | The following endpoints are the standard and expected endpoints for totem and totem-dynamic: 16 | 17 | | Endpoint | Operation | System | 18 | | --- | --- | --- | 19 | | `/` | provide information about the service | Totem and Totem-Dynamic | 20 | | `/analyze/?obj=` | perform tasking and return results | Totem | 21 | | `/feed/?obj=` | submit tasking to the service | Totem-Dynamic | 22 | | `/check/?taskid=` | check to see if the tasking is complete | Totem-Dynamic | 23 | | `/results/?taskid=` | receive service results | Totem-Dynamic | 24 | | `/status/` | retrieve status | Totem-Dynamic | 25 | 26 | #### Docker 27 | We uses Docker and Docker-Compose to manage services. This provides a few nice benefits: keeps most issues from replicating, allowing for easier restart, easier status information, etc. However, to manage the overhead we request that the following DockerFile templates are used. This is because it speeds up the container build time and reduces the on-disk size. 28 | 29 | For Go: 30 | ```dockerfile 31 | FROM golang:alpine 32 | 33 | # create folder 34 | RUN mkdir -p /service 35 | WORKDIR /service 36 | 37 | # get go dependencies 38 | RUN apk add --no-cache \ 39 | git \ 40 | && go get github.com/julienschmidt/httprouter \ 41 | && rm -rf /var/cache/apk/* 42 | 43 | ### 44 | # [Service] specific options 45 | ### 46 | ... 47 | ``` 48 | 49 | For Python: 50 | ```dockerfile 51 | FROM python:alpine 52 | 53 | # add tornado 54 | RUN pip3 install tornado 55 | 56 | # create folder 57 | RUN mkdir -p /service 58 | WORKDIR /service 59 | 60 | # add holmeslibrary 61 | RUN apk add --no-cache \ 62 | wget \ 63 | && wget https://github.com/HolmesProcessing/Holmes-Totem-Service-Library/archive/v0.1.tar.gz \ 64 | && tar xf v0.1.tar.gz \ 65 | && mv Holmes-Totem-Service* holmeslibrary \ 66 | && rm -rf /var/cache/apk/* v0.1.tar.gz 67 | 68 | ### 69 | # [Service] specific options 70 | ### 71 | ... 72 | ``` 73 | 74 | ### Configuration 75 | 76 | #### Configuration File 77 | The Service configuration file should be written in JSON and named `service.conf.example`. 78 | 79 | Totem-Dynamic should be configured. Details on how to do this can be found in the official [documentation](http://holmes-processing.readthedocs.io/en/latest/). 80 | 81 | #### Port Selection 82 | Internal ports should always be `8080` when using Docker. 83 | 84 | External ports should be listed alphabetically starting with the following range: 85 | 86 | | Range | Service Type | 87 | | --- | --- | 88 | | 97xx | No File | 89 | | 72xx | File Based | 90 | 91 | ### Code Standards 92 | Services can be written in any language. However, we recommend using Go (with [httprouter](https://godoc.org/github.com/julienschmidt/httprouter)) or Python (with [Tornado](http://www.tornadoweb.org/en/stable/)) for the entire Service or at least the interface. The example Docker Files will provide both packages. 93 | 94 | #### Standard Libraries 95 | The [Holmes Processing standard library](https://github.com/HolmesProcessing/Holmes-Totem-Service-Library) should be used when appropriate. It provides helpful functions for go and python. 96 | 97 | #### Language Style 98 | The code base of a Service should conform to the recommended style for the programming language. 99 | 100 | | Language | Style Documentation | Checking Tool | 101 | | --- | --- | --- | 102 | | Go | [Effective Go](https://golang.org/doc/effective_go.html) | `go fmt` | 103 | | Python | [PEP 9](http://pep8.org/) | [pycodestyle](https://github.com/PyCQA/pycodestyle) | 104 | 105 | ### Output 106 | The Service should return two outputs: HTTP codes and the results of the Service. 107 | 108 | #### HTTP Error Codes 109 | *work in progress* 110 | 111 | #### Results 112 | Results should be returned as sane JSON. All care should be given to return the data in a native format. For example, a Service for VirusTotal should return the original response by VirusTotal as it is already sane JSON. However, in cases when modification is needed, please provide an example in the README.md file. 113 | -------------------------------------------------------------------------------- /services/cuckoo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "os" 9 | "strconv" 10 | "time" 11 | 12 | "cuckoo/cuckoo" 13 | ) 14 | 15 | type Config struct { 16 | HTTPBinding string 17 | VerifySSL bool 18 | CheckFreeSpace bool 19 | CuckooURL string 20 | MaxPending int 21 | MaxAPICalls int 22 | LogFile string 23 | LogLevel string 24 | } 25 | 26 | type Ctx struct { 27 | Config *Config 28 | Cuckoo *cuckoo.Cuckoo 29 | } 30 | 31 | type RespStatus struct { 32 | Degraded bool 33 | Error string 34 | FreeSlots int 35 | } 36 | 37 | type RespNewTask struct { 38 | Error string 39 | TaskID string 40 | } 41 | 42 | type RespCheckTask struct { 43 | Error string 44 | Done bool 45 | } 46 | 47 | type RespTaskResults struct { 48 | Error string 49 | Results interface{} 50 | } 51 | 52 | // TODO: Replace this with our own schema es soon as we have one 53 | type CrtResult struct { 54 | Subtype string 55 | Result string 56 | Data map[string]interface{} 57 | } 58 | 59 | var ( 60 | ctx *Ctx 61 | ) 62 | 63 | func main() { 64 | // prepare context 65 | ctx = &Ctx{ 66 | Config: &Config{}, 67 | } 68 | 69 | cFile, err := os.Open("./service.conf") 70 | if err != nil { 71 | panic(err.Error()) 72 | } 73 | 74 | decoder := json.NewDecoder(cFile) 75 | err = decoder.Decode(ctx.Config) 76 | if err != nil { 77 | panic(err.Error()) 78 | } 79 | 80 | cuckoo, err := cuckoo.New(ctx.Config.CuckooURL, ctx.Config.VerifySSL) 81 | if err != nil { 82 | panic(err.Error()) 83 | } 84 | ctx.Cuckoo = cuckoo 85 | 86 | // prepare routing 87 | r := http.NewServeMux() 88 | r.HandleFunc("/status/", HTTPStatus) 89 | r.HandleFunc("/feed/", HTTPFeed) 90 | r.HandleFunc("/check/", HTTPCheck) 91 | r.HandleFunc("/results/", HTTPResults) 92 | 93 | srv := &http.Server{ 94 | Handler: r, 95 | Addr: ctx.Config.HTTPBinding, 96 | WriteTimeout: 15 * time.Second, 97 | ReadTimeout: 15 * time.Second, 98 | } 99 | 100 | log.Fatal(srv.ListenAndServe()) 101 | } 102 | 103 | func HTTPStatus(w http.ResponseWriter, r *http.Request) { 104 | resp := &RespStatus{ 105 | Degraded: false, 106 | Error: "", 107 | FreeSlots: 0, 108 | } 109 | 110 | s, err := ctx.Cuckoo.GetStatus() 111 | if err != nil { 112 | resp.Error = err.Error() 113 | HTTP500(w, r, resp) 114 | return 115 | } 116 | 117 | resp.FreeSlots = ctx.Config.MaxPending - s.Tasks.Pending 118 | 119 | if ctx.Config.CheckFreeSpace { 120 | if s.Diskspace != nil && 121 | s.Diskspace.Analyses != nil && 122 | s.Diskspace.Analyses.Free <= 256*1024*1024 { 123 | resp.Degraded = true 124 | resp.Error = "Disk is full!" 125 | } 126 | } 127 | 128 | json.NewEncoder(w).Encode(resp) 129 | } 130 | 131 | func HTTPFeed(w http.ResponseWriter, r *http.Request) { 132 | resp := &RespNewTask{ 133 | Error: "", 134 | TaskID: "", 135 | } 136 | 137 | sample := r.URL.Query().Get("obj") 138 | if sample == "" { 139 | resp.Error = "No sample given" 140 | HTTP500(w, r, resp) 141 | return 142 | } 143 | 144 | sampleBytes, err := ioutil.ReadFile("/tmp/" + sample) 145 | if err != nil { 146 | resp.Error = err.Error() 147 | HTTP500(w, r, resp) 148 | return 149 | } 150 | 151 | // TODO: actually fill payload, but therefore the payload 152 | // has to be send by totem-dyn in the first place. 153 | payload := make(map[string]string) 154 | 155 | taskID, err := ctx.Cuckoo.NewTask(sampleBytes, sample, payload) 156 | if err != nil { 157 | resp.Error = err.Error() 158 | HTTP500(w, r, resp) 159 | return 160 | } 161 | 162 | resp.TaskID = strconv.Itoa(taskID) 163 | 164 | json.NewEncoder(w).Encode(resp) 165 | } 166 | 167 | func HTTPCheck(w http.ResponseWriter, r *http.Request) { 168 | resp := &RespCheckTask{ 169 | Error: "", 170 | Done: false, 171 | } 172 | 173 | taskIDstr := r.URL.Query().Get("taskid") 174 | if taskIDstr == "" { 175 | resp.Error = "No taskID given" 176 | HTTP500(w, r, resp) 177 | return 178 | } 179 | taskID, _ := strconv.Atoi(taskIDstr) 180 | 181 | s, err := ctx.Cuckoo.TaskStatus(taskID) 182 | if err != nil { 183 | resp.Error = err.Error() 184 | HTTP500(w, r, resp) 185 | return 186 | } 187 | 188 | resp.Done = (s == "reported") 189 | 190 | json.NewEncoder(w).Encode(resp) 191 | } 192 | 193 | func HTTPResults(w http.ResponseWriter, r *http.Request) { 194 | resp := &RespTaskResults{ 195 | Error: "", 196 | } 197 | 198 | taskIDstr := r.URL.Query().Get("taskid") 199 | if taskIDstr == "" { 200 | resp.Error = "No taskID given" 201 | HTTP500(w, r, resp) 202 | return 203 | } 204 | taskID, _ := strconv.Atoi(taskIDstr) 205 | 206 | /// 207 | 208 | // get report 209 | report, err := ctx.Cuckoo.TaskReport(taskID) 210 | if err != nil { 211 | resp.Error = err.Error() 212 | HTTP500(w, r, resp) 213 | return 214 | } 215 | 216 | /// 217 | 218 | // build result 219 | resStructs := []*CrtResult{} 220 | 221 | // info 222 | resStructs = processReportInfo(report.Info) 223 | 224 | // signatures 225 | resStructs = append(resStructs, processReportSignatures(report.Signatures)...) 226 | 227 | // behavior 228 | resStructs = append(resStructs, processReportBehavior(report.Behavior)...) 229 | 230 | // dropped files 231 | /* 232 | // support for dropped files will be added later 233 | dResStructs, err := processDropped(m, cuckoo, nil, false) 234 | //if c.NackOnError(err, "processDropped failed", msg) { 235 | // return 236 | //} 237 | if err != nil { 238 | c.Warning.Println("processDropped () exited with", err, "after dropping", len(dResStructs)) 239 | } 240 | resStructs = append(resStructs, dResStructs...) 241 | */ 242 | 243 | if err = ctx.Cuckoo.DeleteTask(taskID); err != nil { 244 | log.Println("Cleaning cuckoo up failed for task", strconv.Itoa(taskID), err.Error()) 245 | } 246 | 247 | json.NewEncoder(w).Encode(resp) 248 | } 249 | 250 | func HTTP500(w http.ResponseWriter, r *http.Request, response interface{}) { 251 | w.WriteHeader(http.StatusInternalServerError) 252 | json.NewEncoder(w).Encode(response) 253 | return 254 | } 255 | -------------------------------------------------------------------------------- /services/cuckoo/processing.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strconv" 7 | 8 | "cuckoo/cuckoo" 9 | ) 10 | 11 | // processReportInfo extracts all the data from the info 12 | // section of the cuckoo report struct. 13 | func processReportInfo(i *cuckoo.TasksReportInfo) []*CrtResult { 14 | if i == nil { 15 | return []*CrtResult{} 16 | } 17 | 18 | // i.machine can be string or struct so we 19 | // have to determin where to get our info 20 | machineString := "" 21 | err := json.Unmarshal(i.Machine, &machineString) 22 | if err != nil || machineString == "" { 23 | machineString = "FAILED" 24 | } 25 | 26 | mStruct := &cuckoo.TasksReportInfoMachine{} 27 | err = json.Unmarshal(i.Machine, mStruct) 28 | if err == nil && mStruct != nil { 29 | machineString = mStruct.Name 30 | } 31 | 32 | resMap := make(map[string]interface{}) 33 | resMap["started"] = i.Started 34 | resMap["ended"] = i.Ended 35 | resMap["analysis_id"] = strconv.Itoa(i.Id) 36 | 37 | return []*CrtResult{&CrtResult{"info", machineString, resMap}} 38 | } 39 | 40 | // processReportSignatures extracts all the data from the signatures 41 | // section of the cuckoo report struct. 42 | func processReportSignatures(sigs []*cuckoo.TasksReportSignature) []*CrtResult { 43 | if sigs == nil { 44 | return []*CrtResult{} 45 | } 46 | 47 | l := len(sigs) 48 | res := make([]*CrtResult, l, l) 49 | resMap := make(map[string]interface{}) 50 | 51 | for k, sig := range sigs { 52 | resMap["severity"] = strconv.Itoa(sig.Severity) 53 | resMap["name"] = sig.Name 54 | 55 | res[k] = &CrtResult{ 56 | "signature", 57 | sig.Description, 58 | resMap, 59 | } 60 | } 61 | 62 | return res 63 | } 64 | 65 | // processReportBehavior extracts all the data from the behavior 66 | // section of the cuckoo report struct. 67 | func processReportBehavior(behavior *cuckoo.TasksReportBehavior) []*CrtResult { 68 | if behavior == nil { 69 | return []*CrtResult{} 70 | } 71 | 72 | var res []*CrtResult 73 | resMap := make(map[string]interface{}) 74 | 75 | if behavior.Processes != nil { 76 | for _, p := range behavior.Processes { 77 | resMap["process_id"] = strconv.Itoa(p.Id) 78 | resMap["parent_id"] = strconv.Itoa(p.ParentId) 79 | resMap["first_seen"] = p.FirstSeen 80 | 81 | res = append(res, &CrtResult{ 82 | "process", 83 | p.Name, 84 | resMap, 85 | }) 86 | } 87 | } 88 | 89 | // push api calls 90 | // not mixed in with upper loop so we can make it optional later 91 | if behavior.Processes != nil { 92 | pushCounter := 0 93 | 94 | for _, p := range behavior.Processes { 95 | 96 | procDescription := fmt.Sprintf("%s (%d)", p.Name, p.Id) 97 | for _, c := range p.Calls { 98 | 99 | if pushCounter >= ctx.Config.MaxAPICalls { 100 | break 101 | } 102 | 103 | resMap := make(map[string]interface{}) 104 | resMap["category"] = c.Category 105 | resMap["status"] = c.Status 106 | resMap["return"] = c.Return 107 | resMap["timestamp"] = c.Timestamp 108 | resMap["thread_id"] = c.ThreadId 109 | resMap["repeated"] = c.Repeated 110 | resMap["api"] = c.Api 111 | resMap["id"] = c.Id 112 | resMap["process"] = procDescription 113 | resMap["arguments"] = c.Arguments 114 | 115 | res = append(res, &CrtResult{ 116 | "api_call", 117 | c.Api, 118 | resMap, 119 | }) 120 | pushCounter += 1 121 | } 122 | } 123 | } 124 | 125 | if behavior.Summary != nil { 126 | if behavior.Summary.Files != nil { 127 | for _, b := range behavior.Summary.Files { 128 | res = append(res, &CrtResult{ 129 | "file", 130 | b, 131 | nil, 132 | }) 133 | } 134 | } 135 | 136 | if behavior.Summary.Keys != nil { 137 | for _, b := range behavior.Summary.Keys { 138 | res = append(res, &CrtResult{ 139 | "registry_key", 140 | b, 141 | nil, 142 | }) 143 | } 144 | } 145 | 146 | if behavior.Summary.Mutexes != nil { 147 | for _, b := range behavior.Summary.Mutexes { 148 | res = append(res, &CrtResult{ 149 | "mutex", 150 | b, 151 | nil, 152 | }) 153 | } 154 | } 155 | } 156 | 157 | return res 158 | } 159 | 160 | /* 161 | // support for dropped files will be added later 162 | func processDropped(m *lib.CheckResultsReq, cuckoo *lib.CuckooConn, crits *lib.CritsConn, upload bool) ([]*CrtResult, error) { 163 | start := time.Now() 164 | 165 | resp, err := cuckoo.GetDropped(m.TaskId) 166 | if err != nil { 167 | return []*CrtResult{}, err 168 | } 169 | 170 | results := []*CrtResult{} 171 | 172 | respReader := bytes.NewReader(resp) 173 | unbzip2 := bzip2.NewReader(respReader) 174 | untar := tar.NewReader(unbzip2) 175 | 176 | for { 177 | hdr, err := untar.Next() 178 | if err == io.EOF { 179 | // end of tar archive 180 | break 181 | } 182 | 183 | if err != nil { 184 | return results, err 185 | } 186 | 187 | if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { 188 | // no real file, might be a dir or symlink 189 | continue 190 | } 191 | 192 | name := filepath.Base(hdr.Name) 193 | fileData, err := ioutil.ReadAll(untar) 194 | 195 | if upload { 196 | 197 | id, err := crits.NewSample(fileData, name) 198 | 199 | // we need to add a short sleep here so tastypie won't crash. 200 | // this is a very ugly work around but sadly necessary 201 | time.Sleep(time.Second * 1) 202 | 203 | if err != nil { 204 | if err.Error() == "empty file" { 205 | continue 206 | } 207 | 208 | return results, err 209 | } 210 | 211 | if err = crits.ForgeRelationship(id); err != nil { 212 | return results, err 213 | } 214 | 215 | // see comment above 216 | time.Sleep(time.Second * 1) 217 | } 218 | 219 | resMap := make(map[string]interface{}) 220 | resMap["md5"] = fmt.Sprintf("%x", md5.Sum(fileData)) 221 | 222 | results = append(results, &CrtResult{ 223 | "file_added", 224 | name, 225 | resMap, 226 | }) 227 | } 228 | 229 | elapsed := time.Since(start) 230 | c.Debug.Printf("Uploaded %d dropped files in %s [%s]\n", len(results), elapsed, m.CritsData.AnalysisId) 231 | 232 | return results, nil 233 | } 234 | */ 235 | -------------------------------------------------------------------------------- /lib/lib.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "errors" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "os" 12 | "path/filepath" 13 | "strings" 14 | "time" 15 | 16 | "github.com/streadway/amqp" 17 | ) 18 | 19 | // general context struct 20 | type Ctx struct { 21 | Config *Config 22 | 23 | Debug *log.Logger 24 | Info *log.Logger 25 | Warning *log.Logger 26 | 27 | AmqpConn *amqp.Connection 28 | Client *http.Client 29 | 30 | Failed *QueueHandler 31 | } 32 | 33 | type Config struct { 34 | Amqp string 35 | QueueSuffix string 36 | ConsumeQueue string 37 | ResultsQueue string 38 | FailedQueue string 39 | 40 | LogFile string 41 | LogLevel string 42 | VerifySSL bool 43 | 44 | Services map[string][]string 45 | 46 | // stuff for feed 47 | FeedPrefetchCount int 48 | 49 | // stuff for check 50 | CheckPrefetchCount int 51 | WaitBetweenRequests int 52 | 53 | // stuff for submit 54 | SubmitPrefetchCount int 55 | } 56 | 57 | // request from the gateway to totem-dynamic 58 | type ExternalRequest struct { 59 | PrimaryURI string `json:"primaryURI"` 60 | SecondaryURI string `json:"secondaryURI"` 61 | Filename string `json:"filename"` 62 | Tasks map[string][]string `json:"tasks"` 63 | Tags []string `json:"tags"` 64 | Comment string `json:"comment"` 65 | Download bool `json:"download"` 66 | Source string `json:"source"` 67 | Attempts int `json:"attempts"` 68 | } 69 | 70 | // request between feed/check/submit 71 | type InternalRequest struct { 72 | Service string 73 | URL string 74 | TaskID string 75 | FilePath string 76 | Started time.Time 77 | OriginalRequest *ExternalRequest 78 | } 79 | 80 | // Init prepares all fields of the given Ctx sturct and 81 | // returns an error if something went wrong. By default 82 | // you should panic if an error is returned. 83 | func (c *Ctx) Init(cPath string) error { 84 | var err error 85 | 86 | c.Config, err = loadConfig(cPath) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | c.setupLogging() 92 | 93 | c.Info.Println("Connecting to amqp server...") 94 | c.AmqpConn, err = amqp.Dial(c.Config.Amqp) 95 | if err != nil { 96 | return err 97 | } 98 | 99 | c.Failed, err = c.SetupQueue(c.Config.FailedQueue) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | c.setupClient() 105 | 106 | return nil 107 | } 108 | 109 | // loadConfig loads the config file from the given path and 110 | // returns a pointer to an populated Config struct. 111 | func loadConfig(cPath string) (*Config, error) { 112 | cPath = strings.TrimSpace(cPath) 113 | 114 | // no path given, try to search in the local directory 115 | if cPath == "" { 116 | cPath, _ = filepath.Abs(filepath.Dir(os.Args[0])) 117 | cPath += "/config/totem-dynamic.conf" 118 | } 119 | 120 | conf := &Config{} 121 | cFile, err := os.Open(cPath) 122 | if err != nil { 123 | return conf, err 124 | } 125 | 126 | decoder := json.NewDecoder(cFile) 127 | err = decoder.Decode(&conf) 128 | if err != nil { 129 | return conf, err 130 | } 131 | 132 | // validate the suffix 133 | if conf.QueueSuffix == "" { 134 | err = errors.New("Suffix is missing") 135 | } 136 | 137 | return conf, err 138 | } 139 | 140 | // setupLogging populates the debug, info and warning logger of the context. 141 | func (c *Ctx) setupLogging() error { 142 | // default: only log to stdout 143 | handler := io.MultiWriter(os.Stdout) 144 | 145 | if c.Config.LogFile != "" { 146 | // log to file 147 | if _, err := os.Stat(c.Config.LogFile); os.IsNotExist(err) { 148 | err := ioutil.WriteFile(c.Config.LogFile, []byte(""), 0600) 149 | if err != nil { 150 | return err 151 | } 152 | } 153 | 154 | f, err := os.OpenFile(c.Config.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | handler = io.MultiWriter(f, os.Stdout) 160 | } 161 | 162 | // TODO: clean this mess up... 163 | empty := io.MultiWriter() 164 | if c.Config.LogLevel == "warning" { 165 | c.Warning = log.New(handler, "WARNING: ", log.Ldate|log.Ltime|log.Lshortfile) 166 | c.Info = log.New(empty, "INFO: ", log.Ldate|log.Ltime) 167 | c.Debug = log.New(empty, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfile) 168 | } else if c.Config.LogLevel == "info" { 169 | c.Warning = log.New(handler, "WARNING: ", log.Ldate|log.Ltime|log.Lshortfile) 170 | c.Info = log.New(handler, "INFO: ", log.Ldate|log.Ltime) 171 | c.Debug = log.New(empty, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfile) 172 | } else { 173 | c.Warning = log.New(handler, "WARNING: ", log.Ldate|log.Ltime|log.Lshortfile) 174 | c.Info = log.New(handler, "INFO: ", log.Ldate|log.Ltime) 175 | c.Debug = log.New(handler, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfile) 176 | } 177 | 178 | return nil 179 | } 180 | 181 | // setupClient populates the http client so there is only one client 182 | // in the context which can keep connections open to improve preformance. 183 | func (c *Ctx) setupClient() { 184 | tr := &http.Transport{} 185 | if !c.Config.VerifySSL { 186 | tr = &http.Transport{ 187 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 188 | } 189 | } 190 | 191 | c.Client = &http.Client{Transport: tr} 192 | } 193 | 194 | // FastGet is a wrapper for http.Get which returns only 195 | // the important data from the request and makes sure 196 | // everyting is closed properly. 197 | func FastGet(c *http.Client, url string, structPointer interface{}) ([]byte, int, error) { 198 | resp, err := c.Get(url) 199 | if err != nil { 200 | return nil, 0, err 201 | } 202 | defer SafeResponseClose(resp) 203 | 204 | respBody, err := ioutil.ReadAll(resp.Body) 205 | if err != nil { 206 | return nil, 0, err 207 | } 208 | 209 | if structPointer != nil { 210 | err = json.Unmarshal(respBody, structPointer) 211 | } 212 | 213 | return respBody, resp.StatusCode, err 214 | } 215 | 216 | func SafeResponseClose(r *http.Response) { 217 | if r == nil { 218 | return 219 | } 220 | 221 | io.Copy(ioutil.Discard, r.Body) 222 | r.Body.Close() 223 | } 224 | -------------------------------------------------------------------------------- /services/cuckoo/cuckoo/cuckoo.go: -------------------------------------------------------------------------------- 1 | package cuckoo 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "mime/multipart" 12 | "net/http" 13 | ) 14 | 15 | type Cuckoo struct { 16 | URL string 17 | Client *http.Client 18 | } 19 | 20 | type Status struct { 21 | Tasks *StatusTasks `json:"tasks"` 22 | Diskspace *StatusDiskspace `json:"diskspace"` 23 | } 24 | 25 | type StatusTasks struct { 26 | Running int `json:"running"` 27 | Pending int `json:"pending"` 28 | } 29 | 30 | type StatusDiskspace struct { 31 | Analyses *StatusSamples `json:"analyses"` 32 | } 33 | 34 | type StatusSamples struct { 35 | Total int `json:"total"` 36 | Free int `json:"free"` 37 | Used int `json:"used"` 38 | } 39 | 40 | type TasksCreateResp struct { 41 | TaskID int `json:"task_id"` 42 | } 43 | 44 | type TasksViewResp struct { 45 | Message string `json:"message"` 46 | Task *TasksViewTask `json:"task"` 47 | } 48 | 49 | type TasksViewTask struct { 50 | Status string `json:"status"` 51 | } 52 | 53 | type TasksReport struct { 54 | Info *TasksReportInfo `json:"info"` 55 | Signatures []*TasksReportSignature `json;"signatures"` 56 | Behavior *TasksReportBehavior `json:"behavior"` 57 | } 58 | 59 | type TasksReportInfo struct { 60 | Started string `json:"started"` 61 | Ended string `json:"ended"` 62 | Id int `json:"id"` 63 | Machine json.RawMessage `json:"machine"` //can be TasksReportInfoMachine OR string 64 | } 65 | 66 | type TasksReportInfoMachine struct { 67 | Name string `json:"name"` 68 | } 69 | 70 | type TasksReportSignature struct { 71 | Severity int `json:"severity"` 72 | Description string `json:"description"` 73 | Name string `json:"name"` 74 | } 75 | 76 | type TasksReportBehavior struct { 77 | Processes []*TasksReportBhvPcs `json:"processes"` 78 | Summary *TasksReportBhvSummary `json:"summary"` 79 | } 80 | 81 | type TasksReportBhvPcs struct { 82 | Name string `json:"process_name"` 83 | Id int `json:"process_id"` 84 | ParentId int `json:"parent_id"` 85 | FirstSeen float64 `json:"first_seen"` 86 | Calls []*TasksReportBhvPcsCall `json:"calls"` 87 | } 88 | 89 | type TasksReportBhvPcsCall struct { 90 | Category string `json:"category"` 91 | Status int `json:"status"` 92 | Return string `json:"return"` 93 | Timestamp string `json:"timestamp"` 94 | ThreadId string `json:"thread_id"` 95 | Repeated int `json:"repeated"` 96 | Api string `json:"api"` 97 | Arguments json.RawMessage `json:"arguments"` 98 | Id int `json:"id"` 99 | } 100 | 101 | type TasksReportBhvPcsCallArg struct { 102 | Name string `json:"name"` 103 | Value string `json:"value"` 104 | } 105 | 106 | type TasksReportBhvSummary struct { 107 | Files []string `json:"files"` 108 | Keys []string `json:"keys"` 109 | Mutexes []string `json:"mutexes"` 110 | } 111 | 112 | type FilesView struct { 113 | Sample *FilesViewSample `json:"sample"` 114 | } 115 | 116 | type FilesViewSample struct { 117 | SHA1 string `json:"sha1"` 118 | FileType string `json:"file_type"` 119 | FileSize int `json:"file_size"` 120 | CRC32 string `json:"crc32"` 121 | SSDeep string `json:"ssdeep"` 122 | SHA256 string `json:"sha256"` 123 | SHA512 string `json:"sha512"` 124 | Id int `json:"id"` 125 | MD5 string `json:"md5"` 126 | } 127 | 128 | func New(URL string, verifySSL bool) (*Cuckoo, error) { 129 | tr := &http.Transport{} 130 | if !verifySSL { 131 | tr = &http.Transport{ 132 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 133 | } 134 | } 135 | 136 | return &Cuckoo{ 137 | URL: URL, 138 | Client: &http.Client{Transport: tr}, 139 | }, nil 140 | } 141 | 142 | func (c *Cuckoo) GetStatus() (*Status, error) { 143 | r := &Status{} 144 | resp, status, err := c.fastGet("/cuckoo/status", r) 145 | if err != nil || status != 200 { 146 | if err == nil { 147 | err = errors.New("no-200 ret") 148 | } 149 | 150 | if resp != nil { 151 | err = errors.New(fmt.Sprintf("%s -> [%d] %s", err.Error(), status, resp)) 152 | } 153 | 154 | return nil, err 155 | } 156 | 157 | return r, nil 158 | } 159 | 160 | // submitTask submits a new task to the cuckoo api. 161 | func (c *Cuckoo) NewTask(fileBytes []byte, fileName string, params map[string]string) (int, error) { 162 | // add the file to the request 163 | body := new(bytes.Buffer) 164 | writer := multipart.NewWriter(body) 165 | part, err := writer.CreateFormFile("file", fileName) 166 | if err != nil { 167 | return 0, err 168 | } 169 | part.Write(fileBytes) 170 | 171 | // add the extra payload to the request 172 | for key, val := range params { 173 | err = writer.WriteField(key, val) 174 | if err != nil { 175 | return 0, err 176 | } 177 | } 178 | 179 | err = writer.Close() 180 | if err != nil { 181 | return 0, err 182 | } 183 | 184 | // finalize request 185 | request, err := http.NewRequest("POST", c.URL+"/tasks/create/file", body) 186 | if err != nil { 187 | return 0, err 188 | } 189 | request.Header.Add("Content-Type", writer.FormDataContentType()) 190 | 191 | // perform request 192 | resp, err := c.Client.Do(request) 193 | if err != nil { 194 | return 0, err 195 | } 196 | defer resp.Body.Close() 197 | 198 | if resp.StatusCode != 200 { 199 | return 0, errors.New(resp.Status) 200 | } 201 | 202 | // parse response 203 | respBody, err := ioutil.ReadAll(resp.Body) 204 | if err != nil { 205 | return 0, err 206 | } 207 | 208 | r := &TasksCreateResp{} 209 | if err := json.Unmarshal(respBody, r); err != nil { 210 | return 0, err 211 | } 212 | 213 | return r.TaskID, nil 214 | } 215 | 216 | func (c *Cuckoo) TaskStatus(id int) (string, error) { 217 | r := &TasksViewResp{} 218 | resp, status, err := c.fastGet(fmt.Sprintf("/tasks/view/%d", id), r) 219 | if err != nil || status != 200 { 220 | if err == nil { 221 | err = errors.New("no-200 ret") 222 | } 223 | 224 | if resp != nil { 225 | err = errors.New(fmt.Sprintf("%s -> [%d] %s", err.Error(), status, resp)) 226 | } 227 | 228 | return "", err 229 | } 230 | 231 | if r.Message != "" { 232 | return "", errors.New(r.Message) 233 | } 234 | 235 | return r.Task.Status, nil 236 | } 237 | 238 | func (c *Cuckoo) TaskReport(id int) (*TasksReport, error) { 239 | r := &TasksReport{} 240 | resp, status, err := c.fastGet(fmt.Sprintf("/tasks/report/%d", id), r) 241 | if err != nil || status != 200 { 242 | if err == nil { 243 | err = errors.New("no-200 ret") 244 | } 245 | 246 | if resp != nil { 247 | err = errors.New(fmt.Sprintf("%s -> [%d] %s", err.Error(), status, resp)) 248 | } 249 | 250 | return nil, err 251 | } 252 | 253 | return r, nil 254 | } 255 | 256 | func (c *Cuckoo) GetFileInfoByMD5(md5 string) (*FilesViewSample, error) { 257 | r := &FilesView{} 258 | resp, status, err := c.fastGet("/files/view/md5/"+md5, r) 259 | if err != nil || status != 200 { 260 | if err == nil { 261 | err = errors.New("no-200 ret") 262 | } 263 | 264 | if resp != nil { 265 | err = errors.New(fmt.Sprintf("%s -> [%d] %s", err.Error(), status, resp)) 266 | } 267 | 268 | return nil, err 269 | } 270 | 271 | return r.Sample, nil 272 | } 273 | 274 | func (c *Cuckoo) GetFileInfoByID(id string) (*FilesViewSample, error) { 275 | r := &FilesView{} 276 | resp, status, err := c.fastGet("/files/view/id/"+id, r) 277 | if err != nil || status != 200 { 278 | if err == nil { 279 | err = errors.New("no-200 ret") 280 | } 281 | 282 | if resp != nil { 283 | err = errors.New(fmt.Sprintf("%s -> [%d] %s", err.Error(), status, resp)) 284 | } 285 | 286 | return nil, err 287 | } 288 | 289 | return r.Sample, nil 290 | } 291 | 292 | func (c *Cuckoo) DeleteTask(id int) error { 293 | resp, status, err := c.fastGet(fmt.Sprintf("/tasks/delete/%d", id), nil) 294 | if err != nil { 295 | if resp != nil { 296 | err = errors.New(fmt.Sprintf("%s -> [%d] %s", err.Error(), status, resp)) 297 | } 298 | 299 | return err 300 | } 301 | 302 | if status != 200 { 303 | return errors.New(fmt.Sprintf("%d - Response code not 200", status)) 304 | } 305 | 306 | return nil 307 | } 308 | 309 | // FastGet is a wrapper for http.Get which returns only 310 | // the important data from the request and makes sure 311 | // everyting is closed properly. 312 | func (c *Cuckoo) fastGet(url string, structPointer interface{}) ([]byte, int, error) { 313 | resp, err := c.Client.Get(c.URL + url) 314 | if err != nil { 315 | return nil, 0, err 316 | } 317 | defer safeResponseClose(resp) 318 | 319 | respBody, err := ioutil.ReadAll(resp.Body) 320 | if err != nil { 321 | return nil, 0, err 322 | } 323 | 324 | if structPointer != nil { 325 | err = json.Unmarshal(respBody, structPointer) 326 | } 327 | 328 | return respBody, resp.StatusCode, err 329 | } 330 | 331 | func safeResponseClose(r *http.Response) { 332 | if r == nil { 333 | return 334 | } 335 | 336 | io.Copy(ioutil.Discard, r.Body) 337 | r.Body.Close() 338 | } 339 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2017 Holmes Group LLC 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------