├── config
├── mkcert.sh
└── gateway.conf.example
├── .travis.yml
├── .gitignore
├── error.go
├── submit_task.html
├── amqp.go
├── main.go
├── object.go
├── README.md
├── LICENSE
└── tasking.go
/config/mkcert.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | openssl req -x509 -newkey rsa:2048 -keyout cert-key.pem -out cert.pem -days 360 -nodes
3 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: go
2 |
3 | go:
4 | - 1.6.2
5 |
6 | # this fixes go imports
7 | before_install:
8 | - RepoName=`basename $PWD`; SrcDir=`dirname $PWD`; DestDir="`dirname $SrcDir`/HolmesProcessing"
9 | - if [[ "$SrcDir" != "$DestDir" ]]; then mv "$SrcDir" "$DestDir"; cd ../../HolmesProcessing/$RepoName; export TRAVIS_BUILD_DIR=`dirname $TRAVIS_BUILD_DIR`/$RepoName; fi
10 |
11 | install:
12 | - go get github.com/streadway/amqp
13 | - go get golang.org/x/crypto/bcrypt
14 | - go get github.com/howeyc/fsnotify
15 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Compiled Object files, Static and Dynamic libs (Shared Objects)
2 | *.o
3 | *.a
4 | *.so
5 |
6 | # Folders
7 | _obj
8 | _test
9 |
10 | # Architecture specific extensions/prefixes
11 | *.[568vq]
12 | [568vq].out
13 |
14 | *.cgo1.go
15 | *.cgo2.c
16 | _cgo_defun.c
17 | _cgo_gotypes.go
18 | _cgo_export.*
19 |
20 | _testmain.go
21 |
22 | *.exe
23 | *.test
24 | *.prof
25 |
26 | # Output of the go coverage tool, specifically when used with LiteIDE
27 | *.out
28 |
29 | ###
30 | # Holmes Processing Specific
31 | ###
32 |
33 | # Configuration files
34 | *.conf
35 |
36 | # Executables
37 | Holmes-Gateway
38 | keys/keys
39 |
40 | # Encryption Keys
41 | keys/*.pub
42 | keys/*.priv
43 | *.pem
44 |
--------------------------------------------------------------------------------
/error.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "log"
7 | )
8 |
9 | type ErrCode int
10 |
11 | const (
12 | ERR_NONE ErrCode = 1 + iota
13 | ERR_KEY_UNKNOWN = iota
14 | ERR_ENCRYPTION = iota
15 | ERR_TASK_INVALID = iota
16 | ERR_NOT_ALLOWED = iota
17 | ERR_OTHER_UNRECOVERABLE = iota
18 | ERR_OTHER_RECOVERABLE = iota
19 | )
20 |
21 | type MyError struct {
22 | Error error
23 | Code ErrCode
24 | }
25 |
26 | func (me MyError) MarshalJSON() ([]byte, error) {
27 | return json.Marshal(
28 | struct {
29 | Error string
30 | Code ErrCode
31 | }{
32 | Error: me.Error.Error(),
33 | Code: me.Code,
34 | })
35 | }
36 |
37 | func (me *MyError) UnmarshalJSON(data []byte) error {
38 | var s struct {
39 | Error string
40 | Code ErrCode
41 | }
42 | err := json.Unmarshal(data, &s)
43 | me.Error = errors.New(s.Error)
44 | me.Code = s.Code
45 | return err
46 | }
47 |
48 | func FailOnError(err error, msg string) {
49 | if err != nil {
50 | log.Fatalf("%s: %s", msg, err)
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/config/gateway.conf.example:
--------------------------------------------------------------------------------
1 | {
2 | "HTTP": ":8090",
3 | "StorageSampleURI": "https://localhost:8016/api/v2/raw_data/",
4 | "Organizations": [{"Name": "Org1", "Sources": ["src1", "src2"]},{"Name": "Org2", "Uri": "https://localhost:8090", "Sources": ["src1", "src2"]}],
5 | "OwnOrganization": "Org1",
6 | "AllowedUsers": [{"name": "test", "pw":"$2a$06$fLcXyZd6xs60iPj8sBXf8exGfcIMnxZWHH5Eyf1.fwkSnuNq0h6Aa", "id":0},
7 | {"name": "test2", "pw":"$2a$06$fLcXyZd6xs60iPj8sBXf8exGfcIMnxZWHH5Eyf1.fwkSnuNq0h6Aa", "id":1}],
8 | "AutoTasks": {"PE":{"PEID":[]}, "":{"YARA":[]}},
9 | "DisableStorageVerify": true,
10 | "CertificateKeyPath":"config/cert-key.pem",
11 | "CertificatePath": "config/cert.pem",
12 | "MaxUploadSize": 200,
13 | "AllowForeignTasks": false,
14 |
15 | "AMQP": "amqp://guest:guest@localhost:5672/",
16 | "AMQPDefault": {"Queue": "totem_input", "Exchange": "totem", "RoutingKey": "work.static.totem"},
17 | "AMQPSplitting": {"CUCKOO": {"Queue": "totem_dynamic_input", "Exchange": "totem_dynamic", "RoutingKey": "work.static.totem"},
18 | "DRAKVUF": {"Queue": "totem_dynamic_input", "Exchange": "totem_dynamic", "RoutingKey": "work.static.totem"},
19 | "VIRUSTOTAL": {"Queue": "totem_dynamic_input", "Exchange": "totem_dynamic", "RoutingKey": "work.static.totem"}}
20 | }
21 |
--------------------------------------------------------------------------------
/submit_task.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
38 |
39 |
56 |
59 |
60 |
61 |
--------------------------------------------------------------------------------
/amqp.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "github.com/streadway/amqp"
7 | "log"
8 | "time"
9 | )
10 |
11 | var (
12 | AMQPChannel *amqp.Channel
13 | )
14 |
15 | type AMQPConf struct {
16 | Queue string
17 | Exchange string
18 | RoutingKey string
19 | }
20 |
21 | func pushToAMQP(task *TaskRequest, aconf *AMQPConf) *MyError {
22 | // Pushes the given TaskRequest in JSON-form to the given AMQP-queue.
23 | // On error retries three times before giving up
24 |
25 | msgBody, err := json.Marshal(task)
26 | if err != nil {
27 | log.Println("Error while Marshalling: ", err)
28 | return &MyError{Error: err, Code: ERR_OTHER_RECOVERABLE}
29 | }
30 | pub := amqp.Publishing{DeliveryMode: amqp.Persistent, ContentType: "text/plain", Body: msgBody}
31 | log.Printf("Pushing to %s: \x1b[0;32m%s\x1b[0m\n", aconf.Exchange, msgBody)
32 | err = AMQPChannel.Publish(aconf.Exchange, aconf.RoutingKey, false, false, pub)
33 |
34 | if err != nil {
35 | log.Println("Error while pushing to transport: ", err)
36 | // try to recover three times
37 | try := 0
38 | for try < 3 {
39 | try++
40 | log.Println("Trying to restore the connection... #", try)
41 | err = connectAMQP()
42 | if err == nil {
43 | break
44 | }
45 | // sleep 3 seconds
46 | time.Sleep(time.Duration(3000000000))
47 | }
48 | if err != nil {
49 | // could not recover the connection after third try => give up
50 | return &MyError{Error: err, Code: ERR_OTHER_RECOVERABLE}
51 | }
52 | log.Println("Connection restored")
53 |
54 | // retry pushing
55 | err = AMQPChannel.Publish(aconf.Exchange, aconf.RoutingKey, false, false, pub)
56 | if err != nil {
57 | return &MyError{Error: err, Code: ERR_OTHER_RECOVERABLE}
58 | }
59 | }
60 | return nil
61 | }
62 |
63 | func connectAMQP() error {
64 | // Create connection to AMQP-server in config
65 | // Create all Queues and Exchanges from the config
66 |
67 | conn, err := amqp.Dial(conf.AMQP)
68 | if err != nil {
69 | return errors.New("Failed to connect to AMQPMQ: " + err.Error())
70 | }
71 | //defer conn.Close()
72 |
73 | AMQPChannel, err = conn.Channel()
74 | if err != nil {
75 | return errors.New("Failed to open a channel: " + err.Error())
76 | }
77 | //defer AMQPChannel.Close()
78 | addAMQPConf(conf.AMQPDefault)
79 |
80 | for c := range conf.AMQPSplitting {
81 | err = addAMQPConf(conf.AMQPSplitting[c])
82 | if err != nil {
83 | return err
84 | }
85 | }
86 |
87 | log.Println("Connected to AMQP")
88 | return nil
89 | }
90 |
91 | func addAMQPConf(c AMQPConf) error {
92 | // Create Queue and Exchange, and bind Queue to Exchange
93 |
94 | queue, err := AMQPChannel.QueueDeclare(
95 | c.Queue, //name
96 | true, // durable
97 | false, // delete when unused
98 | false, // exclusive
99 | false, // no-wait
100 | nil, // arguments
101 | )
102 | if err != nil {
103 | return errors.New("Failed to declare a queue: " + err.Error())
104 | }
105 |
106 | err = AMQPChannel.ExchangeDeclare(
107 | c.Exchange, // name
108 | "topic", // type
109 | true, // durable
110 | false, // auto-deleted
111 | false, // internal
112 | false, // no-wait
113 | nil, // arguments
114 | )
115 | if err != nil {
116 | return errors.New("Failed to declare an exchange: " + err.Error())
117 | }
118 |
119 | err = AMQPChannel.QueueBind(
120 | queue.Name, // queue name
121 | c.RoutingKey, // routing key
122 | c.Exchange, // exchange
123 | false, // nowait
124 | nil, // arguments
125 | )
126 | if err != nil {
127 | return errors.New("Failed to bind queue: " + err.Error())
128 | }
129 | return nil
130 | }
131 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "crypto/tls"
5 | "encoding/json"
6 | "errors"
7 | "flag"
8 | "golang.org/x/crypto/bcrypt"
9 | "log"
10 | "net/http"
11 | "net/http/httputil"
12 | "net/url"
13 | "os"
14 | "path/filepath"
15 | )
16 |
17 | type User struct {
18 | Name string `json:"name"`
19 | Id int `json:"id"`
20 | PasswordHash string `json:"pw"`
21 | }
22 |
23 | type config struct {
24 | HTTP string // The HTTP-binding for listening (IP+Port)
25 | Organizations []Organization // All the known organizations
26 | OwnOrganization string // The name of the own organization (Should also be present in the list "Organizations")
27 | StorageSampleURI string // URI of HolmesStorage
28 | AutoTasks map[string](map[string][]string) // Tasks that should be automatically executed on new objects mimetype -> taskname -> args
29 | MaxUploadSize uint32 // The maximum size of a sample-upload in Megabyte
30 | CertificatePath string
31 | CertificateKeyPath string
32 | AllowedUsers []User
33 |
34 | DisableStorageVerify bool
35 |
36 | AllowForeignTasks bool
37 |
38 | AMQP string
39 | AMQPDefault AMQPConf
40 | AMQPSplitting map[string]AMQPConf
41 | }
42 |
43 | var (
44 | conf *config // The configuration struct
45 | storageURIStoreSample url.URL // The URL to storage for redirecting object-storage requests
46 | users map[string]*User // Map: Username -> User-struct (TODO: Move to storage)
47 | )
48 |
49 | func initHTTP() {
50 | // build a secure tls configuration for the http server
51 | cfg := &tls.Config{
52 | MinVersion: tls.VersionTLS12,
53 | CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256},
54 | PreferServerCipherSuites: true,
55 | CipherSuites: []uint16{
56 | tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
57 | tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
58 | tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
59 | tls.TLS_RSA_WITH_AES_256_CBC_SHA,
60 | },
61 | }
62 |
63 | // necessary to enable strict transport security via header
64 | mux := http.NewServeMux()
65 | mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
66 | w.Header().Add("Strict-Transport-Security", "max-age=63072000; includeSubDomains")
67 | w.Header().Set("Access-Control-Allow-Origin", "*")
68 | w.Header().Set("Access-Control-Allow-Headers", "origin, content-type, accept")
69 | w.Header().Set("Content-Type", "application/json")
70 |
71 | w.Write([]byte("Holmes-Gateway.\n"))
72 | })
73 |
74 | // productive routes
75 | mux.HandleFunc("/task/", httpRequestIncomingTask)
76 | if conf.AllowForeignTasks {
77 | mux.HandleFunc("/task_foreign/", httpRequestIncomingTaskForeign)
78 | }
79 | mux.HandleFunc("/samples/", httpRequestIncomingSample)
80 |
81 | // storage proxy
82 | uri, _ := url.Parse(conf.StorageSampleURI)
83 | storageURIStoreSample = *uri
84 | proxy = httputil.NewSingleHostReverseProxy(uri)
85 | proxy.Transport = &myTransport{}
86 |
87 | // server settings
88 | srv := &http.Server{
89 | Addr: conf.HTTP,
90 | Handler: mux,
91 | TLSConfig: cfg,
92 | TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0),
93 | }
94 |
95 | log.Printf("Listening on %s\n", conf.HTTP)
96 | log.Fatal(srv.ListenAndServeTLS(conf.CertificatePath, conf.CertificateKeyPath))
97 | }
98 |
99 | func initUsers() {
100 | users = make(map[string]*User)
101 | for u := range conf.AllowedUsers {
102 | user := &(conf.AllowedUsers[u])
103 | users[user.Name] = user
104 | }
105 | }
106 |
107 | func authenticate(username string, password string) (*User, error) {
108 | // TODO: Ask storage instead of configuration file for credentials
109 | user, exists := users[username]
110 | if !exists {
111 | // compare some dummy value to prevent timing based attack
112 | bcrypt.CompareHashAndPassword([]byte("$2a$06$fLcXyZd6xs60iPj8sBXf8exGfcIMnxZWHH5Eyf1.fwkSnuNq0h6Aa"), []byte(password))
113 | log.Printf("User '%s' does not exist", username)
114 | return nil, errors.New("Authentication failed")
115 | }
116 | err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(password))
117 | if err != nil {
118 | log.Printf("\x1b[0;31m%s\x1b[0m\n", err)
119 | return nil, errors.New("Authentication failed")
120 | } else {
121 | log.Printf("Authenticated as %s\n", username)
122 | }
123 | return user, nil
124 | }
125 |
126 | func main() {
127 | // Parse arguments
128 | var confPath string
129 | flag.StringVar(&confPath, "config", "", "Path to the config file")
130 | flag.Parse()
131 | if confPath == "" {
132 | confPath, _ = filepath.Abs(filepath.Dir(os.Args[0]))
133 | confPath = filepath.Join(confPath, "/config/gateway.conf")
134 | }
135 |
136 | // Read config
137 | conf = &config{MaxUploadSize: 200, DisableStorageVerify: false, AllowForeignTasks: false}
138 | cfile, _ := os.Open(confPath)
139 | err := json.NewDecoder(cfile).Decode(&conf)
140 | FailOnError(err, "Couldn't read config file")
141 |
142 | err = connectAMQP()
143 | FailOnError(err, "Failed while connecting to AMQP")
144 | initSourceRouting()
145 | initUsers()
146 | initHTTP()
147 | }
148 |
--------------------------------------------------------------------------------
/object.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "crypto/tls"
6 | "encoding/json"
7 | "io"
8 | "io/ioutil"
9 | "log"
10 | "net/http"
11 | "net/http/httputil"
12 | "net/url"
13 | "strconv"
14 | "strings"
15 | "time"
16 | )
17 |
18 | var proxy *httputil.ReverseProxy // The proxy object for redirecting object-storage requests
19 |
20 | type storageResult struct {
21 | Type string `json:"type"`
22 | CreationDateTime time.Time `json:"creation_date_time"`
23 | Submissions []string `json:"submissions"`
24 | Source []string `json:"source"`
25 |
26 | MD5 string `json:"md5"`
27 | SHA1 string `json:"sha1"`
28 | SHA256 string `json:"sha256"`
29 |
30 | FileMime string `json:"file_mime"`
31 | FileName []string `json:file_name"`
32 |
33 | DomainFQDN string `json:"domain_fqdn"`
34 | DomainTLD string `json:"domain_tld"`
35 | DomainSubDomain string `json:"domain_sub_domain"`
36 |
37 | IPAddress string `json:"ip_address"`
38 | IPv6 bool `json:"ip_v6"`
39 |
40 | EmailAddress string `json:"email_address"`
41 | EmailLocalPart string `json:"email_local_part"`
42 | EmailDomainPart string `json:"email_domain_part"`
43 | EmailSubAddressing string `json:"email_sub_addressing"`
44 |
45 | GenericIdentifier string `json:"generic_identifier"`
46 | GenericType string `json:"generic_type"`
47 | GenericDataRelAddress string `json:"generic_data_rel_address"`
48 | }
49 |
50 | type storageResponse struct {
51 | ResponseCode int
52 | Failure string
53 | Result storageResult
54 | }
55 |
56 | type myTransport struct {
57 | }
58 |
59 | func (t *myTransport) RoundTrip(request *http.Request) (*http.Response, error) {
60 | // Forwards the given request to storage and executes auto-tasking, if successfull
61 |
62 | // Since accessing the Form-values of the request changes the reader,
63 | // which cannot be rewinded / seeked, an error would be thrown, if the
64 | // request was forwarded with the reader at the wrong position.
65 | // For this reason, the whole body is read and a new reader is created,
66 | // which can be rewinded.
67 | var err error
68 | if request.ContentLength > 1024*1024*int64(conf.MaxUploadSize) {
69 | respBody := ioutil.NopCloser(bytes.NewBufferString("Upload too large"))
70 | resp := &http.Response{StatusCode: 413, Body: respBody}
71 | respBody.Close()
72 | log.Println("Upload too large")
73 | return resp, nil
74 | }
75 | reqbuf := make([]byte, request.ContentLength)
76 | _, err = io.ReadFull(request.Body, reqbuf)
77 | if err != nil {
78 | log.Printf("Error reading body!", err)
79 | return nil, err
80 | }
81 |
82 | reader := bytes.NewReader(reqbuf)
83 | reqrdr := ioutil.NopCloser(reader)
84 | request.Body = reqrdr
85 |
86 | defer func() {
87 | request.Body.Close()
88 | reqrdr.Close()
89 | }()
90 |
91 | request.ParseMultipartForm(1024 * 1024 * int64(conf.MaxUploadSize))
92 | // Read the name and the source from the request, because they can not be
93 | // reconstructed from storage's response.
94 | name := request.FormValue("name")
95 | source := request.FormValue("source")
96 |
97 | username := request.FormValue("username")
98 | password := request.FormValue("password")
99 |
100 | *request.URL = storageURIStoreSample
101 |
102 | // restore the reader for the body
103 | reader.Seek(0, 0)
104 |
105 | user, err := authenticate(username, password)
106 | if err != nil {
107 | return nil, err
108 | }
109 |
110 | form, _ := url.ParseQuery(request.URL.RawQuery)
111 | form.Set("user_id", strconv.Itoa(user.Id))
112 | request.URL.RawQuery = form.Encode()
113 |
114 | // Do the proxy-request
115 | trans := http.DefaultTransport.(*http.Transport)
116 | trans.TLSClientConfig = &tls.Config{InsecureSkipVerify: conf.DisableStorageVerify}
117 |
118 | response, err := http.DefaultTransport.RoundTrip(request)
119 | if err != nil {
120 | log.Printf("Error performing proxy-request!", err)
121 | return nil, err
122 | }
123 |
124 | // Parse the response. If it was successful, execute automatic tasks
125 | var resp storageResponse
126 | buf := make([]byte, response.ContentLength)
127 |
128 | _, err = io.ReadFull(response.Body, buf)
129 | if err != nil {
130 | log.Printf("Error reading body!", err)
131 | return nil, err
132 | }
133 | rdr := ioutil.NopCloser(bytes.NewReader(buf))
134 | defer func() {
135 | rdr.Close()
136 | response.Body.Close()
137 | }()
138 | json.Unmarshal(buf, &resp)
139 | //log.Printf("%+v\n", resp)
140 | if resp.ResponseCode == 0 {
141 | log.Printf("\x1b[0;32mSuccessfully uploaded sample with SHA256: %s\x1b[0m", resp.Result.SHA256)
142 | // Execute automatic tasks
143 | for t := range conf.AutoTasks {
144 | if strings.Contains(resp.Result.FileMime, t) {
145 | autotasks := conf.AutoTasks[t]
146 | if len(autotasks) != 0 {
147 | task := TaskRequest{
148 | PrimaryURI: resp.Result.SHA256,
149 | SecondaryURI: "",
150 | Filename: name,
151 | Tasks: autotasks,
152 | Tags: []string{},
153 | Attempts: 0,
154 | Source: source,
155 | Download: true,
156 | }
157 |
158 | log.Printf("\x1b[0;33mAutomatically executing %+v\x1b[0m\n", task)
159 | handleOwnTasks([]TaskRequest{task})
160 | }
161 | }
162 | }
163 | } else {
164 | log.Printf("\x1b[0;31mUpload failed\x1b[0m\n")
165 | if response.StatusCode == 200 {
166 | response.StatusCode = 400 // bad request
167 | }
168 | }
169 |
170 | // restore the reader for the body
171 | response.Body = rdr
172 | return response, err
173 | }
174 |
175 | func httpRequestIncomingSample(w http.ResponseWriter, r *http.Request) {
176 | proxy.ServeHTTP(w, r)
177 | }
178 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Holmes-Gateway [](https://travis-ci.org/HolmesProcessing/Holmes-Gateway)
2 |
3 | ## Overview
4 | Holmes-Gateway orchestrates the submission of objects and tasks to HolmesProcessing. Foremost, this greatly simplifies the tasking and enables the ability to automatically route tasks to [Holmes-Totem](https://github.com/HolmesProcessing/Holmes-Totem) and [Holmes-Totem-Dynamic](https://github.com/HolmesProcessing/Holmes-Totem-Dynamic) at a Service level. In addition, Holmes-Gateway provides validation and authentication. Finally, Holmes-Gateway provides the technical foundation for collaboration between organizations.
5 |
6 | Holmes-Gateway is meant to prevent a user from directly connecting to [Holmes-Storage](https://github.com/HolmesProcessing/Holmes-Storage) or RabbitMQ.
7 | Instead tasking-requests and object upload pass through Holmes-Gateway, which performs validity checking, enforces ACL, and forwards the requests.
8 |
9 | If the user wants to upload samples, he sends the request to `/samples/` along with his credentials, and the request will be forwarded to storage.
10 |
11 | If the user wants to task the system, he sends the request to `/task/` along with his credentials. Holmes-Gateway will parse the submitted tasks and find partnering organizations (or the user's own organization) which have access to the sources that are specified by the tasks.
12 | It will then forward the tasking-requests to the corresponding Gateways and these will check the task and forward it to to their AMQP queues.
13 | Gateway can be configured to push different services into different queues.
14 |
15 | This way Gateway can push long-lasting tasks (usually those that perform dynamic analysis) into different queues than quick tasks and thus distribute those tasks among different instances of [Holmes-Totem](https://github.com/HolmesProcessing/Holmes-Totem) and [Holmes-Totem-Dynamic](https://github.com/HolmesProcessing/Holmes-Totem-Dynamic).
16 |
17 | ### Highlights
18 | * Collaborative tasking: Holmes-Gateway allows organizations to enable other organizations to execute analysis-tasks on their samples without actually giving them access to these samples.
19 | * ACL enforcement: Users who want to submit tasks or new objects need to authenticate before they can do so.
20 | * Central point for tasking and sample upload: Without Holmes-Gateway, a user who wants to task the system needs access to RabbitMQ, while a user who wants to upload samples needs access to Holmes-Storage.
21 |
22 |
23 | ## USAGE
24 | ### Setup
25 | #### Building
26 | First build Gateway. Make sure to fetch all missing dependencies with `go get`:
27 |
28 | ```sh
29 | go get ./...
30 | go build
31 | ```
32 | #### Configuration
33 | Gateway uses an SSL-Connection and therefore needs to have a valid SSL-Certificate.
34 | If you don't have one, you can create one by executing
35 | ```sh
36 | cd config
37 | ./mkcert.sh
38 | ```
39 | Doing this will create the files `cert-key.pem` and `cert.pem`. You will need to refer to these files later from the configuration.
40 |
41 | Copy the file `config/gateway.conf.example` to `config/gateway.conf` and edit it to suit your needs.
42 | The following configuration options are available:
43 | * **HTTP**: The binding for the http-listener
44 | * **StorageSampleURI**: The URI to where storage resides for uploading samples. This URI is also prepended to the URIs for tasking-requests.
45 | * **Organizations**: The list of all known partnering organizations. For each Organization a Name, the URI of the organization's gateway, and the list of sources must be given. For the own organization, no URI must be given.
46 | * **OwnOrganization**: The name of the own organization. An organization with this name must also be present in the list of organizations. This organization is used for automatic tasking.
47 | * **AllowedUsers**: A dict mapping the usernames of allowed users to their bcrypt-password-hash. In the future, the credentials will be stored in Holmes-Storage instead.
48 | **NOTE:** The library used for checking the passwords does not support all the possible algorithms for bcrypt. To make sure that your passwords are accepted, it is recommended to use blowfish (hash starts with "$2a$").
49 | * **AutoTasks**: A dict mapping the mimetype to a dict of tasks, that should be executed automatically whenever a sample is uploaded. The mimetype returned by storage is checked against every value in the dict. If the value from the dict is contained in the returned value, all the corresponding tasks are executed. e.g.: `{"PE32":{"PEID":[]}, "":{"YARA":[]}}` means that for every uploaded file the service "YARA" is executed (since every string contains ""). Additionally, files with a memetype, which contains "PE32", the service "PEID" is executed.
50 | * **DisableStorageVerify**: If set to true, the certificate of Holmes-Storage is not checked for validity
51 | * **AllowForeignTasks**: If set to true, tasks from other gateways will be accepted, otherwise only tasks sent from authenticated users will be accepted. Note that at the moment, it is not possible to configure a more fine grained ACL-concept
52 | * **CertificateKeyPath**: The path to the key of the HTTPS-certificate
53 | * **CertificatePath**: The path to the HTTPS-certificate
54 | * **MaxUploadSize**: The maximum allowed size in MB for uploading samples. Defaults to 200 MB, if no value is configured
55 | * **AMQP**: The AMQP-connection-information
56 | * **AMQPDefault**: The Queue, Exchange, and RoutingKey that are used, if no more specific match is found in *AMQPSplitting*.
57 | * **AMQPSplitting**: A dict mapping service names to different Queues, Exchanges, and RoutingKeys.
58 |
59 |
60 | #### Starting
61 | Make sure, your AMQP-server (e.g. RabbitMQ) is running.
62 | You can start RabbitMQ by executing:
63 | ```sh
64 | sudo rabbitmq-server
65 | ```
66 | Once the AMQP-server is running, you can start up gateway:
67 | ```sh
68 | ./Holmes-Gateway --config config/gateway.conf
69 | ```
70 |
71 | ### Example: Routing Different Services To Different Queues:
72 | By modifying gateway's config-file, it is possible to push different services into different AMQP-queues / exchanges.
73 | This way, it is possible to route some services to Holmes-Totem-Dynamic.
74 | The keys **AMQPDefault** and **AMQPSplitting** are used for this purpose. **AMQPSplitting** consists of a dict mapping service-names to Queues, Exchanges, and RoutingKeys. If the service is not found in this dict, the values from AMQPDefault are taken.
75 | e.g.
76 | ```json
77 | "AMQPDefault": {"Queue": "totem_input", "Exchange": "totem", "RoutingKey": "work.static.totem"},
78 | "AMQPSplitting": {"CUCKOO": {"Queue": "totem_dynamic_input", "Exchange": "totem_dynamic", "RoutingKey": "work.static.totem"},
79 | "DRAKVUF": {"Queue": "totem_dynamic_input", "Exchange": "totem_dynamic", "RoutingKey": "work.static.totem"},
80 | "VIRUSTOTAL": {"Queue": "totem_dynamic_input", "Exchange": "totem_dynamic", "RoutingKey": "work.static.totem"}}
81 | ```
82 | This configuration will route services CUCKOO and DRAKVUF to the queue "totem_dynamic_input", while every other service is routed to "totem_input".
83 |
84 |
85 | ### Uploading Samples:
86 | In order to upload samples to storage, the user sends an HTTPS-encrypted POST request
87 | to `/samples/` of the Holmes-Gateway.
88 | Gateway will forward every request for this URI directly to storage.
89 | If storage signals a success, Gateway will immediately issue a tasking-request
90 | for the new samples, if the configuration-option **AutoTasks** is not empty.
91 |
92 | You can use [Holmes-Toolbox](https://github.com/HolmesProcessing/Holmes-Toolbox)
93 | for this purpose. Just replace the storage-URI with the URI of Gateway.
94 | Also make sure, your SSL-Certificate is accepted. You can do so either by adding it to your system's certificate store or by using the command-line option `--insecure`.
95 | The following command uploads all files from the directory $dir to the Gateway instance residing at 127.0.0.1:8090 using 5 threads.
96 | ```sh
97 | ./Holmes-Toolbox --gateway https://127.0.0.1:8090 --user test --pw test --dir $dir --src foo --comment something --workers 5 --insecure
98 | ```
99 |
100 | ### Requesting a Task:
101 | In order to request a task, a user sends an HTTPS-request (GET/POST) to Gateway
102 | containing the following form-fields:
103 | * **username**: The user's login-name
104 | * **password**: The user's password
105 | * **task**: The task which should be executed in json-form, as described below.
106 |
107 | A task consists of the following attributes:
108 | * **primaryURI**: The user enters only the sha256-sum here, as Gateway will
109 | prepend this with the URI to its version of Holmes-Storage
110 | * **secondaryURI** (optional): A secondary URI to the sample, if the primaryURI isn't found
111 | * **filename**: The name of the sample
112 | * **tasks**: All the tasks that should be executed, as a dict
113 | * **tags**: A list of tags associated with this task
114 | * **attempts**: The number of attempts. Should be zero
115 | * **source**: The source this sample belongs to. The executing organization is chosen mainly based on this value
116 | * **download**: A boolean specifying, whether totem has to download the file given as PrimaryURI
117 |
118 | For this purpose any webbrowser or commandline utility can be used.
119 | The following demonstrates an exemplary evocation using CURL.
120 | The `--insecure` parameter is used, to disable certificate checking.
121 |
122 | ```sh
123 | curl --data 'username=test&password=test&task=[{"primaryURI":"3a12f43eeb0c45d241a8f447d4661d9746d6ea35990953334f5ec675f60e36c5","secondaryURI":"","filename":"myfile","tasks":{"PEID":[],"YARA":[]},"tags":["test1"],"attempts":0,"source":"src1","download":true}]' --insecure https://localhost:8090/task/
124 | ```
125 |
126 | Alternatively, it is possible to use Holmes-Toolbox for this task, as well. First a file must be prepared containing a line with the sha256-sum, the filename, and the source (separated by single spaces) for each sample.
127 | ```sh
128 | ./Holmes-Toolbox --gateway https://127.0.0.1:8090 --tasking --file sampleFile --user test --pw test --tasks '{"PEID":[], "YARA":[]}' --tags '["mytag"]' --comment 'mycomment' --insecure
129 | ```
130 |
131 | If no error occured, nothing or an empty list will be returned. Otherwise a list containing the
132 | faulty tasks, as well as a description of the errors will be returned.
133 |
134 | You can also use the Web-Interface by opening the file `submit_task.html` in your browser. However, you will need to create an exception for the certificate by visiting the website of the Gateway manually, before you can use the web interface.
135 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright 2017 Holmes Group LLC
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/tasking.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "crypto/tls"
5 | "encoding/json"
6 | "errors"
7 | "io/ioutil"
8 | "log"
9 | "net/http"
10 | )
11 |
12 | var (
13 | srcRouter map[string][]*Organization // Which source should be routed to which organization
14 | ownOrganization *Organization // Pointer to the own organization in the list of organizations
15 | )
16 |
17 | type TaskError struct {
18 | TaskStruct TaskRequest
19 | Error MyError
20 | }
21 |
22 | type GatewayAnswer struct {
23 | Error *MyError
24 | TskErrors []TaskError
25 | }
26 |
27 | type TaskRequest struct {
28 | PrimaryURI string `json:"primaryURI"`
29 | SecondaryURI string `json:"secondaryURI"`
30 | Filename string `json:"filename"`
31 | Tasks map[string][]string `json:"tasks"`
32 | Tags []string `json:"tags"`
33 | Attempts int `json:"attempts"`
34 | Source string `json:"source"`
35 | Download bool `json:"download"`
36 | Comment string `json:"comment"`
37 | }
38 |
39 | type Organization struct {
40 | Name string
41 | Uri string
42 | Sources []string
43 | }
44 |
45 | func pushToTransport(task TaskRequest) *MyError {
46 | // Splits the task-request into services and pushes them to their corresponding queues
47 | log.Printf("%+v\n", task)
48 |
49 | // split task:
50 | tasks := task.Tasks
51 |
52 | // since each task (e.g. CUCKOO, PEID, ...) can have a special destination defined
53 | // in the config we go trough all tasks in this task struct and check it.
54 | // If the task had a special destination we cut it out of the original task struct and
55 | // send it seperately.
56 | // If the task is sent using RabbitDefault we just leave it in the struct and send the
57 | // whole task struct after we went trough it completly.
58 | for t := range tasks {
59 | log.Println(t)
60 |
61 | // check if special routing is defined in the config
62 | rconf, exists := conf.AMQPSplitting[t]
63 | if !exists {
64 | continue
65 | }
66 |
67 | // build a seperate task struct
68 | task.Tasks = map[string][]string{t: tasks[t]}
69 | if err := pushToAMQP(&task, &rconf); err != nil {
70 | return err
71 | }
72 |
73 | // delete the task from the tasks list of the struct
74 | delete(tasks, t)
75 | }
76 |
77 | // If there are tasks left we send them all as one big pack to the default destination.
78 | if len(tasks) == 0 {
79 | return nil
80 | }
81 |
82 | task.Tasks = tasks
83 | if err := pushToAMQP(&task, &conf.AMQPDefault); err != nil {
84 | return err
85 | }
86 |
87 | return nil
88 | }
89 |
90 | func requestTaskList(tasks []TaskRequest, org *Organization) (error, []byte) {
91 | req, err := http.NewRequest("GET", org.Uri+"/task_foreign/", nil)
92 | if err != nil {
93 | return err, nil
94 | }
95 | tasks_json, err := json.Marshal(tasks)
96 | if err != nil {
97 | return err, nil
98 | }
99 | q := req.URL.Query()
100 | q.Add("task", string(tasks_json))
101 | req.URL.RawQuery = q.Encode()
102 | log.Println(req.URL)
103 | //TODO: REMOVE!!!
104 | //tr := &http.Transport{}
105 | tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
106 | client := &http.Client{Transport: tr}
107 | resp, err := client.Do(req)
108 | if err != nil {
109 | return err, nil
110 | }
111 | answer, err := ioutil.ReadAll(resp.Body)
112 | log.Printf("Received: %+v\n", string(answer))
113 | if err != nil {
114 | log.Println("Error requesting task: ", err)
115 | }
116 | return err, answer
117 | }
118 |
119 | func handleTask(tasks []TaskRequest, username string, password string) (error, []TaskError) {
120 | tskerrors := make([]TaskError, 0)
121 | // TODO: Maybe we want to store the UID in the task
122 | _, err := authenticate(username, password)
123 | if err != nil {
124 | return err, nil
125 | }
126 |
127 | // Submit all the tasks, until all of them are either accepted
128 | // or unrecoverably rejected
129 | unrecoverableErrors := make([]TaskError, 0, len(tasks))
130 | iteration := 0
131 | for len(tasks) > 0 {
132 | tskerrors = tskerrors[0:0]
133 | log.Printf("\x1b[0;32mSending tasks try #%d\x1b[0m\n", iteration)
134 |
135 | // Sort the tasks based on their sources
136 | tasklists := make(map[string][]TaskRequest)
137 | for _, task := range tasks {
138 | tasklist, found := tasklists[task.Source]
139 | if !found {
140 | tasklists[task.Source] = []TaskRequest{task}
141 | } else {
142 | tasklists[task.Source] = append(tasklist, task)
143 | }
144 | }
145 |
146 | // Send the tasks and handle the answers
147 | for source, tasklist := range tasklists {
148 | orgs, found := srcRouter[source]
149 | if !found {
150 | for _, task := range tasklist {
151 | log.Printf("No route for source %s!\n", task.Source)
152 | unrecoverableErrors = append(unrecoverableErrors, TaskError{
153 | TaskStruct: task,
154 | Error: MyError{Error: errors.New("No route for source!")}})
155 | }
156 | continue
157 | }
158 | if len(orgs) <= iteration {
159 | for _, task := range tasklist {
160 | unrecoverableErrors = append(unrecoverableErrors, TaskError{
161 | TaskStruct: task,
162 | Error: MyError{Error: errors.New("Task rejected by all organizations!")}})
163 | }
164 | continue
165 | }
166 |
167 | var answer GatewayAnswer
168 | // send the tasks to the given organization and parse the answer
169 | var org *Organization
170 | org = orgs[iteration]
171 | if org == ownOrganization {
172 | err, tskerrors := handleOwnTasks(tasklist)
173 | answer = GatewayAnswer{
174 | Error: err,
175 | TskErrors: tskerrors,
176 | }
177 | } else {
178 | err, answerString := requestTaskList(tasklist, org)
179 | if err != nil {
180 | log.Println("Error while sending tasks: ", err)
181 | for task := range tasklist {
182 | tskerrors = append(tskerrors, TaskError{
183 | TaskStruct: tasklist[task],
184 | Error: MyError{Error: errors.New("Error while sending task! " + err.Error())}})
185 | }
186 | continue
187 | }
188 | err = json.Unmarshal(answerString, &answer)
189 | if err != nil {
190 | log.Printf("Error while parsing result", err)
191 | for task := range tasklist {
192 | tskerrors = append(tskerrors, TaskError{
193 | TaskStruct: tasklist[task],
194 | Error: MyError{Error: errors.New("Error while parsing result! " + err.Error())}})
195 | }
196 | continue
197 |
198 | }
199 | }
200 |
201 | // The answer contained an error
202 | if answer.Error != nil {
203 | log.Printf("Error: ", answer.Error)
204 | for task := range tasklist {
205 | tskerrors = append(tskerrors, TaskError{
206 | TaskStruct: tasklist[task],
207 | Error: *answer.Error,
208 | })
209 | }
210 | continue
211 | }
212 | tskerrors = append(tskerrors, answer.TskErrors...)
213 | }
214 | // go through list of tskerrors and reissue those with a recoverable error-code
215 | log.Printf("\x1b[0;33mreceived errors: %+v\x1b[0m", tskerrors)
216 | iteration += 1
217 | tasks = make([]TaskRequest, 0, len(tskerrors))
218 | for _, e := range tskerrors {
219 | switch e.Error.Code {
220 | case ERR_OTHER_UNRECOVERABLE, ERR_TASK_INVALID:
221 | // These tasks are not recoverable and won't be reissued
222 | unrecoverableErrors = append(unrecoverableErrors, e)
223 | break
224 | default:
225 | tasks = append(tasks, e.TaskStruct)
226 | }
227 | }
228 | }
229 | log.Printf("\x1b[0;31mUnrecoverable Errors: %+v\x1b[0m", unrecoverableErrors)
230 |
231 | return nil, unrecoverableErrors
232 | }
233 |
234 | func stringPrintable(s string) bool {
235 | for i := 0; i < len(s); i++ {
236 | c := int(s[i])
237 | if c < 0x9 || (c > 0x0d && c < 0x20) || (c > 0x7e) {
238 | return false
239 | }
240 | }
241 | return true
242 | }
243 |
244 | func checkTask(task *TaskRequest) error {
245 | log.Printf("Validating %+v\n", task)
246 | if task.PrimaryURI == "" || !stringPrintable(task.PrimaryURI) {
247 | return errors.New("Invalid Task (PrimaryURI invalid)")
248 | }
249 | if !stringPrintable(task.SecondaryURI) {
250 | return errors.New("Invalid Task (SecondaryURI invalid)")
251 | }
252 | if task.Filename == "" || !stringPrintable(task.Filename) {
253 | return errors.New("Invalid Task (Filename invalid)")
254 | }
255 | if len(task.Tasks) == 0 {
256 | return errors.New("Invalid Task")
257 | }
258 | for k := range task.Tasks {
259 | if k == "" || !stringPrintable(k) {
260 | return errors.New("Invalid Task")
261 | }
262 | }
263 | for j := 0; j < len(task.Tags); j++ {
264 | if !stringPrintable(task.Tags[j]) {
265 | return errors.New("Invalid Task (Tag invalid)")
266 | }
267 | }
268 | if task.Attempts < 0 {
269 | return errors.New("Invalid Task (Negative number of attempts)")
270 | }
271 | if !stringPrintable(task.Comment) {
272 | return errors.New("Invalid Task (Comment invalid)")
273 | }
274 | return nil
275 | }
276 |
277 | func handleOwnTasks(tasks []TaskRequest) (*MyError, []TaskError) {
278 | // Handles tasks destined for the own organization
279 | tskerrors := make([]TaskError, 0)
280 |
281 | // Check for required fields; Check whether strings are in printable ascii-range
282 | for i := 0; i < len(tasks); i++ {
283 | task := tasks[i]
284 | e := checkTask(&task)
285 | if e != nil {
286 | e2 := MyError{Error: e, Code: ERR_TASK_INVALID}
287 | tskerrors = append(tskerrors, TaskError{
288 | TaskStruct: task,
289 | Error: e2})
290 | } else {
291 | savedPrimaryURI := task.PrimaryURI
292 | savedSecondaryURI := task.SecondaryURI
293 | task.PrimaryURI = conf.StorageSampleURI + task.PrimaryURI
294 | if task.SecondaryURI != "" {
295 | task.SecondaryURI = conf.StorageSampleURI + task.SecondaryURI
296 | }
297 | myerr := pushToTransport(task)
298 | if myerr != nil {
299 | task.PrimaryURI = savedPrimaryURI
300 | task.SecondaryURI = savedSecondaryURI
301 | tskerrors = append(tskerrors, TaskError{
302 | TaskStruct: task,
303 | Error: *myerr})
304 | }
305 | }
306 | }
307 |
308 | return nil, tskerrors
309 | }
310 |
311 | func httpRequestIncomingTask(w http.ResponseWriter, r *http.Request) {
312 | taskStr := r.FormValue("task")
313 | username := r.FormValue("username")
314 | password := r.FormValue("password")
315 | var tasks []TaskRequest
316 | log.Println("Task: ", taskStr)
317 | err := json.Unmarshal([]byte(taskStr), &tasks)
318 | if err != nil {
319 | log.Println("Error while unmarshalling tasks: ", err)
320 | http.Error(w, err.Error(), 500)
321 | }
322 |
323 | err, tskerrors := handleTask(tasks, username, password)
324 | if err != nil {
325 | log.Println(err)
326 | http.Error(w, err.Error(), 500)
327 | } else if len(tskerrors) != 0 {
328 | // TODO: For automatical decoding it might be better to Marshal the whole slice
329 | // instead of the individual elements. For readability, here every element is
330 | // individually marshalled and newlines are appended.
331 | //x, _ := json.Marshal(tskerrors)
332 | //w.Write(x)
333 | for _, j := range tskerrors {
334 | x, _ := json.Marshal(j)
335 | w.Write(x)
336 | w.Write([]byte("\n\n"))
337 | }
338 | }
339 | }
340 |
341 | func httpRequestIncomingTaskForeign(w http.ResponseWriter, r *http.Request) {
342 | taskStr := r.FormValue("task")
343 | var tasks []TaskRequest
344 | log.Println("Task: ", taskStr)
345 | err := json.Unmarshal([]byte(taskStr), &tasks)
346 | if err != nil {
347 | log.Println("Error while unmarshalling tasks: ", err)
348 | http.Error(w, err.Error(), 500)
349 | }
350 | //TODO: Some form of ACL-checking...
351 | myerr, tskerrors := handleOwnTasks(tasks)
352 | answer := GatewayAnswer{
353 | Error: myerr,
354 | TskErrors: tskerrors,
355 | }
356 | answer_json, _ := json.Marshal(answer)
357 | w.Write(answer_json)
358 | log.Println(string(answer_json))
359 |
360 | }
361 |
362 | func initSourceRouting() {
363 | // Goes through all the organizations in the configuration-file, looks, what sources
364 | // they have, and creates the mapping srcRouter (source -> []organization)
365 | // Own organization is always first
366 |
367 | //TODO: make this dynamically configurable
368 | ownOrganization = nil
369 | srcRouter = make(map[string][]*Organization)
370 |
371 | log.Println("=====")
372 | for num, org := range conf.Organizations {
373 | log.Println(org)
374 | if org.Name == conf.OwnOrganization {
375 | ownOrganization = &conf.Organizations[num]
376 | }
377 |
378 | for _, src := range org.Sources {
379 | routes, exists := srcRouter[src]
380 | if !exists {
381 | srcRouter[src] = []*Organization{&conf.Organizations[num]}
382 | } else {
383 | if &org == ownOrganization {
384 | // prepend
385 | srcRouter[src] = append([]*Organization{&conf.Organizations[num]}, routes...)
386 | } else {
387 | // append
388 | srcRouter[src] = append(routes, &conf.Organizations[num])
389 | }
390 | }
391 | }
392 | }
393 | log.Println("=====")
394 | log.Println(srcRouter)
395 | if ownOrganization == nil {
396 | log.Fatal("Own organization was not found")
397 | }
398 | }
399 |
--------------------------------------------------------------------------------