' + group + '
';
38 |
39 | this.container.appendChild(entry);
40 | }
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/assets/js/users.js:
--------------------------------------------------------------------------------
1 | /*
2 | * User management.
3 | * Administrators can reset passwords, delete users, etc.
4 | */
5 | class Users {
6 | constructor(admin) {
7 | // Pointer to the admin page core
8 | this.admin = admin;
9 |
10 | // Bind Web GUI buttons
11 | this.bind();
12 |
13 | // Check whether server has returned an error during the page load
14 | this.checkLoadError();
15 | }
16 |
17 | /*
18 | * Bind Web GUI buttons
19 | */
20 | bind() {
21 | // Access 'this' from 'function()'
22 | const users = this;
23 |
24 | // Button to clear user's password
25 | $('.ui.button.reset').on('click', (e) => {
26 | users.apply(e.target.dataset.username, 'reset-password');
27 | });
28 |
29 | // Button to delete account
30 | $('.ui.button.delete').on('click', (e) => {
31 | users.apply(e.target.dataset.username, 'delete');
32 | });
33 |
34 | // Toggle admin rights
35 | $('.ui.checkbox.admin').checkbox({
36 | onChange: function() {
37 | users.apply(this.dataset.username, 'admin-' + this.checked);
38 | }
39 | });
40 | }
41 |
42 | /*
43 | * Check whether server has returned an error during the page load
44 | */
45 | checkLoadError() {
46 | if (MSG !== '')
47 | this.admin.modal.error('Something went wrong!', MSG);
48 | }
49 |
50 | /*
51 | * Modify selected user.
52 | * Receives a user name and an action to apply
53 | */
54 | apply(username, action) {
55 | this.admin.websocket.send('users', username+','+action);
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/assets/tmpl/credits.html:
--------------------------------------------------------------------------------
1 | {{ define "credits" }}
2 |
3 |
6 |
7 | {{ end }}
--------------------------------------------------------------------------------
/assets/tmpl/modal.html:
--------------------------------------------------------------------------------
1 | {{ define "modal" }}
2 |
3 |
6 | ' + count[group] + '
7 |
12 |
13 |
14 |
17 |
18 |
19 |
20 |
27 |
28 |
31 |
21 |
25 |
26 |
32 |
33 |
34 |
35 |
36 | {{ end }}
--------------------------------------------------------------------------------
/assets/tmpl/signin.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
13 |
43 |
44 |
47 | {{ template "credits" . }}
48 |
49 |
50 |
--------------------------------------------------------------------------------
/assets/tmpl/topbar.html:
--------------------------------------------------------------------------------
1 | {{ define "topbar" }}
2 |
3 |
14 | 
15 |
42 |
15 | Graphoscope 16 | {{ if ne .Environment "prod" }} 17 | dev 18 | {{ end }} 19 |
20 | 21 | 22 | 23 | {{ if ne .Error ""}} 24 | 25 | {{ end }} 26 | 27 | 41 |
4 |
52 |
53 | {{ end }}
--------------------------------------------------------------------------------
/certs/graphoscope.crt:
--------------------------------------------------------------------------------
1 | -----BEGIN CERTIFICATE-----
2 | MIIDuzCCAqOgAwIBAgIULDVxLlO0RIPOmmCWbpOXjq5rUj4wDQYJKoZIhvcNAQEL
3 | BQAwbDELMAkGA1UEBhMCTFYxDTALBgNVBAgMBFJpZ2ExDTALBgNVBAcMBFJpZ2Ex
4 | EDAOBgNVBAoMB0NFUlQuTFYxEDAOBgNVBAMMB2NlcnQubHYxGzAZBgkqhkiG9w0B
5 | CQEWDGNlcnRAY2VydC5sdjAgFw0yMDAzMTIxMDU3MjlaGA8yMTIwMDIxNzEwNTcy
6 | OVowbDELMAkGA1UEBhMCTFYxDTALBgNVBAgMBFJpZ2ExDTALBgNVBAcMBFJpZ2Ex
7 | EDAOBgNVBAoMB0NFUlQuTFYxEDAOBgNVBAMMB2NlcnQubHYxGzAZBgkqhkiG9w0B
8 | CQEWDGNlcnRAY2VydC5sdjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
9 | AMdPK8cpDtRs2LqtCO7uMvQcQ9rSixoWMysbjEnimWbL8ua3eTmK2zF3fjnkQ5Tz
10 | Ch18A38veheZOGJmCS5laeeff0QCPmOOWlwaLr2q3RUWT7uBnWSthgM2/02RZqb3
11 | 9TPD9ZFUFMp5EEFnkZowNaBg4txJrm4gN1DPHJwAMajASk/q2xaMJsNfwWNJCZM8
12 | xCKLgoI1xEweZv/wWco2y3k5IlBjsX/CtXuWq7SZM+o6LA6W4vGW2t7TQ9Fr+Z7T
13 | 0JjqH538IXwW4iSdOItq9WY0LHLFMz1IPkaK+clo8GB9kS7iEWRtmEx5PAvS0AAJ
14 | QZUGqvShMKaxdOcphNFHq6UCAwEAAaNTMFEwHQYDVR0OBBYEFLD1Dzi3xnasLnOE
15 | c0eY39xqB9rnMB8GA1UdIwQYMBaAFLD1Dzi3xnasLnOEc0eY39xqB9rnMA8GA1Ud
16 | EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAIRPX//IeazsGKrEfu3lWBBE
17 | 29IGcGFb6KSoJTqalBvePS+1JIQMVSNvM1A9dyFLEthG5Xsyg+N1A/tP2Xu/8U1Z
18 | GtYUBhriy5RXWSVSrLpd7zbEs2CjfgkSq2TXg/3V5HzxSwZx6XQW3DmXf3NQFxJe
19 | q+VmqeETMUprndRGFFbSarxYuFP0bHP4z3GFCFCBc/Or8cZ+IDHKZAudFN3eW7uD
20 | CxFuLvWrBeuGEV/Xw81git5z1CMMV5+xCOVGLJNSHNvXqCZ8UT0pKZiZt5tyFuC1
21 | VfMOp7gENnN6AdPkq1/7n8pVSp0ixeQ/2oc/5pnR0LoXlIp1cOup4/dYdxJsbwk=
22 | -----END CERTIFICATE-----
23 |
--------------------------------------------------------------------------------
/certs/graphoscope.key:
--------------------------------------------------------------------------------
1 | -----BEGIN RSA PRIVATE KEY-----
2 | MIIEowIBAAKCAQEAx08rxykO1GzYuq0I7u4y9BxD2tKLGhYzKxuMSeKZZsvy5rd5
3 | OYrbMXd+OeRDlPMKHXwDfy96F5k4YmYJLmVp559/RAI+Y45aXBouvardFRZPu4Gd
4 | ZK2GAzb/TZFmpvf1M8P1kVQUynkQQWeRmjA1oGDi3EmubiA3UM8cnAAxqMBKT+rb
5 | Fowmw1/BY0kJkzzEIouCgjXETB5m//BZyjbLeTkiUGOxf8K1e5artJkz6josDpbi
6 | 8Zba3tND0Wv5ntPQmOofnfwhfBbiJJ04i2r1ZjQscsUzPUg+Ror5yWjwYH2RLuIR
7 | ZG2YTHk8C9LQAAlBlQaq9KEwprF05ymE0UerpQIDAQABAoIBAFbRwfwrgm4+S9pl
8 | bbLGyCNV/Kjhdf6TFQ7+HQpCTxhcVx7xZTkPp5PQvYdyS44ioJFfaBaLE+AbulgC
9 | opU3T/65l7KEV7D+XZYpQZsVRuDcqza+q1Uj0XCtEGE1qUWqVYGLJvl7auMYAWC8
10 | QMytm26VRb03y2flWLM2xPufigI7m/NVMm0pOgEJLaDD3zGu1jM46rsGvfa2daln
11 | g4VdVBDiDopjbVI6GE5Dl6xoc3lB5o+tgMDYLD3cn+WJd2i1I+TuHsBmvLpOXp16
12 | qs8Kf06Apy2xvHUnB6u5NXLn9TxdWf8Jgudqw85ljsH+qif5141VXVfU/gjOPfd2
13 | ZpsbPTECgYEA8KY22nG4a4d48hQrq4czfEP0Ah29bpvFVHCpQDaZbb0iHfVJ7ORT
14 | cTREWTETGBVZgxPICBbwAb633Li9cZExJODD0ZNGUjKoAIaj3TiCUOsrYl2PAKXb
15 | 6qNTcd2x1leuu7PQGEFobFcJi+/nxponWu2f6lNEXjZ8ZPZYTTvlvF8CgYEA1AXg
16 | koobg3j+mmuLs7mgQZLqTnZ2mqcW/2gZCQhvC8DXbPuibt1LzhXuKFTSHffo4DRX
17 | jKNC1aC6G+48bCCa/USPcOJhLwNRB5qpnc94tXlG7535PHcuX3Mk3ksUUzUVXizz
18 | d38U/0FJsR4OubVVTBBrNk6gdHruBBan/BAKFnsCgYAFSpJQMUnty1fEct8W8W0X
19 | YWMfHMpKgVBQb/24tLqg6BS09ey/MbIH/i82itaxo96I/ElcrCxwzWG7j7BSq++Z
20 | sPt9QzC7o/N/t3Yo6hIrd1BH5Gi9iegQ+7BdA5Pic6Ea7XQ45E9Ieo1yLz84ZbFR
21 | 1YG7pEMPk0Ee8y+z2wpNHwKBgFbkrbwA4/PG47mPt+qJefdF6ccMX+FT92XnWNNN
22 | 5IzRlLhyjIiZI1crv7ZBxPdJQeSZLwRRaLO6smt+AL9jwYFo1syxypiE6HGQXlFx
23 | 1Quyz3KmsJ2qTpQJ0aNU69iKGd7F12Yy6/0M2dG/+tL7USDiXb4dDT+PnfqI+oGg
24 | ZTH/AoGBAM8k86vdY8JsXRYGfTvpsNPS/v21TtF1a93WvvOxTTFQQ/HZiXn1MD4D
25 | GFmNdA52Fq2cZkqzKyHjf+1aqPnmBFjEp1527GsOxJ6d/yN2SL3sEgutpblcQk5I
26 | SQh0jE7jLkR8sQE80ARFUC2UYsWI0joU5nJY12nkuXUX28ijE4MU
27 | -----END RSA PRIVATE KEY-----
28 |
--------------------------------------------------------------------------------
/config.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 |
7 | "github.com/rs/zerolog"
8 | yaml "gopkg.in/yaml.v3"
9 | )
10 |
11 | /*
12 | * Structure to store all the service settings.
13 | * Check "graphoscope.yaml.example" file for a detailed all fields description
14 | */
15 | type Config struct {
16 | Server *struct {
17 | Host string `yaml:"host"`
18 | Port string `yaml:"port"`
19 | CertFile string `yaml:"certFile"`
20 | KeyFile string `yaml:"keyFile"`
21 | ReadTimeout int `yaml:"readTimeout"`
22 | ReadHeaderTimeout int `yaml:"readHeaderTimeout"`
23 | } `yaml:"server"`
24 |
25 | Environment string `yaml:"environment"`
26 | Definitions string `yaml:"definitions"`
27 | Plugins string `yaml:"plugins"`
28 | Limit int `yaml:"limit"`
29 | StabilizationTime int `yaml:"stabilizationTime"`
30 |
31 | Log *struct {
32 | File string `yaml:"file"`
33 | MaxSize int `yaml:"maxSize"`
34 | MaxBackups int `yaml:"maxBackups"`
35 | MaxAge int `yaml:"maxAge"`
36 | Level zerolog.Level `yaml:"level"`
37 | } `yaml:"log"`
38 |
39 | Upload *struct {
40 | Path string `yaml:"path"`
41 | MaxSize int64 `yaml:"maxSize"`
42 | MaxIndicators int `yaml:"maxIndicators"`
43 | DeleteInterval int `yaml:"deleteInterval"`
44 | DeleteExpiration int `yaml:"deleteExpiration"`
45 | } `yaml:"upload"`
46 |
47 | Groups string `yaml:"groups"`
48 | Formats string `yaml:"formats"`
49 | Features string `yaml:"features"`
50 | Docs string `yaml:"docs"`
51 |
52 | Database struct {
53 | URL string `yaml:"url"`
54 | Name string `yaml:"name"`
55 | User string `yaml:"user"`
56 | Password string `yaml:"password"`
57 | Users string `yaml:"users"`
58 | Dashboards string `yaml:"dashboards"`
59 | Notes string `yaml:"notes"`
60 | Sessions string `yaml:"sessions"`
61 | Cache string `yaml:"cache"`
62 | Settings string `yaml:"settings"`
63 | Timeout int `yaml:"timeout"`
64 | CacheTTL int32 `yaml:"cacheTTL"`
65 | } `yaml:"database"`
66 |
67 | Sessions *struct {
68 | TTL int `yaml:"ttl"`
69 | CookieName string `yaml:"cookieName"`
70 | AuthenticationKey string `yaml:"authenticationKey"`
71 | EncryptionKey string `yaml:"encryptionKey"`
72 | } `yaml:"sessions"`
73 | }
74 |
75 | /*
76 | * Load configuration from a YAML file.
77 | *
78 | * Service searches for the "./graphoscope.yaml" file by default.
79 | * however, "CONFIG" environment variable can be set to use a different file
80 | */
81 | func loadConfig() error {
82 | path := "graphoscope.yaml"
83 |
84 | if os.Getenv("CONFIG") != "" {
85 | path = os.Getenv("CONFIG")
86 | }
87 |
88 | buffer, err := loadFileIntoString(path)
89 | if err != nil {
90 | return fmt.Errorf("Failed to open configuration file '%s': %s", path, err.Error())
91 | }
92 |
93 | err = yaml.Unmarshal([]byte(buffer), &config)
94 | if err != nil {
95 | return fmt.Errorf("Invalid configuration YAML file '%s': %s", path, err.Error())
96 | }
97 |
98 | return nil
99 | }
100 |
--------------------------------------------------------------------------------
/definitions/outputs/output.yaml.example:
--------------------------------------------------------------------------------
1 | # Name of the data output
2 | name: application
3 |
4 | # Plugin to use
5 | plugin: example
6 |
7 | # Acceptable actions (connecting, search, etc.) timeout.
8 | # String type, not integer. 60s if not specified
9 | timeout: 60s
10 |
--------------------------------------------------------------------------------
/definitions/processors/processor.yaml.example:
--------------------------------------------------------------------------------
1 | # Name of the data processor
2 | name: application
3 |
4 | # Plugin to use
5 | plugin: taxonomy
6 |
7 | # Acceptable actions (connecting, search, etc.) timeout.
8 | # String type, not integer. 60s if not specified
9 | timeout: 60s
10 |
11 |
12 | # Unique data needed by the current processor.
13 | # Can be a map of any types, as plugins can be very different
14 | data:
15 | ...
16 |
--------------------------------------------------------------------------------
/definitions/sources/demo.yaml.example:
--------------------------------------------------------------------------------
1 | name: demo
2 | label: Demo
3 | icon: database
4 |
5 | plugin: file-csv
6 | inGlobal: false
7 | includeDatetime: false
8 | supportsSQL: true
9 |
10 | access:
11 | path: files/demo.csv
12 |
13 |
14 | relations:
15 | -
16 | from:
17 | id: name
18 | group: name
19 | search: name
20 | attributes: [ "age", "language" ]
21 |
22 | to:
23 | id: country
24 | group: country
25 | search: country
26 |
27 | edge:
28 | label: lives in
29 | attributes: []
30 |
31 | -
32 | from:
33 | id: name
34 | group: name
35 | search: name
36 |
37 | to:
38 | id: friend
39 | group: name
40 | search: name
41 |
42 | -
43 | from:
44 | id: language
45 | group: language
46 | search: language
47 |
48 | to:
49 | id: name
50 | group: name
51 | search: name
--------------------------------------------------------------------------------
/docs.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "net"
6 | "net/http"
7 | "os"
8 | )
9 |
10 | var (
11 | // Built-in documentation content, N sections
12 | docs [3]string
13 | )
14 |
15 | /*
16 | * Serve '/docs' page with a built-in documentation
17 | */
18 | func docsHandler(w http.ResponseWriter, r *http.Request) {
19 | // Get client's IP
20 | ip, _, err := net.SplitHostPort(r.RemoteAddr)
21 | if err != nil {
22 | log.Error().Msg("Can't get IP for the docs page: " + err.Error())
23 | return
24 | }
25 |
26 | // Check existing session
27 | username, err := sessions.exists(w, r)
28 | if err != nil {
29 | log.Error().
30 | Str("ip", ip).
31 | Msg(err.Error())
32 |
33 | http.Redirect(w, r, "/signin", http.StatusSeeOther)
34 | return
35 | }
36 |
37 | // Get account from a database
38 | account, err := db.getAccount(username)
39 | if err != nil {
40 | log.Error().
41 | Str("ip", ip).
42 | Str("username", username).
43 | Msg("Can't GetAccount for the docs page: " + err.Error())
44 |
45 | http.Redirect(w, r, "/signin", http.StatusSeeOther)
46 | return
47 | }
48 |
49 | // All are admins in service's DEV mode
50 | if config.Environment != "prod" {
51 | account.Admin = true
52 | }
53 |
54 | templateData := &TemplateData{
55 | Account: account,
56 | Docs: docs,
57 | }
58 |
59 | renderTemplate(w, "docs", templateData, nil)
60 |
61 | log.Info().
62 | Str("ip", ip).
63 | Str("username", username).
64 | Msg("Docs page requested")
65 | }
66 |
67 | /*
68 | * Load all documentation from the markdown files.
69 | * One file for one documentation section
70 | */
71 | func loadDocs() error {
72 | for i, name := range []string{"ui", "search", "admin"} {
73 | md, err := loadDoc(name)
74 | if err != nil {
75 | return err
76 | }
77 |
78 | docs[i] = md
79 | }
80 |
81 | log.Debug().Msg("Documentation is parsed")
82 | return nil
83 | }
84 |
85 | /*
86 | * Load documentation from a specific markdown file.
87 | * Receives a name of file to load, its extension has to be ".md"
88 | */
89 | func loadDoc(name string) (string, error) {
90 | mdFile, err := os.Open(config.Docs + "/" + name + ".md")
91 | if err != nil {
92 | return "", fmt.Errorf("Failed to open '%s/%s.md' doc: %s", config.Docs, name, err.Error())
93 | }
94 |
95 | fi, _ := mdFile.Stat()
96 | buffer := make([]byte, fi.Size())
97 | _, err = mdFile.Read(buffer)
98 | if err != nil {
99 | return "", fmt.Errorf("Failed to read '%s/%s.md' doc: %s", config.Docs, name, err.Error())
100 | }
101 |
102 | return string(buffer), nil
103 | }
104 |
--------------------------------------------------------------------------------
/features.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | yaml "gopkg.in/yaml.v3"
7 | )
8 |
9 | var (
10 | // A list of new features for the current service's version.
11 | // Will be displayed once for each user
12 | features = []string{}
13 | )
14 |
15 | /*
16 | * Load new features from a YAML file
17 | */
18 | func loadFeatures() error {
19 | buffer, err := loadFileIntoString(config.Features)
20 | if err != nil {
21 | return fmt.Errorf("Failed to read new features file '%s': %s", config.Features, err.Error())
22 | }
23 |
24 | err = yaml.Unmarshal([]byte(buffer), &features)
25 | if err != nil {
26 | return fmt.Errorf("Failed unmarshalling new features yaml: %s", err.Error())
27 | }
28 |
29 | if len(features) > 0 {
30 | log.Debug().Msgf("New features loaded: %v", features)
31 | }
32 |
33 | return nil
34 | }
35 |
36 | /*
37 | * Prevent future notifications after the first one
38 | */
39 | func (a *Account) hideFeatures() {
40 | if a.SeenFeatures == features[0] {
41 | log.Debug().
42 | Str("username", a.Username).
43 | Msg("No new features notifications to disable")
44 | return
45 | }
46 |
47 | err := a.update("seenFeatures", features[0])
48 | if err != nil {
49 | log.Error().Msg("Can't update account to hide new features notifications: " + err.Error())
50 | return
51 | }
52 |
53 | log.Debug().
54 | Str("username", a.Username).
55 | Msg("New features notifications are hidden")
56 | }
57 |
--------------------------------------------------------------------------------
/files/demo.csv:
--------------------------------------------------------------------------------
1 | name,country,age,friend,language
2 | John,Canada,25,Kate,en
3 | John,Canada,25,Jennifer,
4 | Kate,USA,23,Monica,es
5 | Kate,USA,23,John,es
6 | Monica,Canada,35,Kate,
7 | Ben,Japan,41,Kate,en
8 | Ben,Japan,41,John,fr
9 | Ben,Japan,41,Chin,en
10 | Laura,Italy,11,Polina,fr
11 | Polina,Italy,12,Laura,en
12 | Jennifer,Germany,25,John,fr
13 | Sofy,Sweden,51,Tom,en
14 | Tom,Sweden,55,Sofy,fr
15 | Tom,Sweden,55,Chin,fr
16 | Chin,China,61,Tom,
17 | Chin,China,61,Ben,en
--------------------------------------------------------------------------------
/files/features.yaml:
--------------------------------------------------------------------------------
1 | [
2 | "20.12.2024",
3 |
4 | "An option to show partial search results when limit exceeded",
5 | "Latest plugin: Shodan"
6 | ]
--------------------------------------------------------------------------------
/files/formats.yaml.example:
--------------------------------------------------------------------------------
1 | # Query formatting rules.
2 | # Syntax:
3 | #
4 | # indicator's group:
5 | # - regexp 1 to detect it
6 | # - regexp 2 to detect it
7 | # - ...
8 |
9 | ip:
10 | # IPv4
11 | - ^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$
12 | # IPv6
13 | - "^\\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}\
14 | |((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa\
15 | -f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0\
16 | -4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-\
17 | Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:\
18 | ))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2\
19 | [0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){\
20 | 2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\
21 | \\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){\
22 | 1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d\
23 | |[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0\
24 | -4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?\\s*$"
25 |
26 | domain:
27 | - ^([\%\w-]+\.)+[\%a-zA-Z]+$
28 |
29 | email:
30 | - ^[\%\w\.+-]+@[\%\w\.]+\.[\%a-zA-Z]{1,9}$
--------------------------------------------------------------------------------
/files/graphoscope.service:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=Graph project
3 |
4 | [Service]
5 | #User=graphoscope
6 | Environment=CONFIG=/etc/graphoscope/graphoscope.yaml
7 | WorkingDirectory=/opt/graphoscope
8 | ExecStart=/opt/graphoscope/graphoscope
9 |
10 | [Install]
11 | WantedBy=multi-user.target
--------------------------------------------------------------------------------
/files/groups.json.example:
--------------------------------------------------------------------------------
1 | {
2 | "ip": {
3 | "shape": "dot",
4 | "color": {
5 | "background": "#ccc",
6 | "border": "#aaa"
7 | }
8 | },
9 | "domain": {
10 | "shape": "dot",
11 | "color": {
12 | "background": "#09f",
13 | "border": "#07d"
14 | },
15 | "font": {
16 | "color": "#05b"
17 | }
18 | },
19 | "email": {
20 | "shape": "icon",
21 | "icon": {
22 | "face": "Icons",
23 | "weight": "bold",
24 | "code": "\uf0e0",
25 | "size": "30",
26 | "color": "#0a6"
27 | },
28 | "font": {
29 | "color": "#062"
30 | }
31 | },
32 | "identifier": {
33 | "shape": "diamond",
34 | "color": {
35 | "background": "#f75",
36 | "border": "#b44"
37 | },
38 | "font": {
39 | "color": "#b44"
40 | }
41 | },
42 | "institution": {
43 | "shape": "square",
44 | "color": {
45 | "background": "#c7e",
46 | "border": "#95b"
47 | },
48 | "font": {
49 | "color": "#84a"
50 | }
51 | },
52 | "person": {
53 | "shape": "dot",
54 | "color": {
55 | "background": "#0c8",
56 | "border": "#084"
57 | },
58 | "font": {
59 | "color": "#084"
60 | }
61 | },
62 | "taxonomy": {
63 | "shape": "square",
64 | "color": {
65 | "background": "#08e",
66 | "border": "#05b"
67 | },
68 | "font": {
69 | "color": "#05b"
70 | }
71 | },
72 | "rtir": {
73 | "shape": "triangle",
74 | "color": {
75 | "background": "#fa5",
76 | "border": "#c70"
77 | },
78 | "font": {
79 | "color": "#a50"
80 | }
81 | },
82 | "application": {
83 | "shape": "hexagon",
84 | "color": {
85 | "background": "#fc5",
86 | "border": "#ca2"
87 | }
88 | },
89 | "md5": {
90 | "shape": "icon",
91 | "icon": {
92 | "face": "Icons",
93 | "weight": "bold",
94 | "code": "\uf0fd",
95 | "size": "30",
96 | "color": "#f65"
97 | },
98 | "font": {
99 | "color": "#833"
100 | }
101 | },
102 | "sha1": {
103 | "shape": "icon",
104 | "icon": {
105 | "face": "Icons",
106 | "weight": "bold",
107 | "code": "\uf0fd",
108 | "size": "30",
109 | "color": "#58f"
110 | },
111 | "font": {
112 | "color": "#037"
113 | }
114 | },
115 | "sha256": {
116 | "shape": "icon",
117 | "icon": {
118 | "face": "Icons",
119 | "weight": "bold",
120 | "code": "\uf0fd",
121 | "size": "30",
122 | "color": "#a7f"
123 | },
124 | "font": {
125 | "color": "#537"
126 | }
127 | },
128 | "cluster": {
129 | "shape": "hexagon",
130 | "size": 25,
131 | "color": {
132 | "background": "#777b7b",
133 | "border": "#566"
134 | },
135 | "font": {
136 | "color": "#ccc"
137 | }
138 | }
139 | }
140 |
--------------------------------------------------------------------------------
/filters.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | )
6 |
7 | /*
8 | * Structure to hold search filter's state
9 | */
10 | type Filter struct {
11 | // When filter is enabled - related request will be launched
12 | // to retrieve the data when web page is loaded
13 | Enabled bool `json:"enabled" bson:"enabled"`
14 |
15 | // User's original query
16 | Query string `json:"query" bson:"query"`
17 | }
18 |
19 | /*
20 | * Handle 'filters' websocket command to save all current user's filters
21 | */
22 | func (a *Account) filtersHandler(filters string) {
23 | var list []map[string]*Filter
24 | err := json.Unmarshal([]byte(filters), &list)
25 | if err != nil {
26 | a.send("error", "Can't parse filters data.", "Can't save filters!")
27 |
28 | log.Error().
29 | Str("ip", a.Session.IP).
30 | Str("username", a.Username).
31 | Msg("Can't unmarshal filters data: " + err.Error())
32 | return
33 | }
34 |
35 | err = a.update("filters", list)
36 | if err != nil {
37 | a.send("error", err.Error(), "Can't save filters!")
38 | return
39 | }
40 |
41 | log.Debug().
42 | Str("ip", a.Session.IP).
43 | Str("username", a.Username).
44 | Msg("Filters are saved")
45 | }
46 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/cert-lv/graphoscope
2 |
3 | go 1.23
4 |
5 | toolchain go1.23.1
6 |
7 | require (
8 | github.com/0xrawsec/golang-utils v1.3.2
9 | github.com/Jeffail/gabs/v2 v2.7.0
10 | github.com/blastrain/vitess-sqlparser v0.0.0-20201030050434-a139afbb1aba
11 | github.com/elastic/go-elasticsearch/v7 v7.17.10
12 | github.com/elastic/go-elasticsearch/v8 v8.15.0
13 | github.com/georgysavva/scany v1.2.2
14 | github.com/go-sql-driver/mysql v1.8.1
15 | github.com/google/uuid v1.6.0
16 | github.com/gorilla/securecookie v1.1.2
17 | github.com/gorilla/sessions v1.4.0
18 | github.com/gorilla/websocket v1.5.3
19 | github.com/jackc/pgx/v4 v4.18.3
20 | github.com/mattn/go-sqlite3 v1.14.22
21 | github.com/mithrandie/csvq-driver v1.7.0
22 | github.com/ns3777k/go-shodan/v4 v4.2.0
23 | github.com/olekukonko/tablewriter v0.0.5
24 | github.com/redis/go-redis/v9 v9.7.0
25 | github.com/rs/zerolog v1.33.0
26 | github.com/umpc/go-sortedmap v0.0.0-20180422175548-64ab94c482f4
27 | github.com/yukithm/json2csv v0.1.2
28 | go.mongodb.org/mongo-driver v1.17.1
29 | golang.org/x/crypto v0.31.0
30 | golang.org/x/sync v0.10.0
31 | gopkg.in/natefinch/lumberjack.v2 v2.2.1
32 | gopkg.in/yaml.v3 v3.0.1
33 | )
34 |
35 | require (
36 | filippo.io/edwards25519 v1.1.0 // indirect
37 | github.com/cespare/xxhash/v2 v2.3.0 // indirect
38 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
39 | github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
40 | github.com/go-logr/logr v1.4.1 // indirect
41 | github.com/go-logr/stdr v1.2.2 // indirect
42 | github.com/golang/snappy v0.0.4 // indirect
43 | github.com/google/go-querystring v1.0.0 // indirect
44 | github.com/jackc/chunkreader/v2 v2.0.1 // indirect
45 | github.com/jackc/pgconn v1.14.3 // indirect
46 | github.com/jackc/pgio v1.0.0 // indirect
47 | github.com/jackc/pgpassfile v1.0.0 // indirect
48 | github.com/jackc/pgproto3/v2 v2.3.3 // indirect
49 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
50 | github.com/jackc/pgtype v1.14.3 // indirect
51 | github.com/jackc/puddle v1.3.0 // indirect
52 | github.com/juju/errors v1.0.0 // indirect
53 | github.com/klauspost/compress v1.17.11 // indirect
54 | github.com/mattn/go-colorable v0.1.13 // indirect
55 | github.com/mattn/go-isatty v0.0.20 // indirect
56 | github.com/mattn/go-runewidth v0.0.16 // indirect
57 | github.com/mitchellh/go-homedir v1.1.0 // indirect
58 | github.com/mithrandie/csvq v1.18.1 // indirect
59 | github.com/mithrandie/go-file/v2 v2.1.0 // indirect
60 | github.com/mithrandie/go-text v1.6.0 // indirect
61 | github.com/mithrandie/ternary v1.1.1 // indirect
62 | github.com/montanaflynn/stats v0.7.1 // indirect
63 | github.com/rivo/uniseg v0.4.7 // indirect
64 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect
65 | github.com/xdg-go/scram v1.1.2 // indirect
66 | github.com/xdg-go/stringprep v1.0.4 // indirect
67 | github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
68 | go.opentelemetry.io/otel v1.24.0 // indirect
69 | go.opentelemetry.io/otel/metric v1.24.0 // indirect
70 | go.opentelemetry.io/otel/trace v1.24.0 // indirect
71 | golang.org/x/sys v0.28.0 // indirect
72 | golang.org/x/term v0.27.0 // indirect
73 | golang.org/x/text v0.21.0 // indirect
74 | )
75 |
--------------------------------------------------------------------------------
/gui.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "errors"
5 | "net/http"
6 | )
7 |
8 | var (
9 | // Nodes styling definition.
10 | // Will be applied to the JavaScript engine
11 | groups string
12 | )
13 |
14 | /*
15 | * Setup Web GUI handlers
16 | */
17 | func setupGUI() error {
18 | var err error
19 |
20 | // Parse graph elements style groups definitions
21 | groups, err = loadFileIntoString(config.Groups)
22 | if err != nil {
23 | return errors.New("Can't load groups file: " + err.Error())
24 | }
25 |
26 | // Load query formatting rules,
27 | // which help to format comma/space separated indicators to a valid SQL query
28 | err = loadFormats()
29 | if err != nil {
30 | return errors.New("Can't load query formatting rules: " + err.Error())
31 | }
32 |
33 | // Create a sessions store
34 | err = setupSessions()
35 | if err != nil {
36 | return errors.New("Can't setup sessions store: " + err.Error())
37 | }
38 |
39 | // Parse documentation files
40 | err = loadDocs()
41 | if err != nil {
42 | return errors.New("Can't load documentation: " + err.Error())
43 | }
44 |
45 | // Notify users about new features
46 | err = loadFeatures()
47 | if err != nil {
48 | return errors.New("Can't show new features: " + err.Error())
49 | }
50 |
51 | // Setup the indicators upload feature
52 | err = setupUpload()
53 | if err != nil {
54 | return errors.New("Can't setup the indicators upload feature: " + err.Error())
55 | }
56 |
57 | // Setup additional HTTPS handlers
58 | // which are required by a Web GUI
59 | http.HandleFunc("/", indexHandler)
60 | http.HandleFunc("/signin", signinHandler)
61 | http.HandleFunc("/signup", signupHandler)
62 | http.HandleFunc("/signout", signoutHandler)
63 | http.HandleFunc("/profile", profileHandler)
64 | http.HandleFunc("/admin", adminHandler)
65 | http.HandleFunc("/docs", docsHandler)
66 | http.HandleFunc("/upload", uploadHandler)
67 | http.HandleFunc("/download", downloadHandler)
68 | http.HandleFunc("/ws", wsHandler)
69 |
70 | http.Handle("/assets/", http.StripPrefix("/assets", http.FileServer(http.Dir("assets"))))
71 |
72 | return nil
73 | }
74 |
--------------------------------------------------------------------------------
/loaders.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "io/ioutil"
7 | "os"
8 | "regexp"
9 |
10 | yaml "gopkg.in/yaml.v3"
11 | )
12 |
13 | var (
14 | // Query formatting rules,
15 | // which help to format comma/space separated indicators to a valid SQL query
16 | formats string
17 | )
18 |
19 | /*
20 | * Return content of the requested file by its path
21 | */
22 | func loadFileIntoString(path string) (string, error) {
23 | file, err := ioutil.ReadFile(path)
24 | if err != nil {
25 | return "", err
26 | }
27 |
28 | return string(file), nil
29 | }
30 |
31 | /*
32 | * Load query formatting rules
33 | */
34 | func loadFormats() error {
35 | buffer, err := loadFileIntoString(config.Formats)
36 | if err != nil {
37 | return fmt.Errorf("Failed to read rules file '%s': %s", config.Formats, err.Error())
38 | }
39 |
40 | var f map[string][]string
41 |
42 | err = yaml.Unmarshal([]byte(buffer), &f)
43 | if err != nil {
44 | return fmt.Errorf("Failed unmarshalling rules yaml: %s", err.Error())
45 | }
46 |
47 | // Validate regexps
48 | for group, res := range f {
49 | for _, re := range res {
50 | _, err = regexp.Compile(re)
51 | if err != nil {
52 | return fmt.Errorf("Invalid %s's regular expression '%s' : %s", group, re, err.Error())
53 | }
54 | }
55 | }
56 |
57 | b, err := json.Marshal(f)
58 | if err != nil {
59 | return fmt.Errorf("Failed to marshal rules struct: %s", err.Error())
60 | }
61 |
62 | formats = string(b)
63 | return nil
64 | }
65 |
66 | /*
67 | * Load service's version
68 | */
69 | func loadVersion() error {
70 | path := "VERSION"
71 | var err error
72 |
73 | // Try to get from the environment variable first
74 | if os.Getenv(path) != "" {
75 | version = os.Getenv(path)
76 | return nil
77 | }
78 |
79 | version, err = loadFileIntoString(path)
80 | if err != nil {
81 | return err
82 | }
83 |
84 | return nil
85 | }
86 |
--------------------------------------------------------------------------------
/logger.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "time"
6 |
7 | "github.com/rs/zerolog"
8 | "gopkg.in/natefinch/lumberjack.v2"
9 | )
10 |
11 | /*
12 | * Setup logger to the file and stdout.
13 | *
14 | * In a production environment log events to the file only,
15 | * in a development environment log to the stdout only
16 | */
17 | func setupLogger() error {
18 |
19 | // For the production server
20 | if config.Environment == "prod" {
21 | // Lumberjack provides log files rotation
22 | log = zerolog.New(&lumberjack.Logger{
23 | Filename: config.Log.File,
24 | MaxSize: config.Log.MaxSize, // Size in MB before file gets rotated
25 | MaxBackups: config.Log.MaxBackups, // Max number of files kept before being overwritten
26 | MaxAge: config.Log.MaxAge, // Max number of days to keep the files
27 | Compress: true, // Whether to compress log files using gzip
28 | }).With().Timestamp().Logger()
29 |
30 | zerolog.SetGlobalLevel(config.Log.Level)
31 |
32 | return nil
33 | }
34 |
35 | // For the development
36 | stdout := zerolog.ConsoleWriter{
37 | Out: os.Stdout,
38 | TimeFormat: time.RFC3339,
39 | }
40 |
41 | log = zerolog.New(stdout).With().Timestamp().Logger()
42 | zerolog.SetGlobalLevel(config.Log.Level)
43 |
44 | return nil
45 | }
46 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "crypto/tls"
5 | "fmt"
6 | "net/http"
7 | "os"
8 | "time"
9 |
10 | //_ "net/http/pprof"
11 |
12 | "github.com/rs/zerolog"
13 | )
14 |
15 | var (
16 | // Holder all service's configuration
17 | config *Config
18 |
19 | // Instance of the global logger
20 | log zerolog.Logger
21 |
22 | // Current service's version
23 | version string
24 | )
25 |
26 | /*
27 | * Use recommended TLS settings
28 | */
29 | func setupTLSserver() *http.Server {
30 | cfg := &tls.Config{
31 | MinVersion: tls.VersionTLS12, // At least TLS v1.2 is recommended
32 | }
33 |
34 | // Enable secure ciphers only
35 | for _, cipherSuite := range tls.CipherSuites() {
36 | cfg.CipherSuites = append(cfg.CipherSuites, cipherSuite.ID)
37 | }
38 |
39 | return &http.Server{
40 | Addr: config.Server.Host + ":" + config.Server.Port,
41 | TLSConfig: cfg,
42 | ReadTimeout: time.Duration(config.Server.ReadTimeout) * time.Second,
43 | ReadHeaderTimeout: time.Duration(config.Server.ReadHeaderTimeout) * time.Second,
44 | }
45 | }
46 |
47 | func main() {
48 | /*
49 | * Parse configuration file
50 | */
51 | err := loadConfig()
52 | if err != nil {
53 | fmt.Fprintf(os.Stderr, "Can't load configuration: %s", err.Error())
54 | os.Exit(1)
55 | }
56 |
57 | /*
58 | * Setup a global logger to the file or stdout
59 | */
60 | err = setupLogger()
61 | if err != nil {
62 | fmt.Fprintf(os.Stderr, "Can't setup a logger: %s", err.Error())
63 | os.Exit(1)
64 | }
65 |
66 | /*
67 | * Setup a database
68 | */
69 | err = setupDatabase()
70 | if err != nil {
71 | log.Fatal().Msg("Can't setup a database: " + err.Error())
72 | }
73 |
74 | /*
75 | * Setup Web GUI handlers
76 | */
77 | err = setupGUI()
78 | if err != nil {
79 | log.Fatal().Msg("Can't start Web GUI components: " + err.Error())
80 | }
81 |
82 | /*
83 | * Load plugins
84 | */
85 | err = loadPlugins()
86 | if err != nil {
87 | log.Fatal().Msg("Can't load plugins: " + err.Error())
88 | }
89 |
90 | /*
91 | * Setup collectors for the predefined data sources
92 | */
93 | err = setupCollectors()
94 | if err != nil {
95 | log.Fatal().Msg("Can't load collectors: " + err.Error())
96 | }
97 |
98 | /*
99 | * Setup processors of the data sources received data
100 | */
101 | err = setupProcessors()
102 | if err != nil {
103 | log.Fatal().Msg("Can't load processors: " + err.Error())
104 | }
105 |
106 | /*
107 | * Stop collectors on service exit
108 | */
109 | defer func() {
110 | for name, collector := range collectors {
111 | err := collector.Stop()
112 |
113 | if err != nil {
114 | log.Error().
115 | Str("source", name).
116 | Msg("Can't stop the collector: " + err.Error())
117 | } else {
118 | log.Debug().
119 | Str("source", name).
120 | Msg("Collector stopped")
121 | }
122 | }
123 | }()
124 |
125 | // Load service's version
126 | err = loadVersion()
127 | if err != nil {
128 | log.Fatal().Msg("Can't load version: " + err.Error())
129 | }
130 |
131 | /*
132 | * Start an HTTPS server
133 | */
134 | http.HandleFunc("/api", apiHandler)
135 |
136 | log.Info().Msgf("Graphoscope v%s. Starting the service listening on %s:%s", version, config.Server.Host, config.Server.Port)
137 | server := setupTLSserver()
138 |
139 | err = server.ListenAndServeTLS(config.Server.CertFile, config.Server.KeyFile)
140 | if err != nil {
141 | log.Fatal().Msg("Can't ListenAndServeTLS: " + err.Error())
142 | }
143 | }
144 |
--------------------------------------------------------------------------------
/notifications.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "time"
5 | )
6 |
7 | /*
8 | * Structure of one Web UI notification
9 | */
10 | type Notification struct {
11 | // Type of notification.
12 | // Currently possible values: "err", "info"
13 | Type string `bson:"type" json:"type"`
14 |
15 | // Message text
16 | Message string `bson:"message" json:"message"`
17 |
18 | // Timestamp of the creation time
19 | Ts string `bson:"ts" json:"ts"`
20 | }
21 |
22 | /*
23 | * Add new notification to the user.
24 | *
25 | * If user is online - notification will be received in a browser,
26 | * If offline - it will be stored in a database.
27 | *
28 | * Receives its type and message text
29 | */
30 | func (a *Account) addNotification(typ, message string) {
31 | n := &Notification{
32 | Type: typ,
33 | Message: message,
34 | Ts: time.Now().Format("2 Jan 2006 15:04:05"),
35 | }
36 |
37 | // Check whether user is online first
38 | if a.Session != nil {
39 | a.send("notification", message, typ)
40 | return
41 | }
42 |
43 | // Store the notification in a database otherwise
44 | a.Notifications = append(a.Notifications, n)
45 |
46 | // Leave the last 5 notifications only
47 | if len(a.Notifications) > 5 {
48 | a.Notifications = a.Notifications[len(a.Notifications)-5:]
49 | }
50 |
51 | err := a.update("notifications", a.Notifications)
52 | if err != nil {
53 | log.Error().
54 | Str("username", a.Username).
55 | Msg("Can't add a notification to the user: " + err.Error())
56 | }
57 |
58 | log.Info().
59 | Str("ip", a.Session.IP).
60 | Str("username", a.Username).
61 | Msg("Notification added")
62 | }
63 |
64 | /*
65 | * Handle 'notifications' websocket command
66 | * to clean user's notifications
67 | */
68 | func (a *Account) notificationsHandler() {
69 |
70 | a.Notifications = []*Notification{}
71 |
72 | err := a.update("notifications", a.Notifications)
73 | if err != nil {
74 | log.Error().
75 | Str("ip", a.Session.IP).
76 | Str("username", a.Username).
77 | Msg("Can't clean notifications: " + err.Error())
78 |
79 | a.send("error", err.Error(), "Can't clean notifications!")
80 | return
81 | }
82 |
83 | log.Info().
84 | Str("ip", a.Session.IP).
85 | Str("username", a.Username).
86 | Msg("Notifications cleaned")
87 | }
88 |
--------------------------------------------------------------------------------
/pdk/plugin.go:
--------------------------------------------------------------------------------
1 | package pdk
2 |
3 | import (
4 | "github.com/blastrain/vitess-sqlparser/sqlparser"
5 | )
6 |
7 | /*
8 | * Plugin interface to be implemented by the data source plugins
9 | */
10 | type SourcePlugin interface {
11 | // Return data source instance configuration
12 | Conf() *Source
13 |
14 | // Set specific parameters for the data source instance,
15 | // establish connection, etc.
16 | Setup(*Source, int) error
17 |
18 | // Get a list of all known data source's fields
19 | // for the Web GUI autocomplete
20 | Fields() ([]string, error)
21 |
22 | // Execute the given query.
23 | // Returns results, statistics, debug info & error
24 | Search(*sqlparser.Select) ([]map[string]interface{}, map[string]interface{}, map[string]interface{}, error)
25 |
26 | // Stop the collector when the core service stops,
27 | // gracefully disconnect from the data source if needed
28 | Stop() error
29 | }
30 |
31 | /*
32 | * Plugin interface to be implemented by the processor plugins
33 | */
34 | type ProcessorPlugin interface {
35 | // Return instance configuration
36 | Conf() *Processor
37 |
38 | // Set specific parameters for the instance,
39 | // establish connection, etc.
40 | Setup(*Processor) error
41 |
42 | // Process data source's received data in a background
43 | Process([]map[string]interface{}) ([]map[string]interface{}, error)
44 |
45 | // Stop the processor when the core service stops,
46 | // gracefully disconnect if needed
47 | Stop() error
48 | }
49 |
--------------------------------------------------------------------------------
/pdk/processor.go:
--------------------------------------------------------------------------------
1 | /*
2 | * Data processing definition.
3 | * For YAML files in "../processor" by default.
4 | *
5 | * Check "../definitions/processors/processor.yaml.example" for the fields description
6 | */
7 |
8 | package pdk
9 |
10 | import (
11 | "time"
12 | )
13 |
14 | type Processor struct {
15 | Name string `yaml:"name"`
16 | Plugin string `yaml:"plugin"`
17 | Timeout time.Duration `yaml:"timeout"`
18 | Data map[string]interface{} `yaml:"data"`
19 | }
20 |
--------------------------------------------------------------------------------
/pdk/source.go:
--------------------------------------------------------------------------------
1 | /*
2 | * Data source definition.
3 | * For YAML files in "../sources" by default.
4 | *
5 | * Check "../sources/source.yaml.example" for the fields description
6 | */
7 |
8 | package pdk
9 |
10 | import (
11 | "regexp"
12 | "time"
13 | )
14 |
15 | type Source struct {
16 | Name string `yaml:"name"`
17 | Label string `yaml:"label"`
18 | Icon string `yaml:"icon"`
19 | Plugin string `yaml:"plugin"`
20 | InGlobal bool `yaml:"inGlobal"`
21 | IncludeDatetime bool `yaml:"includeDatetime"`
22 | SupportsSQL bool `yaml:"supportsSQL"`
23 | Timeout time.Duration `yaml:"timeout"`
24 | Access map[string]string `yaml:"access"`
25 | QueryFields []string `yaml:"queryFields"`
26 | IncludeFields []string `yaml:"includeFields"`
27 | StatsFields []string `yaml:"statsFields"`
28 | ReplaceFields map[string]string `yaml:"replaceFields"`
29 | Relations []*Relation `yaml:"relations"`
30 | }
31 |
32 | type Relation struct {
33 | From *Node `yaml:"from"`
34 | To *Node `yaml:"to"`
35 | Edge *struct {
36 | Label string `yaml:"label"`
37 | Attributes []string `yaml:"attributes"`
38 | } `yaml:"edge"`
39 | }
40 |
41 | type Node struct {
42 | ID string `yaml:"id"`
43 | Group string `yaml:"group"`
44 | Search string `yaml:"search"`
45 | Attributes []string `yaml:"attributes"`
46 | VarTypes []*struct {
47 | Regex string `yaml:"regex"`
48 | RegexCompiled *regexp.Regexp `yaml:"-"`
49 | Group string `yaml:"group"`
50 | Search string `yaml:"search"`
51 | Label string `yaml:"label"`
52 | } `yaml:"varTypes"`
53 | }
54 |
--------------------------------------------------------------------------------
/pdk/stats.go:
--------------------------------------------------------------------------------
1 | package pdk
2 |
3 | import (
4 | "fmt"
5 | "sync"
6 |
7 | "github.com/umpc/go-sortedmap"
8 | )
9 |
10 | /*
11 | * Structure to contain statistics data
12 | * if some data source has too many entries to return
13 | * (above the preconfigured limit)
14 | */
15 | type Stats struct {
16 | Fields map[string]*sortedmap.SortedMap
17 | mx sync.Mutex
18 | }
19 |
20 | func NewStats() *Stats {
21 | return &Stats{
22 | Fields: make(map[string]*sortedmap.SortedMap),
23 | }
24 | }
25 |
26 | /*
27 | * Update statistics of the received entries from some data source.
28 | * When the amount of returned entries becomes too large
29 | * users will receive the statistics info instead of the graph relations data.
30 | *
31 | * Receives:
32 | * entry - single entry from a data source
33 | * key - statistics chart field to update,
34 | * one entry increases the value by 1
35 | */
36 | func (s *Stats) Update(entry map[string]interface{}, key string) {
37 | // Skip if value is missing
38 | value := entry[key]
39 | if value == nil || fmt.Sprint(value) == "" {
40 | return
41 | }
42 |
43 | s.mx.Lock()
44 |
45 | if val, ok := s.Fields[key].Get(value); ok {
46 | s.Fields[key].Replace(value, val.(int)+1)
47 | } else {
48 | s.Fields[key].Insert(value, 1)
49 | }
50 |
51 | s.mx.Unlock()
52 | }
53 |
54 | /*
55 | * Convert sorted-map object to the native map,
56 | * converted to the JSON later,
57 | * so the Web GUI can draw interactive charts.
58 | *
59 | * Receives a data source name
60 | */
61 | func (s *Stats) ToJSON(source string) (map[string]interface{}, error) {
62 |
63 | // Map to store Top 10 entries
64 | json := make(map[string]interface{})
65 |
66 | // Identifier of the source data belongs to
67 | json["source"] = source
68 |
69 | for k, v := range s.Fields {
70 | i := 1
71 |
72 | iterCh, err := v.IterCh()
73 | if err != nil && len(v.Keys()) != 0 {
74 | return nil, err
75 |
76 | } else if len(v.Keys()) != 0 {
77 | defer iterCh.Close()
78 |
79 | group := make(map[string]int)
80 |
81 | for rec := range iterCh.Records() {
82 | //fmt.Printf("%+v\n", rec)
83 |
84 | group[fmt.Sprint(rec.Key)] = rec.Val.(int)
85 |
86 | // We want Top 10 here and started from i == 1
87 | if i > 9 {
88 | break
89 | }
90 |
91 | i++
92 | }
93 |
94 | json[k] = group
95 | }
96 | }
97 |
98 | return json, nil
99 | }
100 |
--------------------------------------------------------------------------------
/plugins/src/abuseipdb/README.md:
--------------------------------------------------------------------------------
1 | # AbuseIPDB plugin
2 |
3 | AbuseIPDB connector sends IP address in a GET request to the `abuseipdb.com` API and expects a list of reports back.
4 | More info: https://docs.abuseipdb.com/#check-endpoint
5 |
6 |
7 | Simple request to the API:
8 | ```
9 | curl -G https://api.abuseipdb.com/api/v2/check \
10 | --data-urlencode "ipAddress=8.8.8.8" \
11 | -d maxAgeInDays=90 \
12 | -d verbose \
13 | -H "Key: YOUR_OWN_API_KEY" \
14 | -H "Accept: application/json"
15 | ```
16 | where `YOUR_OWN_API_KEY` is your personal/unique API key.
17 |
18 |
19 | `curl` to test plugin:
20 | ```sh
21 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+abuseipdb+WHERE+ip=%278.8.8.8%27'
22 | ```
23 |
24 | Compile with:
25 | ```sh
26 | go build -buildmode=plugin -ldflags="-w" -o abuseipdb.so ./*.go
27 | ```
28 |
29 | # Limitations
30 |
31 | Does not support complex SQL queries and datetime range selection.
32 |
33 |
34 | # Access details
35 |
36 | Source YAML definition's `access` fields:
37 | - **url**: HTTPS access point, `https://api.abuseipdb.com/api/v2/check` at the moment
38 | - **maxAgeInDays**: how far back in time we go to fetch reports, max 365
39 | - **key**: unique API key
40 |
41 |
42 | # Definition file example
43 |
44 | Replace API key with your own:
45 | ```yaml
46 | name: abuseipdb
47 | label: AbuseIPDB
48 | icon: clipboard list
49 |
50 | plugin: abuseipdb
51 | inGlobal: true
52 | includeDatetime: false
53 | supportsSQL: false
54 |
55 | access:
56 | url: https://api.abuseipdb.com/api/v2/check
57 | maxAgeInDays: 180
58 | key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
59 |
60 | queryFields:
61 | - ip
62 |
63 | replaceFields:
64 | ip: ipAddress
65 |
66 |
67 | relations:
68 | -
69 | from:
70 | id: domain
71 | group: domain
72 | search: domain
73 |
74 | to:
75 | id: ipAddress
76 | group: ip
77 | search: ip
78 | attributes: [ "countryCode", "countryName", "hostnames", "isPublic", "isWhitelisted", "isp", "usageType", "totalReports", "lastReportedAt" ]
79 |
80 | ```
--------------------------------------------------------------------------------
/plugins/src/abuseipdb/abuseipdb_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted [][2]string
21 | }{
22 | {`SELECT * WHERE ipAddress='10.10.10.10' LIMIT 5,1`, [][2]string{
23 | [2]string{"ipAddress", "10.10.10.10"},
24 | }},
25 | {`SELECT * WHERE ipAddress='8.8.8.8' or domain='google.com'`, [][2]string{
26 | [2]string{"ipAddress", "8.8.8.8"},
27 | }},
28 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,1`, [][2]string{}},
29 | }
30 |
31 | for _, table := range tables {
32 | // Executed by the main service
33 | ast, err := sqlparser.Parse(table.sql)
34 | if err != nil {
35 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
36 | continue
37 | }
38 |
39 | stmt, ok := ast.(*sqlparser.Select)
40 | if !ok {
41 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
42 | continue
43 | }
44 |
45 | // Executed by the plugin
46 | result, err := c.convert(stmt)
47 | if err != nil {
48 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
49 | continue
50 | }
51 |
52 | if !equal(result, table.converted) {
53 | t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted)
54 | }
55 | }
56 | }
57 |
58 | /*
59 | * Check whether a and b slices contain the same elements
60 | */
61 | func equal(a, b [][2]string) bool {
62 | if len(a) != len(b) {
63 | return false
64 | }
65 | for i, v := range a {
66 | if v != b[i] {
67 | return false
68 | }
69 | }
70 | return true
71 | }
72 |
--------------------------------------------------------------------------------
/plugins/src/abuseipdb/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to the field/value list convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL query to the list of [field,value]
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) ([][2]string, error) {
15 |
16 | // Handle WHERE.
17 | // Top level node pass in an empty interface
18 | // to tell the children this is root.
19 | // Is there any better way?
20 | var rootParent sqlparser.Expr
21 |
22 | // List of requested fields & values
23 | fields, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
24 | if err != nil {
25 | return nil, err
26 | }
27 |
28 | i := 0
29 | for _, field := range fields {
30 | if field[0] != "ipAddress" {
31 | fields = append(fields[:i], fields[i+1:]...)
32 | i -= 1
33 | }
34 |
35 | i += 1
36 | }
37 |
38 | return fields, nil
39 | }
40 |
--------------------------------------------------------------------------------
/plugins/src/abuseipdb/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "abuseipdb"
12 | Version = "1.0.1"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | url string
26 | maxAgeInDays int
27 | key string
28 | limit int
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/circl_passive_ssl/README.md:
--------------------------------------------------------------------------------
1 | # CIRCL Passive SSL plugin
2 |
3 | Data source: https://www.circl.lu/services/passive-ssl/
4 |
5 | Connector sends a GET request to the `https://www.circl.lu/v2pssl/` and expects a JSON back.
6 | To the preconfigured URL `field/value` will be attached as query.
7 |
8 | `curl` to test:
9 | ```sh
10 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+service+WHERE+ip=%278.8.8.8%27'
11 | ```
12 |
13 | Compile with:
14 | ```sh
15 | go build -buildmode=plugin -ldflags="-w" -o circl_passive_ssl.so ./*.go
16 | ```
17 |
18 | # Limitations
19 |
20 | Does not support complex SQL queries and datetime range selection.
21 |
22 |
23 | # Access details
24 |
25 | Source YAML definition's `access` fields:
26 | - **url**: REST API access point, `https://www.circl.lu/v2pssl/`
27 | - **username**: Username
28 | - **password**: User's password
29 |
30 |
31 | # YAML definition example
32 |
33 | ```yaml
34 | name: circl_passive_ssl
35 | label: CIRCL Passive SSL
36 | icon: retweet
37 |
38 | plugin: circl_passive_ssl
39 | inGlobal: false
40 | includeDatetime: false
41 | supportsSQL: false
42 |
43 | access:
44 | url: https://www.circl.lu/v2pssl
45 | username: user
46 | password: password_
47 |
48 | queryFields:
49 | - ip
50 | - network
51 | - sha1
52 |
53 | statsFields:
54 | - ip
55 | - sha1
56 | - subject
57 |
58 |
59 | relations:
60 | -
61 | from:
62 | id: sha1
63 | group: sha1
64 | search: sha1
65 | attributes: ["subject"]
66 |
67 | to:
68 | id: ip
69 | group: ip
70 | search: ip
71 | ```
72 |
--------------------------------------------------------------------------------
/plugins/src/circl_passive_ssl/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to the field/value list convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL query to the list of [field,value]
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) ([2]string, error) {
15 |
16 | // Handle WHERE.
17 | // Top level node pass in an empty interface
18 | // to tell the children this is root.
19 | // Is there any better way?
20 | var rootParent sqlparser.Expr
21 |
22 | // List of requested fields & values
23 | fields, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
24 | if err != nil {
25 | return [2]string{}, err
26 | }
27 |
28 | return fields, nil
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/circl_passive_ssl/passive_ssl_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted [2]string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,1`, [2]string{"ip", "10.10.10.10"}},
23 | {`select * where sha1='00000'`, [2]string{"sha1", "00000"}},
24 | }
25 |
26 | for _, table := range tables {
27 | // Executed by the main service
28 | ast, err := sqlparser.Parse(table.sql)
29 | if err != nil {
30 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
31 | continue
32 | }
33 |
34 | stmt, ok := ast.(*sqlparser.Select)
35 | if !ok {
36 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
37 | continue
38 | }
39 |
40 | // Executed by the plugin
41 | result, err := c.convert(stmt)
42 | if err != nil {
43 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
44 | continue
45 | }
46 |
47 | if !equal(result, table.converted) {
48 | t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted)
49 | }
50 | }
51 | }
52 |
53 | /*
54 | * Check whether a and b slices contain the same elements
55 | */
56 | func equal(a, b [2]string) bool {
57 | for i, v := range a {
58 | if v != b[i] {
59 | return false
60 | }
61 | }
62 | return true
63 | }
64 |
--------------------------------------------------------------------------------
/plugins/src/circl_passive_ssl/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "circl_passive_ssl"
12 | Version = "1.0.1"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | url string
26 | username string
27 | password string
28 | limit int
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/elasticsearch.v7/README.md:
--------------------------------------------------------------------------------
1 | # Elasticsearch plugin
2 |
3 | Plugin to query Elasticsearch (https://www.elastic.co/elasticsearch) v7 as a data source.
4 |
5 | SQl convertor's base: https://github.com/blastrain/vitess-sqlparser/tree/develop/sqlparser
6 |
7 |
8 | Compile with:
9 | ```sh
10 | go build -buildmode=plugin -ldflags="-w" -o elasticsearch.v7.so ./*.go
11 | ```
12 |
13 | # Access details
14 |
15 | Source YAML definition's `access` fields:
16 | - **url**: HTTP access point, for example - `http://localhost:9200`
17 | - **username**: username for the basic auth
18 | - **password**: password for the basic auth
19 | - **key**: authorization key
20 | - **indices**: comma separated indices patterns to query, for example - `apps-*`
21 | - **ca**: CA certificate path
22 |
23 | Only `username/password` or `key` can be used at once.
24 |
25 |
26 | ## Limitations
27 |
28 | - Go package supports specific Elasticsearch major version only,
29 | so version number is included in a plugin's name
30 |
--------------------------------------------------------------------------------
/plugins/src/elasticsearch.v7/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to Elasticsearch query convertor
3 | * Based on: https://github.com/blastrain/vitess-sqlparser/tree/develop/sqlparser
4 | */
5 |
6 | package main
7 |
8 | import (
9 | "errors"
10 | "fmt"
11 | "strings"
12 |
13 | "github.com/blastrain/vitess-sqlparser/sqlparser"
14 | )
15 |
16 | /*
17 | * Convert SQL query to the Elasticsearch JSON query
18 | */
19 | func (p *plugin) convert(sel *sqlparser.Select, fields []string) (string, error) {
20 |
21 | // Handle WHERE.
22 | // Top level node pass in an empty interface
23 | // to tell the children this is root.
24 | // Is there any better way?
25 | var rootParent sqlparser.Expr
26 |
27 | queryMapStr, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
28 | if err != nil {
29 | return "", err
30 | }
31 |
32 | // Handle GROUP BY
33 | if len(sel.GroupBy) > 0 || checkNeedAgg(sel.SelectExprs) {
34 | return "", errors.New("'GROUP BY' & aggregation are not supported")
35 | }
36 |
37 | resultMap := make(map[string]interface{})
38 | resultMap["query"] = queryMapStr
39 |
40 | // Handle ORDER BY
41 | orderByArr := []string{}
42 | for _, orderByExpr := range sel.OrderBy {
43 | orderByStr := fmt.Sprintf(`{"%v": "%v"}`, strings.Replace(sqlparser.String(orderByExpr.Expr), "`", "", -1), orderByExpr.Direction)
44 | orderByArr = append(orderByArr, orderByStr)
45 | }
46 |
47 | if len(orderByArr) > 0 {
48 | resultMap["sort"] = fmt.Sprintf("[%v]", strings.Join(orderByArr, ", "))
49 | }
50 |
51 | // Handle LIMIT
52 | if sel.Limit != nil {
53 | resultMap["from"] = sqlparser.String(sel.Limit.Offset)
54 | resultMap["size"] = sqlparser.String(sel.Limit.Rowcount)
55 | }
56 |
57 | if len(fields) != 0 {
58 | resultMap["_source"] = "[\"" + strings.Join(fields, "\",\"") + "\"]"
59 | }
60 |
61 | // Fields of the JSON to return.
62 | // Keep the traversal in order, avoid unpredicted JSON
63 | keySlice := []string{"query", "from", "size", "sort", "_source"}
64 | resultArr := []string{}
65 |
66 | for _, mapKey := range keySlice {
67 | if val, ok := resultMap[mapKey]; ok {
68 | resultArr = append(resultArr, fmt.Sprintf(`"%v" : %v`, mapKey, val))
69 | }
70 | }
71 |
72 | dsl := "{" + strings.Join(resultArr, ", ") + "}"
73 | //fmt.Println(dsl)
74 |
75 | // Return a JSON formatted query
76 | return dsl, nil
77 | }
78 |
--------------------------------------------------------------------------------
/plugins/src/elasticsearch.v7/elasticsearch_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10'`, `{"query" : {"bool" : {"must" : [{"match_phrase" : {"ip" : "10.10.10.10"}}]}}}`},
23 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,10`, `{"query" : {"bool" : {"must" : [{"match_phrase" : {"ip" : "10.10.10.10"}}]}}, "from" : 5, "size" : 10}`},
24 | {`SELECT * WHERE size>100 ORDER BY name LIMIT 0,1`, `{"query" : {"bool" : {"must" : [{"range" : {"size" : {"gt" : 100}}}]}}, "from" : 0, "size" : 1, "sort" : [{"name": "asc"}]}`},
25 | {`SELECT * WHERE size=10 ORDER BY name DESC LIMIT 0,1`, `{"query" : {"bool" : {"must" : [{"match_phrase" : {"size" : 10}}]}}, "from" : 0, "size" : 1, "sort" : [{"name": "desc"}]}`},
26 | {`SELECT * WHERE size>=100`, `{"query" : {"bool" : {"must" : [{"range" : {"size" : {"from" : 100}}}]}}}`},
27 | {`SELECT * WHERE name LIKE 's%'`, `{"query" : {"bool" : {"must" : [{"query_string": { "default_field": "name.keyword", "query": "s*" }}]}}}`},
28 | {`SELECT * WHERE name NOT LIKE 's%'`, `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"query_string": { "default_field": "name.keyword", "query": "s*" }}}}]}}}`},
29 | {`SELECT * WHERE size BETWEEN 100 AND 300`, `{"query" : {"bool" : {"must" : [{"range" : {"size" : {"from" : 100, "to" : 300}}}]}}}`},
30 | {`SELECT * WHERE size NOT BETWEEN 1 AND 10`, `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"range" : {"size" : {"from" : 1, "to" : 10}}}}}]}}}`},
31 | {`SELECT * WHERE size IN (100,300)`, `{"query" : {"bool" : {"must" : [{"terms" : {"size" : [100, 300]}}]}}}`},
32 | {`SELECT * WHERE size NOT IN (100,300)`, `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"terms" : {"size" : [100, 300]}}}}]}}}`},
33 | {`select * where name='sarah' and age!=40 and (country='LV' or country='AU') limit 0,1`, `{"query" : {"bool" : {"must" : [{"match_phrase" : {"name" : "sarah"}}, {"bool" : {"must_not" : [{"match_phrase" : {"age" : 40}}]}}, {"bool" : {"should" : [{"match_phrase" : {"country" : "LV"}}, {"match_phrase" : {"country" : "AU"}}]}}]}}, "from" : 0, "size" : 1}`},
34 | }
35 |
36 | for _, table := range tables {
37 | // Executed by the main service
38 | ast, err := sqlparser.Parse(table.sql)
39 | if err != nil {
40 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
41 | continue
42 | }
43 |
44 | stmt, ok := ast.(*sqlparser.Select)
45 | if !ok {
46 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
47 | continue
48 | }
49 |
50 | // Executed by the plugin
51 | result, err := c.convert(stmt, nil)
52 | if err != nil {
53 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
54 | continue
55 | }
56 |
57 | if result != table.converted {
58 | t.Errorf("Invalid conversion of '%s': %s, expected: %s", table.sql, result, table.converted)
59 | }
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/plugins/src/elasticsearch.v7/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | "github.com/elastic/go-elasticsearch/v7"
6 | )
7 |
8 | /*
9 | * Export symbols
10 | */
11 | var (
12 | Name = "elasticsearch.v7"
13 | Version = "1.0.9"
14 | Plugin plugin
15 | )
16 |
17 | /*
18 | * Structure to be imported by the core as a plugin
19 | */
20 | type plugin struct {
21 |
22 | // Inherit default configuration fields
23 | source *pdk.Source
24 |
25 | // Custom fields
26 | client *elasticsearch.Client
27 | index string
28 | limit int
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/elasticsearch.v8/README.md:
--------------------------------------------------------------------------------
1 | # Elasticsearch plugin
2 |
3 | Plugin to query Elasticsearch (https://www.elastic.co/elasticsearch) v8 as a data source.
4 |
5 | SQl convertor's base: https://github.com/blastrain/vitess-sqlparser/tree/develop/sqlparser
6 |
7 |
8 | Compile with:
9 | ```sh
10 | go build -buildmode=plugin -ldflags="-w" -o elasticsearch.v8.so ./*.go
11 | ```
12 |
13 | # Access details
14 |
15 | Source YAML definition's `access` fields:
16 | - **url**: HTTP access point, for example - `http://localhost:9200`
17 | - **username**: username for the basic auth
18 | - **password**: password for the basic auth
19 | - **key**: authorization key
20 | - **indices**: comma separated indices patterns to query, for example - `apps-*`
21 | - **ca**: CA certificate path
22 |
23 | Only `username/password` or `key` can be used at once.
24 |
25 |
26 | ## Limitations
27 |
28 | - Go package supports specific Elasticsearch major version only,
29 | so version number is included in a plugin's name
30 |
--------------------------------------------------------------------------------
/plugins/src/elasticsearch.v8/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to Elasticsearch query convertor
3 | * Based on: https://github.com/blastrain/vitess-sqlparser/tree/develop/sqlparser
4 | */
5 |
6 | package main
7 |
8 | import (
9 | "errors"
10 | "fmt"
11 | "strings"
12 |
13 | "github.com/blastrain/vitess-sqlparser/sqlparser"
14 | )
15 |
16 | /*
17 | * Convert SQL query to the Elasticsearch JSON query
18 | */
19 | func (p *plugin) convert(sel *sqlparser.Select, fields []string) (string, error) {
20 |
21 | // Handle WHERE.
22 | // Top level node pass in an empty interface
23 | // to tell the children this is root.
24 | // Is there any better way?
25 | var rootParent sqlparser.Expr
26 |
27 | queryMapStr, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
28 | if err != nil {
29 | return "", err
30 | }
31 |
32 | // Handle GROUP BY
33 | if len(sel.GroupBy) > 0 || checkNeedAgg(sel.SelectExprs) {
34 | return "", errors.New("'GROUP BY' & aggregation are not supported")
35 | }
36 |
37 | resultMap := make(map[string]interface{})
38 | resultMap["query"] = queryMapStr
39 |
40 | // Handle ORDER BY
41 | orderByArr := []string{}
42 | for _, orderByExpr := range sel.OrderBy {
43 | orderByStr := fmt.Sprintf(`{"%v": "%v"}`, strings.Replace(sqlparser.String(orderByExpr.Expr), "`", "", -1), orderByExpr.Direction)
44 | orderByArr = append(orderByArr, orderByStr)
45 | }
46 |
47 | if len(orderByArr) > 0 {
48 | resultMap["sort"] = fmt.Sprintf("[%v]", strings.Join(orderByArr, ", "))
49 | }
50 |
51 | // Handle LIMIT
52 | if sel.Limit != nil {
53 | resultMap["from"] = sqlparser.String(sel.Limit.Offset)
54 | resultMap["size"] = sqlparser.String(sel.Limit.Rowcount)
55 | }
56 |
57 | if len(fields) != 0 {
58 | resultMap["_source"] = "[\"" + strings.Join(fields, "\",\"") + "\"]"
59 | }
60 |
61 | // Fields of the JSON to return.
62 | // Keep the traversal in order, avoid unpredicted JSON
63 | keySlice := []string{"query", "from", "size", "sort", "_source"}
64 | resultArr := []string{}
65 |
66 | for _, mapKey := range keySlice {
67 | if val, ok := resultMap[mapKey]; ok {
68 | resultArr = append(resultArr, fmt.Sprintf(`"%v" : %v`, mapKey, val))
69 | }
70 | }
71 |
72 | dsl := "{" + strings.Join(resultArr, ", ") + "}"
73 | //fmt.Println(dsl)
74 |
75 | // Return a JSON formatted query
76 | return dsl, nil
77 | }
78 |
--------------------------------------------------------------------------------
/plugins/src/elasticsearch.v8/elasticsearch_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10'`, `{"query" : {"bool" : {"must" : [{"match_phrase" : {"ip" : "10.10.10.10"}}]}}}`},
23 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,10`, `{"query" : {"bool" : {"must" : [{"match_phrase" : {"ip" : "10.10.10.10"}}]}}, "from" : 5, "size" : 10}`},
24 | {`SELECT * WHERE size>100 ORDER BY name LIMIT 0,1`, `{"query" : {"bool" : {"must" : [{"range" : {"size" : {"gt" : 100}}}]}}, "from" : 0, "size" : 1, "sort" : [{"name": "asc"}]}`},
25 | {`SELECT * WHERE size=10 ORDER BY name DESC LIMIT 0,1`, `{"query" : {"bool" : {"must" : [{"match_phrase" : {"size" : 10}}]}}, "from" : 0, "size" : 1, "sort" : [{"name": "desc"}]}`},
26 | {`SELECT * WHERE size>=100`, `{"query" : {"bool" : {"must" : [{"range" : {"size" : {"from" : 100}}}]}}}`},
27 | {`SELECT * WHERE name LIKE 's%'`, `{"query" : {"bool" : {"must" : [{"query_string": { "default_field": "name.keyword", "query": "s*" }}]}}}`},
28 | {`SELECT * WHERE name NOT LIKE 's%'`, `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"query_string": { "default_field": "name.keyword", "query": "s*" }}}}]}}}`},
29 | {`SELECT * WHERE size BETWEEN 100 AND 300`, `{"query" : {"bool" : {"must" : [{"range" : {"size" : {"from" : 100, "to" : 300}}}]}}}`},
30 | {`SELECT * WHERE size NOT BETWEEN 1 AND 10`, `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"range" : {"size" : {"from" : 1, "to" : 10}}}}}]}}}`},
31 | {`SELECT * WHERE size IN (100,300)`, `{"query" : {"bool" : {"must" : [{"terms" : {"size" : [100, 300]}}]}}}`},
32 | {`SELECT * WHERE size NOT IN (100,300)`, `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"terms" : {"size" : [100, 300]}}}}]}}}`},
33 | {`select * where name='sarah' and age!=40 and (country='LV' or country='AU') limit 0,1`, `{"query" : {"bool" : {"must" : [{"match_phrase" : {"name" : "sarah"}}, {"bool" : {"must_not" : [{"match_phrase" : {"age" : 40}}]}}, {"bool" : {"should" : [{"match_phrase" : {"country" : "LV"}}, {"match_phrase" : {"country" : "AU"}}]}}]}}, "from" : 0, "size" : 1}`},
34 | }
35 |
36 | for _, table := range tables {
37 | // Executed by the main service
38 | ast, err := sqlparser.Parse(table.sql)
39 | if err != nil {
40 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
41 | continue
42 | }
43 |
44 | stmt, ok := ast.(*sqlparser.Select)
45 | if !ok {
46 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
47 | continue
48 | }
49 |
50 | // Executed by the plugin
51 | result, err := c.convert(stmt, nil)
52 | if err != nil {
53 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
54 | continue
55 | }
56 |
57 | if result != table.converted {
58 | t.Errorf("Invalid conversion of '%s': %s, expected: %s", table.sql, result, table.converted)
59 | }
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/plugins/src/elasticsearch.v8/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | "github.com/elastic/go-elasticsearch/v8"
6 | )
7 |
8 | /*
9 | * Export symbols
10 | */
11 | var (
12 | Name = "elasticsearch.v8"
13 | Version = "1.0.2"
14 | Plugin plugin
15 | )
16 |
17 | /*
18 | * Structure to be imported by the core as a plugin
19 | */
20 | type plugin struct {
21 |
22 | // Inherit default configuration fields
23 | source *pdk.Source
24 |
25 | // Custom fields
26 | client *elasticsearch.Client
27 | index string
28 | limit int
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/file/csv/README.md:
--------------------------------------------------------------------------------
1 | # CSV file plugin
2 |
3 | Plugin to query CSV file as a data source.
4 |
5 | SQL doesn't allow to query missing columns, like Elasticsearch does.
6 | An error `field X does not exist` will be received. That means you must be very
7 | careful with designing a data source and creating a YAML config file to be able
8 | to combine it with data source types other than SQL.
9 |
10 | The easiest solution is to exclude data source from the `global` namespace
11 | and query it independently, to make sure all columns exist.
12 |
13 | `curl` to test:
14 | ```sh
15 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+csvfile+WHERE+ip=%278.8.8.8%27'
16 | ```
17 |
18 | Compile with:
19 | ```sh
20 | go build -buildmode=plugin -ldflags="-w" -o file-csv.so ./*.go
21 | ```
22 |
23 | # Access details
24 |
25 | Source YAML definition's `access` fields:
26 | - **path**: CSV file to use, for example - `/data/test.csv`
27 |
--------------------------------------------------------------------------------
/plugins/src/file/csv/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to CSV query convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL statement to the CSV query
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) (string, error) {
15 |
16 | // Handle WHERE
17 | query := sqlparser.String(sel.Where.Expr)
18 |
19 | // Handle GROUP BY
20 | if len(sel.GroupBy) > 0 {
21 | query += sqlparser.String(sel.GroupBy)
22 | }
23 |
24 | // Handle ORDER BY
25 | if sel.OrderBy != nil {
26 | query += sqlparser.String(sel.OrderBy)
27 | }
28 |
29 | // Handle LIMIT
30 | if sel.Limit != nil {
31 | // Handle rowcount
32 | query += " LIMIT " + sqlparser.String(sel.Limit.Rowcount)
33 | // Handle offset
34 | if sel.Limit.Offset != nil {
35 | query += " OFFSET " + sqlparser.String(sel.Limit.Offset)
36 | } else {
37 | query += " OFFSET 0"
38 | }
39 | }
40 |
41 | return query, nil
42 | }
43 |
--------------------------------------------------------------------------------
/plugins/src/file/csv/csv_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10'`, `ip = '10.10.10.10'`},
23 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,10`, `ip = '10.10.10.10' LIMIT 10 OFFSET 5`},
24 | {`SELECT * WHERE size>100 ORDER BY name LIMIT 0,1`, `size > 100 order by name asc LIMIT 1 OFFSET 0`},
25 | {`SELECT * WHERE size=10 ORDER BY name DESC LIMIT 0,1`, `size = 10 order by name desc LIMIT 1 OFFSET 0`},
26 | {`SELECT * WHERE size>=100`, `size >= 100`},
27 | {`SELECT * WHERE name LIKE 's%'`, `name like 's%'`},
28 | {`SELECT * WHERE name NOT LIKE 's%'`, `name not like 's%'`},
29 | {`SELECT * WHERE size BETWEEN 100 AND 300`, `size between 100 and 300`},
30 | {`SELECT * WHERE size IN (100,300)`, `size in (100, 300)`},
31 | {`SELECT * WHERE size NOT IN (100,300)`, `size not in (100, 300)`},
32 | {`SELECT * WHERE name='sarah' and age!=40 and (country='LV' OR country='AU') ORDER BY age DESC limit 1`,
33 | `name = 'sarah' and age != 40 and (country = 'LV' or country = 'AU') order by age desc LIMIT 1 OFFSET 0`},
34 | }
35 |
36 | for _, table := range tables {
37 | // Executed by the main service
38 | ast, err := sqlparser.Parse(table.sql)
39 | if err != nil {
40 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
41 | continue
42 | }
43 |
44 | stmt, ok := ast.(*sqlparser.Select)
45 | if !ok {
46 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
47 | continue
48 | }
49 |
50 | // Executed by the plugin
51 | result, err := c.convert(stmt)
52 | if err != nil {
53 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
54 | continue
55 | }
56 |
57 | if result != table.converted {
58 | t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted)
59 | }
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/plugins/src/file/csv/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "database/sql"
5 |
6 | "github.com/cert-lv/graphoscope/pdk"
7 | )
8 |
9 | /*
10 | * Export symbols
11 | */
12 | var (
13 | Name = "file-csv"
14 | Version = "1.0.7"
15 | Plugin plugin
16 | )
17 |
18 | /*
19 | * Structure to be imported by the core as a plugin
20 | */
21 | type plugin struct {
22 |
23 | // Inherit default configuration fields
24 | source *pdk.Source
25 |
26 | // Custom fields
27 | db *sql.DB
28 | dir string
29 | base string
30 | limit int
31 | }
32 |
--------------------------------------------------------------------------------
/plugins/src/hashlookup/README.md:
--------------------------------------------------------------------------------
1 | # Hashlookup plugin
2 |
3 | Plugin to query [hashlookup services](https://github.com/hashlookup).
4 | For instance hashlookup.circl.lu
5 |
6 | Sample command to use plugin:
7 | ```sh
8 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+hashlookup+WHERE+sha1=%27deac5aeda66017c25a9e21f36f5ee618d2ad9d3d%27'
9 | ```
10 |
11 | Compile with:
12 | ```sh
13 | go build -buildmode=plugin -ldflags="-w" -o hashlookup.so ./*.go
14 | ```
15 | Or use the Makefile command:
16 | `make plugins-local`
17 |
18 |
19 | # Limitations
20 |
21 | Does not support complex SQL queries and datetime range selection.
22 |
23 |
24 | # Access details
25 |
26 | Source YAML definition's `access` fields:
27 | - **url**: hashlookup API endpoint, for example - `https://hashlookup.circl.lu`
28 | - **apiKey**: optional hashlookup API key
29 |
30 |
31 | # Definition file example
32 |
33 | Replace API key with your own:
34 | ```yaml
35 | name: hashlookup
36 | label: Hashlookup
37 | icon: database
38 |
39 | plugin: hashlookup
40 | inGlobal: true
41 | includeDatetime: false
42 | supportsSQL: false
43 |
44 | access:
45 | url: https://hashlookup.circl.lu
46 | apiKey: .
47 |
48 | queryFields:
49 | - md5
50 | - sha1
51 | - sha256
52 |
53 |
54 | relations:
55 | -
56 | from:
57 | id: SHA-1
58 | group: sha1
59 | search: sha1
60 | attributes: ["FileName", "FileSize", "source-url", "MD5", "SHA-512", "SHA-256", "SSDEEP", "TLSH", "insert-timestamp", "mimetype", "source", "hashlookup-parent-total", "snap-authority", "hashlookup:trust"]
61 |
62 | to:
63 | id: parent
64 | group: sha1
65 | search: sha1
66 | attributes: ["FileName", "FileSize", "source-url", "MD5", "SHA-512", "SHA-256", "SSDEEP", "TLSH", "insert-timestamp", "mimetype", "source", "hashlookup-parent-total", "snap-authority", "hashlookup:trust"]
67 |
68 | edge:
69 | label: ChildOf
70 |
71 | -
72 | from:
73 | id: SHA-1
74 | group: sha1
75 | search: sha1
76 | attributes: ["FileName", "FileSize", "source-url", "MD5", "SHA-512", "SHA-256", "SSDEEP", "TLSH", "insert-timestamp", "mimetype", "source", "hashlookup-parent-total", "snap-authority", "hashlookup:trust"]
77 |
78 | to:
79 | id: children
80 | group: sha1
81 | search: sha1
82 | attributes: ["FileName", "FileSize", "source-url", "MD5", "SHA-512", "SHA-256", "SSDEEP", "TLSH", "insert-timestamp", "mimetype", "source", "hashlookup-parent-total", "snap-authority", "hashlookup:trust"]
83 |
84 | edge:
85 | label: ParentOf
86 |
87 | -
88 | from:
89 | id: SHA-1
90 | group: sha1
91 | search: sha1
92 | attributes: ["FileName", "FileSize", "source-url", "SHA-512", "SHA-256", "SSDEEP", "TLSH", "insert-timestamp", "mimetype", "source", "hashlookup-parent-total", "snap-authority", "hashlookup:trust"]
93 |
94 | to:
95 | id: MD5
96 | group: md5
97 | search: md5
98 | attributes: ["FileName", "FileSize", "source-url", "SHA-512", "SHA-256", "SSDEEP", "TLSH", "insert-timestamp", "mimetype", "source", "hashlookup-parent-total", "snap-authority", "hashlookup:trust"]
99 |
100 | -
101 | from:
102 | id: SHA-1
103 | group: sha1
104 | search: sha1
105 | attributes: ["FileName", "FileSize", "source-url", "MD5", "SHA-512", "SSDEEP", "TLSH", "insert-timestamp", "mimetype", "source", "hashlookup-parent-total", "snap-authority", "hashlookup:trust"]
106 |
107 | to:
108 | id: SHA-256
109 | group: sha256
110 | search: sha256
111 | attributes: ["FileName", "FileSize", "source-url", "MD5", "SHA-512", "SSDEEP", "TLSH", "insert-timestamp", "mimetype", "source", "hashlookup-parent-total", "snap-authority", "hashlookup:trust"]
112 | ```
113 |
--------------------------------------------------------------------------------
/plugins/src/hashlookup/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to MongoDB query convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "errors"
9 | "github.com/blastrain/vitess-sqlparser/sqlparser"
10 | )
11 |
12 | /*
13 | * Convert SQL statement to the object expected by the data source
14 | */
15 | func (p *plugin) convert(sel *sqlparser.Select) ([][2]string, error) {
16 |
17 | // Handle WHERE.
18 | // Top level node pass in an empty interface
19 | // to tell the children this is root.
20 | // Is there any better way?
21 | var rootParent sqlparser.Expr
22 |
23 | // List of requested fields & values
24 | fields, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
25 | if err != nil {
26 | return nil, err
27 | }
28 |
29 | // Handle GROUP BY
30 | if len(sel.GroupBy) > 0 || checkNeedAgg(sel.SelectExprs) {
31 | return nil, errors.New("'GROUP BY' & aggregation are not supported")
32 | }
33 |
34 | // Set LIMIT
35 | if sel.Limit != nil {
36 | fields = append(fields, [2]string{"limit", sqlparser.String(sel.Limit.Rowcount)})
37 | }
38 |
39 | return fields, nil
40 | }
41 |
--------------------------------------------------------------------------------
/plugins/src/hashlookup/hashlookup_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted [][2]string
21 | }{
22 | {`SELECT * WHERE sha1='DEAC5AEDA66017C25A9E21F36F5EE618D2AD9D3D'`, [][2]string{[2]string{"sha1", "DEAC5AEDA66017C25A9E21F36F5EE618D2AD9D3D"}}},
23 | }
24 |
25 | for _, table := range tables {
26 | // Executed by the main service
27 | ast, err := sqlparser.Parse(table.sql)
28 | if err != nil {
29 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
30 | continue
31 | }
32 |
33 | stmt, ok := ast.(*sqlparser.Select)
34 | if !ok {
35 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
36 | continue
37 | }
38 |
39 | // Executed by the plugin
40 | result, err := c.convert(stmt)
41 | if err != nil {
42 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
43 | continue
44 | }
45 |
46 | if !equal(result, table.converted) {
47 | t.Errorf("Invalid conversion of \"%s\": \"%s\", expected: \"%s\"", table.sql, result, table.converted)
48 | }
49 | }
50 | }
51 |
52 | /*
53 | * Check whether a and b slices contain the same elements
54 | */
55 | func equal(a, b [][2]string) bool {
56 | if len(a) != len(b) {
57 | return false
58 | }
59 |
60 | for i, v := range a {
61 | if v != b[i] {
62 | return false
63 | }
64 | }
65 |
66 | return true
67 | }
68 |
--------------------------------------------------------------------------------
/plugins/src/hashlookup/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "hashlookup"
12 | Version = "1.0.1"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | url string
26 | apiKey string
27 | limit int
28 | }
29 |
--------------------------------------------------------------------------------
/plugins/src/http/README.md:
--------------------------------------------------------------------------------
1 | # HTTP plugin
2 |
3 | HTTP connector sends a GET/POST request and expects a `[{...},{...},{...}]` list of flat JSON objects back.
4 |
5 | Request contains:
6 | - a list of `field=value` in case HTTP API doesn't support SQL queries
7 | - `sql` field with a complete SQL query in case API supports it
8 |
9 | `curl` to test:
10 | ```sh
11 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+service+WHERE+ip=%278.8.8.8%27'
12 | ```
13 |
14 | Compile with:
15 | ```sh
16 | go build -buildmode=plugin -ldflags="-w" -o http.so ./*.go
17 | ```
18 |
19 | # Access details
20 |
21 | Source YAML definition's `access` fields:
22 | - **url**: HTTP access point, for example - `http://localhost:8000`
23 | - **method**: `GET` or `POST`, GET by default
24 |
--------------------------------------------------------------------------------
/plugins/src/http/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to the field/value list convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "errors"
9 |
10 | "github.com/blastrain/vitess-sqlparser/sqlparser"
11 | )
12 |
13 | /*
14 | * Convert SQL query to the list of [field,value]
15 | */
16 | func (p *plugin) convert(sel *sqlparser.Select) ([][2]string, error) {
17 |
18 | // Handle WHERE.
19 | // Top level node pass in an empty interface
20 | // to tell the children this is root.
21 | // Is there any better way?
22 | var rootParent sqlparser.Expr
23 |
24 | // List of requested fields & values
25 | fields, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
26 | if err != nil {
27 | return nil, err
28 | }
29 |
30 | // Handle GROUP BY
31 | if len(sel.GroupBy) > 0 || checkNeedAgg(sel.SelectExprs) {
32 | return nil, errors.New("'GROUP BY' & aggregation are not supported")
33 | }
34 |
35 | // Handle WHERE,
36 | // will be a textual representation of the whole query
37 | query := sqlparser.String(sel.Where.Expr)
38 |
39 | // Handle ORDER BY
40 | if sel.OrderBy != nil {
41 | query += sqlparser.String(sel.OrderBy)
42 | }
43 |
44 | // Set selection OFFSET and ROWCOUNT
45 | if sel.Limit != nil {
46 | if sel.Limit.Offset != nil {
47 | fields = append(fields, [2]string{"offset", sqlparser.String(sel.Limit.Offset)})
48 | query += " LIMIT " + sqlparser.String(sel.Limit.Offset) + "," + sqlparser.String(sel.Limit.Rowcount)
49 | } else {
50 | fields = append(fields, [2]string{"offset", "0"})
51 | query += " LIMIT 0," + sqlparser.String(sel.Limit.Rowcount)
52 | }
53 |
54 | fields = append(fields, [2]string{"rowcount", sqlparser.String(sel.Limit.Rowcount)})
55 | }
56 |
57 | fields = append(fields, [2]string{"sql", query})
58 | return fields, nil
59 | }
60 |
--------------------------------------------------------------------------------
/plugins/src/http/http_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted [][2]string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,1`, [][2]string{
23 | [2]string{"ip", "10.10.10.10"},
24 | [2]string{"offset", "5"},
25 | [2]string{"rowcount", "1"},
26 | [2]string{"sql", "ip = '10.10.10.10' LIMIT 5,1"},
27 | }},
28 | {`SELECT * WHERE size BETWEEN 100 AND 300`, [][2]string{
29 | [2]string{"size_from", "100"},
30 | [2]string{"size_to", "300"},
31 | [2]string{"sql", "size between 100 and 300"},
32 | }},
33 | {`select * where name='sarah' and age=40 limit 1`, [][2]string{
34 | [2]string{"name", "sarah"},
35 | [2]string{"age", "40"},
36 | [2]string{"operator", "and"},
37 | [2]string{"offset", "0"},
38 | [2]string{"rowcount", "1"},
39 | [2]string{"sql", "name = 'sarah' and age = 40 LIMIT 0,1"},
40 | }},
41 | }
42 |
43 | for _, table := range tables {
44 | // Executed by the main service
45 | ast, err := sqlparser.Parse(table.sql)
46 | if err != nil {
47 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
48 | continue
49 | }
50 |
51 | stmt, ok := ast.(*sqlparser.Select)
52 | if !ok {
53 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
54 | continue
55 | }
56 |
57 | // Executed by the plugin
58 | result, err := c.convert(stmt)
59 | if err != nil {
60 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
61 | continue
62 | }
63 |
64 | if !equal(result, table.converted) {
65 | t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted)
66 | }
67 | }
68 | }
69 |
70 | /*
71 | * Check whether a and b slices contain the same elements
72 | */
73 | func equal(a, b [][2]string) bool {
74 | if len(a) != len(b) {
75 | return false
76 | }
77 | for i, v := range a {
78 | if v != b[i] {
79 | return false
80 | }
81 | }
82 | return true
83 | }
84 |
--------------------------------------------------------------------------------
/plugins/src/http/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "http"
12 | Version = "1.0.5"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | url string
26 | method string
27 | limit int
28 | }
29 |
--------------------------------------------------------------------------------
/plugins/src/ipinfo/README.md:
--------------------------------------------------------------------------------
1 | # ipinfo.io plugin
2 |
3 | Connector sends a GET request to the `ipinfo.io` API and expects an JSON response with IP details back.
4 | Request can contain `ip` field only with an IP address to check.
5 |
6 | `curl` to test:
7 | ```sh
8 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+ipinfo+WHERE+ip=%278.8.8.8%27'
9 | ```
10 |
11 | Compile with:
12 | ```sh
13 | go build -buildmode=plugin -ldflags="-w" -o ipinfo.so ./*.go
14 | ```
15 |
16 | # Limitations
17 |
18 | Does not support complex SQL queries and datetime range selection.
19 |
20 |
21 | # Access details
22 |
23 | Source YAML definition's `access` fields:
24 | - **server**: API server, for example - `https://ipinfo.io`
25 | - **token**: User's access token, for extended queries limit
26 |
27 |
28 | # Definition file example
29 |
30 | Replace API token with your own:
31 | ```yaml
32 | name: ipinfo
33 | label: ipinfo.io
34 | icon: database
35 |
36 | plugin: ipinfo
37 | inGlobal: true
38 | includeDatetime: false
39 | supportsSQL: false
40 |
41 | access:
42 | server: https://ipinfo.io
43 | token: xxxxxxxxxxxxxx
44 |
45 | queryFields:
46 | - ip
47 |
48 |
49 | relations:
50 | -
51 | from:
52 | id: ip
53 | group: ip
54 | search: ip
55 | attributes: ["anycast", "city", "loc", "postal", "region"]
56 |
57 | to:
58 | id: hostname
59 | group: domain
60 | search: domain
61 |
62 | -
63 | from:
64 | id: ip
65 | group: ip
66 | search: ip
67 | attributes: ["anycast", "city", "loc", "postal", "region"]
68 |
69 | to:
70 | id: org
71 | group: institution
72 | search: institution
73 | attributes: ["asn", "city", "loc", "postal", "region"]
74 | ```
--------------------------------------------------------------------------------
/plugins/src/ipinfo/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to the field/value list convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL query to the list of [field,value]
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) ([2]string, error) {
15 |
16 | // Handle WHERE.
17 | // Top level node pass in an empty interface
18 | // to tell the children this is root.
19 | // Is there any better way?
20 | var rootParent sqlparser.Expr
21 |
22 | // Requested field & value
23 | field, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
24 | if err != nil {
25 | return [2]string{}, err
26 | }
27 |
28 | return field, nil
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/ipinfo/ipinfo_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted [2]string
21 | }{
22 | {`SELECT * WHERE ip='8.8.8.8'`, [2]string{"ip", "8.8.8.8"}},
23 | }
24 |
25 | for _, table := range tables {
26 | // Executed by the main service
27 | ast, err := sqlparser.Parse(table.sql)
28 | if err != nil {
29 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
30 | continue
31 | }
32 |
33 | stmt, ok := ast.(*sqlparser.Select)
34 | if !ok {
35 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
36 | continue
37 | }
38 |
39 | // Executed by the plugin
40 | result, err := c.convert(stmt)
41 | if err != nil {
42 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
43 | continue
44 | }
45 |
46 | if !equal(result, table.converted) {
47 | t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted)
48 | }
49 | }
50 | }
51 |
52 | /*
53 | * Check whether a and b slices contain the same elements
54 | */
55 | func equal(a, b [2]string) bool {
56 | if a[0] != b[0] || a[1] != b[1] {
57 | return false
58 | }
59 |
60 | return true
61 | }
62 |
--------------------------------------------------------------------------------
/plugins/src/ipinfo/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "ipinfo"
12 | Version = "1.0.1"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | server string
26 | token string
27 | limit int
28 | }
29 |
--------------------------------------------------------------------------------
/plugins/src/ipinfo/select.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "strings"
7 |
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Handle single "field operator value" expression.
13 | *
14 | * Receives:
15 | * expr - SQL expression to process
16 | * topLevel - whether it's a top level expression
17 | * parent - container of the expression
18 | */
19 | func handleSelectWhereComparisonExpr(expr *sqlparser.Expr, topLevel bool, parent *sqlparser.Expr) ([2]string, error) {
20 | comparisonExpr := (*expr).(*sqlparser.ComparisonExpr)
21 | colName, ok := comparisonExpr.Left.(*sqlparser.ColName)
22 |
23 | if !ok {
24 | return [2]string{}, errors.New("Invalid comparison expression, the left must be a column name")
25 | }
26 |
27 | colNameStr := sqlparser.String(colName)
28 | colNameStr = strings.Replace(colNameStr, "`", "", -1)
29 | rightIntf, err := buildComparisonExprRightStr(comparisonExpr.Right)
30 | if err != nil {
31 | return [2]string{}, err
32 | }
33 |
34 | if comparisonExpr.Operator == "=" {
35 | return [2]string{colNameStr, fmt.Sprintf("%s", rightIntf)}, nil
36 | }
37 |
38 | return [2]string{}, errors.New("'=' operator is supported only")
39 | }
40 |
41 | /*
42 | * Handle top level or groups of expressions.
43 | *
44 | * Receives:
45 | * expr - SQL expression to process
46 | * topLevel - whether it's a top level expression
47 | * parent - container of the expression
48 | */
49 | func handleSelectWhereParenExpr(expr *sqlparser.Expr, topLevel bool, parent *sqlparser.Expr) ([2]string, error) {
50 | parentBoolExpr := (*expr).(*sqlparser.ParenExpr)
51 | boolExpr := parentBoolExpr.Expr
52 |
53 | // If parent is the top level, bool must is needed
54 | var isThisTopLevel = false
55 | if topLevel {
56 | isThisTopLevel = true
57 | }
58 |
59 | return handleSelectWhere(&boolExpr, isThisTopLevel, parent)
60 | }
61 |
62 | /*
63 | * Check the right part of the expression
64 | * and return its value of specific type.
65 | *
66 | * Receives SQL expression to process
67 | */
68 | func buildComparisonExprRightStr(expr sqlparser.Expr) (interface{}, error) {
69 | var rightStr string
70 | var err error
71 |
72 | switch expr := expr.(type) {
73 | case *sqlparser.SQLVal:
74 | // Use string value type only
75 | rightStr = sqlparser.String(expr)
76 | rightStr = strings.Trim(rightStr, "'")
77 |
78 | case *sqlparser.BoolVal, sqlparser.BoolVal:
79 | rightStr = sqlparser.String(expr)
80 |
81 | case *sqlparser.GroupConcatExpr:
82 | return nil, errors.New("group_concat not supported")
83 |
84 | case *sqlparser.FuncExpr:
85 | return nil, errors.New("functions are not supported")
86 |
87 | case *sqlparser.ColName:
88 | if sqlparser.String(expr) == "exist" {
89 | return nil, errors.New("'exist' expression currently not supported")
90 | }
91 | return nil, errors.New("Column name on the right side of compare operator is not supported")
92 |
93 | case sqlparser.ValTuple:
94 | rightStr = sqlparser.String(expr)
95 |
96 | default:
97 | return nil, fmt.Errorf("Unexpected SQL expression right part's type: %T", expr)
98 | }
99 |
100 | return rightStr, err
101 | }
102 |
103 | /*
104 | * Handle WHERE statement.
105 | *
106 | * Receives:
107 | * expr - SQL expression to process
108 | * topLevel - whether it's a top level expression
109 | * parent - container of the expression
110 | */
111 | func handleSelectWhere(expr *sqlparser.Expr, topLevel bool, parent *sqlparser.Expr) ([2]string, error) {
112 | if expr == nil {
113 | return [2]string{}, errors.New("SQL expression cannot be nil here")
114 | }
115 |
116 | switch (*expr).(type) {
117 | case *sqlparser.ComparisonExpr:
118 | return handleSelectWhereComparisonExpr(expr, topLevel, parent)
119 |
120 | case *sqlparser.ParenExpr:
121 | return handleSelectWhereParenExpr(expr, topLevel, parent)
122 | }
123 |
124 | return [2]string{}, fmt.Errorf("Unexpected SQL expression type received: %T", *expr)
125 | }
126 |
--------------------------------------------------------------------------------
/plugins/src/misp/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to the field/value list convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL query to the [field, value] or
13 | * [field, value, datetime from, datetime to] if datetime exists
14 | */
15 | func (p *plugin) convert(sel *sqlparser.Select) ([]string, error) {
16 |
17 | // Handle WHERE.
18 | // Top level node pass in an empty interface
19 | // to tell the children this is root.
20 | // Is there any better way?
21 | var rootParent sqlparser.Expr
22 |
23 | // List of requested fields & values
24 | fields, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
25 | if err != nil {
26 | return nil, err
27 | }
28 |
29 | return fields, nil
30 | }
31 |
--------------------------------------------------------------------------------
/plugins/src/misp/misp_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted []string
21 | }{
22 | {
23 | "SELECT * WHERE `domain|ip`='10.10.10.10' LIMIT 5,1",
24 | []string{"domain|ip", "10.10.10.10"},
25 | },
26 | {
27 | `SELECT * WHERE ip='10.10.10.10' and datetime BETWEEN '2024-05-04T11:30:14.000Z' AND '2024-06-04T11:30:14.000Z'`,
28 | []string{"ip", "10.10.10.10", "2024-05-04T11:30:14.000Z", "2024-06-04T11:30:14.000Z"},
29 | },
30 | {
31 | `select * where name='sarah'`,
32 | []string{"name", "sarah"},
33 | },
34 | }
35 |
36 | for _, table := range tables {
37 | // Executed by the main service
38 | ast, err := sqlparser.Parse(table.sql)
39 | if err != nil {
40 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
41 | continue
42 | }
43 |
44 | stmt, ok := ast.(*sqlparser.Select)
45 | if !ok {
46 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
47 | continue
48 | }
49 |
50 | // Executed by the plugin
51 | result, err := c.convert(stmt)
52 | if err != nil {
53 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
54 | continue
55 | }
56 |
57 | if !equal(result, table.converted) {
58 | t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted)
59 | }
60 | }
61 | }
62 |
63 | /*
64 | * Check whether a and b slices contain the same elements
65 | */
66 | func equal(a, b []string) bool {
67 | if len(a) != len(b) {
68 | return false
69 | }
70 |
71 | for i, v := range a {
72 | if v != b[i] {
73 | return false
74 | }
75 | }
76 |
77 | return true
78 | }
79 |
--------------------------------------------------------------------------------
/plugins/src/misp/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "misp"
12 | Version = "1.0.1"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | protocol string
26 | host string
27 | apiKey string
28 | caCertPath string
29 | certPath string
30 | keyPath string
31 | types map[string]bool
32 | limit int
33 | }
34 |
--------------------------------------------------------------------------------
/plugins/src/modify/README.md:
--------------------------------------------------------------------------------
1 | # Modify plugin
2 |
3 | Allows to modify parameters of existing graph elements.
4 |
5 |
6 | Compile with:
7 | ```sh
8 | go build -buildmode=plugin -ldflags="-w" -o modify.so ./*.go
9 | ```
10 |
11 | # Configuration
12 |
13 | YAML definition's `data` fields:
14 | - **group**: as all graph nodes belong to some group/type, it can be used to apply modifications to the specific group only
15 | - **modify**: list of replacements. If **regex** matches node/edge's **field** value - put **replacement** instead
16 |
17 | More info about used function: https://pkg.go.dev/regexp#Regexp.ReplaceAllString,
18 | where:
19 | - `MustCompile` receives regex
20 | - `ReplaceAllString` receives graph field value and replacement
21 |
22 | List of replacements example:
23 | ```yaml
24 | modify:
25 | - field: field_name
26 | regex: a(x*)b
27 | replacement: y
28 |
29 | # Anonymize prices
30 | - field: price
31 | regex: (?i)\d{1,3}(?:[.,]\d{3})*(?:[.,]\d{2}) *(eur|€)
32 | replacement: *** Eur
33 | ```
34 |
--------------------------------------------------------------------------------
/plugins/src/modify/modify.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "regexp"
6 |
7 | "github.com/cert-lv/graphoscope/pdk"
8 | )
9 |
10 | /*
11 | * Check "pdk/plugin.go" for the built-in plugin functions description
12 | */
13 |
14 | func (p *plugin) Conf() *pdk.Processor {
15 | return p.processor
16 | }
17 |
18 | func (p *plugin) Setup(processor *pdk.Processor) error {
19 |
20 | // Store settings
21 | p.processor = processor
22 |
23 | // Convert string regexs into actual regexps
24 | if p.processor.Data["modify"] != nil {
25 | for i, entry := range p.processor.Data["modify"].([]interface{}) {
26 | p.processor.Data["modify"].([]interface{})[i].(map[string]interface{})["regex"] = regexp.MustCompile(entry.(map[string]interface{})["regex"].(string))
27 | }
28 | }
29 |
30 | return nil
31 | }
32 |
33 | func (p *plugin) Process(relations []map[string]interface{}) ([]map[string]interface{}, error) {
34 |
35 | if p.processor.Data["modify"] != nil {
36 | for _, relation := range relations {
37 | for _, m := range p.processor.Data["modify"].([]interface{}) {
38 | for _, part := range []string{"from", "to", "edge"} {
39 | rp := relation[part]
40 |
41 | if rp != nil {
42 | rpt := relation[part].(map[string]interface{})
43 |
44 | if p.processor.Data["group"] != nil && rpt["group"] != p.processor.Data["group"].(string) {
45 | continue
46 | }
47 |
48 | mt := m.(map[string]interface{})
49 |
50 | if mt["field"].(string) == "id" {
51 | rpt["id"] = mt["regex"].(*regexp.Regexp).ReplaceAllString(rpt["id"].(string), fmt.Sprint(mt["replacement"]))
52 | }
53 |
54 | if rpt["attributes"] != nil && rpt["attributes"].(map[string]interface{})[mt["field"].(string)] != nil {
55 | rpt["attributes"].(map[string]interface{})[mt["field"].(string)] = mt["regex"].(*regexp.Regexp).ReplaceAllString(rpt["attributes"].(map[string]interface{})[mt["field"].(string)].(string), fmt.Sprint(mt["replacement"]))
56 | }
57 | }
58 | }
59 | }
60 | }
61 | }
62 |
63 | return relations, nil
64 | }
65 |
66 | func (p *plugin) Stop() error {
67 | return nil
68 | }
69 |
--------------------------------------------------------------------------------
/plugins/src/modify/modify_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/cert-lv/graphoscope/pdk"
7 | )
8 |
9 | /*
10 | * Test attributes modifications
11 | */
12 | func TestProcess(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := &plugin{}
16 |
17 | processor := &pdk.Processor{
18 | Data: map[string]interface{}{
19 | "group": "name",
20 |
21 | "modify": []interface{}{
22 | map[string]interface{}{
23 | "field": "age",
24 | "regex": "\\d*",
25 | "replacement": "***",
26 | },
27 | },
28 | },
29 | }
30 |
31 | err := c.Setup(processor)
32 | if err != nil {
33 | t.Errorf("Can't setup a modify plugin: %s", err.Error())
34 | }
35 |
36 | // Pairs of data source plugins responses and the expected processing results
37 | table := []struct {
38 | entry map[string]interface{}
39 | modified map[string]interface{}
40 | }{
41 | // No modifications should be made, because group is different
42 | {map[string]interface{}{
43 | "from": map[string]interface{}{
44 | "id": "John",
45 | "group": "neighbor",
46 | "search": "name",
47 | "attributes": map[string]interface{}{
48 | "age": "25",
49 | },
50 | },
51 | }, map[string]interface{}{
52 | "from": map[string]interface{}{
53 | "id": "John",
54 | "group": "name",
55 | "search": "name",
56 | "attributes": map[string]interface{}{
57 | "age": "25",
58 | },
59 | },
60 | }},
61 |
62 | // Age should be anonymized
63 | {map[string]interface{}{
64 | "from": map[string]interface{}{
65 | "id": "John",
66 | "group": "name",
67 | "search": "name",
68 | "attributes": map[string]interface{}{
69 | "age": "25",
70 | },
71 | },
72 | }, map[string]interface{}{
73 | "from": map[string]interface{}{
74 | "id": "John",
75 | "group": "name",
76 | "search": "name",
77 | "attributes": map[string]interface{}{
78 | "age": "***",
79 | },
80 | },
81 | }},
82 | }
83 |
84 | for _, row := range table {
85 | result, err := c.Process([]map[string]interface{}{row.entry})
86 | if err != nil {
87 | t.Errorf("Can't process '%s': %s", row.entry, err.Error())
88 | continue
89 | }
90 |
91 | modified := row.modified["from"].(map[string]interface{})
92 | from := result[0]["from"].(map[string]interface{})
93 |
94 | if from["id"] != modified["id"] {
95 | t.Errorf("Invalid modification of ID in \"%v\": \"%v\", expected: \"%v\"",
96 | row.entry["from"], from, modified)
97 | break
98 | }
99 |
100 | for k, v := range from["attributes"].(map[string]interface{}) {
101 | if modified["attributes"].(map[string]interface{})[k] != v {
102 | t.Errorf("Invalid modification of \"%v\": \"%v\", expected: \"%v\"",
103 | row.entry["from"], from, modified)
104 | break
105 | }
106 | }
107 | }
108 | }
109 |
--------------------------------------------------------------------------------
/plugins/src/modify/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "modify"
12 | Version = "1.0.0"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | processor *pdk.Processor
23 | }
24 |
--------------------------------------------------------------------------------
/plugins/src/mongodb/README.md:
--------------------------------------------------------------------------------
1 | # MongoDB plugin
2 |
3 | Plugin to query MongoDB (https://www.mongodb.com) as a data source.
4 |
5 |
6 | Compile with:
7 | ```sh
8 | go build -buildmode=plugin -ldflags="-w" -o mongodb.so ./*.go
9 | ```
10 |
11 | # Access details
12 |
13 | Source YAML definition's `access` fields:
14 | - **addr**: HOST:PORT database's access point, for example - `localhost:27017`
15 | - **db**: database name to use
16 | - **collection**: collection name to query
17 |
18 |
19 | # Get all the possible fields
20 |
21 | As MongoDB doesn't provide a built-in way to get all the possible collection's
22 | fields without requesting all documents plugin will try:
23 | 1. To return a manually filled `queryFields` - useful when some fields rarely appear
24 | 2. To request 1000 documents and get all their unique fields, including nested
25 |
26 |
27 | # Golang driver
28 |
29 | The **mongo-go-driver** contains four object types:
30 |
31 | - **bson.D**: A BSON document. This type should be used in situations where order matters, such as MongoDB commands
32 | - **bson.M**: An unordered map. It is the same as D, except it does not preserve order
33 | - **bson.A**: A BSON array
34 | - **bson.E**: A single element inside a D
35 |
--------------------------------------------------------------------------------
/plugins/src/mongodb/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to MongoDB query convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "errors"
9 | "strconv"
10 | "strings"
11 |
12 | "github.com/blastrain/vitess-sqlparser/sqlparser"
13 | "go.mongodb.org/mongo-driver/bson"
14 | "go.mongodb.org/mongo-driver/mongo/options"
15 | )
16 |
17 | /*
18 | * Convert SQL statement to the MongoDB filter & options
19 | */
20 | func (p *plugin) convert(sel *sqlparser.Select, fields []string) (bson.M, *options.FindOptions, error) {
21 |
22 | // Handle WHERE.
23 | // Top level node pass in an empty interface
24 | // to tell the children this is root.
25 | // Is there any better way?
26 | var rootParent sqlparser.Expr
27 |
28 | // Values to search for
29 | filter, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
30 | if err != nil {
31 | return nil, nil, err
32 | }
33 |
34 | // Handle GROUP BY
35 | if len(sel.GroupBy) > 0 || checkNeedAgg(sel.SelectExprs) {
36 | return nil, nil, errors.New("'GROUP BY' & aggregation are not supported")
37 | }
38 |
39 | // Include required fields only
40 | projection := bson.D{}
41 | for _, field := range fields {
42 | projection = append(projection, bson.E{Key: field, Value: 1})
43 | }
44 |
45 | // Pass these options to the Find method
46 | options := options.Find().SetProjection(projection)
47 |
48 | // Offset & Rowcount validation is done by the core service
49 | if sel.Limit != nil {
50 | // Handle offset
51 | if sel.Limit.Offset != nil {
52 | offset := sqlparser.String(sel.Limit.Offset)
53 | queryFrom, _ := strconv.ParseInt(offset, 10, 64)
54 | options.SetSkip(queryFrom)
55 | } else {
56 | options.SetSkip(0)
57 | }
58 |
59 | // Handle limit
60 | rowcount := sqlparser.String(sel.Limit.Rowcount)
61 | querySize, _ := strconv.ParseInt(rowcount, 10, 64)
62 | options.SetLimit(querySize)
63 | }
64 |
65 | // Handle ORDER BY
66 | var orderByArr bson.D
67 | for _, orderByExpr := range sel.OrderBy {
68 | direction := 1
69 | if orderByExpr.Direction == "desc" {
70 | direction = -1
71 | }
72 |
73 | orderByArr = append(orderByArr, bson.E{Key: strings.Replace(sqlparser.String(orderByExpr.Expr), "`", "", -1), Value: direction})
74 | }
75 |
76 | if len(orderByArr) > 0 {
77 | options.SetSort(orderByArr)
78 | }
79 |
80 | return filter, options, nil
81 | }
82 |
--------------------------------------------------------------------------------
/plugins/src/mongodb/mongodb_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "testing"
6 |
7 | "github.com/blastrain/vitess-sqlparser/sqlparser"
8 | "go.mongodb.org/mongo-driver/bson/primitive"
9 | )
10 |
11 | /*
12 | * Test SQL conversion to the data source's expected format
13 | */
14 | func TestConvert(t *testing.T) {
15 |
16 | // Empty plugin's instance to test
17 | c := plugin{}
18 |
19 | // Pairs of SQLs and the expected results
20 | tables := []struct {
21 | sql string
22 | filter string
23 | sort string
24 | skip int64
25 | limit int64
26 | }{
27 | {`SELECT * WHERE ip='10.10.10.10'`, `primitive.M{"ip":"10.10.10.10"}`, `nil`, 0, 0},
28 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,10`, `primitive.M{"ip":"10.10.10.10"}`, `nil`, 5, 10},
29 | {`SELECT * WHERE size>100 ORDER BY name LIMIT 0,1`, `primitive.M{"size":primitive.M{"$gt":100}}`, `primitive.M{"name":1}`, 0, 1},
30 | {`SELECT * WHERE size=10 ORDER BY name DESC LIMIT 0,1`, `primitive.M{"size":10}`, `primitive.M{"name":-1}`, 0, 1},
31 | {`SELECT * WHERE size>=100`, `primitive.M{"size":primitive.M{"$gte":100}}`, `nil`, 0, 1},
32 | {`SELECT * WHERE name LIKE 's%'`, `primitive.M{"name":primitive.M{"$regex":primitive.Regex{Pattern:"s.*", Options:"i"}}}`, `nil`, 0, 1},
33 | {`SELECT * WHERE name NOT LIKE 's%'`, `primitive.M{"name":primitive.M{"$not":primitive.M{"$regex":primitive.Regex{Pattern:"s.*", Options:"i"}}}}`, `nil`, 0, 1},
34 | {`SELECT * WHERE size BETWEEN 100 AND 300`, `primitive.M{"size":primitive.M{"$gte":100, "$lte":300}}`, `nil`, 0, 1},
35 | {`SELECT * WHERE size NOT BETWEEN 1 AND 10`, `primitive.M{"size":primitive.M{"$gt":10, "$lt":1}}`, `nil`, 0, 1},
36 | {`SELECT * WHERE size IN (100,300)`, `primitive.M{"size":primitive.M{"$in":primitive.A{100, 300}}}`, `nil`, 0, 1},
37 | {`SELECT * WHERE size NOT IN (100,300)`, `primitive.M{"size":primitive.M{"$nin":primitive.A{100, 300}}}`, `nil`, 0, 1},
38 | {`select * where name='sarah' and age!=40 and (country='LV' or country='AU') limit 1`, `primitive.M{"$and":primitive.A{primitive.M{"$and":primitive.A{primitive.M{"name":"sarah"}, primitive.M{"age":primitive.M{"$ne":40}}}}, primitive.M{"$or":primitive.A{primitive.M{"country":"LV"}, primitive.M{"country":"AU"}}}}}`, `nil`, 0, 1},
39 | }
40 |
41 | for _, table := range tables {
42 | // Executed by the main service
43 | ast, err := sqlparser.Parse(table.sql)
44 | if err != nil {
45 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
46 | continue
47 | }
48 |
49 | stmt, ok := ast.(*sqlparser.Select)
50 | if !ok {
51 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
52 | continue
53 | }
54 |
55 | // Executed by the plugin
56 | filter, options, err := c.convert(stmt, nil)
57 | if err != nil {
58 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
59 | continue
60 | }
61 |
62 | if fmt.Sprintf("%#v", filter) != table.filter {
63 | t.Errorf("Invalid converted filter of '%s': %#v, expected: %s", table.sql, filter, table.filter)
64 | }
65 | if options.Sort != nil && fmt.Sprintf("%#v", options.Sort.(primitive.D).Map()) != table.sort {
66 | t.Errorf("Invalid converted order of '%s': %#v, expected: %s", table.sql, options.Sort.(primitive.D).Map(), table.sort)
67 | }
68 |
69 | if options.Limit != nil {
70 | if *options.Limit != table.limit || *options.Skip != table.skip {
71 | t.Errorf("Invalid converted options of '%s': '%v,%v', expected: '%v,%v'", table.sql, *options.Skip, *options.Limit, table.skip, table.limit)
72 | }
73 | }
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/plugins/src/mongodb/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | "go.mongodb.org/mongo-driver/mongo"
6 | )
7 |
8 | /*
9 | * Export symbols
10 | */
11 | var (
12 | Name = "mongodb"
13 | Version = "1.0.6"
14 | Plugin plugin
15 | )
16 |
17 | /*
18 | * Structure to be imported by the core as a plugin
19 | */
20 | type plugin struct {
21 |
22 | // Inherit default configuration fields
23 | source *pdk.Source
24 |
25 | // Custom fields
26 | client *mongo.Client
27 | collection *mongo.Collection
28 | limit int
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/mysql/README.md:
--------------------------------------------------------------------------------
1 | # MySQL plugin
2 |
3 | Plugin to query MySQL (https://www.mysql.com/) as a data source.
4 |
5 |
6 | Compile with:
7 | ```sh
8 | go build -buildmode=plugin -ldflags="-w" -o mysql.so ./*.go
9 | ```
10 |
11 | **Warning**
12 |
13 | SQL doesn't allow to query missing columns, like Elasticsearch does.
14 | An error `column "X" does not exist` will be received. That means you must be
15 | very careful with designing a data source and creating a YAML config file to be
16 | able to combine it with data source types other than SQL.
17 |
18 | The easiest solution is to exclude MySQL DB from the `global` namespace
19 | and query it independently, to make sure all columns exist.
20 |
21 |
22 | # Access details
23 |
24 | Source YAML definition's `access` fields:
25 | - **addr**: HOST:PORT database's access point, for example - `localhost:3306`
26 | - **user**: username to connect to the database
27 | - **password**: user's password
28 | - **db**: database name to use
29 | - **table**: table name to query
30 |
31 |
32 | # Usage
33 |
34 | Simple example of a new MySQL data source:
35 | ```sql
36 | mysql -u root -p
37 |
38 | CREATE DATABASE mydb;
39 | CREATE USER 'graphoscope'@'%' IDENTIFIED BY 'password';
40 | GRANT SELECT ON mydb.* TO 'graphoscope'@'%';
41 | FLUSH PRIVILEGES;
42 | USE mydb;
43 | CREATE TABLE mycoll (id SERIAL PRIMARY KEY, email VARCHAR(255) NOT NULL, username VARCHAR(255) NOT NULL, fqdn VARCHAR(255) NOT NULL, count integer NOT NULL, seen TIMESTAMP);
44 | INSERT INTO mycoll (email, username, fqdn, count, seen) VALUES ('a@example.com', 'a', 'example.com', 13, now());
45 | INSERT INTO mycoll (email, username, fqdn, count, seen) VALUES ('b@example.com', 'b', 'example.com', 13, now());
46 | INSERT INTO mycoll (email, username, fqdn, count, seen) VALUES ('c@example.com', 'c', 'example.com', 13, now());
47 | INSERT INTO mycoll (email, username, fqdn, count, seen) VALUES ('d@example.com', 'd', 'example.com', 13, now());
48 | INSERT INTO mycoll (email, username, fqdn, count, seen) VALUES ('e@example.com', 'e', 'example.com', 13, now());
49 | ```
50 |
51 | Access data will be used by the source's YAML definition. Example:
52 | ```yaml
53 | name: mytest
54 | label: MYTest
55 | icon: database
56 |
57 | plugin: mysql
58 | inGlobal: false
59 | includeDatetime: false
60 | supportsSQL: true
61 |
62 | access:
63 | addr: 127.0.0.1:3306
64 | user: graphoscope
65 | password: password
66 | db: mydb
67 | table: mycoll
68 |
69 | statsFields:
70 | - domain
71 |
72 | replaceFields:
73 | datetime: seen
74 | domain: fqdn
75 |
76 |
77 | relations:
78 | -
79 | from:
80 | id: email
81 | group: email
82 | search: email
83 | attributes: [ "seen", "fqdn" ]
84 |
85 | to:
86 | id: username
87 | group: domain
88 | search: username
89 |
90 | edge:
91 | attributes: [ "count" ]
92 | ```
93 |
94 | Test with a query:
95 | ```sh
96 | curl -XGET 'https://localhost:443/api?uuid=auth-key&sql=FROM+mytest+WHERE+email+like+%27a%25%27'
97 | ```
98 |
--------------------------------------------------------------------------------
/plugins/src/mysql/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to MySQL query convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL statement to the MySQL query
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) (string, error) {
15 |
16 | // Handle WHERE
17 | query := sqlparser.String(sel.Where.Expr)
18 |
19 | // Handle GROUP BY
20 | if len(sel.GroupBy) > 0 {
21 | query += sqlparser.String(sel.GroupBy)
22 | }
23 |
24 | // Handle ORDER BY
25 | if sel.OrderBy != nil {
26 | query += sqlparser.String(sel.OrderBy)
27 | }
28 |
29 | // Handle LIMIT offset,rowcount
30 | if sel.Limit != nil {
31 | if sel.Limit.Offset != nil {
32 | query += " LIMIT " + sqlparser.String(sel.Limit.Offset) + "," + sqlparser.String(sel.Limit.Rowcount)
33 | } else {
34 | query += " LIMIT 0," + sqlparser.String(sel.Limit.Rowcount)
35 | }
36 | }
37 |
38 | return query, nil
39 | }
40 |
--------------------------------------------------------------------------------
/plugins/src/mysql/mysql_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10'`, `ip = '10.10.10.10'`},
23 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,10`, `ip = '10.10.10.10' LIMIT 5,10`},
24 | {`SELECT * WHERE size>100 ORDER BY name LIMIT 0,1`, `size > 100 order by name asc LIMIT 0,1`},
25 | {`SELECT * WHERE size=10 ORDER BY name DESC LIMIT 0,1`, `size = 10 order by name desc LIMIT 0,1`},
26 | {`SELECT * WHERE size>=100`, `size >= 100`},
27 | {`SELECT * WHERE name LIKE 's%'`, `name like 's%'`},
28 | {`SELECT * WHERE name NOT LIKE 's%'`, `name not like 's%'`},
29 | {`SELECT * WHERE size BETWEEN 100 AND 300`, `size between 100 and 300`},
30 | {`SELECT * WHERE size IN (100,300)`, `size in (100, 300)`},
31 | {`SELECT * WHERE size NOT IN (100,300)`, `size not in (100, 300)`},
32 | {`SELECT * WHERE name='sarah' and age!=40 AND (country='LV' OR country='AU') order by age desc limit 1`,
33 | `name = 'sarah' and age != 40 and (country = 'LV' or country = 'AU') order by age desc LIMIT 0,1`},
34 | }
35 |
36 | for _, table := range tables {
37 | // Executed by the main service
38 | ast, err := sqlparser.Parse(table.sql)
39 | if err != nil {
40 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
41 | continue
42 | }
43 |
44 | stmt, ok := ast.(*sqlparser.Select)
45 | if !ok {
46 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
47 | continue
48 | }
49 |
50 | // Executed by the plugin
51 | result, err := c.convert(stmt)
52 | if err != nil {
53 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
54 | continue
55 | }
56 |
57 | if result != table.converted {
58 | t.Errorf("Invalid conversion of \"%s\": \"%s\", expected: \"%s\"", table.sql, result, table.converted)
59 | }
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/plugins/src/mysql/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "database/sql"
5 |
6 | "github.com/cert-lv/graphoscope/pdk"
7 | )
8 |
9 | /*
10 | * Export symbols
11 | */
12 | var (
13 | Name = "mysql"
14 | Version = "1.0.5"
15 | Plugin plugin
16 | )
17 |
18 | /*
19 | * Structure to be imported by the core as a plugin
20 | */
21 | type plugin struct {
22 |
23 | // Inherit default configuration fields
24 | source *pdk.Source
25 |
26 | // Custom fields
27 | db *sql.DB
28 | limit int
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/pastelyzer/README.md:
--------------------------------------------------------------------------------
1 | # Pastelyzer plugin
2 |
3 | Plugin to query Pastelyzer (https://github.com/cert-lv/pastelyzer) as a data source.
4 |
5 | Sample command to use plugin:
6 | ```sh
7 | # Get paste IDs where IP 8.8.8.8 was mentioned
8 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+pastelyzer+WHERE+ip=%278.8.8.8%27'
9 | # Get all artefacts of the given paste ID
10 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+pastelyzer+WHERE+source=35853628'
11 | ```
12 |
13 | As there is no way to get automatically all the possible fields to query (for the Web GUI autocomplete) - such artefacts are:
14 | - cc-number
15 | - credential
16 | - domain
17 | - email
18 | - ip
19 | - onion
20 | - sha1
21 | - any
22 |
23 | Compile with:
24 | ```sh
25 | go build -buildmode=plugin -ldflags="-w" -o pastelyzer.so ./*.go
26 | ```
27 |
28 | # Limitations
29 |
30 | Does not support complex SQL queries and datetime range selection.
31 |
32 |
33 | # Access details
34 |
35 | Source YAML definition's `access` fields:
36 | - **url**: HTTP access point, for example - `http://localhost:7000`
37 |
38 |
39 | # Definition file example
40 |
41 | ```yaml
42 | name: pastelyzer
43 | label: Pastelyzer
44 | icon: copy outline
45 |
46 | plugin: pastelyzer
47 | inGlobal: true
48 | includeDatetime: false
49 | supportsSQL: false
50 |
51 | access:
52 | url: http://127.0.0.1:7000
53 |
54 | queryFields:
55 | - source
56 | - cc-number
57 | - credential
58 | - domain
59 | - email
60 | - ip
61 | - onion
62 | - sha1
63 | - any
64 |
65 | statsFields:
66 | - ip
67 | - domain
68 | - type
69 |
70 |
71 | relations:
72 | -
73 | from:
74 | id: domain
75 | group: domain
76 | search: domain
77 |
78 | to:
79 | id: source
80 | group: paste
81 | search: source
82 |
83 | edge:
84 | label: was published
85 |
86 | -
87 | from:
88 | id: ip
89 | group: ip
90 | search: ip
91 |
92 | to:
93 | id: source
94 | group: paste
95 | search: source
96 |
97 | edge:
98 | label: was published
99 |
100 | -
101 | from:
102 | id: address
103 | group: ip
104 | search: ip
105 |
106 | to:
107 | id: source
108 | group: paste
109 | search: source
110 |
111 | edge:
112 | label: was published
113 |
114 | -
115 | from:
116 | id: cc-number
117 | group: cc-number
118 | search: cc-number
119 |
120 | to:
121 | id: source
122 | group: paste
123 | search: source
124 |
125 | edge:
126 | label: was published
127 |
128 | -
129 | from:
130 | id: credential
131 | group: credentials
132 | search: credential
133 |
134 | to:
135 | id: source
136 | group: paste
137 | search: source
138 |
139 | edge:
140 | label: was published
141 |
142 | -
143 | from:
144 | id: email
145 | group: email
146 | search: email
147 |
148 | to:
149 | id: source
150 | group: paste
151 | search: source
152 |
153 | edge:
154 | label: was published
155 |
156 | -
157 | from:
158 | id: onion
159 | group: onion
160 | search: onion
161 |
162 | to:
163 | id: source
164 | group: paste
165 | search: source
166 |
167 | edge:
168 | label: was published
169 |
170 | -
171 | from:
172 | id: sha1
173 | group: sha1
174 | search: sha1
175 |
176 | to:
177 | id: source
178 | group: paste
179 | search: source
180 |
181 | edge:
182 | label: was published
183 | ```
--------------------------------------------------------------------------------
/plugins/src/pastelyzer/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to Pastelyzer query convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "errors"
9 |
10 | "github.com/blastrain/vitess-sqlparser/sqlparser"
11 | )
12 |
13 | /*
14 | * Convert SQL query to the Pastelyzer query
15 | */
16 | func (p *plugin) convert(sel *sqlparser.Select) ([][2]string, error) {
17 |
18 | // Handle WHERE.
19 | // Top level node pass in an empty interface
20 | // to tell the children this is root.
21 | // Is there any better way?
22 | var rootParent sqlparser.Expr
23 |
24 | // List of requested fields & values
25 | fields, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
26 | if err != nil {
27 | return nil, err
28 | }
29 |
30 | // Handle GROUP BY
31 | if len(sel.GroupBy) > 0 || checkNeedAgg(sel.SelectExprs) {
32 | return nil, errors.New("'GROUP BY' & aggregation are not supported")
33 | }
34 |
35 | // Set LIMIT
36 | if sel.Limit != nil {
37 | fields = append(fields, [2]string{"limit", sqlparser.String(sel.Limit.Rowcount)})
38 | }
39 |
40 | return fields, nil
41 | }
42 |
--------------------------------------------------------------------------------
/plugins/src/pastelyzer/pastelyzer_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted [][2]string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10'`, [][2]string{[2]string{"ip", "10.10.10.10"}}},
23 | {`select * where name='sarah' and age=40 limit 0,1`, [][2]string{[2]string{"name", "sarah"}, [2]string{"age", "40"}, [2]string{"limit", "1"}}},
24 | }
25 |
26 | for _, table := range tables {
27 | // Executed by the main service
28 | ast, err := sqlparser.Parse(table.sql)
29 | if err != nil {
30 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
31 | continue
32 | }
33 |
34 | stmt, ok := ast.(*sqlparser.Select)
35 | if !ok {
36 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
37 | continue
38 | }
39 |
40 | // Executed by the plugin
41 | result, err := c.convert(stmt)
42 | if err != nil {
43 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
44 | continue
45 | }
46 |
47 | if !equal(result, table.converted) {
48 | t.Errorf("Invalid conversion of \"%s\": \"%s\", expected: \"%s\"", table.sql, result, table.converted)
49 | }
50 | }
51 | }
52 |
53 | /*
54 | * Check whether a and b slices contain the same elements
55 | */
56 | func equal(a, b [][2]string) bool {
57 | if len(a) != len(b) {
58 | return false
59 | }
60 | for i, v := range a {
61 | if v != b[i] {
62 | return false
63 | }
64 | }
65 | return true
66 | }
67 |
--------------------------------------------------------------------------------
/plugins/src/pastelyzer/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "pastelyzer"
12 | Version = "1.0.4"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | url string
26 | limit int
27 | }
28 |
--------------------------------------------------------------------------------
/plugins/src/phishtank/README.md:
--------------------------------------------------------------------------------
1 | # Phishtank plugin
2 |
3 | Connector sends a GET request to the `phishtank.org` API and expects an XML response back.
4 | Request can contain `url` field only with a URL to check.
5 |
6 | `curl` to test:
7 | ```sh
8 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+service+WHERE+url=%27http%3A%2F%2Fexample.com%27'
9 | ```
10 |
11 | Compile with:
12 | ```sh
13 | go build -buildmode=plugin -ldflags="-w" -o phishtank.so ./*.go
14 | ```
15 |
16 | # Limitations
17 |
18 | Does not support complex SQL queries and datetime range selection.
19 |
20 |
21 | # Access details
22 |
23 | Source YAML definition's `access` fields:
24 | - **url**: API access point, for example - `https://checkurl.phishtank.com/checkurl/index.php`
25 | - **agent**: User-Agent to use
26 |
27 |
28 | # Definition file example
29 |
30 | ```yaml
31 | name: phishtank
32 | label: Phishtank
33 | icon: database
34 |
35 | plugin: phishtank
36 | inGlobal: true
37 | includeDatetime: false
38 | supportsSQL: false
39 |
40 | access:
41 | url: https://checkurl.phishtank.com/checkurl/index.php
42 | agent: phishtank/graphoscope
43 |
44 | queryFields:
45 | - url
46 | - domain
47 |
48 |
49 | relations:
50 | -
51 | from:
52 | id: url
53 | group: url
54 | search: url
55 | attributes: ["in_database", "phish_id", "phish_detail_page", "verified", "verified_at", "valid"]
56 |
57 | to:
58 | id: domain
59 | group: domain
60 | search: domain
61 | ```
--------------------------------------------------------------------------------
/plugins/src/phishtank/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to the field/value list convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "net/url"
9 | "strings"
10 |
11 | "github.com/blastrain/vitess-sqlparser/sqlparser"
12 | )
13 |
14 | /*
15 | * Convert SQL query to the list of [field,value]
16 | */
17 | func (p *plugin) convert(sel *sqlparser.Select) ([][2]string, error) {
18 |
19 | // Handle WHERE.
20 | // Top level node pass in an empty interface
21 | // to tell the children this is root.
22 | // Is there any better way?
23 | var rootParent sqlparser.Expr
24 |
25 | // Requested field & value
26 | field, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
27 | if err != nil {
28 | return nil, err
29 | }
30 |
31 | var fields [][2]string
32 |
33 | urlParsed, err := url.Parse(field[1])
34 | if err != nil {
35 | return nil, err
36 | }
37 |
38 | // Phishtank may contain "/" at the end of a domain and may not contain,
39 | // but it is very important for the search, so we include both variants
40 | if field[0] == "url" {
41 | fields = append(fields, [2]string{field[0], field[1]})
42 |
43 | if urlParsed.Path == "" {
44 | fields = append(fields, [2]string{field[0], field[1] + "/"})
45 | } else if urlParsed.Path == "/" {
46 | fields = append(fields, [2]string{field[0], strings.TrimRight(field[1], "/")})
47 | }
48 | }
49 |
50 | // Allow to search for a domain from Web GUI
51 | if field[0] == "domain" {
52 | fields = append(fields, [2]string{"url", "https://" + field[1] + "/"})
53 | fields = append(fields, [2]string{"url", "https://" + field[1]})
54 | fields = append(fields, [2]string{"url", "http://" + field[1] + "/"})
55 | fields = append(fields, [2]string{"url", "http://" + field[1]})
56 | }
57 |
58 | return fields, nil
59 | }
60 |
--------------------------------------------------------------------------------
/plugins/src/phishtank/phishtank_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted [][2]string
21 | }{
22 | {`SELECT * WHERE url='http://example.com'`, [][2]string{
23 | [2]string{"url", "http://example.com"},
24 | [2]string{"url", "http://example.com/"},
25 | }},
26 | {`SELECT * WHERE domain='example.com'`, [][2]string{
27 | [2]string{"url", "https://example.com/"},
28 | [2]string{"url", "https://example.com"},
29 | [2]string{"url", "http://example.com/"},
30 | [2]string{"url", "http://example.com"},
31 | }},
32 | }
33 |
34 | for _, table := range tables {
35 | // Executed by the main service
36 | ast, err := sqlparser.Parse(table.sql)
37 | if err != nil {
38 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
39 | continue
40 | }
41 |
42 | stmt, ok := ast.(*sqlparser.Select)
43 | if !ok {
44 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
45 | continue
46 | }
47 |
48 | // Executed by the plugin
49 | result, err := c.convert(stmt)
50 | if err != nil {
51 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
52 | continue
53 | }
54 |
55 | if !equal(result, table.converted) {
56 | t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted)
57 | }
58 | }
59 | }
60 |
61 | /*
62 | * Check whether a and b slices contain the same elements
63 | */
64 | func equal(a, b [][2]string) bool {
65 | if len(a) != len(b) {
66 | return false
67 | }
68 | for i, v := range a {
69 | if v != b[i] {
70 | return false
71 | }
72 | }
73 | return true
74 | }
75 |
--------------------------------------------------------------------------------
/plugins/src/phishtank/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "phishtank"
12 | Version = "1.0.1"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | url string
26 | agent string
27 | limit int
28 | }
29 |
--------------------------------------------------------------------------------
/plugins/src/postgresql/README.md:
--------------------------------------------------------------------------------
1 | # PostgreSQL plugin
2 |
3 | Plugin to query PostgreSQL (https://www.postgresql.org/) as a data source.
4 |
5 |
6 | Compile with:
7 | ```sh
8 | go build -buildmode=plugin -ldflags="-w" -o postgresql.so ./*.go
9 | ```
10 |
11 | **Warning**
12 |
13 | SQL doesn't allow to query missing columns, like Elasticsearch does.
14 | An error `column "X" does not exist` will be received.
15 | That means you must be very careful with designing a data source
16 | and creating a YAML config file to be able to
17 | combine it with data source types other than SQL.
18 |
19 | The easiest solution is to exclude PostgreSQL DB from the `global` namespace
20 | and query it independently, to make sure all columns exist.
21 |
22 |
23 | # Access details
24 |
25 | Source YAML definition's `access` fields:
26 | - **addr**: HOST:PORT database's access point, for example - `localhost:5432`
27 | - **user**: username to connect to the database
28 | - **password**: user's password
29 | - **db**: database name to use
30 | - **table**: table name to query
31 |
32 |
33 | # Demo
34 |
35 | Simple example of a new PostgreSQL data source:
36 | ```sql
37 | sudo -u postgres psql
38 |
39 | CREATE DATABASE pgdb;
40 | CREATE USER graphoscope WITH ENCRYPTED PASSWORD 'password';
41 | GRANT ALL PRIVILEGES ON DATABASE pgdb TO graphoscope;
42 | \connect pgdb
43 | CREATE TABLE pgcoll (id SERIAL PRIMARY KEY, email VARCHAR(255) NOT NULL, username VARCHAR(255) NOT NULL, fqdn VARCHAR(255) NOT NULL, count integer NOT NULL, seen TIMESTAMP);
44 | GRANT ALL PRIVILEGES ON TABLE pgcoll TO graphoscope;
45 |
46 | INSERT INTO pgcoll (email, username, fqdn, count, seen) VALUES ('a@example.com', 'a', 'example.com', 13, now());
47 | INSERT INTO pgcoll (email, username, fqdn, count, seen) VALUES ('b@example.com', 'b', 'example.com', 13, now());
48 | INSERT INTO pgcoll (email, username, fqdn, count, seen) VALUES ('c@example.com', 'c', 'example.com', 13, now());
49 | INSERT INTO pgcoll (email, username, fqdn, count, seen) VALUES ('d@example.com', 'd', 'example.com', 13, now());
50 | INSERT INTO pgcoll (email, username, fqdn, count, seen) VALUES ('e@example.com', 'e', 'example.com', 13, now());
51 | ```
52 |
53 | Access data will be used by the YAML configs. Example:
54 | ```yaml
55 | name: pgtest
56 | label: PGTest
57 | icon: database
58 |
59 | plugin: postgresql
60 | inGlobal: true
61 | includeDatetime: false
62 | supportsSQL: true
63 |
64 | access:
65 | addr: 127.0.0.1:5432
66 | db: pgdb
67 | table: pgcoll
68 | user: graphoscope
69 | password: password
70 |
71 | statsFields:
72 | - domain
73 |
74 | replaceFields:
75 | datetime: seen
76 | domain: fqdn
77 |
78 |
79 | relations:
80 | -
81 | from:
82 | id: email
83 | group: email
84 | search: email
85 | attributes: [ "username", "fqdn" ]
86 |
87 | to:
88 | id: fqdn
89 | group: domain
90 | search: domain
91 |
92 | edge:
93 | attributes: [ "count" ]
94 | ```
95 |
96 | Test with a query:
97 | ```sh
98 | curl -XGET 'https://localhost:443/api?uuid=auth-key&sql=FROM+pgtest+WHERE+email+like+%27a%25%27'
99 | ```
100 |
101 | # TODO
102 |
103 | - [ ] Check `TODO` in `convert.go`
104 |
--------------------------------------------------------------------------------
/plugins/src/postgresql/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to PostgreSQL query convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL statement to the PostgreSQL filter & options
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) (string, error) {
15 |
16 | // Handle WHERE
17 | query := sqlparser.String(sel.Where.Expr)
18 |
19 | // Handle GROUP BY
20 | if len(sel.GroupBy) > 0 {
21 | query += sqlparser.String(sel.GroupBy)
22 | }
23 |
24 | // Handle ORDER BY
25 | if sel.OrderBy != nil {
26 | query += sqlparser.String(sel.OrderBy)
27 | }
28 |
29 | // Handle LIMIT
30 | if sel.Limit != nil {
31 | // Handle offset
32 | if sel.Limit.Offset != nil {
33 | query += " OFFSET " + sqlparser.String(sel.Limit.Offset)
34 | } else {
35 | query += " OFFSET 0"
36 | }
37 |
38 | // Handle rowcount
39 | query += " LIMIT " + sqlparser.String(sel.Limit.Rowcount)
40 | }
41 |
42 | // Replace ` chars around keywords as PostgreSQL requires.
43 | //
44 | // TODO: Find cases when SQL parser produces backtiks
45 | // and replace them in a correct way. Simple "strings.Replace" will replace
46 | // expected and valid chars in the middle of the string too.
47 | //
48 | //query = strings.Replace(query, "`", "\"", -1)
49 |
50 | return query, nil
51 | }
52 |
--------------------------------------------------------------------------------
/plugins/src/postgresql/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | "github.com/jackc/pgx/v4/pgxpool"
6 | )
7 |
8 | /*
9 | * Export symbols
10 | */
11 | var (
12 | Name = "postgresql"
13 | Version = "1.0.5"
14 | Plugin plugin
15 | )
16 |
17 | /*
18 | * Structure to be imported by the core as a plugin
19 | */
20 | type plugin struct {
21 |
22 | // Inherit default configuration fields
23 | source *pdk.Source
24 |
25 | // Custom fields
26 | connection *pgxpool.Pool
27 | limit int
28 | }
29 |
--------------------------------------------------------------------------------
/plugins/src/postgresql/postgresql_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10'`, `ip = '10.10.10.10'`},
23 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,10`, `ip = '10.10.10.10' OFFSET 5 LIMIT 10`},
24 | {`SELECT * WHERE size>100 ORDER BY name LIMIT 0,1`, `size > 100 order by name asc OFFSET 0 LIMIT 1`},
25 | {`SELECT * WHERE size=10 ORDER BY name DESC LIMIT 0,1`, `size = 10 order by name desc OFFSET 0 LIMIT 1`},
26 | {`SELECT * WHERE size>=100`, `size >= 100`},
27 | {`SELECT * WHERE name LIKE 's%'`, `name like 's%'`},
28 | {`SELECT * WHERE name NOT LIKE 's%'`, `name not like 's%'`},
29 | {`SELECT * WHERE size BETWEEN 100 AND 300`, `size between 100 and 300`},
30 | {`SELECT * WHERE size IN (100,300)`, `size in (100, 300)`},
31 | {`SELECT * WHERE size NOT IN (100,300)`, `size not in (100, 300)`},
32 | {`SELECT * WHERE name='sarah' and age!=40 AND (country='LV' OR country='AU') ORDER BY age DESC limit 1`,
33 | `name = 'sarah' and age != 40 and (country = 'LV' or country = 'AU') order by age desc OFFSET 0 LIMIT 1`},
34 | }
35 |
36 | for _, table := range tables {
37 | // Executed by the main service
38 | ast, err := sqlparser.Parse(table.sql)
39 | if err != nil {
40 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
41 | continue
42 | }
43 |
44 | stmt, ok := ast.(*sqlparser.Select)
45 | if !ok {
46 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
47 | continue
48 | }
49 |
50 | // Executed by the plugin
51 | result, err := c.convert(stmt)
52 | if err != nil {
53 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
54 | continue
55 | }
56 |
57 | if result != table.converted {
58 | t.Errorf("Invalid conversion of \"%s\": \"%s\", expected: \"%s\"", table.sql, result, table.converted)
59 | }
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/plugins/src/redis/README.md:
--------------------------------------------------------------------------------
1 | # Redis plugin
2 |
3 | Plugin to query Redis (https://redis.io) as a data source.
4 |
5 |
6 | Compile with:
7 | ```sh
8 | go build -buildmode=plugin -ldflags="-w" -o redis.so ./*.go
9 | ```
10 |
11 | **Warning**
12 |
13 | Redis does NOT accept complex queries, like SQL databases do.
14 |
15 | The easiest workaround is to exclude Redis DB from the `global` namespace
16 | and query it independently, to execute needed queries one by one.
17 |
18 |
19 | # Limitations
20 |
21 | Does not support complex SQL queries and datetime range selection.
22 |
23 |
24 | # Access details
25 |
26 | Source YAML definition's `access` fields:
27 | - **addr**: HOST:PORT database's access point, for example - `localhost:6379`
28 | - **user**: username to connect to the database
29 | - **password**: user's password
30 | - **db**: database number to use
31 | - **field**: Redis key will be used as this field name
32 |
33 |
34 | # Usage
35 |
36 | Simple example of a new Redis data source. Insert test data:
37 | ```sh
38 | redis-cli -u redis://localhost:6379/8
39 | ACL SETUSER graphoscope on >password allkeys +hset +hget +hgetall +select +ping
40 | ACL SAVE # Or 'CONFIG REWRITE'
41 | AUTH graphoscope password
42 |
43 | HSET 'a@example.com' username 'a' fqdn 'example.com' count 13 seen '18-02-2023T15:34:00.000000Z'
44 | HSET 'b@example.com' username 'b' fqdn 'example.com' count 13 seen '19-02-2023T15:34:00.000000Z'
45 | HSET 'c@example.com' username 'c' fqdn 'example.com' count 13 seen '20-02-2023T15:34:00.000000Z'
46 | HSET 'd@example.com' username 'd' fqdn 'example.com' count 13 seen '21-02-2023T15:34:00.000000Z'
47 | HSET 'e@example.com' username 'e' fqdn 'example.com' count 13 seen '22-02-2023T15:34:00.000000Z'
48 | ```
49 |
50 | Access data will be used by the source's YAML definition. Example:
51 | ```yaml
52 | name: retest
53 | label: RETest
54 | icon: database
55 |
56 | plugin: redis
57 | inGlobal: false
58 | includeDatetime: false
59 | supportsSQL: false
60 |
61 | access:
62 | addr: 127.0.0.1:6379
63 | user: graphoscope
64 | password: password
65 | db: 8
66 | field: email
67 |
68 | queryFields:
69 | - email
70 |
71 | replaceFields:
72 | datetime: seen
73 | domain: fqdn
74 |
75 |
76 | relations:
77 | -
78 | from:
79 | id: email
80 | group: email
81 | search: email
82 | attributes: [ "seen", "fqdn" ]
83 |
84 | to:
85 | id: username
86 | group: username
87 | search: username
88 |
89 | edge:
90 | attributes: [ "count" ]
91 | ```
92 |
93 | Test with a query:
94 | ```sh
95 | curl -XGET 'https://localhost:443/api?uuid=auth-key&sql=FROM+retest+WHERE+email=%27a@example.com%27'
96 | ```
97 |
--------------------------------------------------------------------------------
/plugins/src/redis/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to the field/value list convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL query to the list of [field,value]
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) (string, error) {
15 |
16 | // Handle WHERE.
17 | // Top level node pass in an empty interface
18 | // to tell the children this is root.
19 | // Is there any better way?
20 | var rootParent sqlparser.Expr
21 |
22 | // List of requested fields & values
23 | fields, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
24 | if err != nil {
25 | return "", err
26 | }
27 |
28 | if len(fields) == 1 && fields[0][0] == p.source.Access["field"] {
29 | return fields[0][1], nil
30 | }
31 |
32 | return "", nil
33 | }
34 |
--------------------------------------------------------------------------------
/plugins/src/redis/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | "github.com/redis/go-redis/v9"
6 | )
7 |
8 | /*
9 | * Export symbols
10 | */
11 | var (
12 | Name = "redis"
13 | Version = "1.0.0"
14 | Plugin plugin
15 | )
16 |
17 | /*
18 | * Structure to be imported by the core as a plugin
19 | */
20 | type plugin struct {
21 |
22 | // Inherit default configuration fields
23 | source *pdk.Source
24 |
25 | // Custom fields
26 | client *redis.Client
27 | limit int
28 | }
29 |
--------------------------------------------------------------------------------
/plugins/src/redis/redis_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "testing"
6 |
7 | "github.com/blastrain/vitess-sqlparser/sqlparser"
8 | "github.com/cert-lv/graphoscope/pdk"
9 | )
10 |
11 | /*
12 | * Test SQL conversion to the data source's expected format
13 | */
14 | func TestConvert(t *testing.T) {
15 |
16 | // Empty plugin's instance to test
17 | p := plugin{}
18 |
19 | p.source = &pdk.Source{
20 | Access: map[string]string{
21 | "field": "email",
22 | },
23 | }
24 |
25 | // Pairs of SQLs and the expected results
26 | tables := []struct {
27 | sql string
28 | converted string
29 | }{
30 | {`SELECT * WHERE email='a@example.com'`, "a@example.com"},
31 | {`SELECT * WHERE user='a@example.com'`, ""},
32 | }
33 |
34 | for _, table := range tables {
35 | // Executed by the main service
36 | ast, err := sqlparser.Parse(table.sql)
37 | if err != nil {
38 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
39 | continue
40 | }
41 |
42 | stmt, ok := ast.(*sqlparser.Select)
43 | if !ok {
44 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
45 | continue
46 | }
47 |
48 | // Executed by the plugin
49 | fmt.Println(1, p.source)
50 | result, err := p.convert(stmt)
51 | if err != nil {
52 | t.Errorf("Can't convert \"%s\": %s", table.sql, err.Error())
53 | continue
54 | }
55 |
56 | if result != table.converted {
57 | t.Errorf("Invalid conversion of \"%s\": %v, expected: %v", table.sql, result, table.converted)
58 | }
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/plugins/src/rest/README.md:
--------------------------------------------------------------------------------
1 | # REST API plugin
2 |
3 | Connector sends a GET request and expects a list of flat JSON objects back, one line - one JSON.
4 | To the preconfigured REST API URL `field/value` will be attached as query.
5 |
6 | `curl` to test:
7 | ```sh
8 | curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+service+WHERE+ip=%278.8.8.8%27'
9 | ```
10 |
11 | Compile with:
12 | ```sh
13 | go build -buildmode=plugin -ldflags="-w" -o rest.so ./*.go
14 | ```
15 |
16 | # Access details
17 |
18 | Source YAML definition's `access` fields:
19 | - **url**: REST API access point, for example - `http://localhost:8000/RESTv1`
20 | - **username**: Username if exists
21 | - **password**: User's password if exists
22 |
--------------------------------------------------------------------------------
/plugins/src/rest/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to the field/value list convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL query to the list of [field,value]
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) ([2]string, error) {
15 |
16 | // Handle WHERE.
17 | // Top level node pass in an empty interface
18 | // to tell the children this is root.
19 | // Is there any better way?
20 | var rootParent sqlparser.Expr
21 |
22 | // List of requested fields & values
23 | fields, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
24 | if err != nil {
25 | return [2]string{}, err
26 | }
27 |
28 | return fields, nil
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/rest/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "rest"
12 | Version = "1.0.1"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | url string
26 | username string
27 | password string
28 | limit int
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/rest/rest_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted [2]string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,1`, [2]string{"ip", "10.10.10.10"}},
23 | {`select * where name='sarah'`, [2]string{"name", "sarah"}},
24 | }
25 |
26 | for _, table := range tables {
27 | // Executed by the main service
28 | ast, err := sqlparser.Parse(table.sql)
29 | if err != nil {
30 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
31 | continue
32 | }
33 |
34 | stmt, ok := ast.(*sqlparser.Select)
35 | if !ok {
36 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
37 | continue
38 | }
39 |
40 | // Executed by the plugin
41 | result, err := c.convert(stmt)
42 | if err != nil {
43 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
44 | continue
45 | }
46 |
47 | if !equal(result, table.converted) {
48 | t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted)
49 | }
50 | }
51 | }
52 |
53 | /*
54 | * Check whether a and b slices contain the same elements
55 | */
56 | func equal(a, b [2]string) bool {
57 | for i, v := range a {
58 | if v != b[i] {
59 | return false
60 | }
61 | }
62 | return true
63 | }
64 |
--------------------------------------------------------------------------------
/plugins/src/shodan/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to the field/value list convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL query to the list of [field,value]
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) ([2]string, error) {
15 |
16 | // Handle WHERE.
17 | // Top level node pass in an empty interface
18 | // to tell the children this is root.
19 | // Is there any better way?
20 | var rootParent sqlparser.Expr
21 |
22 | // Requested field & value
23 | field, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent)
24 | if err != nil {
25 | return [2]string{}, err
26 | }
27 |
28 | return field, nil
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/shodan/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "shodan"
12 | Version = "1.0.1"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | source *pdk.Source
23 |
24 | // Custom fields
25 | limit int
26 | pages int
27 | credits int
28 | }
29 |
--------------------------------------------------------------------------------
/plugins/src/shodan/shodan_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted [2]string
21 | }{
22 | {`SELECT * WHERE ip='8.8.8.8'`, [2]string{"ip", "8.8.8.8"}},
23 | }
24 |
25 | for _, table := range tables {
26 | // Executed by the main service
27 | ast, err := sqlparser.Parse(table.sql)
28 | if err != nil {
29 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
30 | continue
31 | }
32 |
33 | stmt, ok := ast.(*sqlparser.Select)
34 | if !ok {
35 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
36 | continue
37 | }
38 |
39 | // Executed by the plugin
40 | result, err := c.convert(stmt)
41 | if err != nil {
42 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
43 | continue
44 | }
45 |
46 | if !equal(result, table.converted) {
47 | t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted)
48 | }
49 | }
50 | }
51 |
52 | /*
53 | * Check whether a and b slices contain the same elements
54 | */
55 | func equal(a, b [2]string) bool {
56 | if a[0] != b[0] || a[1] != b[1] {
57 | return false
58 | }
59 |
60 | return true
61 | }
62 |
--------------------------------------------------------------------------------
/plugins/src/sqlite/README.md:
--------------------------------------------------------------------------------
1 | # SQLite plugin
2 |
3 | Plugin to query SQLite (https://www.sqlite.org) as a data source.
4 |
5 |
6 | Compile with:
7 | ```sh
8 | CGO_EBABLED=1 CGO_CFLAGS="-g -O2 -Wno-return-local-addr" go build -buildmode=plugin -ldflags="-w" -o sqlite.so ./*.go
9 | ```
10 | Test with:
11 | ```sh
12 | CGO_CFLAGS="-g -O2 -Wno-return-local-addr" go test
13 | ```
14 | **CFLAGS** as a temp. solution for the https://github.com/mattn/go-sqlite3/issues/803
15 |
16 |
17 | **Warnings**
18 |
19 | When a `SIGSEGV` occurs while running a C code called via cgo (what SQLite
20 | plugin does), that `SIGSEGV` is not turned into a Go panic. The mechanism that
21 | Go uses to turn a memory error into a panic can only work for a Go code, not
22 | for a C code. That means `segmentation violation` errors in a C code will crash
23 | the API service.
24 |
25 | ---
26 |
27 | SQL doesn't allow to query missing columns, like Elasticsearch does.
28 | An error `no such column: X` will be received. That means you must be very
29 | careful with designing a data source and creating a YAML config file to be able
30 | to combine it with data source types other than SQL.
31 |
32 | The easiest solution is to exclude SQLite DB from the `global` namespace and
33 | query it independently, to make sure all columns exist.
34 |
35 |
36 | # Access details
37 |
38 | Source YAML definition's `access` fields:
39 | - **db**: database file to use, for example - `/data/sqtest.db`
40 | - **table**: table name to query
41 |
42 |
43 | # Demo
44 |
45 | Simple example of creation a new SQLite data source from a CLI:
46 | ```sql
47 | sqlite3 sqtest.db
48 |
49 | CREATE TABLE sqcoll (email VARCHAR(255) NOT NULL, username VARCHAR(255) NOT NULL, fqdn VARCHAR(255) NOT NULL, count integer NOT NULL, seen TIMESTAMP);
50 | INSERT INTO sqcoll (email, username, fqdn, count, seen) VALUES ('a@example.com', 'a', 'example.com', 13, DateTime('now', 'localtime'));
51 | INSERT INTO sqcoll (email, username, fqdn, count, seen) VALUES ('b@example.com', 'b', 'example.com', 13, DateTime('now', 'localtime'));
52 | INSERT INTO sqcoll (email, username, fqdn, count, seen) VALUES ('c@example.com', 'c', 'example.com', 13, DateTime('now', 'localtime'));
53 | INSERT INTO sqcoll (email, username, fqdn, count, seen) VALUES ('d@example.com', 'd', 'example.com', 13, DateTime('now', 'localtime'));
54 | INSERT INTO sqcoll (email, username, fqdn, count, seen) VALUES ('e@example.com', 'e', 'example.com', 13, DateTime('now', 'localtime'));
55 | .quit
56 | ```
57 |
58 | Access data will be used by the YAML configs. Example:
59 | ```yaml
60 | name: sqtest
61 | label: SQTest
62 | icon: database
63 |
64 | plugin: sqlite
65 | inGlobal: true
66 | includeDatetime: false
67 | supportsSQL: true
68 |
69 | access:
70 | db: /data/sqtest.db
71 | table: sqcoll
72 |
73 | statsFields:
74 | - domain
75 |
76 | replaceFields:
77 | datetime: seen
78 | domain: fqdn
79 |
80 |
81 | relations:
82 | -
83 | from:
84 | id: email
85 | group: email
86 | search: email
87 | attributes: [ "username", "fqdn" ]
88 |
89 | to:
90 | id: fqdn
91 | group: domain
92 | search: domain
93 |
94 | edge:
95 | attributes: [ "count" ]
96 | ```
97 |
98 | Test with a query:
99 | ```sh
100 | curl -XGET 'https://localhost:443/api?uuid=auth-key&sql=FROM+sqtest+WHERE+email+like+%27a%25%27'
101 | ```
102 |
--------------------------------------------------------------------------------
/plugins/src/sqlite/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to SQLite query convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL statement to the SQLite query
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) (string, error) {
15 |
16 | // Handle WHERE
17 | query := sqlparser.String(sel.Where.Expr)
18 |
19 | // Handle GROUP BY
20 | if len(sel.GroupBy) > 0 {
21 | query += sqlparser.String(sel.GroupBy)
22 | }
23 |
24 | // Handle ORDER BY
25 | if sel.OrderBy != nil {
26 | query += sqlparser.String(sel.OrderBy)
27 | }
28 |
29 | // Handle LIMIT offset,rowcount
30 | if sel.Limit != nil {
31 | if sel.Limit.Offset != nil {
32 | query += " LIMIT " + sqlparser.String(sel.Limit.Offset) + "," + sqlparser.String(sel.Limit.Rowcount)
33 | } else {
34 | query += " LIMIT 0," + sqlparser.String(sel.Limit.Rowcount)
35 | }
36 | }
37 |
38 | return query, nil
39 | }
40 |
--------------------------------------------------------------------------------
/plugins/src/sqlite/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "database/sql"
5 |
6 | "github.com/cert-lv/graphoscope/pdk"
7 | )
8 |
9 | /*
10 | * Export symbols
11 | */
12 | var (
13 | Name = "sqlite"
14 | Version = "1.0.5"
15 | Plugin plugin
16 | )
17 |
18 | /*
19 | * Structure to be imported by the core as a plugin
20 | */
21 | type plugin struct {
22 |
23 | // Inherit default configuration fields
24 | source *pdk.Source
25 |
26 | // Custom fields
27 | db *sql.DB
28 | limit int
29 | }
30 |
--------------------------------------------------------------------------------
/plugins/src/sqlite/sqlite_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * Test SQL conversion to the data source's expected format
11 | */
12 | func TestConvert(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := plugin{}
16 |
17 | // Pairs of SQLs and the expected results
18 | tables := []struct {
19 | sql string
20 | converted string
21 | }{
22 | {`SELECT * WHERE ip='10.10.10.10'`, `ip = '10.10.10.10'`},
23 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 5,10`, `ip = '10.10.10.10' LIMIT 5,10`},
24 | {`SELECT * WHERE size>100 ORDER BY name LIMIT 0,1`, `size > 100 order by name asc LIMIT 0,1`},
25 | {`SELECT * WHERE size=10 ORDER BY name DESC LIMIT 0,1`, `size = 10 order by name desc LIMIT 0,1`},
26 | {`SELECT * WHERE size>=100`, `size >= 100`},
27 | {`SELECT * WHERE name LIKE 's%'`, `name like 's%'`},
28 | {`SELECT * WHERE name NOT LIKE 's%'`, `name not like 's%'`},
29 | {`SELECT * WHERE size BETWEEN 100 AND 300`, `size between 100 and 300`},
30 | {`SELECT * WHERE size IN (100,300)`, `size in (100, 300)`},
31 | {`SELECT * WHERE size NOT IN (100,300)`, `size not in (100, 300)`},
32 | {`SELECT * WHERE name='sarah' and age!=40 AND (country='LV' OR country='AU') ORDER BY age DESC limit 1`,
33 | `name = 'sarah' and age != 40 and (country = 'LV' or country = 'AU') order by age desc LIMIT 0,1`},
34 | }
35 |
36 | for _, table := range tables {
37 | // Executed by the main service
38 | ast, err := sqlparser.Parse(table.sql)
39 | if err != nil {
40 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
41 | continue
42 | }
43 |
44 | stmt, ok := ast.(*sqlparser.Select)
45 | if !ok {
46 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
47 | continue
48 | }
49 |
50 | // Executed by the plugin
51 | result, err := c.convert(stmt)
52 | if err != nil {
53 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
54 | continue
55 | }
56 |
57 | if result != table.converted {
58 | t.Errorf("Invalid conversion of \"%s\": \"%s\", expected: \"%s\"", table.sql, result, table.converted)
59 | }
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/plugins/src/taxonomy/README.md:
--------------------------------------------------------------------------------
1 | # Taxonomy plugin
2 |
3 | Apply the predefined taxonomy to all nodes. New virtual nodes will be inserted, which will group existing nodes.
4 |
5 |
6 | Compile with:
7 | ```sh
8 | go build -buildmode=plugin -ldflags="-w" -o taxonomy.so ./*.go
9 | ```
10 |
11 | # Configuration
12 |
13 | YAML definition's `data` fields:
14 | - **field**: field to use as a filter
15 | - **group**: as all graph nodes belong to some group/type, it can be used to apply taxonomy to the specific group only
16 | - **taxonomy**: mapping to use. If node/edge's **field** is equal to the "key" - insert a new relation with "value" as a new node
17 |
18 | Mapping example:
19 | ```yaml
20 | taxonomy:
21 | field_value_1: taxonomy_group
22 | field_value_2: taxonomy_group
23 | ```
24 |
--------------------------------------------------------------------------------
/plugins/src/taxonomy/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | Name = "taxonomy"
12 | Version = "1.0.0"
13 | Plugin plugin
14 | )
15 |
16 | /*
17 | * Structure to be imported by the core as a plugin
18 | */
19 | type plugin struct {
20 |
21 | // Inherit default configuration fields
22 | processor *pdk.Processor
23 | }
24 |
--------------------------------------------------------------------------------
/plugins/src/taxonomy/taxonomy.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Check "pdk/plugin.go" for the built-in plugin functions description
9 | */
10 |
11 | func (p *plugin) Conf() *pdk.Processor {
12 | return p.processor
13 | }
14 |
15 | func (p *plugin) Setup(processor *pdk.Processor) error {
16 |
17 | // Store settings
18 | p.processor = processor
19 |
20 | return nil
21 | }
22 |
23 | func (p *plugin) Process(relations []map[string]interface{}) ([]map[string]interface{}, error) {
24 |
25 | for _, relation := range relations {
26 | for k, v := range p.processor.Data["taxonomy"].(map[string]interface{}) {
27 | for _, part := range []string{"from", "to", "edge"} {
28 | rp := relation[part]
29 |
30 | if rp != nil && (rp.(map[string]interface{})[p.processor.Data["field"].(string)] == k ||
31 | (rp.(map[string]interface{})["attributes"] != nil &&
32 | rp.(map[string]interface{})["attributes"].(map[string]interface{})[p.processor.Data["field"].(string)] == k)) {
33 |
34 | if p.processor.Data["group"] != nil &&
35 | rp.(map[string]interface{})["group"] != p.processor.Data["group"].(string) {
36 | continue
37 | }
38 |
39 | taxRelation := p.createRelation(v.(string), k,
40 | rp.(map[string]interface{})["group"].(string),
41 | rp.(map[string]interface{})["search"].(string),
42 | )
43 |
44 | relations = append(relations, taxRelation)
45 | }
46 | }
47 | }
48 | }
49 |
50 | return relations, nil
51 | }
52 |
53 | /*
54 | * Generate new graph relation to display taxonomy info as a new node
55 | */
56 | func (p *plugin) createRelation(tid, fid, group, search string) map[string]interface{} {
57 | from := map[string]interface{}{
58 | "id": fid,
59 | "group": group,
60 | "search": search,
61 | }
62 |
63 | to := map[string]interface{}{
64 | "id": tid,
65 | "group": "taxonomy",
66 | "search": "taxonomy",
67 | }
68 |
69 | // Resulting graph relation to return
70 | result := make(map[string]interface{})
71 |
72 | // Put it together
73 | result["from"] = from
74 | result["to"] = to
75 | result["source"] = p.Conf().Name
76 |
77 | return result
78 | }
79 |
80 | func (p *plugin) Stop() error {
81 | return nil
82 | }
83 |
--------------------------------------------------------------------------------
/plugins/src/taxonomy/taxonomy_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/cert-lv/graphoscope/pdk"
7 | )
8 |
9 | /*
10 | * Test taxonomy nodes creating
11 | */
12 | func TestProcess(t *testing.T) {
13 |
14 | // Empty plugin's instance to test
15 | c := &plugin{}
16 |
17 | processor := &pdk.Processor{
18 | Data: map[string]interface{}{
19 | "field": "id",
20 | "group": "type",
21 |
22 | "taxonomy": map[string]interface{}{
23 | "brute-force": "intrusion-attempts",
24 | },
25 | },
26 | }
27 |
28 | err := c.Setup(processor)
29 | if err != nil {
30 | t.Errorf("Can't setup a taxonomy plugin: %s", err.Error())
31 | }
32 |
33 | // Pairs of data source plugins responses and the expected processing results
34 | table := []struct {
35 | entry map[string]interface{}
36 | inserted map[string]interface{}
37 | }{
38 | // New relation should be added because "id" == "brute-force"
39 | {map[string]interface{}{
40 | "from": map[string]interface{}{
41 | "id": "brute-force",
42 | "group": "type",
43 | "search": "type",
44 | },
45 | }, map[string]interface{}{
46 | "id": "intrusion-attempts",
47 | "group": "taxonomy",
48 | "search": "taxonomy",
49 | }},
50 |
51 | // New relation should be added because "attributes.id" == "brute-force"
52 | {map[string]interface{}{
53 | "from": map[string]interface{}{
54 | "id": "malware",
55 | "group": "type",
56 | "search": "type",
57 | "attributes": map[string]interface{}{
58 | "id": "brute-force",
59 | },
60 | },
61 | }, map[string]interface{}{
62 | "id": "intrusion-attempts",
63 | "group": "taxonomy",
64 | "search": "taxonomy",
65 | }},
66 |
67 | // New relation should NOT be added because "id" value is not mentioned in processor.Data["taxonomy"]
68 | {map[string]interface{}{
69 | "from": map[string]interface{}{
70 | "id": "ddos",
71 | "group": "type",
72 | "search": "type",
73 | },
74 | }, nil},
75 |
76 | // New relation should NOT be added because "group" value is not equal to processor.Data["group"]
77 | {map[string]interface{}{
78 | "from": map[string]interface{}{
79 | "id": "brute-force",
80 | "group": "address",
81 | "search": "type",
82 | },
83 | }, nil},
84 | }
85 |
86 | for _, row := range table {
87 | result, err := c.Process([]map[string]interface{}{row.entry})
88 | if err != nil {
89 | t.Errorf("Can't process '%s': %s", row.entry, err.Error())
90 | continue
91 | }
92 |
93 | if row.inserted == nil && len(result) > 1 {
94 | t.Errorf("Unwanted relation added for \"%s\": \"%s\"", row.entry["from"], result[1])
95 |
96 | } else if len(result) == 1 && row.inserted != nil {
97 | t.Errorf("No new relations added for \"%s\": \"%s\", expected: \"%s\"", row.entry["from"], result, row.inserted)
98 |
99 | } else if len(result) > 1 && row.inserted != nil {
100 | for k, v := range result[1]["to"].(map[string]interface{}) {
101 | if row.inserted[k] != v {
102 | t.Errorf("Invalid taxonomy added for \"%s\": \"%s\", expected: \"%s\"",
103 | row.entry["from"], result[1]["to"], row.inserted)
104 | break
105 | }
106 | }
107 | }
108 | }
109 | }
110 |
--------------------------------------------------------------------------------
/plugins/src/template/README.md:
--------------------------------------------------------------------------------
1 | ### STEP 16.
2 | ### Write plugin description and documentation
3 |
4 |
5 | # Template plugin
6 |
7 | Template to build new plugins.
8 | Check GUI built-in documentation section `Administration` for a complete
9 | step-by-step workflow.
10 |
11 |
12 | Compile with:
13 | ```sh
14 | go build -buildmode=plugin -ldflags="-w" -o template.so ./*.go
15 | ```
16 |
17 | # Access details
18 |
19 | Source YAML definition's `access` fields:
20 | - **url**: PROTO://HOST:PORT database's access point, for example - `127.0.0.1:3000`
21 | - **user**: username to connect with
22 | - **password**: user's password
23 | - **db**: database name to use
24 | - **collection**: collection name to query
25 |
--------------------------------------------------------------------------------
/plugins/src/template/convert.go:
--------------------------------------------------------------------------------
1 | /*
2 | * SQL to X query convertor
3 | */
4 |
5 | package main
6 |
7 | import (
8 | "github.com/blastrain/vitess-sqlparser/sqlparser"
9 | )
10 |
11 | /*
12 | * Convert SQL statement to the object expected by the data source
13 | */
14 | func (p *plugin) convert(sel *sqlparser.Select) (string, error) {
15 |
16 | /*
17 | * STEP 8.
18 | *
19 | * Do the SQL conversion.
20 | * Check, for example, a MongoDB plugin to see how SQL
21 | * can be converted to the hierarchical object.
22 | *
23 | * Here we just return a simple static 'field=value' pair.
24 | *
25 | * File not needed for the processor plugin!
26 | */
27 |
28 | filter := "field=value"
29 | return filter, nil
30 | }
31 |
--------------------------------------------------------------------------------
/plugins/src/template/plugin.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/cert-lv/graphoscope/pdk"
5 | )
6 |
7 | /*
8 | * Export symbols
9 | */
10 | var (
11 | /*
12 | * STEP 15.
13 | *
14 | * Set plugin name and version
15 | */
16 |
17 | Name = "template"
18 | Version = "1.0.0"
19 | Plugin plugin
20 | )
21 |
22 | /*
23 | * Structure to be imported by the core as a plugin
24 | */
25 | type plugin struct {
26 |
27 | /*
28 | * STEP 13.
29 | *
30 | * Inherit default configuration fields for the data source or
31 | * processor plugin
32 | */
33 |
34 | source *pdk.Source
35 | //processor *pdk.Processor
36 |
37 | /*
38 | * STEP 14.
39 | *
40 | * Define all the custom fields needed by the plugin,
41 | * such as "client" object, database/collection name, etc..
42 | */
43 |
44 | //client *package.Client
45 | limit int
46 | }
47 |
--------------------------------------------------------------------------------
/plugins/src/template/template_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/blastrain/vitess-sqlparser/sqlparser"
7 | )
8 |
9 | /*
10 | * STEP 17.
11 | *
12 | * Test plugin's functionality
13 | */
14 |
15 | func TestConvert(t *testing.T) {
16 |
17 | // Empty plugin's instance to test
18 | c := plugin{}
19 |
20 | // Test whether example SQL queries are correctly converted to the expected format.
21 | // Pairs of SQLs and the expected results:
22 | tables := []struct {
23 | sql string
24 | converted string
25 | }{
26 | {`SELECT * WHERE ip='10.10.10.10' LIMIT 0,10`, ``},
27 | {`SELECT * WHERE size>100 ORDER BY name LIMIT 5,1`, ``},
28 | {`SELECT * WHERE size=10 ORDER BY name DESC LIMIT 0,1`, ``},
29 | {`SELECT * WHERE size>=100 LIMIT 0,1`, ``},
30 | {`SELECT * WHERE name LIKE 's%' LIMIT 0,1`, ``},
31 | {`SELECT * WHERE name NOT LIKE 's%' LIMIT 0,1`, ``},
32 | {`SELECT * WHERE size BETWEEN 100 AND 300 LIMIT 0,1`, ``},
33 | {`SELECT * WHERE size IN (100,300) LIMIT 0,1`, ``},
34 | {`SELECT * WHERE size NOT IN (100,300) LIMIT 0,1`, ``},
35 | {`SELECT * WHERE name='sarah' AND age!=40 AND (country='LV' OR country='AU') ORDER BY age DESC LIMIT 1`, ``},
36 | }
37 |
38 | for _, table := range tables {
39 | // Executed by the main service
40 | ast, err := sqlparser.Parse(table.sql)
41 | if err != nil {
42 | t.Errorf("Can't parse '%s': %s", table.sql, err.Error())
43 | continue
44 | }
45 |
46 | stmt, ok := ast.(*sqlparser.Select)
47 | if !ok {
48 | t.Errorf("Only SELECT statement is allowed: %s", table.sql)
49 | continue
50 | }
51 |
52 | // Executed by the plugin
53 | result, err := c.convert(stmt)
54 | if err != nil {
55 | t.Errorf("Can't convert '%s': %s", table.sql, err.Error())
56 | continue
57 | }
58 |
59 | if result != table.converted {
60 | t.Errorf("Invalid conversion of \"%s\": \"%s\", expected: \"%s\"", table.sql, result, table.converted)
61 | }
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
5 | 
6 | Graphoscope
7 | {{ if ne .Environment "prod" }}
8 | dev
9 | {{ end }}
10 |
11 |
12 |
13 |
6 | Graphoscope
7 | {{ if ne .Environment "prod" }}
8 | dev
9 | {{ end }}
10 |
11 |
14 | {{ .Account.Username }}
15 |
16 |
51 |
17 | {{ if .Account.Notifications }}
18 |
21 |
22 |
50 | {{ len .Account.Notifications }}
19 | {{ end }}
20 |