├── .github ├── changelog.json ├── changelog_template.md └── workflows │ ├── build.yml │ ├── docker.yml │ ├── docs.yml │ └── release.yml ├── .gitignore ├── .gitmodules ├── .goreleaser.yml ├── .nojekyll ├── Dockerfile ├── LICENSE.md ├── Makefile ├── README.md ├── cmd └── driplane │ └── main.go ├── config.yaml.example ├── core ├── configuration.go ├── configuration_test.go ├── orchestrator.go ├── pipe_rule.go ├── rule_parser.go ├── rule_parser_test.go ├── ruleset.go └── version.go ├── data ├── event.go └── message.go ├── example.rule ├── feeders ├── apt.go ├── base.go ├── file.go ├── folder.go ├── imap.go ├── rss.go ├── slack.go ├── telegram.go ├── telegram_auth.go ├── timer.go ├── twitter.go └── web.go ├── filters ├── base.go ├── base_test.go ├── cache.go ├── cache_test.go ├── changed.go ├── changed_test.go ├── echo.go ├── echo_test.go ├── elasticsearch.go ├── file.go ├── filter_test.go ├── format.go ├── format_test.go ├── hash.go ├── hash_test.go ├── html.go ├── html_test.go ├── http.go ├── js.go ├── json.go ├── json_test.go ├── mail.go ├── mimetype.go ├── number.go ├── number_test.go ├── override.go ├── override_test.go ├── pdf.go ├── random.go ├── random_test.go ├── ratelimit.go ├── ratelimit_test.go ├── slack.go ├── slack_test.go ├── striptag.go ├── system.go ├── system_test.go ├── telegram.go ├── text.go ├── text_test.go ├── url.go ├── url_test.go └── xls.go ├── go.mod ├── go.sum ├── plugins ├── cache.go ├── cache_test.go ├── file.go ├── file_test.go ├── http.go ├── http_test.go ├── log.go ├── log_test.go ├── strings.go ├── strings_test.go ├── util.go └── util_test.go ├── release.stork ├── src_docs ├── archetypes │ └── default.md ├── config │ └── _default │ │ ├── config.toml │ │ ├── languages.toml │ │ ├── menus.en.toml │ │ └── params.toml ├── content │ ├── _index.md │ └── doc │ │ ├── _index.md │ │ ├── configuration │ │ └── _index.md │ │ ├── feeders │ │ ├── _index.md │ │ ├── apt.md │ │ ├── file.md │ │ ├── folder.md │ │ ├── imap.md │ │ ├── rss.md │ │ ├── slack.md │ │ ├── telegram.md │ │ ├── timer.md │ │ ├── twitter.md │ │ └── web.md │ │ ├── filters │ │ ├── _index.md │ │ ├── cache.md │ │ ├── changed.md │ │ ├── echo.md │ │ ├── elasticsearch.md │ │ ├── file.md │ │ ├── format.md │ │ ├── hash.md │ │ ├── html.md │ │ ├── http.md │ │ ├── js │ │ │ ├── _index.md │ │ │ ├── basics.md │ │ │ ├── entrypoint.md │ │ │ └── packages.md │ │ ├── json.md │ │ ├── mail.md │ │ ├── mimetype.md │ │ ├── number.md │ │ ├── override.md │ │ ├── pdf.md │ │ ├── random.md │ │ ├── ratelimit.md │ │ ├── slack.md │ │ ├── striptag.md │ │ ├── system.md │ │ ├── telegram.md │ │ ├── text.md │ │ ├── url.md │ │ └── xls.md │ │ ├── installation │ │ ├── _index.md │ │ ├── build.md │ │ └── docker.md │ │ └── rules │ │ ├── _index.md │ │ ├── definition.md │ │ └── syntax.md ├── go.sum ├── resources │ └── _gen │ │ └── assets │ │ └── scss │ │ ├── driplane │ │ ├── sass │ │ │ ├── main.scss_b4f67ac5085b89b62b54c1923e5a9145.content │ │ │ └── main.scss_b4f67ac5085b89b62b54c1923e5a9145.json │ │ └── scss │ │ │ └── slate │ │ │ ├── print.css.scss_c14439616ffbc3ae1827507340d6c08b.content │ │ │ ├── print.css.scss_c14439616ffbc3ae1827507340d6c08b.json │ │ │ ├── screen.css.scss_d18af36970f1f09b308ef20ee65e3a03.content │ │ │ └── screen.css.scss_d18af36970f1f09b308ef20ee65e3a03.json │ │ └── scss │ │ └── slate │ │ ├── print.css.scss_c14439616ffbc3ae1827507340d6c08b.content │ │ ├── print.css.scss_c14439616ffbc3ae1827507340d6c08b.json │ │ ├── screen.css.scss_d18af36970f1f09b308ef20ee65e3a03.content │ │ └── screen.css.scss_d18af36970f1f09b308ef20ee65e3a03.json └── static │ ├── favicon │ ├── android-icon-144x144.png │ ├── android-icon-192x192.png │ ├── android-icon-36x36.png │ ├── android-icon-48x48.png │ ├── android-icon-72x72.png │ ├── android-icon-96x96.png │ ├── apple-icon-114x114.png │ ├── apple-icon-120x120.png │ ├── apple-icon-144x144.png │ ├── apple-icon-152x152.png │ ├── apple-icon-180x180.png │ ├── apple-icon-57x57.png │ ├── apple-icon-60x60.png │ ├── apple-icon-72x72.png │ ├── apple-icon-76x76.png │ ├── apple-icon-precomposed.png │ ├── apple-icon.png │ ├── browserconfig.xml │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ ├── favicon-96x96.png │ ├── favicon.ico │ ├── logo.png │ ├── logo192x192.png │ ├── manifest.json │ ├── ms-icon-144x144.png │ ├── ms-icon-150x150.png │ ├── ms-icon-310x310.png │ └── ms-icon-70x70.png │ └── logo.png └── utils ├── apt ├── package.go ├── package_test.go ├── release.go ├── release_test.go ├── repository.go └── repository_test.go ├── cookie.go ├── cookie_test.go ├── global_ttl_map.go ├── global_ttl_map_test.go ├── html.go ├── html_test.go ├── misc.go ├── misc_test.go ├── ttl_map.go └── ttl_map_test.go /.github/changelog.json: -------------------------------------------------------------------------------- 1 | { 2 | "resolve": "commits", 3 | "sort": "asc", 4 | 5 | "template": ".github/changelog_template.md", 6 | 7 | "groupings": [ 8 | { "name": "New Features", "patterns": [ "(?i)^new\\b" ] }, 9 | { "name": "Fixes", "patterns": [ "(?i)^fix\\b" ] }, 10 | { "name": "Misc", "patterns": [ ".*" ] } 11 | ], 12 | "exclude": [ 13 | "^test\\b", 14 | "^docs\\b", 15 | "^(?i)refactor\\b", 16 | "^(?i)release\\s+\\d+\\.\\d+\\.\\d+", 17 | "^(?i)minor fix\\b", 18 | "^(?i)no comment\\b", 19 | "^(?i)wip\\b" 20 | ], 21 | 22 | "local": false, 23 | 24 | "max_commits": 250 25 | } -------------------------------------------------------------------------------- /.github/changelog_template.md: -------------------------------------------------------------------------------- 1 | {{define "GroupTemplate" -}} 2 | {{- range .Grouped}} 3 | ### {{ .Name }} 4 | 5 | {{range .Items -}} 6 | * [{{.CommitHashShort}}]({{.CommitURL}}) {{.Title}} ({{if .IsPull}}[contributed]({{.PullURL}}) by {{end}}[{{.Author}}]({{.AuthorURL}})) 7 | {{end -}} 8 | {{end -}} 9 | {{end -}} 10 | {{define "FlatTemplate" -}} 11 | {{range .Items -}} 12 | * [{{.CommitHashShort}}]({{.CommitURL}}) {{.Title}} ({{if .IsPull}}[contributed]({{.PullURL}}) by {{end}}[{{.Author}}]({{.AuthorURL}})) 13 | {{end -}} 14 | {{end -}} 15 | {{define "DefaultTemplate" -}} 16 | ## Release Note {{.Version}} 17 | {{if len .Grouped -}} 18 | {{template "GroupTemplate" . -}} 19 | {{- else}} 20 | {{template "FlatTemplate" . -}} 21 | {{end}} 22 | {{end -}} 23 | {{template "DefaultTemplate" . -}} -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | 3 | # This workflow will run on master branch and on any pull requests targeting master 4 | on: 5 | push: 6 | branches: 7 | - master 8 | paths-ignore: 9 | - 'docs/**' 10 | - 'src_docs/**' 11 | - '.github/**' 12 | pull_request: 13 | 14 | jobs: 15 | #lint: 16 | # name: Lint 17 | # runs-on: ubuntu-latest 18 | # steps: 19 | # - name: Set up Go 20 | # uses: actions/setup-go@v2 21 | # with: 22 | # go-version: 1.18 23 | # 24 | # - name: Check out code 25 | # uses: actions/checkout@v2 26 | # 27 | # - name: Lint Go Code 28 | # run: | 29 | # go get -u golang.org/x/lint/golint 30 | # make lint 31 | 32 | test: 33 | name: Test 34 | runs-on: ubuntu-latest 35 | steps: 36 | - name: Set up Go 37 | uses: actions/setup-go@v5 38 | with: 39 | go-version: '1.21' 40 | 41 | - name: Check out code 42 | uses: actions/checkout@v2 43 | 44 | - name: Run Unit tests. 45 | run: make test-coverage 46 | 47 | - name: Upload Coverage report to CodeCov 48 | uses: codecov/codecov-action@v1.0.0 49 | with: 50 | token: ${{secrets.CODECOV_TOKEN}} 51 | file: ./coverage.txt 52 | 53 | build: 54 | name: Build 55 | runs-on: ubuntu-latest 56 | needs: [test] 57 | steps: 58 | - name: Set up Go 59 | uses: actions/setup-go@v5 60 | with: 61 | go-version: '1.21' 62 | 63 | - name: Check out code 64 | uses: actions/checkout@v2 65 | 66 | - name: Build 67 | run: make build -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | push_to_registry: 10 | name: Push Docker image to Docker Hub 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Check out the repo 14 | uses: actions/checkout@v2 15 | - name: Push to Docker Hub 16 | uses: docker/build-push-action@v1 17 | with: 18 | username: ${{ secrets.DOCKER_USERNAME }} 19 | password: ${{ secrets.DOCKER_PASSWORD }} 20 | repository: matrix86/driplane 21 | tag_with_ref: true -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Update website 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | paths: 8 | - 'src_docs/**' 9 | 10 | jobs: 11 | deploy: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | with: 16 | submodules: true # Fetch Hugo themes (true OR recursive) 17 | fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod 18 | 19 | - name: Setup Hugo 20 | uses: peaceiris/actions-hugo@v2 21 | with: 22 | hugo-version: '0.75.1' 23 | extended: true 24 | 25 | - name: Build 26 | run: hugo --minify -s src_docs 27 | 28 | - name: Deploy 29 | uses: peaceiris/actions-gh-pages@v3 30 | with: 31 | personal_token: ${{ secrets.PERSONAL_TOKEN }} 32 | commit_message: ${{ github.event.head_commit.message }} 33 | publish_branch: gh-pages 34 | publish_dir: ./src_docs/public -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - 'v*' 6 | 7 | jobs: 8 | release: 9 | name: Release on GitHub 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check out code 13 | uses: actions/checkout@v3 14 | with: 15 | fetch-depth: 0 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v5 19 | 20 | - name: Find Last Tag 21 | id: last 22 | uses: jimschubert/query-tag-action@v1 23 | with: 24 | include: 'v*' 25 | exclude: '*-rc*' 26 | commit-ish: 'HEAD~' 27 | skip-unshallow: 'true' 28 | 29 | - name: Find Current Tag 30 | id: current 31 | uses: jimschubert/query-tag-action@v1 32 | with: 33 | include: 'v*' 34 | exclude: '*-rc*' 35 | commit-ish: '@' 36 | skip-unshallow: 'true' 37 | 38 | - name: Create Changelog 39 | id: changelog 40 | uses: jimschubert/beast-changelog-action@v1 41 | with: 42 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 43 | CONFIG_LOCATION: .github/changelog.json 44 | FROM: ${{steps.last.outputs.tag}} 45 | TO: ${{steps.current.outputs.tag}} 46 | OUTPUT: .github/CHANGELOG.md 47 | 48 | - name: View Changelog 49 | run: cat .github/CHANGELOG.md 50 | 51 | - name: Validates GO releaser config 52 | uses: goreleaser/goreleaser-action@v3 53 | with: 54 | distribution: goreleaser 55 | version: latest 56 | args: check 57 | 58 | - name: Create release on GitHub 59 | uses: goreleaser/goreleaser-action@v3 60 | env: 61 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 62 | with: 63 | args: release --rm-dist --release-notes .github/CHANGELOG.md 64 | #workdir: ./cmd/driplane 65 | 66 | #- name: Create release on GitHub 67 | # uses: docker://goreleaser/goreleaser:latest 68 | # env: 69 | # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 70 | # with: 71 | # args: release --rm-dist --release-notes .github/CHANGELOG.md 72 | # workdir: ./cmd/driplane 73 | 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | *.swp 14 | vendor/* 15 | .idea/ 16 | testing/* 17 | bin/* 18 | # to avoid the error 'error=git is currently in a dirty state, please check in your pipeline what can be changing the following files:' on goreleaser 19 | .github/CHANGELOG.md -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "src_docs/themes/zdoc"] 2 | path = src_docs/themes/zdoc 3 | url = https://github.com/zzossig/hugo-theme-zdoc.git 4 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # This is an example goreleaser.yaml file with some sane defaults. 2 | # Make sure to check the documentation at http://goreleaser.com 3 | before: 4 | hooks: 5 | # You may remove this if you don't use go modules. 6 | - go mod download 7 | # you may remove this if you don't need go generate 8 | - go generate ./... 9 | builds: 10 | - env: 11 | - CGO_ENABLED=0 12 | goos: 13 | - linux 14 | - windows 15 | - darwin 16 | main: ./cmd/driplane/main.go 17 | 18 | checksum: 19 | name_template: 'checksums.txt' 20 | snapshot: 21 | name_template: "{{ .Tag }}-next" 22 | changelog: 23 | sort: asc 24 | filters: 25 | exclude: 26 | - '^docs:' 27 | - 'typo' 28 | - '^test:' 29 | - '^src_docs:' 30 | -------------------------------------------------------------------------------- /.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/.nojekyll -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine AS build-env 2 | 3 | RUN apk add --update ca-certificates 4 | RUN apk add --no-cache --update make 5 | 6 | WORKDIR /go/src/app 7 | 8 | COPY . . 9 | 10 | RUN go get -d -v ./... 11 | 12 | RUN make build 13 | 14 | FROM alpine:latest 15 | 16 | COPY --from=build-env /go/src/app/bin/driplane /app/ 17 | 18 | WORKDIR /app 19 | 20 | ENTRYPOINT ["/app/driplane"] 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT_NAME := "driplane" 2 | TARGET=driplane 3 | LDFLAGS="-s -w" 4 | PKG_LIST := $(shell go list ./... | grep -v /vendor/) 5 | 6 | 7 | all: build 8 | 9 | test: 10 | @go test -short ${PKG_LIST} 11 | 12 | test-coverage: 13 | @go test -short -coverprofile cover.out -covermode=atomic ${PKG_LIST} 14 | @cat cover.out >> coverage.txt 15 | 16 | test-coverage-html: 17 | @go test -short -coverprofile cover.out -covermode=atomic ${PKG_LIST} 18 | @go tool cover -html=cover.out 19 | 20 | lint: 21 | @golint -set_exit_status ${PKG_LIST} 22 | 23 | build: clean 24 | @mkdir -p bin 25 | go build -buildvcs=false -o bin/driplane -trimpath -v -ldflags=${LDFLAGS} cmd/driplane/main.go 26 | 27 | install: build 28 | go install -ldflags=${LDFLAGS} ./cmd/driplane 29 | 30 | clean: 31 | @rm -rf bin 32 | -------------------------------------------------------------------------------- /config.yaml.example: -------------------------------------------------------------------------------- 1 | general: 2 | log_path: "" 3 | rules_path: "rules" 4 | js_path: "js" 5 | templates_path: "templates" 6 | debug: false 7 | 8 | twitter: 9 | consumerKey: "" 10 | consumerSecret: "" 11 | accessToken: "" 12 | accessSecret: "" 13 | keywords: "#italy #coding #malware something" 14 | stallWarnings: "true" -------------------------------------------------------------------------------- /core/configuration.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "reflect" 8 | "sync" 9 | 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | // Configuration contains all the configs read by yaml file 14 | type Configuration struct { 15 | sync.RWMutex 16 | 17 | FilePath string 18 | flat map[string]string 19 | } 20 | 21 | // LoadConfiguration create a Configuration struct from a filename 22 | func LoadConfiguration(path string) (*Configuration, error) { 23 | configuration := &Configuration{ 24 | FilePath: path, 25 | } 26 | 27 | file, err := os.Open(path) 28 | if err != nil { 29 | return configuration, fmt.Errorf("loading configuration: file opening: %s", err) 30 | } 31 | 32 | bytes, _ := ioutil.ReadAll(file) 33 | 34 | var cc map[interface{}]interface{} 35 | 36 | if err := yaml.Unmarshal(bytes, &cc); err != nil { 37 | return configuration, fmt.Errorf("loading configuration: %s", err) 38 | } 39 | configuration.flat = configuration.flatMap(cc) 40 | 41 | return configuration, nil 42 | } 43 | 44 | func (c *Configuration) flatMap(m map[interface{}]interface{}) map[string]string { 45 | flatten := make(map[string]string) 46 | for k, v := range m { 47 | switch reflect.TypeOf(v).Kind() { 48 | case reflect.Map: 49 | mv := c.flatMap(v.(map[interface{}]interface{})) 50 | for kk, vv := range mv { 51 | key := fmt.Sprintf("%s.%s", k, kk) 52 | flatten[key] = vv 53 | } 54 | 55 | case reflect.Array, reflect.Slice: 56 | for i, vv := range v.([]interface{}) { 57 | mv := c.flatMap(vv.(map[interface{}]interface{})) 58 | for kk, vv := range mv { 59 | key := fmt.Sprintf("%s.%s%d.%s", k, k, i, kk) 60 | flatten[key] = vv 61 | } 62 | } 63 | 64 | default: 65 | flatten[k.(string)] = fmt.Sprint(v) 66 | } 67 | } 68 | return flatten 69 | } 70 | 71 | // Get returns the config value with that name, if it exists 72 | func (c *Configuration) Get(name string) string { 73 | c.RLock() 74 | defer c.RUnlock() 75 | if _, ok := c.flat[name]; !ok { 76 | return "" 77 | } 78 | 79 | return c.flat[name] 80 | } 81 | 82 | // Set insert a new config in the Configuration struct 83 | func (c *Configuration) Set(name string, value string) error { 84 | c.Lock() 85 | defer c.Unlock() 86 | 87 | c.flat[name] = value 88 | return nil 89 | } 90 | 91 | // GetConfig returns the complete configuration in a flatten way 92 | func (c *Configuration) GetConfig() map[string]string { 93 | c.RLock() 94 | defer c.RUnlock() 95 | return c.flat 96 | } 97 | -------------------------------------------------------------------------------- /core/orchestrator.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "github.com/Matrix86/driplane/data" 6 | "path/filepath" 7 | "sync" 8 | 9 | "github.com/Matrix86/driplane/feeders" 10 | 11 | "github.com/evilsocket/islazy/fs" 12 | "github.com/evilsocket/islazy/log" 13 | ) 14 | 15 | // Orchestrator handles the pipelines and rules 16 | type Orchestrator struct { 17 | asts map[string]*AST 18 | config *Configuration 19 | 20 | waitFeeder sync.WaitGroup 21 | sync.Mutex 22 | } 23 | 24 | // NewOrchestrator create a new instance of the Orchestrator 25 | func NewOrchestrator(config *Configuration) (*Orchestrator, error) { 26 | o := &Orchestrator{ 27 | config: config, 28 | asts: make(map[string]*AST), 29 | } 30 | 31 | parser, _ := NewParser() 32 | 33 | err := fs.Glob(config.Get("general.rules_path"), "*.rule", func(file string) error { 34 | abs, err := filepath.Abs(file) 35 | if err != nil { 36 | log.Fatal("cannot get absolute path of %s: %s", file, err) 37 | } 38 | file = abs 39 | log.Info("parsing rule file: %s", file) 40 | ast, err := parser.ParseFile(file) 41 | if err != nil { 42 | log.Fatal("rule parsing: file '%s': %s", file, err) 43 | } 44 | o.asts[file] = ast 45 | 46 | _, err = RuleSetInstance().CompileAst(file, ast, o.config) 47 | if err != nil { 48 | return fmt.Errorf("compilation of '%s': %s", file, err) 49 | } 50 | return nil 51 | }) 52 | if err != nil { 53 | return o, fmt.Errorf("%s", err) 54 | } 55 | return o, nil 56 | } 57 | 58 | // StartFeeders opens the gates 59 | func (o *Orchestrator) StartFeeders() { 60 | o.Lock() 61 | defer o.Unlock() 62 | rs := RuleSetInstance() 63 | for _, rulename := range rs.feedRules { 64 | f := rs.rules[rulename].getFirstNode().(feeders.Feeder) 65 | if f.IsRunning() == false { 66 | log.Debug("[%s] Starting %s", rulename, f.Name()) 67 | o.waitFeeder.Add(1) 68 | f.Start() 69 | } 70 | } 71 | } 72 | 73 | // HasRunningFeeder return true if one or more feeders are running 74 | func (o *Orchestrator) HasRunningFeeder() bool { 75 | rs := RuleSetInstance() 76 | for _, rulename := range rs.feedRules { 77 | f := rs.rules[rulename].getFirstNode().(feeders.Feeder) 78 | if f.IsRunning() { 79 | return true 80 | } 81 | } 82 | return false 83 | } 84 | 85 | // WaitFeeders waits until all the feeders are stopped 86 | func (o *Orchestrator) WaitFeeders() { 87 | log.Debug("Waiting") 88 | o.waitFeeder.Wait() 89 | log.Debug("Stop waiting") 90 | } 91 | 92 | // StopFeeders closes the gates 93 | func (o *Orchestrator) StopFeeders() { 94 | o.Lock() 95 | defer o.Unlock() 96 | 97 | rs := RuleSetInstance() 98 | for _, rulename := range rs.feedRules { 99 | f := rs.rules[rulename].getFirstNode().(feeders.Feeder) 100 | if f.IsRunning() { 101 | log.Debug("[%s] Stopping %s", rulename, f.Name()) 102 | f.Stop() 103 | o.waitFeeder.Done() 104 | log.Debug("[%s] Stopped %s", rulename, f.Name()) 105 | } 106 | } 107 | 108 | // sending a shutdown event on the bus 109 | rs.bus.Publish(data.EventTopicName, &data.Event{Type: "shutdown"}) 110 | rs.bus.WaitAsync() 111 | } 112 | -------------------------------------------------------------------------------- /core/ruleset.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "sync" 7 | 8 | bus "github.com/asaskevich/EventBus" 9 | "github.com/evilsocket/islazy/log" 10 | ) 11 | 12 | var ( 13 | instance *Ruleset 14 | once sync.Once 15 | ) 16 | 17 | // Ruleset identifies a set of rules 18 | type Ruleset struct { 19 | rules map[string]*PipeRule 20 | compiledDeps map[string][]string 21 | 22 | feedRules []string 23 | bus bus.Bus 24 | lastID int32 25 | } 26 | 27 | // RuleSetInstance is the singleton for the Ruleset object 28 | func RuleSetInstance() *Ruleset { 29 | once.Do(func() { 30 | instance = &Ruleset{ 31 | rules: make(map[string]*PipeRule), 32 | compiledDeps: make(map[string][]string), 33 | bus: bus.New(), 34 | lastID: 0, 35 | } 36 | }) 37 | return instance 38 | } 39 | 40 | // CompileAst compiles the AST 41 | func (r *Ruleset) CompileAst(filename string, ast *AST, config *Configuration) ([]string, error) { 42 | // file has been already compiled 43 | if c, ok := r.compiledDeps[filename]; ok { 44 | return c, nil 45 | } 46 | 47 | deps := make([]string, 0) 48 | // compile dependencies first 49 | for f, d := range ast.Dependencies { 50 | childDeps, err := r.CompileAst(f, d, config) 51 | if err != nil { 52 | return nil, fmt.Errorf("compile file '%s': %s", f, err) 53 | } 54 | deps = append(deps, f) 55 | deps = append(deps, childDeps...) 56 | } 57 | // track the compiled files and its dependencies 58 | r.compiledDeps[filename] = deps 59 | 60 | for _, rn := range ast.Rules { 61 | //pp.Println(rn) 62 | err := r.AddRule(filename, rn, config, deps) 63 | if err != nil { 64 | err = fmt.Errorf("adding rule '%s': %s", rn.Identifier, err) 65 | return nil, err 66 | } 67 | } 68 | 69 | return deps, nil 70 | } 71 | 72 | // AddRule appends a new rule to the set 73 | func (r *Ruleset) AddRule(filename string, node *RuleNode, config *Configuration, deps []string) error { 74 | if node == nil || node.Identifier == "" { 75 | return fmt.Errorf("Ruleset.AddRule: rules without name are not supported") 76 | } 77 | 78 | // Prepend the filename to the node identifier 79 | name := strings.Join([]string{filename, node.Identifier}, ":") 80 | if _, ok := r.rules[name]; ok { 81 | return fmt.Errorf("Ruleset.AddRule: rule '%s' redefined previously", node.Identifier) 82 | } 83 | 84 | pr, err := NewPipeRule(node, config, filename, deps) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | log.Debug("Added @%s to rules", pr.Name) 90 | r.rules[pr.GetIdentifier()] = pr 91 | if pr.HasFeeder { 92 | r.feedRules = append(r.feedRules, pr.GetIdentifier()) 93 | } 94 | 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /core/version.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | // Info on driplane 4 | const ( 5 | Name = "driplane" 6 | Version = "1.20.0" 7 | Author = "Gianluca Braga aka Matrix86" 8 | ) 9 | -------------------------------------------------------------------------------- /data/event.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | // EventTopicName name of the topic on the bus 4 | const EventTopicName = "#EVENT_TYPE#" 5 | 6 | // Event contains the info about what just happened 7 | type Event struct { 8 | Type string 9 | Content interface{} 10 | } 11 | -------------------------------------------------------------------------------- /data/message.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | html "html/template" 7 | "strings" 8 | "sync" 9 | text "text/template" 10 | ) 11 | 12 | // Message is the data generated from a Feeder and it travels across Filters 13 | type Message struct { 14 | sync.RWMutex 15 | 16 | fields map[string]interface{} 17 | firstRun bool 18 | } 19 | 20 | // NewMessage creates a new Message struct with only the "main" data 21 | func NewMessage(msg interface{}) *Message { 22 | return NewMessageWithExtra(msg, map[string]interface{}{}) 23 | } 24 | 25 | // NewMessageWithExtra creates a Message struct with "main" and extra data 26 | func NewMessageWithExtra(msg interface{}, extra map[string]interface{}) *Message { 27 | extra["main"] = msg 28 | return &Message{ 29 | fields: extra, 30 | } 31 | } 32 | 33 | // SetMessage allows to change the "main" data in the Message struct 34 | func (d *Message) SetMessage(msg interface{}) { 35 | d.Lock() 36 | defer d.Unlock() 37 | d.fields["main"] = msg 38 | } 39 | 40 | // GetMessage returns the "main" data in the Message struct 41 | func (d *Message) GetMessage() interface{} { 42 | d.RLock() 43 | defer d.RUnlock() 44 | return d.fields["main"] 45 | } 46 | 47 | // SetExtra allows to change the "extra" data with key k and value v in the Message struct 48 | func (d *Message) SetExtra(k string, v interface{}) { 49 | d.Lock() 50 | defer d.Unlock() 51 | if k == "main" { 52 | return 53 | } 54 | d.fields[k] = v 55 | } 56 | 57 | // GetExtra returns all the "extra" data in the Message struct 58 | func (d *Message) GetExtra() map[string]interface{} { 59 | d.Lock() 60 | defer d.Unlock() 61 | 62 | clone := make(map[string]interface{}) 63 | for key, value := range d.fields { 64 | if key == "main" { 65 | // Ignoring main content 66 | continue 67 | } 68 | if strings.HasPrefix(key, "_") { 69 | // ignore keys that starts with _ 70 | continue 71 | } 72 | clone[key] = value 73 | } 74 | return clone 75 | } 76 | 77 | // SetTarget is like SetExtra but it can change also the "main" key 78 | func (d *Message) SetTarget(name string, value interface{}) { 79 | d.Lock() 80 | defer d.Unlock() 81 | d.fields[name] = value 82 | } 83 | 84 | // SetFirstRun set the firstRun flag 85 | func (d *Message) SetFirstRun() { 86 | d.Lock() 87 | defer d.Unlock() 88 | d.firstRun = true 89 | } 90 | 91 | // ClearFirstRun clear the firstRun flag 92 | func (d *Message) ClearFirstRun() { 93 | d.Lock() 94 | defer d.Unlock() 95 | d.firstRun = false 96 | } 97 | 98 | // IsFirstRun return the status of the firstRun flag 99 | func (d *Message) IsFirstRun() bool { 100 | d.RLock() 101 | defer d.RUnlock() 102 | return d.firstRun 103 | } 104 | 105 | // GetTarget returns the value of a key in the Message struct. It can return also the "main" data 106 | func (d *Message) GetTarget(name string) interface{} { 107 | d.RLock() 108 | defer d.RUnlock() 109 | if v, ok := d.fields[name]; ok { 110 | return v 111 | } 112 | return nil 113 | } 114 | 115 | // Clone creates a deep copy of the Message struct 116 | func (d *Message) Clone() *Message { 117 | clone := &Message{ 118 | fields: make(map[string]interface{}, 0), 119 | } 120 | 121 | for k, v := range d.fields { 122 | clone.fields[k] = v 123 | } 124 | clone.firstRun = d.firstRun 125 | 126 | return clone 127 | } 128 | 129 | // ApplyPlaceholder executes the template specified using the data in the Message struct 130 | func (d *Message) ApplyPlaceholder(template interface{}) (string, error) { 131 | d.RLock() 132 | defer d.RUnlock() 133 | var writer bytes.Buffer 134 | 135 | switch t := template.(type) { 136 | case *html.Template: 137 | err := t.Execute(&writer, d.fields) 138 | if err != nil { 139 | return "", err 140 | } 141 | return writer.String(), nil 142 | case *text.Template: 143 | err := t.Execute(&writer, d.fields) 144 | if err != nil { 145 | return "", err 146 | } 147 | return writer.String(), nil 148 | } 149 | return "", fmt.Errorf("template type not supported") 150 | } 151 | -------------------------------------------------------------------------------- /example.rule: -------------------------------------------------------------------------------- 1 | # this is a comment 2 | # "name of the rule" => *rule pipe* 3 | 4 | # define a rule with only a feeder to be used from other rules 5 | # Twitter => ; 6 | 7 | #rulehash => @Twitter | hash() | echo(); 8 | #ruleurl => @Twitter | url() | http() | echo(); 9 | 10 | #ruletext => @Twitter | text(regexp="android") | hash(md5="true",extract="true") | echo(); 11 | 12 | # "tail -f" on a file and filtering urls 13 | # RuleFile => | echo(); -------------------------------------------------------------------------------- /feeders/base.go: -------------------------------------------------------------------------------- 1 | package feeders 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/Matrix86/driplane/data" 7 | "github.com/asaskevich/EventBus" 8 | "github.com/evilsocket/islazy/log" 9 | ) 10 | 11 | // FeederFactory identifies a function to instantiate a Feeder using the Factory 12 | type FeederFactory func(conf map[string]string) (Feeder, error) 13 | 14 | var feederFactories = make(map[string]FeederFactory) 15 | 16 | // Feeder defines Base methods of the object 17 | type Feeder interface { 18 | setName(name string) 19 | setBus(bus EventBus.Bus) 20 | setID(id int32) 21 | setRuleName(name string) 22 | 23 | Name() string 24 | Rule() string 25 | Start() 26 | Stop() 27 | IsRunning() bool 28 | GetIdentifier() string 29 | OnEvent(e *data.Event) 30 | } 31 | 32 | // Base is inherited from the feeders 33 | type Base struct { 34 | name string 35 | rule string 36 | id int32 37 | isRunning bool 38 | bus EventBus.Bus 39 | } 40 | 41 | // Propagate sends the Message to the connected Filters 42 | func (f *Base) Propagate(data *data.Message) { 43 | data.SetExtra("source_feeder", f.Name()) 44 | data.SetExtra("source_feeder_rule", f.Rule()) 45 | data.SetExtra("rule_name", f.Rule()) 46 | f.bus.Publish(f.GetIdentifier(), data) 47 | } 48 | 49 | func (f *Base) setID(id int32) { 50 | f.id = id 51 | } 52 | 53 | func (f *Base) setBus(bus EventBus.Bus) { 54 | f.bus = bus 55 | } 56 | 57 | func (f *Base) setName(name string) { 58 | f.name = name 59 | } 60 | 61 | func (f *Base) setRuleName(name string) { 62 | f.rule = name 63 | } 64 | 65 | // GetIdentifier returns the Node identifier ID used in the bus 66 | func (f *Base) GetIdentifier() string { 67 | return fmt.Sprintf("%s:%d", f.name, f.id) 68 | } 69 | 70 | // Name returns the name of the Node 71 | func (f *Base) Name() string { 72 | return f.name 73 | } 74 | 75 | // Rule returns the rule in which the Feeder is found 76 | func (f *Base) Rule() string { 77 | return f.rule 78 | } 79 | 80 | // Start initializes the Node 81 | func (f *Base) Start() {} 82 | 83 | // Stop stops the Node 84 | func (f *Base) Stop() {} 85 | 86 | // IsRunning returns true if the Node is up and running 87 | func (f *Base) IsRunning() bool { 88 | return f.isRunning 89 | } 90 | 91 | func register(name string, f FeederFactory) { 92 | feederName := name + "feeder" 93 | if f == nil { 94 | log.Fatal("Factory method doesn't exists") 95 | } 96 | if _, ok := feederFactories[feederName]; ok { 97 | log.Fatal("Factory method with the same name already exists") 98 | } 99 | feederFactories[feederName] = f 100 | } 101 | 102 | // Init 103 | func init() { 104 | } 105 | 106 | // NewFeeder creates a new registered Feeder from it's name 107 | func NewFeeder(rule string, name string, conf map[string]string, bus EventBus.Bus, id int32) (Feeder, error) { 108 | if _, ok := feederFactories[name]; ok { 109 | f, err := feederFactories[name](conf) 110 | if err == nil && f != nil { 111 | f.setName(name) 112 | f.setRuleName(rule) 113 | f.setBus(bus) 114 | f.setID(id) 115 | } 116 | 117 | return f, err 118 | } 119 | return nil, fmt.Errorf("feeder '%s' doesn't exist", name) 120 | } 121 | -------------------------------------------------------------------------------- /feeders/file.go: -------------------------------------------------------------------------------- 1 | package feeders 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | 8 | "github.com/Matrix86/driplane/data" 9 | 10 | "github.com/evilsocket/islazy/log" 11 | "github.com/hpcloud/tail" 12 | ) 13 | 14 | // File is a Feeder that creates a stream from a file 15 | type File struct { 16 | Base 17 | 18 | filename string 19 | lastLines bool 20 | 21 | fp *tail.Tail 22 | } 23 | 24 | // NewFileFeeder is the registered method to instantiate a FileFeeder 25 | func NewFileFeeder(conf map[string]string) (Feeder, error) { 26 | f := &File{ 27 | lastLines: false, 28 | } 29 | 30 | if val, ok := conf["file.filename"]; ok { 31 | f.filename = val 32 | } 33 | if val, ok := conf["file.toend"]; ok && val == "true" { 34 | f.lastLines = true 35 | } 36 | 37 | info, err := os.Stat(f.filename) 38 | if os.IsNotExist(err) || info.IsDir() { 39 | return nil, fmt.Errorf("file '%s' does not exist", f.filename) 40 | } 41 | 42 | seek := tail.SeekInfo{ 43 | Offset: 0, 44 | Whence: io.SeekStart, 45 | } 46 | if f.lastLines { 47 | seek.Offset = info.Size() 48 | } 49 | 50 | f.fp, err = tail.TailFile(f.filename, tail.Config{ 51 | Logger: tail.DiscardingLogger, 52 | Follow: true, 53 | Location: &seek, 54 | }) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | return f, nil 60 | } 61 | 62 | // Start propagates a message every time a new line is read 63 | func (f *File) Start() { 64 | go func() { 65 | for line := range f.fp.Lines { 66 | msg := data.NewMessage(line.Text) 67 | msg.SetExtra("file_name", f.filename) 68 | f.Propagate(msg) 69 | } 70 | }() 71 | 72 | f.isRunning = true 73 | } 74 | 75 | // Stop handles the Feeder shutdown 76 | func (f *File) Stop() { 77 | log.Debug("feeder '%s' stream stop", f.Name()) 78 | f.fp.Stop() 79 | f.fp.Cleanup() 80 | f.isRunning = false 81 | } 82 | 83 | // OnEvent is called when an event occurs 84 | func (f *File) OnEvent(event *data.Event) {} 85 | 86 | // Auto factory adding 87 | func init() { 88 | register("file", NewFileFeeder) 89 | } 90 | -------------------------------------------------------------------------------- /feeders/telegram_auth.go: -------------------------------------------------------------------------------- 1 | package feeders 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "os" 9 | "strings" 10 | "syscall" 11 | 12 | "golang.org/x/term" 13 | 14 | "github.com/gotd/td/telegram/auth" 15 | "github.com/gotd/td/tg" 16 | ) 17 | 18 | type Terminal struct { 19 | PhoneNumber string // optional, will be prompted if empty 20 | } 21 | 22 | func (Terminal) SignUp(ctx context.Context) (auth.UserInfo, error) { 23 | return auth.UserInfo{}, errors.New("signing up not implemented in Terminal") 24 | } 25 | 26 | func (Terminal) AcceptTermsOfService(ctx context.Context, tos tg.HelpTermsOfService) error { 27 | return &auth.SignUpRequired{TermsOfService: tos} 28 | } 29 | 30 | func (Terminal) Code(ctx context.Context, sentCode *tg.AuthSentCode) (string, error) { 31 | fmt.Print("Enter code: ") 32 | code, err := bufio.NewReader(os.Stdin).ReadString('\n') 33 | if err != nil { 34 | return "", err 35 | } 36 | return strings.TrimSpace(code), nil 37 | } 38 | 39 | func (a Terminal) Phone(_ context.Context) (string, error) { 40 | if a.PhoneNumber != "" { 41 | return a.PhoneNumber, nil 42 | } 43 | fmt.Print("Enter phone in international format (e.g. +1234567890): ") 44 | phone, err := bufio.NewReader(os.Stdin).ReadString('\n') 45 | if err != nil { 46 | return "", err 47 | } 48 | return strings.TrimSpace(phone), nil 49 | } 50 | 51 | func (Terminal) Password(_ context.Context) (string, error) { 52 | fmt.Print("Enter 2FA password: ") 53 | bytePwd, err := term.ReadPassword(syscall.Stdin) 54 | if err != nil { 55 | return "", err 56 | } 57 | return strings.TrimSpace(string(bytePwd)), nil 58 | } 59 | -------------------------------------------------------------------------------- /feeders/timer.go: -------------------------------------------------------------------------------- 1 | package feeders 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/Matrix86/driplane/data" 8 | 9 | "github.com/evilsocket/islazy/log" 10 | ) 11 | 12 | // Timer is a Feeder that triggers the pipeline using a timer 13 | type Timer struct { 14 | Base 15 | 16 | frequency time.Duration 17 | 18 | stopChan chan bool 19 | ticker *time.Ticker 20 | } 21 | 22 | // NewTimerFeeder is the registered method to instantiate a TimerFeeder 23 | func NewTimerFeeder(conf map[string]string) (Feeder, error) { 24 | f := &Timer{ 25 | stopChan: make(chan bool), 26 | frequency: 60 * time.Second, 27 | } 28 | 29 | if val, ok := conf["timer.freq"]; ok { 30 | d, err := time.ParseDuration(val) 31 | if err != nil { 32 | return nil, fmt.Errorf("specified frequency cannot be parsed '%s': %s", val, err) 33 | } 34 | f.frequency = d 35 | } 36 | 37 | return f, nil 38 | } 39 | 40 | // Start propagates a message every time the ticker is fired 41 | func (f *Timer) Start() { 42 | f.ticker = time.NewTicker(f.frequency) 43 | go func() { 44 | for { 45 | select { 46 | case <-f.stopChan: 47 | log.Debug("%s: stop arrived on the channel", f.Name()) 48 | return 49 | case <-f.ticker.C: 50 | t := time.Now() 51 | extra := make(map[string]interface{}) 52 | extra["timestamp"] = t.Unix() 53 | extra["rfc3339"] = t.Format(time.RFC3339) 54 | msg := data.NewMessageWithExtra(extra["rfc3339"], extra) 55 | f.Propagate(msg) 56 | } 57 | } 58 | }() 59 | 60 | f.isRunning = true 61 | } 62 | 63 | // Stop handles the Feeder shutdown 64 | func (f *Timer) Stop() { 65 | log.Debug("feeder '%s' stream stop", f.Name()) 66 | f.stopChan <- true 67 | f.ticker.Stop() 68 | f.isRunning = false 69 | } 70 | 71 | // OnEvent is called when an event occurs 72 | func (f *Timer) OnEvent(event *data.Event) {} 73 | 74 | // Auto factory adding 75 | func init() { 76 | register("timer", NewTimerFeeder) 77 | } 78 | -------------------------------------------------------------------------------- /filters/cache.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/Matrix86/driplane/data" 9 | "github.com/Matrix86/driplane/utils" 10 | 11 | "github.com/evilsocket/islazy/log" 12 | ) 13 | 14 | // Cache handles a cache usable in the rule 15 | type Cache struct { 16 | sync.Mutex 17 | Base 18 | 19 | target string 20 | ttl time.Duration 21 | refreshOnGet bool 22 | global bool 23 | syncTime time.Duration 24 | ignoreFirstRun bool 25 | cacheName string 26 | 27 | persistentFile string 28 | 29 | params map[string]string 30 | cache *utils.TTLMap 31 | } 32 | 33 | // NewCacheFilter is the registered method to instantiate a CacheFilter 34 | func NewCacheFilter(p map[string]string) (Filter, error) { 35 | f := &Cache{ 36 | params: p, 37 | target: "main", 38 | refreshOnGet: true, 39 | global: false, 40 | ttl: 24 * time.Hour, 41 | syncTime: 5 * time.Minute, 42 | ignoreFirstRun: false, 43 | } 44 | f.cbFilter = f.DoFilter 45 | 46 | if v, ok := f.params["target"]; ok { 47 | f.target = v 48 | } 49 | if v, ok := f.params["refresh_on_get"]; ok && v == "false" { 50 | f.refreshOnGet = false 51 | } 52 | if v, ok := f.params["sync_time"]; ok { 53 | // https://golang.org/pkg/time/#ParseDuration 54 | i, err := time.ParseDuration(v) 55 | if err != nil { 56 | return nil, err 57 | } 58 | f.syncTime = i 59 | } 60 | if v, ok := f.params["ttl"]; ok { 61 | // https://golang.org/pkg/time/#ParseDuration 62 | i, err := time.ParseDuration(v) 63 | if err != nil { 64 | return nil, err 65 | } 66 | f.ttl = i 67 | } 68 | if v, ok := f.params["global"]; ok && v == "true" { 69 | f.global = true 70 | f.cacheName = "global" 71 | f.cache = utils.GetNamedTTLMap("global", f.syncTime) 72 | } else if v, ok := f.params["name"]; ok { 73 | f.cacheName = v 74 | f.cache = utils.GetNamedTTLMap(v, f.syncTime) 75 | } else { 76 | f.cache = utils.NewTTLMap(f.syncTime) 77 | } 78 | 79 | if v, ok := f.params["ignore_first_run"]; ok && v == "true" { 80 | f.ignoreFirstRun = true 81 | } 82 | 83 | if v, ok := f.params["file"]; ok { 84 | err := f.cache.SetPersistence(v) 85 | if err != nil { 86 | return nil, fmt.Errorf("cache tried to load %s: %s", v, err) 87 | } 88 | f.persistentFile = v 89 | } 90 | 91 | return f, nil 92 | } 93 | 94 | // DoFilter is the mandatory method used to "filter" the input data.Message 95 | func (f *Cache) DoFilter(msg *data.Message) (bool, error) { 96 | var text interface{} 97 | 98 | if f.target == "main" { 99 | text = msg.GetMessage() 100 | } else if v, ok := msg.GetExtra()[f.target]; ok { 101 | text = v 102 | } 103 | 104 | if text == nil { 105 | return true, nil 106 | } 107 | 108 | hash := utils.MD5Sum(text) 109 | if _, ok := f.cache.Get(hash); !ok { 110 | f.cache.Put(hash, true, int64(f.ttl.Seconds())) 111 | if f.ignoreFirstRun { 112 | log.Debug("ignoring first run") 113 | return true, nil 114 | } 115 | log.Debug("caching '%s' firstRun:%v", text, msg.IsFirstRun()) 116 | // don't propagate the message is it is the first run 117 | return !msg.IsFirstRun(), nil 118 | } else if f.refreshOnGet { 119 | f.cache.Put(hash, true, int64(f.ttl.Seconds())) 120 | } 121 | return false, nil 122 | } 123 | 124 | // OnEvent is called when an event occurs 125 | func (f *Cache) OnEvent(event *data.Event) { 126 | if event.Type == "shutdown" { 127 | f.cache.Close() 128 | } 129 | } 130 | 131 | // Set the name of the filter 132 | func init() { 133 | register("cache", NewCacheFilter) 134 | } 135 | -------------------------------------------------------------------------------- /filters/changed.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/Matrix86/driplane/utils" 5 | "sync" 6 | 7 | "github.com/Matrix86/driplane/data" 8 | ) 9 | 10 | // Changed is a Filter that call the propagation method only if 11 | // the input Message is different from the previous one 12 | type Changed struct { 13 | sync.Mutex 14 | Base 15 | 16 | target string 17 | 18 | params map[string]string 19 | cache string 20 | } 21 | 22 | // NewChangedFilter is the registered method to instantiate a ChangedFilter 23 | func NewChangedFilter(p map[string]string) (Filter, error) { 24 | f := &Changed{ 25 | params: p, 26 | target: "main", 27 | } 28 | f.cbFilter = f.DoFilter 29 | 30 | if v, ok := f.params["target"]; ok { 31 | f.target = v 32 | } 33 | 34 | return f, nil 35 | } 36 | 37 | // DoFilter is the mandatory method used to "filter" the input data.Message 38 | func (f *Changed) DoFilter(msg *data.Message) (bool, error) { 39 | var text interface{} 40 | 41 | if f.target == "main" { 42 | text = msg.GetMessage() 43 | } else if v, ok := msg.GetExtra()[f.target]; ok { 44 | text = v 45 | } else { 46 | return false, nil 47 | } 48 | 49 | hash := utils.MD5Sum(text) 50 | if f.cache != hash { 51 | f.Lock() 52 | defer f.Unlock() 53 | f.cache = hash 54 | return true, nil 55 | } 56 | 57 | return false, nil 58 | } 59 | 60 | // OnEvent is called when an event occurs 61 | func (f *Changed) OnEvent(event *data.Event){} 62 | 63 | // Set the name of the filter 64 | func init() { 65 | register("changed", NewChangedFilter) 66 | } 67 | -------------------------------------------------------------------------------- /filters/echo.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/Matrix86/driplane/data" 7 | 8 | "github.com/evilsocket/islazy/log" 9 | ) 10 | 11 | // Echo is a filter that print the input Message on the logs. 12 | type Echo struct { 13 | Base 14 | 15 | printExtra bool 16 | target string 17 | 18 | params map[string]string 19 | } 20 | 21 | // NewEchoFilter is the registered method to instantiate a EchoFilter 22 | func NewEchoFilter(p map[string]string) (Filter, error) { 23 | f := &Echo{ 24 | params: p, 25 | printExtra: false, 26 | target: "main", 27 | } 28 | f.cbFilter = f.DoFilter 29 | 30 | if v, ok := f.params["extra"]; ok && v == "true" { 31 | f.printExtra = true 32 | } 33 | if v, ok := f.params["target"]; ok { 34 | f.target = v 35 | } 36 | return f, nil 37 | } 38 | 39 | // DoFilter is the mandatory method used to "filter" the input data.Message 40 | func (f *Echo) DoFilter(msg *data.Message) (bool, error) { 41 | var text string 42 | data := msg.GetTarget(f.target ) 43 | text = fmt.Sprintf("%#v", data) 44 | if f.printExtra { 45 | for k, v := range msg.GetExtra() { 46 | text = fmt.Sprintf("%s [%#v: %#v] ", text, k, v) 47 | } 48 | } 49 | log.Info("%s", text) 50 | return true, nil 51 | } 52 | 53 | // OnEvent is called when an event occurs 54 | func (f *Echo) OnEvent(event *data.Event) {} 55 | 56 | // Set the name of the filter 57 | func init() { 58 | register("echo", NewEchoFilter) 59 | } 60 | -------------------------------------------------------------------------------- /filters/echo_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/Matrix86/driplane/data" 5 | "testing" 6 | ) 7 | 8 | func TestNewEchoFilter(t *testing.T) { 9 | filter, err := NewEchoFilter(map[string]string{"none": "none", "extra": "true"}) 10 | if err != nil { 11 | t.Errorf("constructor returned '%s'", err) 12 | } 13 | if e, ok := filter.(*Echo); ok { 14 | if e.printExtra == false { 15 | t.Errorf("'extra' parameter ignored") 16 | } 17 | } else { 18 | t.Errorf("cannot cast to proper Filter...") 19 | } 20 | } 21 | 22 | func TestEchoDoFilter(t *testing.T) { 23 | filter, err := NewEchoFilter(map[string]string{"none": "none", "extra": "true"}) 24 | if err != nil { 25 | t.Errorf("constructor returned '%s'", err) 26 | } 27 | if e, ok := filter.(*Echo); ok { 28 | m := data.NewMessageWithExtra("main message", map[string]interface{}{"extra": "1"}) 29 | b, err := e.DoFilter(m) 30 | if b != true { 31 | t.Errorf("DoFilter cannot return false") 32 | } 33 | if err != nil { 34 | t.Errorf("DoFilter cannot return an error '%s'", err) 35 | } 36 | } else { 37 | t.Errorf("cannot cast to proper Filter...") 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /filters/elasticsearch.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "context" 5 | "crypto/sha256" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "github.com/Matrix86/driplane/data" 12 | 13 | elasticsearch "github.com/elastic/go-elasticsearch/v7" 14 | "github.com/elastic/go-elasticsearch/v7/esapi" 15 | "github.com/evilsocket/islazy/log" 16 | ) 17 | 18 | // ElasticSearch is a filter that imports JSON documents in an Elastsic Search database. 19 | type ElasticSearch struct { 20 | Base 21 | 22 | client *elasticsearch.Client 23 | address string 24 | username string 25 | password string 26 | index string 27 | retries int 28 | target string 29 | 30 | params map[string]string 31 | } 32 | 33 | // NewElasticSearchFilter is the registered method to instantiate a ElasticSearchFilter 34 | func NewElasticSearchFilter(p map[string]string) (Filter, error) { 35 | f := &ElasticSearch{ 36 | params: p, 37 | client: nil, 38 | retries: 1, 39 | address: "localhost:9200", 40 | target: "main", 41 | } 42 | f.cbFilter = f.DoFilter 43 | 44 | if v, ok := f.params["address"]; ok { 45 | f.address = v 46 | } 47 | 48 | if v, ok := f.params["username"]; ok { 49 | f.username = v 50 | } 51 | 52 | if v, ok := f.params["password"]; ok { 53 | f.password = v 54 | } 55 | 56 | if v, ok := f.params["index"]; ok { 57 | f.index = v 58 | } 59 | 60 | if v, ok := f.params["target"]; ok { 61 | f.target = v 62 | } 63 | 64 | if v, ok := f.params["retries"]; ok { 65 | n, err := strconv.Atoi(v) 66 | if err != nil { 67 | return nil, err 68 | } 69 | f.retries = n 70 | } 71 | 72 | return f, nil 73 | } 74 | 75 | func (f *ElasticSearch) connect() (err error) { 76 | wait := time.Duration(5) * time.Second 77 | 78 | log.Debug("connecting to %s ...", f.address) 79 | 80 | for attempt := 0; attempt < f.retries; attempt++ { 81 | cfg := elasticsearch.Config{ 82 | Addresses: []string{f.address}, 83 | Username: f.username, 84 | Password: f.password, 85 | } 86 | 87 | if f.client, err = elasticsearch.NewClient(cfg); err != nil { 88 | return fmt.Errorf("error creating ES client: %v", err) 89 | } 90 | 91 | if res, err := f.client.Info(); err != nil { 92 | log.Debug("waiting for ES to come up ... [%v] (attempt %d of %d, retrying in %s)", err, attempt+1, 93 | f.retries, wait) 94 | time.Sleep(wait) 95 | continue 96 | } else { 97 | defer res.Body.Close() 98 | if res.IsError() { 99 | return fmt.Errorf("error getting information from ES cluster: %v", res) 100 | } 101 | log.Debug("%+v", res) 102 | return nil 103 | } 104 | } 105 | 106 | return fmt.Errorf("could not connect") 107 | } 108 | 109 | // DoFilter is the mandatory method used to "filter" the input data.Message 110 | func (f *ElasticSearch) DoFilter(msg *data.Message) (bool, error) { 111 | if f.client == nil { 112 | if err := f.connect(); err != nil { 113 | return true, err 114 | } 115 | } 116 | 117 | rawJSON := msg.GetTarget(f.target).(string) 118 | // make the document id contents dependent so that if we have multiple 119 | // events for the same object we're not going to create duplicate events 120 | docID := fmt.Sprintf("%x", sha256.Sum256([]byte(rawJSON))) 121 | 122 | req := esapi.IndexRequest{ 123 | Index: f.index, 124 | DocumentID: docID, 125 | Body: strings.NewReader(rawJSON), 126 | Refresh: "true", 127 | } 128 | log.Debug("IndexRequest: %#v", req) 129 | 130 | res, err := req.Do(context.Background(), f.client) 131 | if err != nil { 132 | return true, fmt.Errorf("could not save document %s to index: %v", docID, err) 133 | } 134 | defer res.Body.Close() 135 | 136 | if res.IsError() { 137 | return true, fmt.Errorf("could not index document %s: %v", docID, res) 138 | } 139 | 140 | msg.SetMessage(docID) 141 | 142 | return true, nil 143 | } 144 | 145 | // OnEvent is called when an event occurs 146 | func (f *ElasticSearch) OnEvent(event *data.Event) {} 147 | 148 | // Set the name of the filter 149 | func init() { 150 | register("elasticsearch", NewElasticSearchFilter) 151 | } 152 | -------------------------------------------------------------------------------- /filters/file.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "github.com/Matrix86/driplane/data" 8 | "github.com/evilsocket/islazy/log" 9 | ) 10 | 11 | // File is a filter that interprets the input Message as a file path, reads it and prints it. 12 | type File struct { 13 | Base 14 | 15 | target string 16 | } 17 | 18 | // NewFileFilter is the registered method to instantiate a File filter 19 | func NewFileFilter(params map[string]string) (Filter, error) { 20 | f := File{ 21 | target: "main", 22 | } 23 | f.cbFilter = f.DoFilter 24 | 25 | if v, ok := params["target"]; ok { 26 | f.target = v 27 | } 28 | 29 | return &f, nil 30 | } 31 | 32 | // DoFilter is the mandatory method used to "filter" the input data.Message 33 | func (f *File) DoFilter(msg *data.Message) (bool, error) { 34 | // if the message data is a string 35 | if path, ok := msg.GetMessage().(string); ok { 36 | // if the path exists and it's a file 37 | if stat, err := os.Stat(path); err == nil && !stat.IsDir() { 38 | log.Debug("path='%s' size=%d extra=%v", path, stat.Size(), msg.GetExtra()) 39 | readData, err := ioutil.ReadFile(path) 40 | if err != nil { 41 | return true, err 42 | } 43 | msg.SetTarget(f.target, string(readData)) 44 | return true, nil 45 | } 46 | log.Debug("%s is not a file", path) 47 | } 48 | return false, nil 49 | } 50 | 51 | // OnEvent is called when an event occurs 52 | func (f *File) OnEvent(event *data.Event) {} 53 | 54 | // Set the name of the filter 55 | func init() { 56 | register("file", NewFileFilter) 57 | } 58 | -------------------------------------------------------------------------------- /filters/filter_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import "github.com/Matrix86/driplane/data" 4 | 5 | type FakeBus struct { 6 | Collected []*data.Message 7 | } 8 | 9 | func NewFakeBus() *FakeBus { 10 | return &FakeBus{ 11 | Collected: make([]*data.Message, 0), 12 | } 13 | } 14 | 15 | func (b *FakeBus) Reset() { 16 | b.Collected = make([]*data.Message, 0) 17 | } 18 | 19 | func (b *FakeBus) Publish(topic string, args ...interface{}) { 20 | for _, k := range args { 21 | if v, ok := k.(*data.Message); ok { 22 | b.Collected = append(b.Collected, v) 23 | } 24 | } 25 | } 26 | 27 | func (b *FakeBus) HasCallback(topic string) bool { return true } 28 | func (b *FakeBus) WaitAsync() {} 29 | func (b *FakeBus) Subscribe(topic string, fn interface{}) error { return nil } 30 | func (b *FakeBus) SubscribeAsync(topic string, fn interface{}, transactional bool) error { return nil } 31 | func (b *FakeBus) SubscribeOnce(topic string, fn interface{}) error { return nil } 32 | func (b *FakeBus) SubscribeOnceAsync(topic string, fn interface{}) error { return nil } 33 | func (b *FakeBus) Unsubscribe(topic string, handler interface{}) error { return nil } 34 | -------------------------------------------------------------------------------- /filters/format.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | html "html/template" 6 | "io/ioutil" 7 | "path/filepath" 8 | text "text/template" 9 | 10 | "github.com/Matrix86/driplane/data" 11 | ) 12 | 13 | // Format is a Filter that apply a Golang Template to the input Message 14 | // and propagate it to the next Filter 15 | type Format struct { 16 | Base 17 | 18 | target string 19 | template interface{} 20 | templateType string // html or text 21 | 22 | params map[string]string 23 | } 24 | 25 | // NewFormatFilter is the registered method to instantiate a FormatFilter 26 | func NewFormatFilter(p map[string]string) (Filter, error) { 27 | f := &Format{ 28 | target: "main", 29 | params: p, 30 | templateType: "text", 31 | } 32 | f.cbFilter = f.DoFilter 33 | 34 | if v, ok := f.params["type"]; ok && v == "html" { 35 | f.templateType = "html" 36 | } 37 | 38 | if v, ok := f.params["target"]; ok { 39 | f.target = v 40 | } 41 | 42 | if v, ok := f.params["template"]; ok { 43 | if f.templateType == "html" { 44 | t, err := html.New("formatFilterTemplate").Parse(v) 45 | if err != nil { 46 | return nil, err 47 | } 48 | f.template = t 49 | } else { 50 | t, err := text.New("formatFilterTemplate").Parse(v) 51 | if err != nil { 52 | return nil, err 53 | } 54 | f.template = t 55 | } 56 | } 57 | if v, ok := f.params["file"]; ok { 58 | fpath := v 59 | if v, ok := p["general.templates_path"]; !ok { 60 | r := "" 61 | if r, ok = p["general.rules_path"]; !ok { 62 | return nil, fmt.Errorf("NewJsFilter: rules_path or js_path configs not found") 63 | } 64 | fpath = filepath.Join(r, fpath) 65 | 66 | } else { 67 | fpath = filepath.Join(v, fpath) 68 | } 69 | content, err := ioutil.ReadFile(fpath) 70 | if err != nil { 71 | return nil, err 72 | } 73 | if f.templateType == "html" { 74 | t, err := html.New("formatFilterTemplate").Parse(string(content)) 75 | if err != nil { 76 | return nil, err 77 | } 78 | f.template = t 79 | } else { 80 | t, err := text.New("formatFilterTemplate").Parse(string(content)) 81 | if err != nil { 82 | return nil, err 83 | } 84 | f.template = t 85 | } 86 | } 87 | 88 | return f, nil 89 | } 90 | 91 | // DoFilter is the mandatory method used to "filter" the input data.Message 92 | func (f *Format) DoFilter(msg *data.Message) (bool, error) { 93 | txt, err := msg.ApplyPlaceholder(f.template) 94 | if err != nil { 95 | return false, err 96 | } 97 | msg.SetTarget(f.target, txt) 98 | return true, nil 99 | } 100 | 101 | // OnEvent is called when an event occurs 102 | func (f *Format) OnEvent(event *data.Event) {} 103 | 104 | // Set the name of the filter 105 | func init() { 106 | register("format", NewFormatFilter) 107 | } 108 | -------------------------------------------------------------------------------- /filters/format_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/Matrix86/driplane/data" 5 | "testing" 6 | ) 7 | 8 | func TestNewFormatFilter(t *testing.T) { 9 | filter, err := NewFormatFilter(map[string]string{"none": "none", "template": "test {{.main}}"}) 10 | if err != nil { 11 | t.Errorf("constructor returned '%s'", err) 12 | } 13 | if e, ok := filter.(*Format); ok { 14 | if e.template == nil { 15 | t.Errorf("'template' parameter ignored") 16 | } 17 | } else { 18 | t.Errorf("cannot cast to proper Filter...") 19 | } 20 | } 21 | 22 | func TestFormatDoFilter(t *testing.T) { 23 | filter, err := NewFormatFilter(map[string]string{"template": "main : {{.main}} extra : {{.test}}"}) 24 | if err != nil { 25 | t.Errorf("constructor returned '%s'", err) 26 | } 27 | if e, ok := filter.(*Format); ok { 28 | m := data.NewMessageWithExtra("message", map[string]interface{}{"test": "1"}) 29 | b, err := e.DoFilter(m) 30 | if err != nil { 31 | t.Errorf("DoFilter returned an error '%s'", err) 32 | } 33 | if b == false { 34 | t.Errorf("it should return true") 35 | } 36 | 37 | if m.GetMessage() != "main : message extra : 1" { 38 | t.Errorf("message not formatted correctly") 39 | } 40 | } else { 41 | t.Errorf("cannot cast to proper Filter...") 42 | } 43 | 44 | filter, err = NewFormatFilter(map[string]string{"target": "other", "template": "main : {{.main}} extra : {{.test}}"}) 45 | if err != nil { 46 | t.Errorf("constructor returned '%s'", err) 47 | } 48 | if e, ok := filter.(*Format); ok { 49 | m := data.NewMessageWithExtra("message", map[string]interface{}{"test": "1"}) 50 | b, err := e.DoFilter(m) 51 | if err != nil { 52 | t.Errorf("DoFilter returned an error '%s'", err) 53 | } 54 | if b == false { 55 | t.Errorf("it should return true") 56 | } 57 | 58 | if m.GetTarget("other") != "main : message extra : 1" { 59 | t.Errorf("message not formatted correctly") 60 | } 61 | } else { 62 | t.Errorf("cannot cast to proper Filter...") 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /filters/hash.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | 7 | "github.com/Matrix86/driplane/data" 8 | ) 9 | 10 | // Hash is a Filter that searches for hashes in the Message 11 | type Hash struct { 12 | Base 13 | 14 | regex *regexp.Regexp 15 | 16 | useMd5 bool 17 | useSha1 bool 18 | useSha256 bool 19 | useSha512 bool 20 | extractHash bool 21 | target string 22 | 23 | //filter_extra string 24 | 25 | params map[string]string 26 | } 27 | 28 | // NewHashFilter is the registered method to instantiate a HashFilter 29 | func NewHashFilter(p map[string]string) (Filter, error) { 30 | f := &Hash{ 31 | params: p, 32 | useMd5: true, 33 | useSha1: true, 34 | useSha256: true, 35 | useSha512: true, 36 | extractHash: false, 37 | target: "main", 38 | } 39 | f.cbFilter = f.DoFilter 40 | 41 | f.regex = regexp.MustCompile(`(?i)[a-f0-9]{32,128}`) 42 | 43 | if v, ok := f.params["md5"]; ok && v == "false" { 44 | f.useMd5 = false 45 | } 46 | if v, ok := f.params["sha1"]; ok && v == "false" { 47 | f.useSha1 = false 48 | } 49 | if v, ok := f.params["sha256"]; ok && v == "false" { 50 | f.useSha256 = false 51 | } 52 | if v, ok := f.params["sha512"]; ok && v == "false" { 53 | f.useSha512 = false 54 | } 55 | if v, ok := f.params["extract"]; ok && v == "true" { 56 | f.extractHash = true 57 | } 58 | if v, ok := f.params["target"]; ok { 59 | f.target = v 60 | } 61 | 62 | return f, nil 63 | } 64 | 65 | // DoFilter is the mandatory method used to "filter" the input data.Message 66 | func (f *Hash) DoFilter(msg *data.Message) (bool, error) { 67 | var text string 68 | 69 | if v, ok := msg.GetTarget(f.target).(string); ok { 70 | text = v 71 | } else if v, ok := msg.GetTarget(f.target).([]byte); ok { 72 | text = string(v) 73 | } else { 74 | // ERROR this filter can't be used with different types 75 | return false, fmt.Errorf("received data is not a string") 76 | } 77 | match := f.regex.FindAllStringSubmatch(text, -1) 78 | if match != nil { 79 | for _, m := range match { 80 | length := len(m[0]) 81 | found := false 82 | if f.useMd5 && length == 32 { 83 | found = true 84 | } else if f.useSha1 && length == 40 { 85 | found = true 86 | } else if f.useSha256 && length == 64 { 87 | found = true 88 | } else if f.useSha512 && length == 128 { 89 | found = true 90 | } 91 | 92 | if found { 93 | if f.extractHash { 94 | clone := msg.Clone() 95 | clone.SetMessage(m[0]) 96 | clone.SetExtra("fulltext", text) 97 | f.Propagate(clone) 98 | } else { 99 | return true, nil 100 | } 101 | } 102 | } 103 | } 104 | return false, nil 105 | } 106 | 107 | // OnEvent is called when an event occurs 108 | func (f *Hash) OnEvent(event *data.Event){} 109 | 110 | // Set the name of the filter 111 | func init() { 112 | register("hash", NewHashFilter) 113 | } 114 | -------------------------------------------------------------------------------- /filters/html.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | "github.com/Matrix86/driplane/data" 6 | "github.com/PuerkitoBio/goquery" 7 | "strings" 8 | 9 | "github.com/evilsocket/islazy/log" 10 | ) 11 | 12 | // HTML is a filter to parse the HTML format 13 | type HTML struct { 14 | Base 15 | 16 | selectors string 17 | getType string 18 | attr string 19 | target string 20 | 21 | params map[string]string 22 | } 23 | 24 | // NewHTMLFilter is the registered method to instantiate a HtmlFilter 25 | func NewHTMLFilter(p map[string]string) (Filter, error) { 26 | f := &HTML{ 27 | params: p, 28 | target: "main", 29 | getType: "html", 30 | } 31 | f.cbFilter = f.DoFilter 32 | 33 | if v, ok := f.params["selector"]; ok { 34 | f.selectors = v 35 | } 36 | if v, ok := f.params["get"]; ok { 37 | switch v { 38 | case "attr", 39 | "text", 40 | "html": 41 | f.getType = v 42 | 43 | default: 44 | return nil, fmt.Errorf("get param is not valid") 45 | } 46 | 47 | } 48 | if v, ok := f.params["target"]; ok { 49 | f.target = v 50 | } 51 | if v, ok := f.params["attr"]; ok { 52 | f.attr = v 53 | } 54 | 55 | return f, nil 56 | } 57 | 58 | // DoFilter is the mandatory method used to "filter" the input data.Message 59 | func (f *HTML) DoFilter(msg *data.Message) (bool, error) { 60 | var err error 61 | var text string 62 | 63 | if v, ok := msg.GetTarget(f.target).(string); ok { 64 | text = v 65 | } else if v, ok := msg.GetTarget(f.target).([]byte); ok { 66 | text = string(v) 67 | } else { 68 | // ERROR this filter can't be used with different types 69 | return false, fmt.Errorf("received data is not a string") 70 | } 71 | 72 | stringReader := strings.NewReader(text) 73 | 74 | doc, err := goquery.NewDocumentFromReader(stringReader) 75 | if err != nil { 76 | return false, err 77 | } 78 | 79 | doc.Find(f.selectors).Each(func(i int, s *goquery.Selection) { 80 | message := "" 81 | 82 | switch f.getType { 83 | case "attr": 84 | message, _ = s.Attr(f.attr) 85 | 86 | case "text": 87 | message = s.Text() 88 | 89 | case "html": 90 | message, err = s.Html() 91 | if err != nil { 92 | log.Error("error on parsing: %s", err) 93 | return 94 | } 95 | } 96 | 97 | if message != "" { 98 | clone := msg.Clone() 99 | clone.SetMessage(message) 100 | clone.SetExtra("fulltext", text) 101 | f.Propagate(clone) 102 | } 103 | }) 104 | 105 | return false, nil 106 | } 107 | 108 | // OnEvent is called when an event occurs 109 | func (f *HTML) OnEvent(event *data.Event) {} 110 | 111 | // Set the name of the filter 112 | func init() { 113 | register("html", NewHTMLFilter) 114 | } 115 | -------------------------------------------------------------------------------- /filters/html_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/Matrix86/driplane/data" 5 | "github.com/asaskevich/EventBus" 6 | "testing" 7 | ) 8 | 9 | const htmlTest = ` 10 | 11 | 12 | 13 | Tests for siblings 14 | 15 | 16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 | 32 | 33 | ` 34 | 35 | func TestNewHTMLFilter(t *testing.T) { 36 | filter, err := NewHTMLFilter(map[string]string{"none": "none", "selector": "#selector"}) 37 | if err != nil { 38 | t.Errorf("constructor returned '%s'", err) 39 | } 40 | if e, ok := filter.(*HTML); ok { 41 | if e.target != "main" { 42 | t.Errorf("target should be 'main' if not specified") 43 | } 44 | if e.attr != "" { 45 | t.Errorf("attr should be '' if not specified") 46 | } 47 | if e.getType != "html" { 48 | t.Errorf("get should be 'html' if not specified") 49 | } 50 | if e.selectors != "#selector" { 51 | t.Errorf("selector should be '#selector'") 52 | } 53 | } else { 54 | t.Errorf("cannot cast to proper Filter...") 55 | } 56 | } 57 | 58 | func TestNewHTMLFilterParams(t *testing.T) { 59 | filter, err := NewHTMLFilter(map[string]string{ 60 | "selector": "#selector", 61 | "target": "othertarget", 62 | "get": "text", 63 | "attr": "attr", 64 | }) 65 | if err != nil { 66 | t.Errorf("constructor returned '%s'", err) 67 | } 68 | if e, ok := filter.(*HTML); ok { 69 | if e.target != "othertarget" { 70 | t.Errorf("target should be 'othertarget'") 71 | } 72 | if e.attr != "attr" { 73 | t.Errorf("attr should be 'attr'") 74 | } 75 | if e.getType != "text" { 76 | t.Errorf("get should be 'text'. Received %s", e.getType) 77 | } 78 | if e.selectors != "#selector" { 79 | t.Errorf("selector should be '#selector'") 80 | } 81 | } else { 82 | t.Errorf("cannot cast to proper Filter...") 83 | } 84 | } 85 | 86 | func TestNewHTMLFilterWrongGet(t *testing.T) { 87 | _, err := NewHTMLFilter(map[string]string{ 88 | "selector": "#selector", 89 | "get": "fake", 90 | }) 91 | if err == nil { 92 | t.Errorf("constructor should return an error") 93 | } 94 | if err.Error() != "get param is not valid" { 95 | t.Errorf("constructor should return 'constructor should return an error'") 96 | } 97 | } 98 | 99 | func TestHTML_DoFilter(t *testing.T) { 100 | filter, err := NewHTMLFilter(map[string]string{ 101 | "selector": ".row", 102 | "target": "main", 103 | "get": "attr", 104 | "attr": "id", 105 | }) 106 | if err != nil { 107 | t.Errorf("constructor returned '%s'", err) 108 | } 109 | if e, ok := filter.(*HTML); ok { 110 | fb := NewFakeBus() 111 | filter.setBus(EventBus.Bus(fb)) 112 | 113 | msg := htmlTest 114 | m := data.NewMessage(msg) 115 | _, err := e.DoFilter(m) 116 | if err != nil { 117 | t.Errorf("DoFilter returned an error '%s'", err) 118 | } 119 | if m.GetMessage() != msg { 120 | t.Errorf("the message has been altered by the filter") 121 | } 122 | if len(fb.Collected) == 0 || fb.Collected[0].GetMessage() != "n1" { 123 | t.Errorf("tags have not been extracted correctly") 124 | } 125 | } else { 126 | t.Errorf("cannot cast to proper Filter...") 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /filters/js.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | 7 | "github.com/Matrix86/driplane/data" 8 | 9 | "github.com/evilsocket/islazy/plugin" 10 | "github.com/robertkrimen/otto" 11 | ) 12 | 13 | // Js is a Filter that load a plugin written in Javascript to create a custom Filter 14 | type Js struct { 15 | Base 16 | 17 | filepath string 18 | function string 19 | 20 | p *plugin.Plugin 21 | 22 | params map[string]string 23 | } 24 | 25 | // NewJsFilter is the registered method to instantiate a JsFilter 26 | func NewJsFilter(p map[string]string) (Filter, error) { 27 | f := &Js{ 28 | params: p, 29 | function: "DoFilter", 30 | } 31 | f.cbFilter = f.DoFilter 32 | 33 | if v, ok := p["path"]; ok { 34 | f.filepath = v 35 | } 36 | 37 | // If function is not defined we call DoFilter 38 | if v, ok := p["function"]; ok { 39 | f.function = v 40 | } 41 | 42 | var err error 43 | 44 | // If the path specified is relative, we're resolving it with 'rules_path' config 45 | if !filepath.IsAbs(f.filepath) { 46 | if v, ok := p["general.js_path"]; !ok { 47 | r := "" 48 | if r, ok = p["general.rules_path"]; !ok { 49 | return nil, fmt.Errorf("NewJsFilter: rules_path or js_path configs not found") 50 | } 51 | f.filepath = filepath.Join(r, f.filepath) 52 | 53 | } else { 54 | f.filepath = filepath.Join(v, f.filepath) 55 | } 56 | } 57 | 58 | // load the plugin 59 | f.p, err = plugin.Load(f.filepath) 60 | if err != nil { 61 | return nil, fmt.Errorf("JsFilter '%s': %s", f.filepath, err) 62 | } 63 | 64 | // Check if the JS plugin contains the DoFilter method 65 | if !f.p.HasFunc(f.function) { 66 | return nil, fmt.Errorf("NewJsFilter: %s doesn't contain the %s function", f.filepath, f.function) 67 | } 68 | 69 | return f, nil 70 | } 71 | 72 | func (f *Js) jsResponse2Msg(obj interface{}, msg *data.Message) { 73 | switch t := obj.(type) { 74 | case string: 75 | msg.SetMessage(t) 76 | 77 | case map[string]interface{}: 78 | for key, vi := range t { 79 | if value, ok := vi.(string); ok { 80 | if key == "main" { 81 | msg.SetMessage(value) 82 | } else { 83 | msg.SetExtra(key, value) 84 | } 85 | } 86 | } 87 | } 88 | } 89 | 90 | // DoFilter is the mandatory method used to "filter" the input data.Message 91 | func (f *Js) DoFilter(msg *data.Message) (bool, error) { 92 | triggered := false 93 | text := msg.GetMessage() 94 | extra := msg.GetExtra() 95 | 96 | res, err := f.p.Call(f.function, text, extra, f.params) 97 | if err != nil { 98 | err := err.(*otto.Error) 99 | return false, fmt.Errorf("DoFilter: file '%s': '%s'", f.p.Path, err.String()) 100 | } 101 | 102 | if res != nil { 103 | result, ok := res.(map[string]interface{}) 104 | if ok { 105 | if v, ok := result["filtered"]; ok { 106 | if t, ok := v.(bool); ok { 107 | triggered = t 108 | } else if t, ok := v.(string); ok { 109 | triggered = t == "true" 110 | } 111 | 112 | if triggered { 113 | if v, ok := result["data"]; ok { 114 | if array, ok := v.([]map[string]interface {}); ok { 115 | triggered = false // avoiding to send the original message more than once 116 | for _, x := range array { 117 | clone := msg.Clone() 118 | f.jsResponse2Msg(x, clone) 119 | clone.SetExtra("fulltext", text) 120 | f.Propagate(clone) 121 | } 122 | } else { 123 | f.jsResponse2Msg(v, msg) 124 | msg.SetExtra("fulltext", text) 125 | } 126 | } 127 | } 128 | } 129 | } 130 | } 131 | 132 | return triggered, nil 133 | } 134 | 135 | // OnEvent is called when an event occurs 136 | func (f *Js) OnEvent(event *data.Event){} 137 | 138 | // Set the name of the filter 139 | func init() { 140 | register("js", NewJsFilter) 141 | } 142 | -------------------------------------------------------------------------------- /filters/json.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/Matrix86/driplane/data" 9 | 10 | "github.com/antchfx/jsonquery" 11 | "github.com/evilsocket/islazy/log" 12 | "github.com/evilsocket/islazy/str" 13 | ) 14 | 15 | // JSON is a filter to parse the JSON format 16 | type JSON struct { 17 | Base 18 | 19 | selector string 20 | target string 21 | 22 | params map[string]string 23 | } 24 | 25 | // NewJSONFilter is the registered method to instantiate a JSONFilter 26 | func NewJSONFilter(p map[string]string) (Filter, error) { 27 | f := &JSON{ 28 | params: p, 29 | target: "main", 30 | selector: "", 31 | } 32 | f.cbFilter = f.DoFilter 33 | 34 | if v, ok := f.params["selector"]; ok { 35 | f.selector = v 36 | } 37 | 38 | if f.selector == "" { 39 | return nil, errors.New("no selector specified for JSON filter") 40 | } 41 | if v, ok := f.params["target"]; ok { 42 | f.target = v 43 | } 44 | 45 | return f, nil 46 | } 47 | 48 | // DoFilter is the mandatory method used to "filter" the input data.Message 49 | func (f *JSON) DoFilter(msg *data.Message) (bool, error) { 50 | //var err error 51 | var text string 52 | 53 | if v, ok := msg.GetTarget(f.target).(string); ok { 54 | text = str.Trim(v) 55 | } else if v, ok := msg.GetTarget(f.target).([]byte); ok { 56 | text = string(v) 57 | } else { 58 | // ERROR this filter can't be used with different types 59 | return false, fmt.Errorf("received data is not a string") 60 | } 61 | 62 | if len(text) > 0 { 63 | var jsonData string 64 | 65 | if text[0] == '{' { 66 | // json text 67 | jsonData = str.Trim(text) 68 | } else { 69 | log.Error("'%v' is not a json document", text) 70 | return false, nil 71 | } 72 | 73 | if doc, err := jsonquery.Parse(strings.NewReader(jsonData)); err == nil { 74 | atLeastOne := false 75 | for _, node := range jsonquery.Find(doc, f.selector) { 76 | atLeastOne = true 77 | clone := msg.Clone() 78 | clone.SetMessage(node.Value()) 79 | f.Propagate(clone) 80 | } 81 | 82 | return atLeastOne, nil 83 | 84 | } else { 85 | log.Debug("'%v' could not be parsed as JSON: %v", text, err) 86 | return false, nil 87 | } 88 | 89 | } 90 | 91 | return false, nil 92 | } 93 | 94 | // OnEvent is called when an event occurs 95 | func (f *JSON) OnEvent(event *data.Event) {} 96 | 97 | // Set the name of the filter 98 | func init() { 99 | register("json", NewJSONFilter) 100 | } 101 | -------------------------------------------------------------------------------- /filters/mail.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "html/template" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/Matrix86/driplane/data" 11 | "github.com/evilsocket/islazy/log" 12 | 13 | gomail "gopkg.in/gomail.v2" 14 | ) 15 | 16 | // Mail is a Filter to send e-mail using the input Message 17 | type Mail struct { 18 | Base 19 | 20 | template *template.Template 21 | username string 22 | password string 23 | server string 24 | fromAddr string 25 | fromName string 26 | port int 27 | to []string 28 | subject string 29 | useAuth bool 30 | params map[string]string 31 | } 32 | 33 | // NewMailFilter is the registered method to instantiate a MailFilter 34 | func NewMailFilter(p map[string]string) (Filter, error) { 35 | f := &Mail{ 36 | params: p, 37 | useAuth: false, 38 | } 39 | f.cbFilter = f.DoFilter 40 | 41 | if v, ok := f.params["body"]; ok { 42 | t, err := template.New("MailFilterTemplate").Parse(v) 43 | if err != nil { 44 | return nil, err 45 | } 46 | f.template = t 47 | } 48 | if v, ok := f.params["username"]; ok { 49 | f.username = v 50 | } 51 | if v, ok := f.params["password"]; ok { 52 | f.password = v 53 | } 54 | if v, ok := f.params["host"]; ok { 55 | f.server = v 56 | } 57 | if v, ok := f.params["port"]; ok { 58 | i, err := strconv.Atoi(v) 59 | if err != nil { 60 | return nil, err 61 | } 62 | f.port = i 63 | } 64 | if v, ok := f.params["fromAddr"]; ok { 65 | f.fromAddr = v 66 | } 67 | if v, ok := f.params["fromName"]; ok { 68 | f.fromName = v 69 | } 70 | if v, ok := f.params["to"]; ok { 71 | f.to = strings.Split(v, ",") 72 | } 73 | if v, ok := f.params["subject"]; ok { 74 | f.subject = v 75 | } 76 | if v, ok := f.params["use_auth"]; ok && v == "true" { 77 | f.useAuth = true 78 | } 79 | return f, nil 80 | } 81 | 82 | // DoFilter is the mandatory method used to "filter" the input data.Message 83 | func (f *Mail) DoFilter(msg *data.Message) (bool, error) { 84 | var err error 85 | var text string 86 | 87 | if v, ok := msg.GetMessage().(string); ok { 88 | text = v 89 | } else if v, ok := msg.GetMessage().([]byte); ok { 90 | text = string(v) 91 | } else { 92 | // ERROR this filter can't be used with different types 93 | return false, fmt.Errorf("received data is not a string") 94 | } 95 | 96 | if f.template != nil { 97 | text, err = msg.ApplyPlaceholder(f.template) 98 | if err != nil { 99 | return false, err 100 | } 101 | } 102 | 103 | var d gomail.Dialer 104 | if f.useAuth { 105 | d = gomail.Dialer{Host: f.server, Port: f.port, Username: f.username, Password: f.password} 106 | d.TLSConfig = &tls.Config{InsecureSkipVerify: true} 107 | } else { 108 | d = gomail.Dialer{Host: f.server, Port: f.port} 109 | } 110 | 111 | s, err := d.Dial() 112 | if err != nil { 113 | return false, err 114 | } 115 | 116 | m := gomail.NewMessage() 117 | for _, n := range f.to { 118 | m.SetAddressHeader("From", f.fromAddr, f.fromName) 119 | m.SetHeader("To", n) 120 | m.SetHeader("Subject", f.subject) 121 | m.SetBody("text/html", text) 122 | 123 | if err := gomail.Send(s, m); err != nil { 124 | log.Error("could not send email to %s: %s", n, err) 125 | } 126 | m.Reset() 127 | } 128 | 129 | return true, nil 130 | } 131 | 132 | // OnEvent is called when an event occurs 133 | func (f *Mail) OnEvent(event *data.Event){} 134 | 135 | // Set the name of the filter 136 | func init() { 137 | register("mail", NewMailFilter) 138 | } 139 | -------------------------------------------------------------------------------- /filters/mimetype.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/Matrix86/driplane/data" 7 | "text/template" 8 | 9 | "github.com/gabriel-vasile/mimetype" 10 | ) 11 | 12 | // Mimetype is a Filter to detect the format of an input 13 | type Mimetype struct { 14 | Base 15 | 16 | target string 17 | filename *template.Template 18 | 19 | params map[string]string 20 | } 21 | 22 | // NewMimetypeFilter is the registered method to instantiate a MimetypeFilter 23 | func NewMimetypeFilter(p map[string]string) (Filter, error) { 24 | f := &Mimetype{ 25 | params: p, 26 | target: "main", 27 | } 28 | f.cbFilter = f.DoFilter 29 | 30 | if v, ok := p["filename"]; ok { 31 | t, err := template.New("mimeFilterFilename").Parse(v) 32 | if err != nil { 33 | return nil, err 34 | } 35 | f.filename = t 36 | } else if v, ok := p["target"]; ok { 37 | f.target = v 38 | } 39 | 40 | return f, nil 41 | } 42 | 43 | // DoFilter is the mandatory method used to "filter" the input data.Message 44 | func (f *Mimetype) DoFilter(msg *data.Message) (bool, error) { 45 | if f.filename != nil { 46 | text, err := msg.ApplyPlaceholder(f.filename) 47 | if err != nil { 48 | return false, err 49 | } 50 | 51 | mime, err := mimetype.DetectFile(text) 52 | if err != nil { 53 | return false, err 54 | } 55 | msg.SetMessage(mime.String()) 56 | msg.SetExtra("mimetype_ext", mime.Extension()) 57 | msg.SetExtra("fulltext", text) 58 | } else { 59 | if v, ok := msg.GetTarget(f.target).([]byte); ok { 60 | buf := bytes.NewBuffer(v) 61 | mime := mimetype.Detect(buf.Bytes()) 62 | msg.SetMessage(mime.String()) 63 | msg.SetExtra("mimetype_ext", mime.Extension()) 64 | msg.SetExtra("fulltext", msg.GetTarget("main")) 65 | } else { 66 | return false, fmt.Errorf("data type is not supported") 67 | } 68 | } 69 | 70 | return true, nil 71 | } 72 | 73 | // OnEvent is called when an event occurs 74 | func (f *Mimetype) OnEvent(event *data.Event){} 75 | 76 | // Set the name of the filter 77 | func init() { 78 | register("mime", NewMimetypeFilter) 79 | } 80 | -------------------------------------------------------------------------------- /filters/number.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | 7 | "github.com/Matrix86/driplane/data" 8 | ) 9 | 10 | // Number is a Filter to treat a string from the input Message as numeric value and apply some operator on it 11 | type Number struct { 12 | Base 13 | 14 | target string 15 | operator string 16 | value string 17 | fValue float64 18 | 19 | params map[string]string 20 | } 21 | 22 | // NewNumberFilter is the registered method to instantiate a TextFilter 23 | func NewNumberFilter(p map[string]string) (Filter, error) { 24 | f := &Number{ 25 | params: p, 26 | target: "main", 27 | operator: ">", 28 | value: "0", 29 | fValue: 0.0, 30 | } 31 | f.cbFilter = f.DoFilter 32 | 33 | if v, ok := f.params["value"]; ok { 34 | if value, err := strconv.ParseFloat(v, 64); err != nil { 35 | return nil, fmt.Errorf("value parameter is not numeric: %s", err) 36 | } else { 37 | f.value = v 38 | f.fValue = value 39 | } 40 | } 41 | 42 | if v, ok := f.params["target"]; ok { 43 | f.target = v 44 | } 45 | 46 | if v, ok := f.params["op"]; ok { 47 | switch v { 48 | case ">", ">=", "<", "<=", "!=", "==": 49 | f.operator = v 50 | 51 | default: 52 | return nil, fmt.Errorf("operator '%s' not supported", v) 53 | } 54 | 55 | } 56 | 57 | return f, nil 58 | } 59 | 60 | // DoFilter is the mandatory method used to "filter" the input data.Message 61 | func (f *Number) DoFilter(msg *data.Message) (bool, error) { 62 | var text string 63 | var currentValue float64 64 | 65 | target := msg.GetTarget(f.target) 66 | if target == nil { 67 | return false, nil 68 | } 69 | 70 | switch v := target.(type) { 71 | case string: 72 | text = v 73 | if value, err := strconv.ParseFloat(text, 64); err != nil { 74 | return false, fmt.Errorf("received data is not a numeric value: %s", err) 75 | } else { 76 | currentValue = value 77 | } 78 | 79 | case []byte: 80 | text = string(v) 81 | if value, err := strconv.ParseFloat(text, 64); err != nil { 82 | return false, fmt.Errorf("received data is not a numeric value: %s", err) 83 | } else { 84 | currentValue = value 85 | } 86 | 87 | case int: 88 | currentValue = float64(v) 89 | case int8: 90 | currentValue = float64(v) 91 | case int16: 92 | currentValue = float64(v) 93 | case int32: 94 | currentValue = float64(v) 95 | case int64: 96 | currentValue = float64(v) 97 | case float32: 98 | currentValue = float64(v) 99 | case float64: 100 | currentValue = float64(v) 101 | 102 | default: 103 | // ERROR this filter can't be used with different types 104 | return false, fmt.Errorf("received data is not a string") 105 | } 106 | 107 | switch f.operator { 108 | case ">": 109 | if currentValue > f.fValue { 110 | return true, nil 111 | } 112 | 113 | case ">=": 114 | if currentValue >= f.fValue { 115 | return true, nil 116 | } 117 | 118 | case "<": 119 | if currentValue < f.fValue { 120 | return true, nil 121 | } 122 | 123 | case "<=": 124 | if currentValue <= f.fValue { 125 | return true, nil 126 | } 127 | 128 | case "!=": 129 | if currentValue != f.fValue { 130 | return true, nil 131 | } 132 | 133 | case "==": 134 | if currentValue == f.fValue { 135 | return true, nil 136 | } 137 | } 138 | 139 | return false, nil 140 | } 141 | 142 | // OnEvent is called when an event occurs 143 | func (f *Number) OnEvent(event *data.Event) {} 144 | 145 | // Set the name of the filter 146 | func init() { 147 | register("number", NewNumberFilter) 148 | } 149 | -------------------------------------------------------------------------------- /filters/override.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "text/template" 5 | 6 | "github.com/Matrix86/driplane/data" 7 | 8 | "github.com/evilsocket/islazy/log" 9 | ) 10 | 11 | 12 | // Override is a Filter that allow to change the current Message 13 | // before to send it to the next Filter 14 | type Override struct { 15 | Base 16 | 17 | name *template.Template 18 | value *template.Template 19 | 20 | params map[string]string 21 | } 22 | 23 | // NewOverrideFilter is the registered method to instantiate a OverrideFilter 24 | func NewOverrideFilter(p map[string]string) (Filter, error) { 25 | f := &Override{ 26 | params: p, 27 | } 28 | f.cbFilter = f.DoFilter 29 | 30 | if v, ok := p["name"]; ok { 31 | t, err := template.New("setFilterName").Parse(v) 32 | if err != nil { 33 | return nil, err 34 | } 35 | f.name = t 36 | } 37 | 38 | if v, ok := p["value"]; ok { 39 | t, err := template.New("setFilterValue").Parse(v) 40 | if err != nil { 41 | return nil, err 42 | } 43 | f.value = t 44 | } 45 | 46 | return f, nil 47 | } 48 | 49 | // DoFilter is the mandatory method used to "filter" the input data.Message 50 | func (f *Override) DoFilter(msg *data.Message) (bool, error) { 51 | name, err := msg.ApplyPlaceholder(f.name) 52 | if err != nil { 53 | return false, err 54 | } 55 | value, err := msg.ApplyPlaceholder(f.value) 56 | if err != nil { 57 | return false, err 58 | } 59 | 60 | log.Debug("[setfilter] setting msg[%s]=%s", name, value) 61 | msg.SetTarget(name, value) 62 | 63 | return true, nil 64 | } 65 | 66 | // OnEvent is called when an event occurs 67 | func (f *Override) OnEvent(event *data.Event){} 68 | 69 | // Set the name of the filter 70 | func init() { 71 | register("override", NewOverrideFilter) 72 | } 73 | -------------------------------------------------------------------------------- /filters/override_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/Matrix86/driplane/data" 5 | "testing" 6 | ) 7 | 8 | func TestNewOverrideFilter(t *testing.T) { 9 | filter, err := NewOverrideFilter(map[string]string{"name": "name", "value": "value", "ignorethis": "ignorethis"}) 10 | if err != nil { 11 | t.Errorf("constructor returned '%s'", err) 12 | } 13 | if e, ok := filter.(*Override); ok { 14 | if e.name == nil { 15 | t.Errorf("'name' parameter ignored") 16 | } 17 | if e.value == nil { 18 | t.Errorf("'value' parameter ignored") 19 | } 20 | } else { 21 | t.Errorf("cannot cast to proper Filter...") 22 | } 23 | } 24 | 25 | func TestOverrideDoFilterAddNewField(t *testing.T) { 26 | filter, err := NewOverrideFilter(map[string]string{"name": "newname", "value": "newvalue"}) 27 | if err != nil { 28 | t.Errorf("constructor returned '%s'", err) 29 | } 30 | if e, ok := filter.(*Override); ok { 31 | msg := "main message" 32 | extra := make(map[string]interface{}) 33 | m := data.NewMessageWithExtra(msg, extra) 34 | b, err := e.DoFilter(m) 35 | if err != nil { 36 | t.Errorf("DoFilter returned an error '%s'", err) 37 | } 38 | if b == false { 39 | t.Errorf("it should return true") 40 | } 41 | if m.GetMessage() != msg { 42 | t.Errorf("the main field of the message has been altered by the filter") 43 | } 44 | if m.GetTarget("newname").(string) != "newvalue" { 45 | t.Errorf("the 'newname' field of the message is wrong: expected=newvalue had=%s", m.GetTarget("newname").(string)) 46 | } 47 | } else { 48 | t.Errorf("cannot cast to proper Filter...") 49 | } 50 | 51 | filter2, err := NewOverrideFilter(map[string]string{"name": "{{ .field1 }}", "value": "{{ .field2 }}"}) 52 | if err != nil { 53 | t.Errorf("constructor returned '%s'", err) 54 | } 55 | if e, ok := filter2.(*Override); ok { 56 | msg := "main message" 57 | extra := make(map[string]interface{}) 58 | extra["field1"] = "field1" 59 | extra["field2"] = "field2" 60 | m := data.NewMessageWithExtra(msg, extra) 61 | b, err := e.DoFilter(m) 62 | if err != nil { 63 | t.Errorf("DoFilter returned an error '%s'", err) 64 | } 65 | if b == false { 66 | t.Errorf("it should return true") 67 | } 68 | if m.GetMessage() != msg { 69 | t.Errorf("the main field of the message has been altered by the filter") 70 | } 71 | if m.GetTarget("field1").(string) != "field2" { 72 | t.Errorf("the 'field1' field of the message is wrong: expected=%s had=%s", "field2", m.GetTarget("field1").(string)) 73 | } 74 | } else { 75 | t.Errorf("cannot cast to proper Filter...") 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /filters/pdf.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "text/template" 7 | 8 | "github.com/Matrix86/driplane/data" 9 | 10 | "github.com/ledongthuc/pdf" 11 | ) 12 | 13 | // PDF is a Filter that extract plain text from a PDF file 14 | type PDF struct { 15 | Base 16 | 17 | target string 18 | filename *template.Template 19 | 20 | params map[string]string 21 | } 22 | 23 | // NewPDFFilter is the registered method to instantiate a TextFilter 24 | func NewPDFFilter(p map[string]string) (Filter, error) { 25 | f := &PDF{ 26 | params: p, 27 | target: "main", 28 | } 29 | f.cbFilter = f.DoFilter 30 | 31 | if v, ok := p["filename"]; ok { 32 | t, err := template.New("pdfFilterFilename").Parse(v) 33 | if err != nil { 34 | return nil, err 35 | } 36 | f.filename = t 37 | } else if v, ok := p["target"]; ok { 38 | f.target = v 39 | } 40 | 41 | return f, nil 42 | } 43 | 44 | // DoFilter is the mandatory method used to "filter" the input data.Message 45 | func (f *PDF) DoFilter(msg *data.Message) (bool, error) { 46 | if f.filename != nil { 47 | text, err := msg.ApplyPlaceholder(f.filename) 48 | if err != nil { 49 | return false, err 50 | } 51 | 52 | h, r, err := pdf.Open(text) 53 | if err != nil { 54 | return false, err 55 | } 56 | defer h.Close() 57 | 58 | var buf bytes.Buffer 59 | b, err := r.GetPlainText() 60 | if err != nil { 61 | return false, err 62 | } 63 | buf.ReadFrom(b) 64 | plain := buf.String() 65 | 66 | msg.SetMessage(plain) 67 | msg.SetExtra("fulltext", text) 68 | } else { 69 | if _, ok := msg.GetTarget(f.target).([]byte); !ok { 70 | // ERROR this filter can't be used with different types 71 | return false, fmt.Errorf("received data is not supported") 72 | } 73 | 74 | buf := bytes.NewBuffer(msg.GetTarget(f.target).([]byte)) 75 | 76 | r, err := pdf.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) 77 | if err != nil { 78 | return false, err 79 | } 80 | 81 | var buff bytes.Buffer 82 | b, err := r.GetPlainText() 83 | if err != nil { 84 | return false, err 85 | } 86 | buff.ReadFrom(b) 87 | plain := buff.String() 88 | 89 | msg.SetMessage(plain) 90 | msg.SetExtra("fulltext", msg.GetTarget("main")) 91 | } 92 | 93 | return true, nil 94 | } 95 | 96 | // OnEvent is called when an event occurs 97 | func (f *PDF) OnEvent(event *data.Event) {} 98 | 99 | // Set the name of the filter 100 | func init() { 101 | register("pdf", NewPDFFilter) 102 | } 103 | -------------------------------------------------------------------------------- /filters/random.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/Matrix86/driplane/data" 10 | 11 | "github.com/evilsocket/islazy/log" 12 | ) 13 | 14 | // Random is a Filter to create a random number 15 | type Random struct { 16 | Base 17 | 18 | output string 19 | min int64 20 | max int64 21 | 22 | params map[string]string 23 | } 24 | 25 | // NewRandomFilter is the registered method to instantiate a RandomFilter 26 | func NewRandomFilter(p map[string]string) (Filter, error) { 27 | f := &Random{ 28 | params: p, 29 | output: "main", 30 | min: 0, 31 | max: 999999, 32 | } 33 | f.cbFilter = f.DoFilter 34 | 35 | 36 | if v, ok := p["output"]; ok { 37 | f.output = v 38 | } 39 | if v, ok := p["min"]; ok { 40 | i, err := strconv.ParseInt(v, 10, 64) 41 | if err != nil { 42 | return nil, err 43 | } 44 | f.min = i 45 | } 46 | if v, ok := p["max"]; ok { 47 | i, err := strconv.ParseInt(v, 10, 64) 48 | if err != nil { 49 | return nil, err 50 | } 51 | f.max = i 52 | } 53 | 54 | return f, nil 55 | } 56 | 57 | // DoFilter is the mandatory method used to "filter" the input data.Message 58 | func (f *Random) DoFilter(msg *data.Message) (bool, error) { 59 | s1 := rand.NewSource(time.Now().UnixNano()) 60 | r1 := rand.New(s1) 61 | 62 | random := fmt.Sprintf("%d", r1.Int63n(f.max - f.min) + f.min) 63 | 64 | log.Debug("[randomfilter] picked %s", random) 65 | msg.SetTarget(f.output, random) 66 | 67 | return true, nil 68 | } 69 | 70 | // OnEvent is called when an event occurs 71 | func (f *Random) OnEvent(event *data.Event){} 72 | 73 | // Set the name of the filter 74 | func init() { 75 | register("random", NewRandomFilter) 76 | } 77 | -------------------------------------------------------------------------------- /filters/random_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/Matrix86/driplane/data" 5 | "strconv" 6 | "testing" 7 | ) 8 | 9 | func TestNewRandomFilter(t *testing.T) { 10 | filter, err := NewRandomFilter(map[string]string{"output": "random_number", "min": "10", "max":"40" }) 11 | if err != nil { 12 | t.Errorf("constructor returned '%s'", err) 13 | } 14 | if e, ok := filter.(*Random); ok { 15 | if e.output != "random_number" { 16 | t.Errorf("'output' parameter ignored") 17 | } 18 | if e.min != 10 { 19 | t.Errorf("'min' parameter ignored") 20 | } 21 | if e.max != 40 { 22 | t.Errorf("'max' parameter ignored") 23 | } 24 | } else { 25 | t.Errorf("cannot cast to proper Filter...") 26 | } 27 | 28 | filter, err = NewRandomFilter(map[string]string{"output": "random_number", "min": "x", "max":"40" }) 29 | if err == nil { 30 | t.Errorf("constructor should return error") 31 | } 32 | 33 | filter, err = NewRandomFilter(map[string]string{"output": "random_number", "min": "1", "max":"x" }) 34 | if err == nil { 35 | t.Errorf("constructor should return error") 36 | } 37 | } 38 | 39 | func TestRandom_DoFilter(t *testing.T) { 40 | filter, err := NewRandomFilter(map[string]string{"output": "random_number", "min": "10", "max":"40" }) 41 | if err != nil { 42 | t.Errorf("constructor returned '%s'", err) 43 | } 44 | 45 | if e, ok := filter.(*Random); ok { 46 | msg := "this is a test..." 47 | m := data.NewMessage(msg) 48 | _, err := e.DoFilter(m) 49 | if err != nil { 50 | t.Errorf("DoFilter returned an error '%s'", err) 51 | } 52 | res := m.GetTarget("random_number") 53 | i, err := strconv.Atoi(res.(string)) 54 | if err != nil { 55 | t.Errorf("received data is not a string") 56 | } 57 | if i < 10 && i > 40 { 58 | t.Errorf("random number is out of range") 59 | } 60 | } else { 61 | t.Errorf("cannot cast to proper Filter...") 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /filters/ratelimit.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | 7 | "github.com/Matrix86/driplane/data" 8 | "github.com/evilsocket/islazy/log" 9 | "golang.org/x/time/rate" 10 | ) 11 | 12 | // RateLimit is a Filter to create a RateLimit number 13 | type RateLimit struct { 14 | Base 15 | 16 | objects int64 17 | seconds int64 18 | limiter *rate.Limiter 19 | context context.Context 20 | cancelContext context.CancelFunc 21 | 22 | params map[string]string 23 | } 24 | 25 | // NewRateLimitFilter is the registered method to instantiate a RateLimit Filter 26 | func NewRateLimitFilter(p map[string]string) (Filter, error) { 27 | ctx, cancel := context.WithCancel(context.Background()) 28 | f := &RateLimit{ 29 | params: p, 30 | seconds: 1, 31 | context: ctx, 32 | cancelContext: cancel, 33 | } 34 | f.cbFilter = f.DoFilter 35 | 36 | if v, ok := p["rate"]; ok { 37 | i, err := strconv.ParseInt(v, 10, 64) 38 | if err != nil { 39 | return nil, err 40 | } 41 | f.objects = i 42 | } 43 | 44 | f.limiter = rate.NewLimiter(rate.Limit(f.objects), int(f.seconds)) 45 | 46 | return f, nil 47 | } 48 | 49 | // DoFilter is the mandatory method used to "filter" the input data.Message 50 | func (f *RateLimit) DoFilter(msg *data.Message) (bool, error) { 51 | if f.objects > 0 { 52 | if err := f.limiter.Wait(f.context); err != nil { 53 | return false, nil 54 | } 55 | } 56 | 57 | return true, nil 58 | } 59 | 60 | // OnEvent is called when an event occurs 61 | func (f *RateLimit) OnEvent(event *data.Event) { 62 | if event.Type == "shutdown" { 63 | log.Debug("shutdown event received") 64 | f.cancelContext() 65 | } 66 | } 67 | 68 | // Set the name of the filter 69 | func init() { 70 | register("ratelimit", NewRateLimitFilter) 71 | } 72 | -------------------------------------------------------------------------------- /filters/ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/Matrix86/driplane/data" 7 | ) 8 | 9 | func TestNewRateLimitFilter(t *testing.T) { 10 | filter, err := NewRateLimitFilter(map[string]string{"rate": "2", "min": "10", "max": "40"}) 11 | if err != nil { 12 | t.Errorf("constructor returned '%s'", err) 13 | } 14 | if e, ok := filter.(*RateLimit); ok { 15 | if e.objects != 2 { 16 | t.Errorf("'output' parameter ignored") 17 | } 18 | } else { 19 | t.Errorf("cannot cast to proper Filter...") 20 | } 21 | 22 | _, err = NewRateLimitFilter(map[string]string{"rate": "random text", "min": "x", "max": "40"}) 23 | if err == nil { 24 | t.Errorf("constructor should return error") 25 | } 26 | } 27 | 28 | func TestRateLimit_DoFilter(t *testing.T) { 29 | filter, err := NewRateLimitFilter(map[string]string{"rate": "5"}) 30 | if err != nil { 31 | t.Errorf("constructor returned '%s'", err) 32 | } 33 | 34 | if e, ok := filter.(*RateLimit); ok { 35 | msg := "this is a test..." 36 | m := data.NewMessage(msg) 37 | _, err := e.DoFilter(m) 38 | if err != nil { 39 | t.Errorf("DoFilter returned an error '%s'", err) 40 | } 41 | if m.GetMessage() != msg { 42 | t.Errorf("DoFilter changed the message") 43 | } 44 | 45 | filter.OnEvent(&data.Event{ 46 | Type: "shutdown", 47 | Content: "shutdown", 48 | }) 49 | v, err := e.DoFilter(m) 50 | if err != nil { 51 | t.Errorf("DoFilter returned an error '%s'", err) 52 | } 53 | if v { 54 | t.Errorf("DoFilter has to return false after the shutdown") 55 | } 56 | 57 | } else { 58 | t.Errorf("cannot cast to proper Filter...") 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /filters/striptag.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | "github.com/Matrix86/driplane/utils" 6 | 7 | "github.com/Matrix86/driplane/data" 8 | ) 9 | 10 | // StripTag is a filter that removes HTML tags from a text. 11 | type StripTag struct { 12 | Base 13 | 14 | target string 15 | 16 | params map[string]string 17 | } 18 | 19 | // NewStripTagFilter is the registered method to instantiate a StripTagFilter 20 | func NewStripTagFilter(p map[string]string) (Filter, error) { 21 | f := &StripTag{ 22 | params: p, 23 | target: "main", 24 | } 25 | f.cbFilter = f.DoFilter 26 | 27 | if v, ok := f.params["target"]; ok { 28 | f.target = v 29 | } 30 | 31 | return f, nil 32 | } 33 | 34 | // DoFilter is the mandatory method used to "filter" the input data.Message 35 | func (f *StripTag) DoFilter(msg *data.Message) (bool, error) { 36 | var text string 37 | if v, ok := msg.GetTarget(f.target).(string); ok { 38 | text = v 39 | } else if v, ok := msg.GetTarget(f.target).([]byte); ok { 40 | text = string(v) 41 | } else { 42 | // ERROR this filter can't be used with different types 43 | return false, fmt.Errorf("received data is not a string") 44 | } 45 | stripped := utils.ExtractTextFromHTML(text) 46 | msg.SetExtra("fulltext", text) 47 | msg.SetMessage(stripped) 48 | return true, nil 49 | } 50 | 51 | // OnEvent is called when an event occurs 52 | func (f *StripTag) OnEvent(event *data.Event) {} 53 | 54 | // Set the name of the filter 55 | func init() { 56 | register("striptag", NewStripTagFilter) 57 | } 58 | -------------------------------------------------------------------------------- /filters/system.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "os/exec" 5 | "text/template" 6 | 7 | "github.com/Matrix86/driplane/data" 8 | 9 | "github.com/evilsocket/islazy/log" 10 | ) 11 | 12 | // System is a Filter to exec a command on the host machine using the input Message 13 | type System struct { 14 | Base 15 | 16 | command *template.Template 17 | 18 | params map[string]string 19 | } 20 | 21 | // NewSystemFilter is the registered method to instantiate a SystemFilter 22 | func NewSystemFilter(p map[string]string) (Filter, error) { 23 | f := &System{ 24 | params: p, 25 | } 26 | f.cbFilter = f.DoFilter 27 | 28 | if v, ok := p["cmd"]; ok { 29 | t, err := template.New("systemFilterCommand").Parse(v) 30 | if err != nil { 31 | return nil, err 32 | } 33 | f.command = t 34 | } 35 | 36 | return f, nil 37 | } 38 | 39 | // DoFilter is the mandatory method used to "filter" the input data.Message 40 | func (f *System) DoFilter(msg *data.Message) (bool, error) { 41 | cmd, err := msg.ApplyPlaceholder(f.command) 42 | if err != nil { 43 | return false, err 44 | } 45 | 46 | log.Debug("[systemfilter] command: %s", cmd) 47 | c := exec.Command("sh", "-c", cmd) 48 | output, err := c.CombinedOutput() 49 | if err != nil { 50 | log.Debug("[systemfilter] command failed: %s %s", err, output) 51 | return false, err 52 | } 53 | 54 | msg.SetMessage(string(output)) 55 | 56 | return true, nil 57 | } 58 | 59 | // OnEvent is called when an event occurs 60 | func (f *System) OnEvent(event *data.Event){} 61 | 62 | // Set the name of the filter 63 | func init() { 64 | register("system", NewSystemFilter) 65 | } 66 | -------------------------------------------------------------------------------- /filters/system_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/Matrix86/driplane/data" 5 | "testing" 6 | ) 7 | 8 | func TestNewSystemFilter(t *testing.T) { 9 | filter, err := NewSystemFilter(map[string]string{"cmd": "/bin/echo {{ .main }}" }) 10 | if err != nil { 11 | t.Errorf("constructor returned '%s'", err) 12 | } 13 | if e, ok := filter.(*System); ok { 14 | m := data.NewMessage("test") 15 | cmd, _ := m.ApplyPlaceholder(e.command) 16 | if cmd != "/bin/echo test" { 17 | t.Errorf("'cmd' parameter ignored") 18 | } 19 | } else { 20 | t.Errorf("cannot cast to proper Filter...") 21 | } 22 | } 23 | 24 | func TestSystem_DoFilter(t *testing.T) { 25 | filter, err := NewSystemFilter(map[string]string{"cmd": "/bin/echo -n {{ .main }}" }) 26 | if err != nil { 27 | t.Errorf("constructor returned '%s'", err) 28 | } 29 | 30 | if e, ok := filter.(*System); ok { 31 | m := data.NewMessage("test") 32 | _, err := e.DoFilter(m) 33 | if err != nil { 34 | t.Errorf("DoFilter returned an error '%s'", err) 35 | } 36 | txt := m.GetMessage().(string) 37 | if txt != "test" { 38 | t.Errorf("TestSystem_DoFilter: wrong output: expected=%#v had=%#v", "test", txt) 39 | } 40 | } else { 41 | t.Errorf("cannot cast to proper Filter...") 42 | } 43 | } -------------------------------------------------------------------------------- /filters/text.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/Matrix86/driplane/data" 9 | ) 10 | 11 | // Text is a Filter to search and extract strings from the input Message 12 | type Text struct { 13 | Base 14 | 15 | regexp *regexp.Regexp 16 | 17 | extract bool 18 | pattern string 19 | enableRegexp bool 20 | target string 21 | 22 | params map[string]string 23 | } 24 | 25 | // NewTextFilter is the registered method to instantiate a TextFilter 26 | func NewTextFilter(p map[string]string) (Filter, error) { 27 | var err error 28 | f := &Text{ 29 | params: p, 30 | regexp: nil, 31 | extract: false, 32 | pattern: "", 33 | enableRegexp: false, 34 | target: "main", 35 | } 36 | f.cbFilter = f.DoFilter 37 | 38 | // mandatory field 39 | if v, ok := p["pattern"]; ok { 40 | f.pattern = v 41 | } else { 42 | return nil, fmt.Errorf("pattern field is required on textfilter") 43 | } 44 | 45 | if v, ok := p["regexp"]; ok && v == "true" { 46 | f.regexp, err = regexp.Compile(f.pattern) 47 | if err != nil { 48 | return nil, fmt.Errorf("textfilter: cannot compile the regular expression '%s'", f.pattern) 49 | } 50 | } 51 | if v, ok := f.params["extract"]; ok && v == "true" { 52 | f.extract = true 53 | } 54 | if v, ok := p["target"]; ok { 55 | f.target = v 56 | } 57 | 58 | return f, nil 59 | } 60 | 61 | // DoFilter is the mandatory method used to "filter" the input data.Message 62 | func (f *Text) DoFilter(msg *data.Message) (bool, error) { 63 | var text string 64 | target := msg.GetTarget(f.target) 65 | if target == nil { 66 | return false, nil 67 | } 68 | if v, ok := target.(string); ok { 69 | text = v 70 | } else if v, ok := target.([]byte); ok { 71 | text = string(v) 72 | } else { 73 | // ERROR this filter can't be used with different types 74 | return false, fmt.Errorf("received data is not a string") 75 | } 76 | 77 | found := false 78 | if f.regexp != nil { 79 | if f.extract { 80 | matched := make([]string, 0) 81 | match := f.regexp.FindAllStringSubmatch(text, -1) 82 | if match != nil { 83 | for _, m := range match { 84 | matched = append(matched, m[1:]...) 85 | } 86 | } 87 | if len(matched) == 1 { 88 | msg.SetMessage(matched[0]) 89 | msg.SetExtra("fulltext", text) 90 | return true, nil 91 | } else if len(matched) > 1 { 92 | for _, m := range matched { 93 | clone := msg.Clone() 94 | clone.SetMessage(m) 95 | clone.SetExtra("fulltext", text) 96 | f.Propagate(clone) 97 | } 98 | return false, nil 99 | } 100 | } else if f.regexp.MatchString(text) { 101 | found = true 102 | } 103 | } else if f.pattern != "" && strings.Contains(text, f.pattern) { 104 | found = true 105 | } 106 | 107 | return found, nil 108 | } 109 | 110 | // OnEvent is called when an event occurs 111 | func (f *Text) OnEvent(event *data.Event){} 112 | 113 | // Set the name of the filter 114 | func init() { 115 | register("text", NewTextFilter) 116 | } 117 | -------------------------------------------------------------------------------- /filters/url.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/Matrix86/driplane/data" 9 | ) 10 | 11 | // URL is a Filter to search urls in the input Message 12 | type URL struct { 13 | Base 14 | 15 | rURL *regexp.Regexp 16 | 17 | getHTTP bool 18 | getHTTPS bool 19 | getFTP bool 20 | target string 21 | 22 | extractURL bool 23 | 24 | params map[string]string 25 | } 26 | 27 | // NewURLFilter is the registered method to instantiate a UrlFilter 28 | func NewURLFilter(p map[string]string) (Filter, error) { 29 | f := &URL{ 30 | params: p, 31 | getHTTP: true, 32 | getHTTPS: true, 33 | getFTP: true, 34 | extractURL: true, 35 | target: "main", 36 | } 37 | f.cbFilter = f.DoFilter 38 | 39 | f.rURL = regexp.MustCompile(`(?i)((http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/|ftp:\/\/)?(([a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5})|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]{1,5})?(\/[^\s]+)?)`) 40 | 41 | if v, ok := f.params["http"]; ok && v == "false" { 42 | f.getHTTP = false 43 | } 44 | if v, ok := f.params["https"]; ok && v == "false" { 45 | f.getHTTPS = false 46 | } 47 | if v, ok := f.params["ftp"]; ok && v == "false" { 48 | f.getFTP = false 49 | } 50 | if v, ok := f.params["extract"]; ok && v == "false" { 51 | f.extractURL = false 52 | } 53 | if v, ok := f.params["target"]; ok { 54 | f.target = v 55 | } 56 | 57 | return f, nil 58 | } 59 | 60 | // DoFilter is the mandatory method used to "filter" the input data.Message 61 | func (f *URL) DoFilter(msg *data.Message) (bool, error) { 62 | var text string 63 | 64 | if v, ok := msg.GetTarget(f.target).(string); ok { 65 | text = v 66 | } else if v, ok := msg.GetTarget(f.target).([]byte); ok { 67 | text = string(v) 68 | } else { 69 | // ERROR this filter can't be used with different types 70 | return false, fmt.Errorf("received data is not a string") 71 | } 72 | 73 | found := false 74 | match := f.rURL.FindAllStringSubmatch(text, -1) 75 | if match != nil { 76 | for _, m := range match { 77 | mm := m[0] 78 | if f.getHTTP && strings.HasPrefix(strings.ToLower(mm), "http://") { 79 | found = true 80 | } else if f.getHTTPS && strings.HasPrefix(strings.ToLower(mm), "https://") { 81 | found = true 82 | } else if f.getFTP && strings.HasPrefix(strings.ToLower(mm), "ftp://") { 83 | found = true 84 | } 85 | 86 | if f.extractURL && found { 87 | clone := msg.Clone() 88 | clone.SetMessage(mm) 89 | clone.SetExtra("fulltext", text) 90 | f.Propagate(clone) 91 | // We need to stop the propagation of the first message 92 | found = false 93 | } else if found { 94 | break 95 | } 96 | } 97 | } 98 | return found, nil 99 | } 100 | 101 | // OnEvent is called when an event occurs 102 | func (f *URL) OnEvent(event *data.Event){} 103 | 104 | // Set the name of the filter 105 | func init() { 106 | register("url", NewURLFilter) 107 | } 108 | -------------------------------------------------------------------------------- /filters/xls.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | "text/template" 8 | 9 | "github.com/Matrix86/driplane/data" 10 | 11 | "github.com/xuri/excelize/v2" 12 | ) 13 | 14 | // XLS is a Filter that extract all the rows from an Excel file 15 | type XLS struct { 16 | Base 17 | 18 | target string 19 | filename *template.Template 20 | 21 | params map[string]string 22 | } 23 | 24 | // NewXLSFilter is the registered method to instantiate a XLSFilter 25 | func NewXLSFilter(p map[string]string) (Filter, error) { 26 | f := &XLS{ 27 | params: p, 28 | target: "main", 29 | } 30 | f.cbFilter = f.DoFilter 31 | 32 | if v, ok := p["filename"]; ok { 33 | t, err := template.New("xlsFilterFilename").Parse(v) 34 | if err != nil { 35 | return nil, err 36 | } 37 | f.filename = t 38 | } else if v, ok := p["target"]; ok { 39 | f.target = v 40 | } 41 | 42 | return f, nil 43 | } 44 | 45 | // DoFilter is the mandatory method used to "filter" the input data.Message 46 | func (f *XLS) DoFilter(msg *data.Message) (bool, error) { 47 | if f.filename != nil { 48 | text, err := msg.ApplyPlaceholder(f.filename) 49 | if err != nil { 50 | return false, err 51 | } 52 | 53 | x, err := excelize.OpenFile(text) 54 | if err != nil { 55 | return false, err 56 | } 57 | defer x.Close() 58 | 59 | sheets := x.GetSheetList() 60 | for _, sName := range sheets { 61 | rows, err := x.GetRows(sName) 62 | if err != nil { 63 | return false, err 64 | } 65 | for _, row := range rows { 66 | cloned := msg.Clone() 67 | cloned.SetMessage(strings.Join(row, ",")) 68 | cloned.SetExtra("xls_sheet", sName) 69 | cloned.SetExtra("xls_filename", text) 70 | f.Propagate(cloned) 71 | } 72 | } 73 | } else { 74 | if _, ok := msg.GetTarget(f.target).([]byte); !ok { 75 | // ERROR this filter can't be used with different types 76 | return false, fmt.Errorf("received data is not supported") 77 | } 78 | 79 | buf := bytes.NewBuffer(msg.GetTarget(f.target).([]byte)) 80 | 81 | x, err := excelize.OpenReader(bytes.NewReader(buf.Bytes())) 82 | if err != nil { 83 | return false, err 84 | } 85 | defer x.Close() 86 | 87 | sheets := x.GetSheetList() 88 | for _, sName := range sheets { 89 | rows, err := x.GetRows(sName) 90 | if err != nil { 91 | return false, err 92 | } 93 | for _, row := range rows { 94 | cloned := msg.Clone() 95 | cloned.SetMessage(strings.Join(row, ",")) 96 | cloned.SetExtra("xls_sheet", sName) 97 | f.Propagate(cloned) 98 | } 99 | } 100 | } 101 | 102 | return true, nil 103 | } 104 | 105 | // OnEvent is called when an event occurs 106 | func (f *XLS) OnEvent(event *data.Event) {} 107 | 108 | // Set the name of the filter 109 | func init() { 110 | register("xls", NewXLSFilter) 111 | } 112 | -------------------------------------------------------------------------------- /plugins/cache.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/Matrix86/driplane/utils" 7 | ) 8 | 9 | // CachePackage contains methods for use the cache (local or global) 10 | type CachePackage struct{} 11 | 12 | // GetCache returns the FilePackage struct 13 | func GetCache() *CachePackage { 14 | return &CachePackage{} 15 | } 16 | 17 | // CacheResponse contains the return values 18 | type CacheResponse struct { 19 | Error error 20 | Status bool 21 | Value string 22 | } 23 | 24 | // Put add a new value in the cache 25 | func (c *CachePackage) Put(k, v string, ttl int64) { 26 | cache := utils.GetNamedTTLMap("global", 5*time.Minute) 27 | cache.Put(k, v, ttl) 28 | } 29 | 30 | // Get return a cache item if it exists 31 | func (c *CachePackage) Get(k string) *CacheResponse { 32 | cache := utils.GetNamedTTLMap("global", 5*time.Minute) 33 | v, ok := cache.Get(k) 34 | if ok { 35 | return &CacheResponse{ 36 | Error: nil, 37 | Status: true, 38 | Value: v.(string), 39 | } 40 | } 41 | return &CacheResponse{ 42 | Error: nil, 43 | Status: false, 44 | Value: "", 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /plugins/cache_test.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestCachePluginGetPutMethods(t *testing.T) { 9 | cache := GetCache() 10 | 11 | res := cache.Get("key") 12 | if res.Status != false { 13 | t.Errorf("cache.Get should return false" ) 14 | } 15 | 16 | cache.Put("key", "newvalue", 1) 17 | res = cache.Get("key") 18 | if res.Status == false { 19 | t.Errorf("cache.Get should return true" ) 20 | } 21 | if res.Value != "newvalue" { 22 | t.Errorf("cache.Get should return 'newvalue'" ) 23 | } 24 | 25 | time.Sleep(1 * time.Second) 26 | 27 | res = cache.Get("key") 28 | if res.Status != false { 29 | t.Errorf("cache.Get should return false" ) 30 | } 31 | } -------------------------------------------------------------------------------- /plugins/log.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "github.com/evilsocket/islazy/log" 5 | ) 6 | 7 | // LogPackage contains logs methods 8 | type LogPackage struct{} 9 | 10 | // GetLog returns the LogPackage 11 | func GetLog() *LogPackage { 12 | return &LogPackage{} 13 | } 14 | 15 | // Info prints an info line on the logs 16 | func (l *LogPackage) Info(format string, a ...interface{}) { 17 | log.Info(format, a...) 18 | } 19 | 20 | // Error prints an error line on the logs 21 | func (l *LogPackage) Error(format string, a ...interface{}) { 22 | log.Error(format, a...) 23 | } 24 | 25 | // Debug prints a debug line on the logs 26 | func (l *LogPackage) Debug(format string, a ...interface{}) { 27 | log.Debug(format, a...) 28 | } 29 | -------------------------------------------------------------------------------- /plugins/log_test.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "testing" 9 | 10 | "github.com/evilsocket/islazy/log" 11 | ) 12 | 13 | func TestLogPackage_Debug(t *testing.T) { 14 | l := GetLog() 15 | 16 | logfile := path.Join(os.TempDir(), "log.unit_test") 17 | log.Output = logfile 18 | log.Format = "{message}" 19 | log.Level = log.DEBUG 20 | log.Open() 21 | defer log.Close() 22 | defer os.Remove(logfile) 23 | 24 | message := "test debug" 25 | expected := fmt.Sprintf("%s\n", message) 26 | l.Debug("%s", message) 27 | 28 | dat, _ := ioutil.ReadFile(logfile) 29 | if string(dat) != expected { 30 | t.Errorf("wrong string: expected=%#v had=%#v", expected, string(dat)) 31 | } 32 | } 33 | 34 | func TestLogPackage_Info(t *testing.T) { 35 | l := GetLog() 36 | logfile := path.Join(os.TempDir(), "log.unit_test") 37 | log.Output = logfile 38 | log.Format = "{message}" 39 | log.Level = log.DEBUG 40 | log.Open() 41 | defer log.Close() 42 | defer os.Remove(logfile) 43 | 44 | message := "test debug" 45 | expected := fmt.Sprintf("%s\n", message) 46 | l.Info("%s", message) 47 | 48 | dat, _ := ioutil.ReadFile(logfile) 49 | if string(dat) != expected { 50 | t.Errorf("wrong string: expected=%#v had=%#v", expected, string(dat)) 51 | } 52 | } 53 | 54 | 55 | func TestLogPackage_Error(t *testing.T) { 56 | l := GetLog() 57 | 58 | logfile := path.Join(os.TempDir(), "log.unit_test") 59 | log.Output = logfile 60 | log.Format = "{message}" 61 | log.Level = log.DEBUG 62 | log.Open() 63 | defer log.Close() 64 | defer os.Remove(logfile) 65 | 66 | message := "test debug" 67 | expected := fmt.Sprintf("%s\n", message) 68 | l.Error("%s", message) 69 | 70 | dat, _ := ioutil.ReadFile(logfile) 71 | if string(dat) != expected { 72 | t.Errorf("wrong string: expected=%#v had=%#v", expected, string(dat)) 73 | } 74 | } -------------------------------------------------------------------------------- /plugins/strings.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // StringsPackage contains string manipulation methods 8 | type StringsPackage struct {} 9 | 10 | // GetStrings returns a StringsPackage 11 | func GetStrings() *StringsPackage { 12 | return &StringsPackage{} 13 | } 14 | 15 | // StringsResponse contains the return values 16 | type StringsResponse struct { 17 | Error error 18 | Status bool 19 | } 20 | 21 | // StartsWith returns true if a string start with a substring 22 | func (c *StringsPackage) StartsWith(str, substr string) StringsResponse { 23 | ret := strings.HasPrefix(str, substr) 24 | return StringsResponse{ 25 | Error: nil, 26 | Status: ret, 27 | } 28 | } -------------------------------------------------------------------------------- /plugins/strings_test.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import "testing" 4 | 5 | func TestStringsPluginStartsWithMethod(t *testing.T) { 6 | s := GetStrings() 7 | 8 | res := s.StartsWith("testing string", "test") 9 | if res.Status == false { 10 | t.Errorf("bad response: expected=%t had=%t", true, res.Status ) 11 | } 12 | 13 | res = s.StartsWith("no testing string", "test") 14 | if res.Status == true { 15 | t.Errorf("bad response: expected=%t had=%t", false, res.Status ) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /plugins/util.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "github.com/Matrix86/driplane/utils" 5 | "os" 6 | "time" 7 | ) 8 | 9 | // UtilPackage contains useful generic methods 10 | type UtilPackage struct{} 11 | 12 | // GetUtil returns an UtilPackage 13 | func GetUtil() *UtilPackage { 14 | return &UtilPackage{} 15 | } 16 | 17 | // UtilResponse contains the return values 18 | type UtilResponse struct { 19 | Error error 20 | Status bool 21 | Value string 22 | } 23 | 24 | // Sleep call Sleep method for N seconds 25 | func (c *UtilPackage) Sleep(seconds int) UtilResponse { 26 | time.Sleep(time.Duration(seconds) * time.Second) 27 | return UtilResponse{ 28 | Error: nil, 29 | Status: true, 30 | } 31 | } 32 | 33 | // Getenv returns an environment variable if it exists 34 | func (c *UtilPackage) Getenv(name string) UtilResponse { 35 | return UtilResponse{ 36 | Error: nil, 37 | Status: true, 38 | Value: os.Getenv(name), 39 | } 40 | } 41 | 42 | // Md5File returns the MD5 hash of the file 43 | func (c *UtilPackage) Md5File(filename string) UtilResponse { 44 | hash, err := utils.Md5File(filename) 45 | if err != nil { 46 | return UtilResponse{ 47 | Error: err, 48 | Status: false, 49 | Value: "", 50 | } 51 | } 52 | return UtilResponse{ 53 | Error: nil, 54 | Status: true, 55 | Value: hash, 56 | } 57 | } 58 | 59 | // Sha1File returns the SHA1 hash of the file 60 | func (c *UtilPackage) Sha1File(filename string) UtilResponse { 61 | hash, err := utils.Sha1File(filename) 62 | if err != nil { 63 | return UtilResponse{ 64 | Error: err, 65 | Status: false, 66 | Value: "", 67 | } 68 | } 69 | return UtilResponse{ 70 | Error: nil, 71 | Status: true, 72 | Value: hash, 73 | } 74 | } 75 | 76 | // Sha256File returns the SHA256 hash of the file 77 | func (c *UtilPackage) Sha256File(filename string) UtilResponse { 78 | hash, err := utils.Sha256File(filename) 79 | if err != nil { 80 | return UtilResponse{ 81 | Error: err, 82 | Status: false, 83 | Value: "", 84 | } 85 | } 86 | return UtilResponse{ 87 | Error: nil, 88 | Status: true, 89 | Value: hash, 90 | } 91 | } 92 | 93 | // Sha512File returns the SHA512 hash of the file 94 | func (c *UtilPackage) Sha512File(filename string) UtilResponse { 95 | hash, err := utils.Sha512File(filename) 96 | if err != nil { 97 | return UtilResponse{ 98 | Error: err, 99 | Status: false, 100 | Value: "", 101 | } 102 | } 103 | return UtilResponse{ 104 | Error: nil, 105 | Status: true, 106 | Value: hash, 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /plugins/util_test.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestUtilPluginSleepMethod(t *testing.T) { 9 | u := GetUtil() 10 | 11 | res := u.Sleep(0) 12 | if res.Status == false { 13 | t.Errorf("bad response: expected=%t had=%t", true, res.Status) 14 | } 15 | } 16 | 17 | func TestUtilPluginGetEnvMethod(t *testing.T) { 18 | u := GetUtil() 19 | 20 | res := u.Getenv("ENVTESTVAR") 21 | if res.Value != "" { 22 | t.Errorf("the env var should be empty") 23 | } 24 | 25 | os.Setenv("ENVTESTVAR", "1") 26 | 27 | res = u.Getenv("ENVTESTVAR") 28 | if res.Value != "1" { 29 | t.Errorf("the env var should contain '1'") 30 | } 31 | } 32 | 33 | func TestUtilPluginMd5Method(t *testing.T) { 34 | u := GetUtil() 35 | 36 | file, err := os.CreateTemp(os.TempDir(), "prefix") 37 | if err != nil { 38 | t.Errorf("cannot create a temporary file") 39 | } 40 | defer os.Remove(file.Name()) 41 | 42 | res := u.Md5File("/tmp/notexistentfile_") 43 | if res.Status == true { 44 | t.Errorf("Status should be false") 45 | } 46 | 47 | res = u.Md5File(file.Name()) 48 | if res.Status == false { 49 | t.Errorf("Status should be true") 50 | } 51 | 52 | expected := "d41d8cd98f00b204e9800998ecf8427e" 53 | if res.Value != expected { 54 | t.Errorf("Value has a bad value: expected=%s had=%s", expected, res.Value) 55 | } 56 | } 57 | 58 | func TestUtilPluginSha1Method(t *testing.T) { 59 | u := GetUtil() 60 | 61 | file, err := os.CreateTemp(os.TempDir(), "prefix") 62 | if err != nil { 63 | t.Errorf("cannot create a temporary file") 64 | } 65 | defer os.Remove(file.Name()) 66 | 67 | res := u.Sha1File("/tmp/notexistentfile_") 68 | if res.Status == true { 69 | t.Errorf("Status should be false") 70 | } 71 | 72 | res = u.Sha1File(file.Name()) 73 | if res.Status == false { 74 | t.Errorf("Status should be true") 75 | } 76 | 77 | expected := "da39a3ee5e6b4b0d3255bfef95601890afd80709" 78 | if res.Value != expected { 79 | t.Errorf("Value has a bad value: expected=%s had=%s", expected, res.Value) 80 | } 81 | } 82 | 83 | func TestUtilPluginSha256Method(t *testing.T) { 84 | u := GetUtil() 85 | 86 | file, err := os.CreateTemp(os.TempDir(), "prefix") 87 | if err != nil { 88 | t.Errorf("cannot create a temporary file") 89 | } 90 | defer os.Remove(file.Name()) 91 | 92 | res := u.Sha256File("/tmp/notexistentfile_") 93 | if res.Status == true { 94 | t.Errorf("Status should be false") 95 | } 96 | 97 | res = u.Sha256File(file.Name()) 98 | if res.Status == false { 99 | t.Errorf("Status should be true") 100 | } 101 | 102 | expected := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" 103 | if res.Value != expected { 104 | t.Errorf("Value has a bad value: expected=%s had=%s", expected, res.Value) 105 | } 106 | } 107 | 108 | func TestUtilPluginSha512Method(t *testing.T) { 109 | u := GetUtil() 110 | 111 | file, err := os.CreateTemp(os.TempDir(), "prefix") 112 | if err != nil { 113 | t.Errorf("cannot create a temporary file") 114 | } 115 | defer os.Remove(file.Name()) 116 | 117 | res := u.Sha512File("/tmp/notexistentfile_") 118 | if res.Status == true { 119 | t.Errorf("Status should be false") 120 | } 121 | 122 | res = u.Sha512File(file.Name()) 123 | if res.Status == false { 124 | t.Errorf("Status should be true") 125 | } 126 | 127 | expected := "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" 128 | if res.Value != expected { 129 | t.Errorf("Value has a bad value: expected=%s had=%s", expected, res.Value) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /release.stork: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env stork -f 2 | 3 | git:changelog 4 | 5 | version:file "core/version.go" 6 | version:from_user 7 | 8 | git:create_tag $VERSION -------------------------------------------------------------------------------- /src_docs/archetypes/default.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "{{ replace .Name "-" " " | title }}" 3 | date: {{ .Date }} 4 | draft: true 5 | --- 6 | 7 | -------------------------------------------------------------------------------- /src_docs/config/_default/config.toml: -------------------------------------------------------------------------------- 1 | baseURL = "https://matrix86.github.io/driplane/" 2 | title = "Driplane documentation" 3 | theme = "zdoc" 4 | 5 | defaultContentLanguage = "en" 6 | defaultContentLanguageInSubdir = true 7 | hasCJKLanguage = true 8 | 9 | [author] 10 | name = "Matrix86" 11 | homepage = "https://github.com/Matrix86/" 12 | 13 | copyright = "©{year}, All Rights Reserved" 14 | timeout = 10000 15 | enableEmoji = true 16 | paginate = 13 17 | rssLimit = 100 18 | 19 | googleAnalytics = "" 20 | 21 | [markup] 22 | [markup.goldmark] 23 | [markup.goldmark.renderer] 24 | hardWraps = true 25 | unsafe = true 26 | xHTML = true 27 | [markup.highlight] 28 | codeFences = true 29 | lineNos = true 30 | lineNumbersInTable = true 31 | noClasses = false 32 | [markup.tableOfContents] 33 | endLevel = 4 34 | ordered = false 35 | startLevel = 2 36 | 37 | [outputs] 38 | home = ["HTML", "RSS", "JSON"] 39 | 40 | [taxonomies] 41 | tag = "tags" 42 | 43 | publishDir = "docs" -------------------------------------------------------------------------------- /src_docs/config/_default/languages.toml: -------------------------------------------------------------------------------- 1 | [en] 2 | title = "Dripline Documentation" 3 | languageName = "English" 4 | weight = 1 5 | -------------------------------------------------------------------------------- /src_docs/config/_default/menus.en.toml: -------------------------------------------------------------------------------- 1 | [[main]] 2 | identifier = "doc" 3 | name = "Docs" 4 | url = "doc" 5 | weight = 1 -------------------------------------------------------------------------------- /src_docs/config/_default/params.toml: -------------------------------------------------------------------------------- 1 | logo = true # Logo that appears in the site navigation bar. 2 | logoText = "Driplane" # Logo text that appears in the site navigation bar. 3 | logoType = "short" # long, short 4 | description = "Driplane documentation website." # for SEO 5 | 6 | themeOptions = ["light", "dark"] # select options for site color theme 7 | 8 | useFaviconGenerator = true # https://www.favicon-generator.org/ 9 | 10 | enableSearch = true 11 | enableLangChange = false 12 | enableDarkMode = true 13 | enableBreadcrumb = true 14 | enableToc = true 15 | enableMenu = true 16 | enableNavbar = true 17 | enableFooter = false 18 | showPoweredBy = false 19 | 20 | paginateWindow = 1 # setting it to 1 gives 7 buttons, 2 gives 9, etc. If set 1: [1 ... 4 5 6 ... 356] [1 2 3 4 5 ... 356] etc 21 | taxoPaginate = 13 # items per page 22 | taxoGroupByDate = "2006" # "2006-01": group by month, "2006": group by year 23 | 24 | github = "https://github.com/matrix86/driplane" -------------------------------------------------------------------------------- /src_docs/content/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | author: Gianluca Braga 3 | title: Driplane 4 | description: Driplane site 5 | date: 2020-01-26T04:15:05+09:00 6 | draft: false 7 | landing: 8 | image: favicon/logo192x192.png 9 | title: 10 | - Driplane 11 | text: 12 | - Create automated tasks and keep an eye on what's going on around you! 13 | titleColor: 14 | textColor: 15 | spaceBetweenTitleText: 30 16 | buttons: 17 | - link: doc 18 | text: GET STARTED 19 | color: primary 20 | - link: https://github.com/Matrix86/driplane 21 | text: DOWNLOAD 22 | color: default 23 | 24 | --- -------------------------------------------------------------------------------- /src_docs/content/doc/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Driplane Documentation" 3 | description: "Documentation of Driplane project" 4 | date: 2020-01-11T14:09:21+09:00 5 | --- 6 | 7 | # Introduction 8 | 9 | `Driplane` allows you to create an automatic alerting system or start automated tasks triggered by events. 10 | You can keep under control a stream source as Twitter, a file, a RSS feed or a website. 11 | It includes a mini language that allows you to define the filtering pipelines (the rules), and it is extensible thanks to an integrated JS plugin system. 12 | 13 | With `driplane` you can create several rules starting from one or more streams, filter the content in the pipeline and launch tasks or send alerts if some event occurred. 14 | 15 | The documentation can be found [HERE](https://matrix86.github.io/driplane/doc/) 16 | 17 | ## How it works 18 | 19 | The user can define one or more rules. Usually a rule contains a source (`feeder`), which takes care of getting the information and sending updates (`Message`) through the pipeline, and one or more `filters`. 20 | The filters' job is to choose whether to propagate or not the `Message` to the next filter in the pipeline relying on a _condition_, or change the `Message` received before to propagate it. The `Message` will be propagated only if it verifies the condition. 21 | 22 | ## Use cases 23 | 24 | Using `driplane` it is possible to: 25 | 26 | * keep track of keywords or users on Twitter, receive the new tweets or quoted tweets from them, search for URLs or particular strings in them and send a Telegram or a Slack message through their webhooks. 27 | * keep track of a RSS feed or a website, and download and store on file all the new changes to them. 28 | * keep track of changes on a file, and launch alert if a particular condition is verified. 29 | 30 | The rules and the JS plugins allow you to create very complex custom logics. 31 | 32 | ## Usage 33 | 34 | ``` 35 | Usage of ./bin/driplane: 36 | -config string 37 | Set configuration file. 38 | -debug 39 | Enable debug logs. 40 | -dry-run 41 | Only test the rules syntax. 42 | -help 43 | This help. 44 | -js string 45 | Path of the js plugins. 46 | -rules string 47 | Path of the rules' directory. 48 | ``` 49 | 50 | {{< alert theme="success" >}} 51 | \> driplane -config /path/to/config.yml 52 | {{< /alert >}} 53 | 54 | -------------------------------------------------------------------------------- /src_docs/content/doc/configuration/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 2 3 | title: "Configuration" 4 | date: 2020-09-16T22:38:03+02:00 5 | draft: false 6 | --- 7 | 8 | ## Configuration 9 | 10 | `Driplane` needs a yaml file to work. An example can be found [here](https://github.com/Matrix86/driplane/blob/master/config.yaml.example). 11 | 12 | The configuration file could have several sections, but the mandatory one is the `general` one. It specifies all the paths needed by `driplane` to find js files, templates and rules. 13 | 14 | > It contains the paths of rules, templates and javascript plugins. 15 | 16 | ```yaml 17 | general: 18 | log_path: "" # if nothing is specified it prints logs on stdout 19 | rules_path: "rules" # path containing the rules 20 | js_path: "js" # path of the js plugins 21 | templates_path: "templates" # path of templates 22 | debug: false # if true enable the debug logs 23 | 24 | update: 25 | enable: "false" # if true it reloads the rules every time a file is updated 26 | ``` 27 | 28 | In the configuration it is possible to define default params for _Feeders_ and _Filters_. In this way we don't need to specify that configuration in the rules. 29 | 30 | > For the twitter feeder we can set the keys one time. 31 | ```yaml 32 | twitter: 33 | bearerToken: "token", 34 | keywords: "#italy #coding #malware something", 35 | stallWarnings: "true" 36 | ``` 37 | 38 | > Same for the Slack feeder and/or feeder. 39 | ```yaml 40 | slack: 41 | token: "xoxb-xxx-xxx-xxx" 42 | verification_token: "xxx" 43 | lt_enable: "true" 44 | lt_subdomain: "mytestdomain" 45 | ``` 46 | > Each config will be visible __ONLY__ to the related filter or feeder. 47 | 48 | We can also define `custom` configurations and they will be available to all the feeders and filters. 49 | 50 | ```yaml 51 | custom: 52 | customKey: "This is a custom configuration" 53 | ``` -------------------------------------------------------------------------------- /src_docs/content/doc/feeders/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 4 3 | title: "Feeders" 4 | date: 2020-09-17T11:30:07+02:00 5 | draft: false 6 | collapsible: true 7 | --- 8 | -------------------------------------------------------------------------------- /src_docs/content/doc/feeders/apt.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Apt" 3 | date: 2022-03-15T11:38:24+01:00 4 | draft: false 5 | --- 6 | 7 | ## Apt feeder 8 | 9 | This feeder can create the stream starting from an apt [repository](https://wiki.debian.org/it/DebianRepository). It supports also flat repositories. 10 | It is possible to specify the frequency of the quering and receive a message every time a new package is published. 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |--------------|----------------------------------------------------------|----------|---------------------------------------------------------------------------------------------| 16 | | **url** | _STRING_ | empty | URL of the apt repo | 17 | | **freq** | _[DURATION](https://golang.org/pkg/time/#ParseDuration)_ | 60s | how often the feed should be parsed | 18 | | **suite** | _STRING_ | "stable" | suite of the repo to keep under control | 19 | | **arch** | _STRING_ | empty" | architecture of the repo, if empty the first arch returned by the Release file will be used | 20 | | **index** | _STRING_ | empty | URL of the Packages file (it overrides the url parameter) | 21 | | **insecure** | _BOOL_ | false | allow repository with insecure certificates | 22 | 23 | {{< notice info "Example" >}} 24 | `... | | ...` 25 | {{< /notice >}} 26 | 27 | ### Output 28 | 29 | #### Text 30 | 31 | The `main` field of the Message will contain the filename of the package and all the other field will be present in extra fields. 32 | 33 | #### Extra 34 | 35 | List of the supported field that will be returned as extra field. 36 | 37 | | Name | 38 | |----------------| 39 | | Filename | 40 | | Size | 41 | | MD5sum | 42 | | SHA1 | 43 | | SHA256 | 44 | | DescriptionMD5 | 45 | | Depends | 46 | | InstalledSize | 47 | | Package | 48 | | Architecture | 49 | | Version | 50 | | Section | 51 | | Maintainer | 52 | | Homepage | 53 | | Description | 54 | | Tag | 55 | | Author | 56 | | Name | 57 | 58 | ### Examples 59 | 60 | {{< alert theme="warning" >}} 61 | Soon... 62 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/feeders/file.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "File" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## File feeder 8 | 9 | This feeder can create the stream starting from a file. Like the `tail -f` command it opens the specified file and propagates a data message if a line is being added to the file. 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |--------------|----------|---------|------------------------------------------------------------------------| 15 | | **filename** | _STRING_ | empty | the path of the file that it has to keep track | 16 | | **toend** | _BOOL_ | "false" | the feeder will start to create data messages only for new added lines | 17 | 18 | {{< notice info "Example" >}} 19 | `... | | ...` 20 | {{< /notice >}} 21 | 22 | ### Output 23 | 24 | #### Text 25 | 26 | The `main` field of the Message will contain the new read line. 27 | 28 | #### Extra 29 | 30 | | Name | Description | 31 | |-----------|---------------------------| 32 | | file_name | the name of the read file | 33 | 34 | ### Examples 35 | 36 | {{< alert theme="warning" >}} 37 | Soon... 38 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/feeders/folder.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Folder" 3 | date: 2021-02-01T13:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Folder feeder 8 | 9 | This feeder can create the stream of fsnotify events for a given folder or for cloud platform storage like Amazon S3, 10 | Google Drive and Dropbox. 11 | 12 | The feeder use [cloudwatcher](https://github.com/Matrix86/cloudwatcher) to keep track of changes on the chosen directory. 13 | 14 | ### Parameters 15 | 16 | | Parameter | Type | Default | Description | 17 | |-----------|----------------------------------------------------------|---------|------------------------------------------------------------------------| 18 | | **name** | _STRING_ | empty | the path of the folder that it has to keep track | 19 | | **type** | _STRING_ | "local" | the type of service to use `local`, `dropbox`, `gdrive`, `s3` or `git` | 20 | | **freq** | _[DURATION](https://golang.org/pkg/time/#ParseDuration)_ | 2s | how often the directory should be checked for updates | 21 | 22 | {{< alert theme="warning" >}} 23 | Some services like Gdrive, S3 and Dropbox require additional configurations (you can check them from [here](https://github.com/Matrix86/cloudwatcher/blob/main/README.md)). 24 | You can pass them using the config file ([more here](https://matrix86.github.io/driplane/doc/configuration/)) OR 25 | define them in the rule itself: `` 26 | {{< /alert >}} 27 | 28 | {{< notice info "Example" >}} 29 | ` | ...` 30 | {{< /notice >}} 31 | 32 | ### Output 33 | 34 | #### Text 35 | 36 | The `main` field of the Message will contain the filename while the `op` extra the type of event. 37 | 38 | #### Extra 39 | 40 | | Name | Description | 41 | |------|----------------------------------------------------------------------------| 42 | | op | type of event: `FileCreated`, `FileChanged`, `FileDeleted`, `TagsChanged` | 43 | | size | for some events you can find the size of the file that triggered the event | 44 | 45 | ### Examples 46 | 47 | {{< alert theme="warning" >}} 48 | Soon... 49 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/feeders/rss.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "RSS" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## RSS 8 | 9 | This feeder creates a stream starting from a feed `RSS`, `ATOM` or `JSON`. 10 | It is based on [gofeed](https://github.com/mmcdole/gofeed) so you can refer to it for more info and supported formats. 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |--------------------------|----------------------------------------------------------|---------|------------------------------------------------------------------------------------------------------------------------| 16 | | **url** | _STRING_ | empty | URL of the feed | 17 | | **freq** | _[DURATION](https://golang.org/pkg/time/#ParseDuration)_ | 60s | how often the feed should be parsed | 18 | | **start_from_beginning** | _BOOL_ | "false" | if "true" it starts to parse the feed from the beginning (the first time it will ignore the pubdate field of the feed) | 19 | | **ignore_pubdate** | _BOOL_ | "false" | if "true" it ignores the pubdate and it returns all the feed content every time | 20 | 21 | {{< notice info "Example" >}} 22 | `... | | ...` 23 | {{< /notice >}} 24 | 25 | ### Output 26 | 27 | #### Text 28 | 29 | The `main` field of the Message will contain the `item.Title` string of the `gofeed.Item` struct. 30 | 31 | #### Extra 32 | 33 | | Name | Description | 34 | |----------------|-------------------------------------------------------------------------------------------------------------| 35 | | feed_title | title of the feed ([feed.Title](https://github.com/mmcdole/gofeed#default-mappings)) | 36 | | feed_feedlink | feed url ([feed.FeedLink](https://github.com/mmcdole/gofeed#default-mappings)) | 37 | | feed_updated | time of the last update ([feed.Updated](https://github.com/mmcdole/gofeed#default-mappings)) | 38 | | feed_published | date of publication ([feed.Published](https://github.com/mmcdole/gofeed#default-mappings)) | 39 | | feed_author | author in the form `name ` ([feed.Author.Name](https://github.com/mmcdole/gofeed#default-mappings)) | 40 | | feed_language | language of the feed ([feed.Language](https://github.com/mmcdole/gofeed#default-mappings)) | 41 | | feed_copyright | copyright ([feed.Copyright](https://github.com/mmcdole/gofeed#default-mappings)) | 42 | | feed_generator | generator used to create the feed ([feed.Generator](https://github.com/mmcdole/gofeed#default-mappings)) | 43 | 44 | In addition to the feed tags, the Extra will also contain the item's fields. Since they could be different from feed to feed and it is possible to configure custom tag, you will find all them in the extra with their name. 45 | 46 | {{< notice warning "ATTENTION" >}} 47 | Not all the Extra field could be filled. If the relative tag is not present on the feed it will be empty. 48 | {{< /notice >}} 49 | 50 | ### Examples 51 | 52 | {{< alert theme="warning" >}} 53 | Soon... 54 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/feeders/timer.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Timer" 3 | date: 2021-02-20T18:06:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Timer feeder 8 | 9 | This feeder trigger a pipeline every time the timer is fired. 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |-----------|----------------------------------------------------------|---------|------------------------------------------------------------------| 15 | | **freq** | _[DURATION](https://golang.org/pkg/time/#ParseDuration)_ | 60s | The intervals (in duration) on how often to execute the pipeline | 16 | 17 | 18 | {{< notice info "Example" >}} 19 | ` | ...` 20 | {{< /notice >}} 21 | 22 | ### Output 23 | 24 | #### Text 25 | 26 | The `main` field of the Message will contain time in rfc3339 format. 27 | 28 | #### Extra 29 | 30 | | Name | Description | 31 | | --- | --- | 32 | | rfc3339 | time in rfc3339 format | 33 | | timestamp | time in Unix timestamp | 34 | 35 | ### Examples 36 | 37 | {{< alert theme="warning" >}} 38 | Soon... 39 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/feeders/web.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Web" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Web 8 | 9 | This feeder creates a stream starting from a web page. It is possible to define how often the page should be downloaded and parsed. 10 | Every time the page is parsed a Message is sent down the lane. 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |---------------|----------------------------------------------------------|---------|------------------------------------------------------------------------------------------------| 16 | | **url** | _STRING_ | empty | URL of the web page | 17 | | **freq** | _[DURATION](https://golang.org/pkg/time/#ParseDuration)_ | 60s | how often the page should be parsed | 18 | | **text_only** | _BOOL_ | "false" | if "true" it removes all the tags from the page | 19 | | **method** | _STRING_ | "GET" | HTTP method to use on the requests | 20 | | **headers** | _JSON_ | empty | Headers to use in the request | 21 | | **data** | _JSON_ | empty | POST fields to send with the requests (it's not possible to use in combination with `rawData`) | 22 | | **rawData** | _STRING_ | empty | raw body of the requests (it's not possible to use in combination with `data`) | 23 | | **status** | _STRING_ | empty | the filter will propagate the Message only if the returned status is this | 24 | | **cookies** | _STRING_ | empty | Path of the JSON file containing the cookies to use | 25 | 26 | {{< notice info "Example" >}} 27 | `... | | ...` 28 | {{< /notice >}} 29 | 30 | ### Output 31 | 32 | #### Text 33 | 34 | The `main` field of the Message will contain the HTML source or the text of the website if the `text_only` parameter is set to true. 35 | 36 | #### Extra 37 | 38 | | Name | Description | 39 | |-------------|------------------------| 40 | | url | URL of the web page | 41 | | title | meta tag `title` | 42 | | description | meta tag `description` | 43 | | image | meta tag `image` | 44 | | sitename | meta tag `sitename` | 45 | 46 | {{< notice warning "ATTENTION" >}} 47 | Not all the Extra field could be filled. If the relative tag is not present on the feed it will be empty. 48 | {{< /notice >}} 49 | 50 | ### Examples 51 | 52 | {{< alert theme="warning" >}} 53 | Soon... 54 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 5 3 | title: "Filters" 4 | date: 2020-09-17T11:30:11+02:00 5 | draft: false 6 | collapsible: true 7 | --- 8 | -------------------------------------------------------------------------------- /src_docs/content/doc/filters/cache.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Cache" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Cache 8 | 9 | This filter introduces a simple cache mechanism in the rule. It is a TTL based cache and it can have a local visibility (cache is only visible to the current filter) or a global visibility (cache shared across **ALL** the rules). 10 | If the target of the Message as input has been cached before, and his TTL is not expired, it will be dropped and not propagated to the next filter. 11 | Otherwise if the target of the Message is new to the cache, it is inserted in it and propagated to the next filter. 12 | 13 | ### Parameters 14 | 15 | | Parameter | Type | Default | Description | 16 | |----------------------|----------------------------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------| 17 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or and extra field) | 18 | | **refresh_on_get** | _BOOL_ | "true" | the TTL is refreshed if the key has been looked up | 19 | | **ttl** | _[DURATION](https://golang.org/pkg/time/#ParseDuration)_ | 24h | how long after the key will be deleted | 20 | | **sync_time** | _[DURATION](https://golang.org/pkg/time/#ParseDuration)_ | 5m | how often the sync on file should be called | 21 | | **name** | _STRING_ | "" | named global cache: this cache will be available for ALL the rules also for different files | 22 | | **global** | _BOOL_ | "false" | make this cache global: like for the `name` param this cache is available to all the rules with the name "global" | 23 | | **file** | _STRING_ | "" | enable cache persistence. It loads and writes the cache from a file | 24 | | **ignore_first_run** | _BOOL_ | "false" | if `true` the messages that arrives to the cache with the firstRun flag enabled will be send to the next filter | 25 | 26 | {{< notice info "Example" >}} 27 | `... | cache(ttl="24h", global="true") | ...` 28 | {{< /notice >}} 29 | 30 | ### Output 31 | 32 | The output is not being changed. This filter can only stop or not the propagation of the Message. 33 | 34 | ### Examples 35 | 36 | {{< alert theme="warning" >}} 37 | Soon... 38 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/changed.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Changed" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Changed 8 | 9 | This filter is similar to the cache. It can only stop the propagation of the Message across the lane, but only if the target of the received Message is different from the previous one. 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |------------|----------|---------|---------------------------------------------------------------------------------------------------| 15 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or and extra field) | 16 | 17 | {{< notice info "Example" >}} 18 | `... | changed(target="original_author") | ...` 19 | {{< /notice >}} 20 | 21 | ### Output 22 | 23 | The output is not being changed. This filter can only stop or not the propagation of the Message. 24 | 25 | ### Examples 26 | 27 | {{< alert theme="warning" >}} 28 | Soon... 29 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/echo.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Echo" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Echo 8 | 9 | This filter prints the Message on the logs. It is mostly used to debug the rules. 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |-----------|--------|---------|---------------------------------| 15 | | **extra** | _BOOL_ | "false" | print also all the extra fields | 16 | 17 | {{< notice info "Example" >}} 18 | `... | echo(extra="false") | ...` 19 | {{< /notice >}} 20 | 21 | ### Output 22 | 23 | The output is not being changed. This filter prints the received Message and send it to the next filter in the rule. 24 | 25 | ### Examples 26 | 27 | {{< alert theme="warning" >}} 28 | Soon... 29 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/elasticsearch.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Elasticsearch" 3 | date: 2022-03-12T15:40:23+01:00 4 | draft: false 5 | --- 6 | 7 | ## ElasticSearch 8 | 9 | This filter writes on ElasticSearch all the message it receives as input and return as output the document ID. 10 | You can also specify what message's field it should use as input. 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |------------|----------|---------|-------------------------------------------------------------------------------------------------------| 16 | | **target** | _STRING_ | "main" | the field of the Message that should be used for filter's output (it could be main or an extra field) | 17 | 18 | {{< notice info "Example" >}} 19 | `... | elasticsearch(target="input_field") | ...` 20 | {{< /notice >}} 21 | 22 | ### Output 23 | 24 | The `main` field will contain the docID returned by ElasticSearch. 25 | 26 | ### Examples 27 | 28 | {{< alert theme="warning" >}} 29 | Soon... 30 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/file.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "File" 3 | date: 2022-03-12T15:33:42+01:00 4 | draft: false 5 | --- 6 | 7 | ## File 8 | 9 | This filter get as input the path of a local file, read it and return the content back to the pipeline. 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |------------|----------|---------|-------------------------------------------------------------------------------------------------------| 15 | | **target** | _STRING_ | "main" | the field of the Message that should be used for filter's output (it could be main or an extra field) | 16 | 17 | {{< notice info "Example" >}} 18 | `... | file(target="file_content") | ...` 19 | {{< /notice >}} 20 | 21 | ### Output 22 | 23 | The output will contain the content of the read file. Using the `target` parameter you can specify the output field's name. 24 | 25 | ### Examples 26 | 27 | {{< alert theme="warning" >}} 28 | Soon... 29 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/format.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Format" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Format 8 | 9 | This filter is used to format the received Message. It is based on 10 | the [Golang templates](https://golang.org/pkg/text/template/) and it can load templates from the `template_path` 11 | directory specified in the configuration file. 12 | 13 | ### Parameters 14 | 15 | | Parameter | Type | Default | Description | 16 | |--------------|----------|---------|--------------------------------------------------------------------------------------------------| 17 | | **type** | _STRING_ | "text" | specify the type of template to use : `"text"` or `"html"` | 18 | | **template** | _STRING_ | empty | a template could be specified directly here, instead of load it from file | 19 | | **file** | _STRING_ | empty | load the template from file | 20 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or an extra field) | 21 | 22 | In the template is allowed to use all the fields of the received Message: main or extra. 23 | 24 | {{< notice info "Example" >}} 25 | `... | format(type="html", template="main : {{.main}} extra : {{.file_name}}") | ...` 26 | {{< /notice >}} 27 | 28 | ### Output 29 | 30 | The new formatted text is sent to the next filter in the `main` field of the Message if the `target` parameter is not specified. 31 | The extra fields do not undergo changes. 32 | 33 | ### Examples 34 | 35 | {{< alert theme="warning" >}} Soon... {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/hash.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Hash" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Hash 8 | 9 | This filter is used to search or extract hashes from a Message. 10 | Supported types of hashes are: 11 | * MD5 12 | * SHA1 13 | * SHA256 14 | * SHA512 15 | 16 | ### Parameters 17 | 18 | | Parameter | Type | Default | Description | 19 | |-------------|----------|---------|--------------------------------------------------------------------------------------------------| 20 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or an extra field) | 21 | | **extract** | _BOOL_ | "false" | if `"true"` the main field of the output Message will be the extracted hash | 22 | | **md5** | _BOOL_ | "true" | if `"false"` md5 hashes will be ignored | 23 | | **sha1** | _BOOL_ | "true" | if `"false"` sha1 hashes will be ignored | 24 | | **sha256** | _BOOL_ | "true" | if `"false"` sha256 hashes will be ignored | 25 | | **sha512** | _BOOL_ | "true" | if `"false"` sha512 hashes will be ignored | 26 | 27 | 28 | {{< notice info "Example" >}} 29 | `... | hash(target="description", extract="true") | ...` 30 | {{< /notice >}} 31 | 32 | ### Output 33 | 34 | If the `extract` parameter is "false", the received Message will be propagated only if at least one hash is found in it. 35 | Otherwise, if `extract` is "true" and the Message contains one or more hashes, the `main` field of the propagated Message will contain only the extracted hash. 36 | 37 | If the `extract` parameter is "true" and the message is propagated, a new extra field is created: `fulltext` will contain the original `target` string. 38 | 39 | {{< notice warning "ATTENTION" >}} 40 | If the targeted field contains multiple hashes, the filter will create and propagate multiple messages, one for each hash. 41 | {{< /notice >}} 42 | 43 | ### Examples 44 | 45 | {{< alert theme="warning" >}} 46 | Soon... 47 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/html.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Html" 3 | date: 2022-07-19T23:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Html 8 | 9 | This filter is used to extract information from an HTML page received through the Message. 10 | Like jQuery for JS, we can set a selector to find in the page, extract text from the tags and the html content (we are using [goquery library](https://github.com/PuerkitoBio/goquery) under the hood). 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |--------------|----------|---------|--------------------------------------------------------------------------------------------------| 16 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or an extra field) | 17 | | **selector** | _STRING_ | "" | the selector to find in the HTML page | 18 | | **get** | _STRING_ | "html" | what do we want to retrieve on the tags found in the selected one: `html`, `text`, `attr` | 19 | | **attr** | _STRING_ | "" | if get is `attr` you can define what attr name it should extract | 20 | 21 | 22 | {{< notice info "Example" >}} 23 | `... | html(selector=".link", get="attr", attr="href") | ...` 24 | {{< /notice >}} 25 | 26 | ### Output 27 | 28 | The filter will generate one or more Messages. It is possible to use more than 1 time this filter. 29 | The field `fulltext` will contain the original `target` string. 30 | 31 | ### Examples 32 | 33 | {{< alert theme="warning" >}} 34 | Soon... 35 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/http.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Http" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## HTTP 8 | 9 | This filter allows you to send HTTP requests. When a `Message` arrives to the filter, we can decide if use the `main` field of the `Message` as URL on the request, or its content for the HTTP data. 10 | This behaviour can be handled with the parameters. 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |-----------------|----------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------| 16 | | **url** | _STRING_ | empty | URL of the web page. It is possible use the [Golang templates](https://golang.org/pkg/text/template/) to use fields of the `Message` | 17 | | **download_to** | _STRING_ | empty | path of where to download the file. It is possible use the [Golang templates](https://golang.org/pkg/text/template/) to use fields of the `Message` | 18 | | **text_only** | _BOOL_ | "false" | if "true" it removes all the tags from the body response | 19 | | **method** | _STRING_ | "GET" | HTTP method to use on the request | 20 | | **headers** | _JSON_ | empty | Headers to use in the request | 21 | | **data** | _JSON_ | empty | POST fields to send with the request (it's not possible to use in combination with `rawData`) | 22 | | **rawData** | _STRING_ | empty | raw body of the request (it's not possible to use in combination with `data`) | 23 | | **status** | _STRING_ | empty | the filter will propagate the Message only if the returned status has the specified value | 24 | | **cookies** | _STRING_ | empty | Path of the JSON file containing the cookies to use | 25 | 26 | 27 | {{< notice info "Example" >}} 28 | `... | http(url="{{ .main }}", cookies="exported.json", headers="{\"Content-type\": \"application/json\"}") | ...` 29 | {{< /notice >}} 30 | 31 | ### Output 32 | 33 | If the request was successful, the output `Message` will have the `main` field set to the HTTP body response. If the `status` is set, and the response http status is different from it, the `Message` will be dropped. 34 | 35 | {{< notice warning "ATTENTION" >}} 36 | The `Message` is dropped if the request is failed. 37 | {{< /notice >}} 38 | 39 | ### Examples 40 | 41 | {{< alert theme="warning" >}} 42 | Soon... 43 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/js/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 12 3 | title: "JS" 4 | date: 2020-09-16T22:38:02+02:00 5 | draft: false 6 | collapsible: true 7 | --- 8 | -------------------------------------------------------------------------------- /src_docs/content/doc/filters/js/basics.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 1 3 | title: "Basics" 4 | date: 2020-09-16T22:38:02+02:00 5 | draft: false 6 | --- 7 | 8 | ## JS 9 | 10 | This filter allows to extend the basic `driplane`'s filter, defining Javascript scripts. It is based on [islazy/plugin](https://github.com/evilsocket/islazy) which in turn is based on [robertkrimen/otto](https://github.com/robertkrimen/otto). 11 | Defining a JS file with our custom logic, it is possible create a complex filter. 12 | 13 | ### Parameters 14 | 15 | | Parameter | Type | Default | Description | 16 | |--------------|----------|---------|--------------------------------------------------------------------------| 17 | | **path** | _STRING_ | empty | path of the Javascript file (it can contains multiple functions) | 18 | | **function** | _STRING_ | empty | name of the function in the JS file to call when a `Message` is received | 19 | 20 | {{< notice info "Example" >}} 21 | `... | js(path="script.js", function="MyFunction") | ...` 22 | {{< /notice >}} 23 | 24 | ### Output 25 | 26 | The output `Message` of this filter depends on the return value of the JS function itself. 27 | 28 | ### Examples 29 | 30 | {{< alert theme="warning" >}} 31 | Soon... 32 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/js/entrypoint.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 2 3 | title: "Entrypoint" 4 | date: 2020-09-16T22:38:02+02:00 5 | draft: false 6 | --- 7 | 8 | ## Entrypoint 9 | 10 | The JS file consists of one or more functions, and they can be used in different filters or in combination between them. 11 | The function must respect some constraints: 12 | 13 | * the function's name contained in the `function` parameter of the filter has to start with a capital letter; 14 | * the function prototype must 3 variables; 15 | * the function must return a JS object with at least the `filtered` field; 16 | 17 | ### Function's prototype 18 | 19 | The function's name specified in the `function` parameter of the filter, receives 3 input parameters: 20 | * main: it is a string with the content of the field `main` of the input Message; 21 | * extra: a JS object that can be seen like an associative array, containing the extra fields of the input Message; 22 | * params: a JS object like the previous, but it contains the configurations from the `custom` and the `general` sections. 23 | 24 | So for example we can define a function like the follow: 25 | 26 | ```javascript 27 | function Entry(main, extra, params) { 28 | ... 29 | } 30 | ``` 31 | 32 | ### Return value 33 | 34 | The JS function has to return a value back to the filter, so that the filter can use it to propagate the Message or drop it. 35 | The method used on `function` parameter can return a JS object containing at least the `filtered` field. 36 | If this field has been set to true, the `Message` will be sent to the next filter, otherwise if the `filtered` field has been set to false, the filter will drop the Message. 37 | 38 | We would change the fields of the `Message`, and to do that we can use the `data` field in the returned object. 39 | It could be an associative array or an array of associative array (for multiple messages to send through the pipeline) and it will be mapped in a map[string]string/[]map[string] object in the Go env. 40 | The key of the array's row is the name of the field to add or change, while the value is the string that field's Message will contain after the return. 41 | 42 | ```javascript 43 | function Entry(mainData, extra, params) { 44 | return { 45 | "filtered": true, 46 | "data": { 47 | "main": "main field of the input Message changed", 48 | "new_field": "new_field will be added as Message's extra field" 49 | } 50 | }; 51 | } 52 | ``` 53 | 54 | ```javascript 55 | function Entry(mainData, extra, params) { 56 | return { 57 | "filtered": true, 58 | "data": [ 59 | { 60 | "main": "main field of the input Message changed", 61 | "new_field": "new_field will be added as Message's extra field" 62 | }, 63 | { 64 | "main": "main field of the secondo Message", 65 | "new_field": "new_field will be added as Message's extra field" 66 | }, 67 | ] 68 | }; 69 | } 70 | ``` -------------------------------------------------------------------------------- /src_docs/content/doc/filters/json.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Json" 3 | date: 2023-01-29T18:57:01+02:00 4 | draft: false 5 | --- 6 | 7 | ## Json 8 | 9 | This filter is used to extract information from a JSON doc received through the Message. 10 | It is possible to use XPath query for JSON specifying a selector to search in the doc and extract data (it uses [jsonquery](https://github.com/antchfx/jsonquery)). 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |--------------|----------|---------|--------------------------------------------------------------------------------------------------| 16 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or an extra field) | 17 | | **selector** | _STRING_ | "" | the selector to find the data in the JSON | 18 | 19 | 20 | {{< notice info "Example" >}} 21 | `... | json(selector="id", target="doc") | ...` 22 | {{< /notice >}} 23 | 24 | ### Output 25 | 26 | The filter will generate one or more Messages. It is possible to use more than 1 time this filter. 27 | The field `fulltext` will contain the original `target` string. 28 | 29 | ### Examples 30 | 31 | {{< alert theme="warning" >}} 32 | Soon... 33 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/mail.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Mail" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Mail 8 | 9 | This filter allows you to send an e-mail. When a `Message` arrives to the filter, we can decide if use the `main` field of the `Message` as URL on the request, or its content for the HTTP data. 10 | This behaviour can be handled with the parameters. 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |--------------|----------|---------|----------------------------------------------------------------------------------------------------------| 16 | | **body** | _STRING_ | empty | the body of the e-mail (supports [Golang templates](https://golang.org/pkg/text/template/)) | 17 | | **username** | _STRING_ | empty | username for the host authentication | 18 | | **password** | _STRING_ | empty | password for the host authentication | 19 | | **host** | _STRING_ | empty | host server used to send the e-mail | 20 | | **port** | _STRING_ | empty | port of the host server | 21 | | **fromAddr** | _STRING_ | empty | source e-mail address | 22 | | **fromName** | _STRING_ | empty | source name address | 23 | | **to** | _STRING_ | empty | destination e-mail address (supports multi-destination, comma separated) | 24 | | **subject** | _STRING_ | empty | subject field of the e-mail to send | 25 | | **use_auth** | _BOOL_ | "false" | if "true" the sendmail server will receive the credentials specified in `username` and `password` fields | 26 | 27 | 28 | {{< notice info "Example" >}} 29 | `... | http(url="{{ .main }}", cookies="exported.json", headers="{\"Content-type\": \"application/json\"}") | ...` 30 | {{< /notice >}} 31 | 32 | {{< notice success "Remember" >}} 33 | Every default's value can be set in the configuration, creating a section with the name of the`Filter`/`Feeder`. 34 | [more info](../../configuration) 35 | {{< /notice >}} 36 | 37 | ### Output 38 | 39 | The input `Message` is always propagated to the next filter without changes. 40 | 41 | ### Examples 42 | 43 | {{< alert theme="warning" >}} 44 | Soon... 45 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/mimetype.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "MIME type" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## MIME type 8 | 9 | This filter allows you to detect the MIME type of a file and its extension. 10 | _Based on the [gabriel-vasile/mimetype](https://github.com/gabriel-vasile/mimetype) library._ 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |--------------|----------|---------|---------------------------------------------------------------------------------------------------------| 16 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be the `main` or and extra field) | 17 | | **filename** | _STRING_ | empty | the filename of the file to detect (supports [Golang templates](https://golang.org/pkg/text/template/)) | 18 | 19 | {{< notice info "Example" >}} 20 | `... | mime(target="{{ .extra_field }}") | ...` 21 | {{< /notice >}} 22 | 23 | {{< notice warning "ATTENTION" >}} 24 | The `filename` field override the `target`. They are mutually exclusive, so you can specify only one of them. 25 | {{< /notice >}} 26 | 27 | 28 | ### Output 29 | 30 | The propagated Message will contain the mimetype's string in the `main` field and the extension in the extra field `mimetype_ext`. 31 | 32 | ### Examples 33 | 34 | {{< alert theme="warning" >}} 35 | Soon... 36 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/number.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Number" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Number 8 | 9 | This filter allows you to treat a string from the input Message as numeric value and apply some operator on it (for example, check if this field contains a number greater than X). 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |-------------|----------|---------|------------------------------------------------------------------------------------------------------------------------------------------------| 15 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or and extra field) | 16 | | **op** | _STRING_ | "" | Compare operator to use for the numeric value (">", ">=", "<", "<=", "!=", "==") | 17 | | **value** | _STRING_ | "" | It has to be a numeric value and it is the number to use for the comparison | 18 | 19 | 20 | {{< notice info "Example" >}} 21 | `... | number(target="num_field", op=">=", value="44") | ...` 22 | {{< /notice >}} 23 | 24 | ### Output 25 | 26 | If the comparison is verified the received Message will be propagated. 27 | 28 | ### Examples 29 | 30 | {{< alert theme="warning" >}} 31 | Soon... 32 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/override.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Override" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Override 8 | 9 | This filter allows you to change a field of a Message, before sending it to the next filter. Template can be used. 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |-----------|----------|---------|---------------------------------------------------------------------------------------------------------------------------| 15 | | **name** | _STRING_ | empty | name of the field to change (supports [Golang templates](https://golang.org/pkg/text/template/)) | 16 | | **value** | _STRING_ | empty | new value to assign to the Message's field specified (supports [Golang templates](https://golang.org/pkg/text/template/)) | 17 | 18 | 19 | {{< notice info "Example" >}} 20 | `... | override(name="description", value="{{ .title }}") | ...` 21 | {{< /notice >}} 22 | 23 | ### Output 24 | 25 | The propagated Message will be identical to the original, with only the specified field changed. 26 | 27 | ### Examples 28 | 29 | {{< alert theme="warning" >}} 30 | Soon... 31 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/pdf.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Pdf" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Pdf 8 | 9 | This filter allows you to extract plain text from a PDF file. 10 | _Based on the [ledongthuc/pdf](https://github.com/ledongthuc/pdf) library._ 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |--------------|----------|---------|------------------------------------------------------------------------------------------------------------| 16 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be the `main` or and extra field) | 17 | | **filename** | _STRING_ | empty | the filename of the PDF file to parse (supports [Golang templates](https://golang.org/pkg/text/template/)) | 18 | 19 | {{< notice info "Example" >}} 20 | `... | pdf(target="{{ .extra_field }}") | ...` 21 | {{< /notice >}} 22 | 23 | {{< notice warning "ATTENTION" >}} 24 | The `filename` field override the `target`. They are mutually exclusive, so you can specify only one of them. 25 | {{< /notice >}} 26 | 27 | ### Output 28 | 29 | The propagated Message contains the plain text of the input PDF file (`fulltext` will be set to the file name received as input). 30 | 31 | ### Examples 32 | 33 | {{< alert theme="warning" >}} 34 | Soon... 35 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/random.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Random" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Random 8 | 9 | This filter is used to inject an extra field with a random number in the propagated `Message`. 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |------------|----------|----------|---------------------------------------------------------------------------| 15 | | **output** | _STRING_ | "main" | the field of the propagated `Message` that will contain the random number | 16 | | **min** | _STRING_ | "0" | the min value of the extracted number is `min` | 17 | | **max** | _STRING_ | "999999" | the max value of the extracted number is `max` | 18 | 19 | {{< notice info "Example" >}} 20 | `... | random(output="random_field", min="9", max="90") | ...` 21 | {{< /notice >}} 22 | 23 | ### Output 24 | 25 | The output `Message` will be equal to the input, but it will also include the new random field. 26 | 27 | ### Examples 28 | 29 | {{< alert theme="warning" >}} 30 | Soon... 31 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/ratelimit.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "RateLimit" 3 | date: 2023-01-31T18:50:23+02:00 4 | draft: false 5 | --- 6 | 7 | ## RateLimit 8 | 9 | This filter allows you to set a rate limit on the messages that can go through it. So for example if we don't want to limit the number of messages in a pipe to 5 messages per second we just to set the parameter `rate` to 5. 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |--------------|----------|---------|--------------------------------------------------------------------------------------------------| 15 | | **rate** | _STRING_ | "0" | how many event per second you want to have as rate limiter | 16 | 17 | 18 | {{< notice info "Example" >}} 19 | `... | ratelimit(rate="5") | ...` 20 | {{< /notice >}} 21 | 22 | ### Output 23 | 24 | The filter will slow down the rate of the messages as specified on the parameter `rate`. 25 | 26 | ### Examples 27 | 28 | {{< alert theme="warning" >}} 29 | Soon... 30 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/striptag.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Striptag" 3 | date: 2021-07-01T15:35:53+02:00 4 | draft: false 5 | --- 6 | 7 | ## StripTag 8 | 9 | This filter is used to remove all the HTML tags from a string. 10 | 11 | ### Parameters 12 | 13 | | Parameter | Type | Default | Description | 14 | |------------|----------|---------|--------------------------------------------------------------------------------------------------| 15 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or an extra field) | 16 | 17 | 18 | {{< notice info "Example" >}} 19 | `... | striptag(target="main") | ...` 20 | {{< /notice >}} 21 | 22 | ### Output 23 | 24 | The `main` of the output `Message` will have the text extracted from the `target` config, stripped by all the HTML tags. 25 | 26 | A new extra field is created: `fulltext` will contain the original `target` string. 27 | 28 | ### Examples 29 | 30 | {{< alert theme="warning" >}} 31 | Soon... 32 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/system.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "System" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## System 8 | 9 | This filter allows you to exec a command on the host machine. The received Message can be used to create the command to launch. 10 | It supports [Golang templates](https://golang.org/pkg/text/template/) (only text.Template). 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |-----------|----------|---------|---------------------------------------------------------------------------------------------------------------------| 16 | | **cmd** | _STRING_ | empty | command line to exec for each received Message (supports [Golang templates](https://golang.org/pkg/text/template/)) | 17 | 18 | 19 | {{< notice info "Example" >}} 20 | `... | system(cmd="echo '{{ .author }} wrote {{ .main }}' >> logs.txt") | ...` 21 | {{< /notice >}} 22 | 23 | ### Output 24 | 25 | The propagated Message will contain the output of the command if it is provided, and it is not failed. 26 | 27 | ### Examples 28 | 29 | {{< alert theme="warning" >}} 30 | Soon... 31 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/telegram.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Telegram" 3 | date: 2024-02-10T23:02:24+02:00 4 | draft: false 5 | --- 6 | 7 | ## Telegram 8 | 9 | This filter allows you to download files received from the Telegram feeder or send messages. 10 | Note: it can be used only in a rule with the Telegram Feeder. 11 | 12 | ### Parameters 13 | 14 | The following parameters are required from this filter: 15 | 16 | | Parameter | Type | Default | Description | 17 | |------------|----------|----------------|------------------------------------------------------------------------------| 18 | | **action** | _STRING_ | "send_message" | action to perform: "send_message", "download_file" | 19 | | **to** | _STRING_ | "" | used with action `send_message`, it has to contain the recipient of the message: username, @username, phone number with the country code (supports [Golang templates](https://golang.org/pkg/text/template/)) | 20 | | **to_chatid** | _STRING_ | empty | the recipient of the message will be a chat | 21 | | **filename** | _STRING_ | "" | specified the path of where to store the downloaded file: `msg_filename` in the extra will contain the name of the file (supports [Golang templates](https://golang.org/pkg/text/template/)) | 22 | | **text** | _STRING_ | "" | the text of the file (supports [Golang templates](https://golang.org/pkg/text/template/)) | 23 | 24 | Each `action` can use different parameters: 25 | 26 | #### action = send_message 27 | 28 | | Parameter | Type | Default | Description | 29 | |------------|----------|---------|----------------------------------------------------------------------------------------------------------------------------| 30 | | **to** | _STRING_ | "" | used with action `send_message`, it has to contain the recipient of the message: username, @username, phone number with the country code (supports [Golang templates](https://golang.org/pkg/text/template/)) | 31 | | **to_chatid** | _STRING_ | empty | the recipient of the message will be a chat | 32 | | **text** | _STRING_ | "" | the text of the file (supports [Golang templates](https://golang.org/pkg/text/template/)) | 33 | 34 | #### action = download_file 35 | 36 | | Parameter | Type | Default | Description | 37 | |--------------|----------|----------|-------------| 38 | | **filename** | _STRING_ | "" | specified the path of where to store the downloaded file: `msg_filename` in the extra will contain the name of the file (supports [Golang templates](https://golang.org/pkg/text/template/)) | 39 | 40 | {{< notice info "Example" >}} 41 | `... | telegram(action="send_message", to="@username", text="the file '{{ .msg_filename }}' received from {{ .user_username }} has been downloaded.") | ...` 42 | {{< /notice >}} 43 | 44 | ### Examples 45 | 46 | {{< notice info "Download file from a specific channel and send a private message to the user @username" >}} 47 | `telegramRule => | text(target="chan_id", pattern="123456") | telegram(action="download_file", filename="/tmp/{{ .msg_filename }}") | telegram(action="send_message", to="@username", text="the file '{{ .msg_filename }}' received from {{ .user_username }} has been downloaded.")` 48 | {{< /notice >}} 49 | 50 | {{< notice info "Reply to a private message" >}} 51 | `telegramRule => | text(target="main", pattern="help") | telegram(action="send_message", to="{{ .user_username }}", text="Hi {{ .user_username }}! You wrote the following message: {{ .main }}.")` 52 | {{< /notice >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/text.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Text" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Text 8 | 9 | This filter searches or extracts strings from the received Message. It can be used with a regular expression or a simple string. 10 | If the string is found, the condition is matched and the Message is propagated to the next filter. 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |-------------|----------|---------|------------------------------------------------------------------------------------------------------------------------------------------------| 16 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or and extra field) | 17 | | **regexp** | _BOOL_ | false | the pattern field is a regular expression | 18 | | **extract** | _BOOL_ | "false" | if "true" the `main` field of the propagated Message will contain the extracted string (it can be used only if `regexp` parameter is set true) | 19 | | **pattern** | _STRING_ | empty | specifies the pattern that should be matched on the Message to check the condition | 20 | 21 | 22 | {{< notice info "Example" >}} 23 | `... | text(target="description", pattern="(#[^\\s]+)", regexp="true", extract="true") | ...` 24 | {{< /notice >}} 25 | 26 | ### Output 27 | 28 | If the `extract` parameter is "false", the received Message will be propagated only if the specified `pattern` is matched in the `target` field of the Message. 29 | Otherwise if `extract` is "true" (only `regexp` can be used in this case), and one or more strings matches with the pattern, the `main` field of the propagated Message will contain only the matched string. 30 | 31 | If the `extract` parameter is "true" and the message is propagated, a new extra field is created: `fulltext` will contain the original `target` string. 32 | 33 | {{< notice warning "ATTENTION" >}} 34 | If the targeted field contains multiple matches, the filter will create and propagate multiple Messages, one for each matched string. 35 | {{< /notice >}} 36 | 37 | ### Examples 38 | 39 | {{< alert theme="warning" >}} 40 | Soon... 41 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/url.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Url" 3 | date: 2020-09-16T22:38:02+02:00 4 | draft: false 5 | --- 6 | 7 | ## Url 8 | 9 | This filter is used to search or extract URLs from a Message. 10 | Currently supported types of URLs are: 11 | * http/s 12 | * ftp 13 | 14 | ### Parameters 15 | 16 | | Parameter | Type | Default | Description | 17 | |-------------|----------|---------|--------------------------------------------------------------------------------------------------| 18 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be main or an extra field) | 19 | | **http** | _BOOL_ | "true" | if "false", `http` scheme urls are ignored | 20 | | **https** | _BOOL_ | "true" | if "false", `https` scheme urls are ignored | 21 | | **ftp** | _BOOL_ | "true" | if "false", `ftp` scheme urls are ignored | 22 | | **extract** | _BOOL_ | "true" | if "true", the `main` field of the propagated Message will contain the found URL | 23 | 24 | 25 | {{< notice info "Example" >}} 26 | `... | url(extract="true") | ...` 27 | {{< /notice >}} 28 | 29 | ### Output 30 | 31 | If the `extract` parameter is "false", the received Message will be propagated only if at least one URL is found in it. 32 | Otherwise, if `extract` is "true" and the Message contains one or more URLs, the `main` field of the propagated Message will contain only the extracted URLs. 33 | 34 | If the `extract` parameter is "true" and the message is propagated, a new extra field is created: `fulltext` will contain the original `target` string. 35 | 36 | {{< notice warning "ATTENTION" >}} 37 | If the targeted field contains multiple URLs, the filter will create and propagate multiple messages, one for each URL. 38 | {{< /notice >}} 39 | 40 | ### Examples 41 | 42 | {{< alert theme="warning" >}} 43 | Soon... 44 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/filters/xls.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "XLS" 3 | date: 2023-11-16T20:08:45+02:00 4 | draft: false 5 | --- 6 | 7 | ## XLS 8 | 9 | This filter allows you to extract all the rows from a Excel file. 10 | _Based on the [qax-os/excelize](https://github.com/qax-os/excelize) library._ 11 | 12 | ### Parameters 13 | 14 | | Parameter | Type | Default | Description | 15 | |--------------|----------|---------|------------------------------------------------------------------------------------------------------------| 16 | | **target** | _STRING_ | "main" | the field of the Message that should be used for the filter (it could be the `main` or and extra field) | 17 | | **filename** | _STRING_ | empty | the filename of the XLS file to parse (supports [Golang templates](https://golang.org/pkg/text/template/)) | 18 | 19 | {{< notice info "Example" >}} 20 | `... | xls(target="{{ .extra_field }}") | ...` 21 | {{< /notice >}} 22 | 23 | {{< notice warning "ATTENTION" >}} 24 | The `filename` field override the `target`. They are mutually exclusive, so you can specify only one of them. 25 | {{< /notice >}} 26 | 27 | ### Output 28 | 29 | The filter produces one Message for each row of the XLS file. 30 | 31 | ### Examples 32 | 33 | {{< alert theme="warning" >}} 34 | Soon... 35 | {{< /alert >}} -------------------------------------------------------------------------------- /src_docs/content/doc/installation/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 1 3 | title: "Installation" 4 | date: 2020-09-16T22:38:02+02:00 5 | draft: false 6 | collapsible: true 7 | --- 8 | 9 | -------------------------------------------------------------------------------- /src_docs/content/doc/installation/build.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 1 3 | title: "Build and install" 4 | date: 2020-09-16T22:38:02+02:00 5 | draft: false 6 | --- 7 | 8 | ## Build and install 9 | 10 | Launching the follow command go will take care of the downloading, the dependency resolving, the building and finally of the installation of the binary in the _$GOBIN_ directory. 11 | 12 | {{< alert theme="success" >}} 13 | go get -u github.com/Matrix86/driplane/... 14 | {{< /alert >}} 15 | -------------------------------------------------------------------------------- /src_docs/content/doc/installation/docker.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 2 3 | title: "Docker" 4 | date: 2020-09-16T22:38:02+02:00 5 | draft: false 6 | --- 7 | 8 | ## Docker 9 | 10 | `driplane` is containerized using a really lightweight Linux distribution called **Alpine Linux**. 11 | 12 | To pull the latest image version: 13 | 14 | {{< alert theme="success" >}} 15 | docker pull matrix86/driplane 16 | {{< /alert >}} 17 | 18 | To run it: 19 | 20 | {{< alert theme="success" >}} 21 | docker run --rm -v config:/app/config -it matrix86/driplane:latest -config config/config.yaml 22 | {{< /alert >}} 23 | 24 | where the `config` directory contains the `config.yaml` file, the `rule` directory, the `js` directory and the `templates` directory. 25 | 26 | 27 | [Link to the Docker repository](https://hub.docker.com/repository/docker/matrix86/driplane) 28 | -------------------------------------------------------------------------------- /src_docs/content/doc/rules/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 3 3 | title: "Rules" 4 | date: 2020-09-17T11:29:50+02:00 5 | draft: false 6 | collapsible: true 7 | --- -------------------------------------------------------------------------------- /src_docs/content/doc/rules/definition.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 1 3 | title: "First look" 4 | date: 2020-09-16T22:38:02+02:00 5 | draft: false 6 | --- 7 | 8 | ## First look 9 | 10 | `Driplane` include a simple language to define where to get the stream and what operation to exec, or how to filter the data. 11 | 12 | In the rule could be defined 3 type of nodes: 13 | 14 | * `FEEDER` : it is the node responsible for creating of a stream of data (it reads every changes on a file, gets tweets from Twitter, etc..) 15 | 16 | * `FILTER` : it receives data from a feeder or another filter and checks some conditions on them or makes some changes on them. It sends the data to the next filter **ONLY** if the condition is verified. 17 | 18 | * `RULE CALL` : every rule has a name, so you can define a rule with a preset feeder or a pipe of filters and connect them to another filter/feeder. 19 | 20 | It is possible to define custom parameters for feeders or filters. Each one of them has a different type of parameters that can change their behaviour and you can find a list of them in the related section. -------------------------------------------------------------------------------- /src_docs/content/doc/rules/syntax.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 2 3 | title: "Syntax" 4 | date: 2020-09-16T22:38:02+02:00 5 | draft: false 6 | --- 7 | 8 | ## Syntax 9 | 10 | The syntax of the `driplane`'s rules is very simple. As for the BASH every filter's output is sent to the next filter's input concatenating the two of them with the `|` character. 11 | All the rules have to start with a name and end with a `;` char. 12 | 13 | ### Import 14 | Using the directive `import` it is possible to include one or more files and their rules in another one. All the rules defined in an included file 15 | are available for the file that imports them. 16 | 17 | To use this directive the path of the file can be absolute or relative to the `rules directory` specified in the configuration: 18 | 19 | > Example: 20 | > `#import "file_to_import.rule"` 21 | 22 | ### Rule Name and Rule Call 23 | 24 | Each rule has to start with an identifier follow by `=>`. This identifier identifies _rule name_ and it could be used in another rule to concatenate 2 rules together. 25 | 26 | {{< notice warning "ATTENTION" >}} 27 | This name must be **unique** in the rules. 28 | {{< /notice >}} 29 | 30 | > Example: 31 | > `IDENTIFIER => ... ;` 32 | 33 | It can be included in another rule (**rule call**) prepending the `@` to it. 34 | 35 | > Example: 36 | > `IDENTIFIER1 => ... ;` 37 | > `IDENTIFIER2 => ... | @IDENTIFIER1 | ... ;` 38 | 39 | ### Feeder 40 | 41 | The feeder creates the stream, so they don't accept inputs. For this reason, they can be positioned **ONLY** to the beginning of a rule. 42 | 43 | The feeder definition starts with a `<` char followed by an identifier. That's the type of the feeder we want to use. 44 | 45 | After the type we found a `:` char followed by a list of **parameters** comma separated and a `>`. 46 | 47 | {{< notice info "Parameters" >}} 48 | The parameters are in the form of key/value where the value is between double quotes `key="value"`. 49 | {{< /notice >}} 50 | 51 | > Example: 52 | > `IDENTIFIER => | ... ;` 53 | 54 | ### Filter 55 | 56 | The filters are the main operators of a rule, because they decide if a data is interesting and perform operations. 57 | 58 | The definition of a filter start with his name and it is followed by parameters contained between `(` and `)`. 59 | According to the settings a Filter can change his behaviour and can modify the data passing through it. 60 | 61 | > Example: 62 | > `IDENTIFIER => ... | FILTER_TYPE( param1="value1", ... ) | ... ;` 63 | 64 | {{< notice info "NOT" >}} 65 | The operator **NOT** `!` can be used on filter to negate his result (propagate the data if the condition is not verified). 66 | It has to be put before the filter definition: `!FILTER_TYPE(...)` 67 | {{< /notice >}} 68 | 69 | {{< notice warning "ATTENTION" >}} 70 | All the parameters have to be enclosed in quotes. 71 | JSON requires **double quotes** to encode strings, so in order to define a JSON string you need to escape the quotes `\"`. 72 | {{< /notice >}} 73 | 74 | ### Data message and Extra 75 | 76 | The data stream in `driplane` is based on text and the basic object that is part of it is the _Message_. 77 | The _Message_ is an object that contains the text that needs to be filtered and extra. 78 | The main string is identified as `text` in the filters, whereas the extra data are identified by a key. 79 | 80 | There are fixed extra, created from `driplane` itself and other extras related to a feeder or filter. 81 | 82 | | Name | Description | 83 | |---------------|------------------------------------------------| 84 | | source_feeder | the name of the feeder creates this Message | 85 | | rule_name | the name of the rule that contains this filter | -------------------------------------------------------------------------------- /src_docs/go.sum: -------------------------------------------------------------------------------- 1 | github.com/bep/empty-hugo-module v1.0.0/go.mod h1:whohinbSjMoFi/Skivj9kvdPs1tEgzYpZ4rXoQk/0/I= 2 | github.com/jquery/jquery-dist v0.0.0-20190501211928-15bc73803f76/go.mod h1:/lTfttEqFU0GWTaOOMIeNTzLGQ7yTIgyzjtkS/pYIoc= 3 | github.com/jquery/jquery-dist v0.0.0-20200504225046-4c0e4becb826 h1:JO+IS20NQ+R4nFQT4doRfL3hE75WJK/+XZbpSlBW92Y= 4 | github.com/jquery/jquery-dist v0.0.0-20200504225046-4c0e4becb826/go.mod h1:/lTfttEqFU0GWTaOOMIeNTzLGQ7yTIgyzjtkS/pYIoc= 5 | github.com/olivernn/lunr.js v2.3.8+incompatible/go.mod h1:yEkQ1DUSMtNsn8n2CqvQXZd0ErWPEG8g9QRmblR+KS8= 6 | github.com/olivernn/lunr.js v2.3.9+incompatible h1:eH8iBnjlR4mwlYDdNuqy9PCNLjp2bEs6aoNnTSaccx0= 7 | github.com/olivernn/lunr.js v2.3.9+incompatible/go.mod h1:yEkQ1DUSMtNsn8n2CqvQXZd0ErWPEG8g9QRmblR+KS8= 8 | github.com/slatedocs/slate v2.3.1+incompatible/go.mod h1:n698aXLkExWIlF7prJey0Kq6wlKIKvj/stVb5oouZDM= 9 | github.com/slatedocs/slate v2.7.1+incompatible h1:U0wXBtrCSJq8PAePS4Gxf6yLdXsYRMXEHpaZ6Is4fTg= 10 | github.com/slatedocs/slate v2.7.1+incompatible/go.mod h1:n698aXLkExWIlF7prJey0Kq6wlKIKvj/stVb5oouZDM= 11 | -------------------------------------------------------------------------------- /src_docs/resources/_gen/assets/scss/driplane/sass/main.scss_b4f67ac5085b89b62b54c1923e5a9145.json: -------------------------------------------------------------------------------- 1 | {"Target":"css/main.min.css","MediaType":"text/css","Data":{}} -------------------------------------------------------------------------------- /src_docs/resources/_gen/assets/scss/driplane/scss/slate/print.css.scss_c14439616ffbc3ae1827507340d6c08b.json: -------------------------------------------------------------------------------- 1 | {"Target":"styles/print.css","MediaType":"text/css","Data":{}} -------------------------------------------------------------------------------- /src_docs/resources/_gen/assets/scss/driplane/scss/slate/screen.css.scss_d18af36970f1f09b308ef20ee65e3a03.json: -------------------------------------------------------------------------------- 1 | {"Target":"styles/screen.css","MediaType":"text/css","Data":{}} -------------------------------------------------------------------------------- /src_docs/resources/_gen/assets/scss/scss/slate/print.css.scss_c14439616ffbc3ae1827507340d6c08b.json: -------------------------------------------------------------------------------- 1 | {"Target":"styles/print.css","MediaType":"text/css","Data":{}} -------------------------------------------------------------------------------- /src_docs/resources/_gen/assets/scss/scss/slate/screen.css.scss_d18af36970f1f09b308ef20ee65e3a03.json: -------------------------------------------------------------------------------- 1 | {"Target":"styles/screen.css","MediaType":"text/css","Data":{}} -------------------------------------------------------------------------------- /src_docs/static/favicon/android-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/android-icon-144x144.png -------------------------------------------------------------------------------- /src_docs/static/favicon/android-icon-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/android-icon-192x192.png -------------------------------------------------------------------------------- /src_docs/static/favicon/android-icon-36x36.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/android-icon-36x36.png -------------------------------------------------------------------------------- /src_docs/static/favicon/android-icon-48x48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/android-icon-48x48.png -------------------------------------------------------------------------------- /src_docs/static/favicon/android-icon-72x72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/android-icon-72x72.png -------------------------------------------------------------------------------- /src_docs/static/favicon/android-icon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/android-icon-96x96.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-114x114.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-114x114.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-120x120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-120x120.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-144x144.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-152x152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-152x152.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-180x180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-180x180.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-57x57.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-57x57.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-60x60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-60x60.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-72x72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-72x72.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-76x76.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-76x76.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon-precomposed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon-precomposed.png -------------------------------------------------------------------------------- /src_docs/static/favicon/apple-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/apple-icon.png -------------------------------------------------------------------------------- /src_docs/static/favicon/browserconfig.xml: -------------------------------------------------------------------------------- 1 | 2 | #ffffff -------------------------------------------------------------------------------- /src_docs/static/favicon/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/favicon-16x16.png -------------------------------------------------------------------------------- /src_docs/static/favicon/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/favicon-32x32.png -------------------------------------------------------------------------------- /src_docs/static/favicon/favicon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/favicon-96x96.png -------------------------------------------------------------------------------- /src_docs/static/favicon/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/favicon.ico -------------------------------------------------------------------------------- /src_docs/static/favicon/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/logo.png -------------------------------------------------------------------------------- /src_docs/static/favicon/logo192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/logo192x192.png -------------------------------------------------------------------------------- /src_docs/static/favicon/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "App", 3 | "icons": [ 4 | { 5 | "src": "\/android-icon-36x36.png", 6 | "sizes": "36x36", 7 | "type": "image\/png", 8 | "density": "0.75" 9 | }, 10 | { 11 | "src": "\/android-icon-48x48.png", 12 | "sizes": "48x48", 13 | "type": "image\/png", 14 | "density": "1.0" 15 | }, 16 | { 17 | "src": "\/android-icon-72x72.png", 18 | "sizes": "72x72", 19 | "type": "image\/png", 20 | "density": "1.5" 21 | }, 22 | { 23 | "src": "\/android-icon-96x96.png", 24 | "sizes": "96x96", 25 | "type": "image\/png", 26 | "density": "2.0" 27 | }, 28 | { 29 | "src": "\/android-icon-144x144.png", 30 | "sizes": "144x144", 31 | "type": "image\/png", 32 | "density": "3.0" 33 | }, 34 | { 35 | "src": "\/android-icon-192x192.png", 36 | "sizes": "192x192", 37 | "type": "image\/png", 38 | "density": "4.0" 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /src_docs/static/favicon/ms-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/ms-icon-144x144.png -------------------------------------------------------------------------------- /src_docs/static/favicon/ms-icon-150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/ms-icon-150x150.png -------------------------------------------------------------------------------- /src_docs/static/favicon/ms-icon-310x310.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/ms-icon-310x310.png -------------------------------------------------------------------------------- /src_docs/static/favicon/ms-icon-70x70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/favicon/ms-icon-70x70.png -------------------------------------------------------------------------------- /src_docs/static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Matrix86/driplane/b30f1de611a491bb969d8288cc3cb8c33bc58e8c/src_docs/static/logo.png -------------------------------------------------------------------------------- /utils/apt/package.go: -------------------------------------------------------------------------------- 1 | package apt 2 | 3 | import ( 4 | "compress/bzip2" 5 | "compress/gzip" 6 | "fmt" 7 | "io" 8 | "pault.ag/go/debian/control" 9 | ) 10 | 11 | // Index contains the content of Packages files 12 | type Index struct { 13 | Type string 14 | Binaries []BinaryPackage 15 | // TODO: implement also the SourcePackages 16 | } 17 | 18 | // BinaryPackage represents all the entry of the Packages file 19 | type BinaryPackage struct { 20 | // mandatory 21 | Filename string 22 | Size string 23 | 24 | // optional 25 | BinaryPackage string 26 | MD5sum string 27 | SHA1 string 28 | SHA256 string 29 | DescriptionMD5 string 30 | Depends []string `delim:", " strip:"\n\r\t "` 31 | InstalledSize string `control:"Installed-Size"` 32 | Package string 33 | Architecture string 34 | Version string 35 | Section string 36 | Maintainer string 37 | Homepage string 38 | Description string 39 | Tag string 40 | Author string 41 | Name string 42 | } 43 | 44 | // ParsePackageIndex parses the Packages file (also if compressed) 45 | func ParsePackageIndex(r io.Reader, mtype string) (*Index, error) { 46 | reader := r 47 | var err error 48 | if mtype == "bz2" { 49 | reader = bzip2.NewReader(r) 50 | } else if mtype == "gz" { 51 | reader, err = gzip.NewReader(r) 52 | if err != nil { 53 | return nil, fmt.Errorf("gzip error: %s", err) 54 | } 55 | } 56 | p := &Index{Type: mtype} 57 | if err := control.Unmarshal(&p.Binaries, reader); err != nil { 58 | return nil, err 59 | } 60 | return p, nil 61 | } 62 | -------------------------------------------------------------------------------- /utils/apt/release.go: -------------------------------------------------------------------------------- 1 | package apt 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "pault.ag/go/debian/control" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | // Release represents the Release file 12 | type Release struct { 13 | // Optional 14 | Description string `control:"Description"` 15 | Origin string `control:"Origin"` 16 | Label string `control:"Label"` 17 | Version string `control:"Version"` 18 | Suite string `control:"Suite"` 19 | Codename string `control:"Codename"` 20 | 21 | Components []string `control:"Components"` 22 | Architectures []string `control:"Architectures"` 23 | 24 | Date string `control:"Date"` 25 | ValidUntil string `control:"Valid-Until"` 26 | PackagePaths []string `control:"-"` 27 | MD5Sum []IndexHash `control:"MD5Sum" delim:"\n" strip:"\n\r\t "` 28 | SHA1 []IndexHash `control:"SHA1" delim:"\n" strip:"\n\r\t "` 29 | SHA256 []IndexHash `control:"SHA256" delim:"\n" strip:"\n\r\t "` 30 | } 31 | 32 | // IndexHash contains hash and size of a Packages file 33 | type IndexHash struct { 34 | Hash string 35 | Size int64 36 | Path string 37 | } 38 | 39 | // UnmarshalControl tells how to unmarshal the control files 40 | func (i *IndexHash) UnmarshalControl(data string) error { 41 | splitter := func(r rune) bool { 42 | return r == '\t' || r == ' ' 43 | } 44 | parts := strings.FieldsFunc(data, splitter) 45 | if len(parts) != 3 { 46 | return nil 47 | } 48 | i.Hash = parts[0] 49 | s, err := strconv.ParseInt(parts[1], 10, 64) 50 | if err != nil { 51 | return fmt.Errorf("can't unmarshal size: %s", err) 52 | } 53 | i.Size = s 54 | i.Path = parts[2] 55 | return nil 56 | } 57 | 58 | // ParseRelease parses the Release file of a repository 59 | func ParseRelease(r io.Reader) (*Release, error) { 60 | release := &Release{} 61 | if err := control.Unmarshal(release, r); err != nil { 62 | return nil, err 63 | } 64 | 65 | if len(release.MD5Sum) != 0 { 66 | release.PackagePaths = make([]string, len(release.MD5Sum)) 67 | for i, h := range release.MD5Sum { 68 | release.PackagePaths[i] = h.Path 69 | } 70 | } else if len(release.SHA1) != 0 { 71 | release.PackagePaths = make([]string, len(release.SHA1)) 72 | for i, h := range release.SHA1 { 73 | release.PackagePaths[i] = h.Path 74 | } 75 | } else if len(release.SHA256) != 0 { 76 | release.PackagePaths = make([]string, len(release.SHA256)) 77 | for i, h := range release.SHA256 { 78 | release.PackagePaths[i] = h.Path 79 | } 80 | } 81 | return release, nil 82 | } 83 | -------------------------------------------------------------------------------- /utils/apt/release_test.go: -------------------------------------------------------------------------------- 1 | package apt 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | func TestParseRelease(t *testing.T) { 9 | txt := `Origin: Debian 10 | Label: Debian 11 | Suite: stable 12 | Version: 11.2 13 | Codename: bullseye 14 | Changelogs: https://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog 15 | Date: Sat, 18 Dec 2021 10:38:58 UTC 16 | Acquire-By-Hash: yes 17 | No-Support-for-architecture-all: PackagePaths 18 | Architectures: all amd64 arm64 armel armhf i386 mips64el mipsel ppc64el s390x 19 | Components: main contrib non-free 20 | Description: Debian 11.2 Released 18 December 2021 21 | MD5Sum: 22 | 7fdf4db15250af5368cc52a91e8edbce 738242 contrib/Contents-all 23 | cbd7bc4d3eb517ac2b22f929dfc07b47 57319 contrib/Contents-all.gz 24 | 37d6231ff08b9f383fba5134e90c1246 786460 contrib/Contents-amd64 25 | f862fd63c5e4927f91c1cc77a27c89eb 54567 contrib/Contents-amd64.gz 26 | 098f43776cd6b43334b2c1eb867a4d64 370054 contrib/Contents-arm64 27 | 014eca050cdd0b30df0d5a72df217c5b 29661 contrib/Contents-arm64.gz 28 | b6d2673f17fbdb3a5ce92404a62c2d7e 359292 contrib/Contents-armel 29 | d02d94be587d56a1246b407669d2a24c 28039 contrib/Contents-armel.gz 30 | SHA256: 31 | 3957f28db16e3f28c7b34ae84f1c929c567de6970f3f1b95dac9b498dd80fe63 738242 contrib/Contents-all 32 | 3e9a121d599b56c08bc8f144e4830807c77c29d7114316d6984ba54695d3db7b 57319 contrib/Contents-all.gz 33 | 425a90016c3b1b64fd560b0f4524d53d5b3ee3aa0835859408e8413aa1145dc9 786460 contrib/Contents-amd64 34 | 1c336a418784bb0eb78318bab337b4df34b34e56683e3a7887f319a2a8985c6b 54567 contrib/Contents-amd64.gz 35 | d1301db9f59f4baf78398a8e123e76088b9963b612274e030e8c1d52720e0151 370054 contrib/Contents-arm64 36 | 1a01ec345569da804ff98ed259b2135845130a30e434909c4e69c88bf9cb8d9a 29661 contrib/Contents-arm64.gz 37 | b4985377d670dbc4ab9bf0f7fb15d11b100c442050dee7c1e9203d3f0cfd3f37 359292 contrib/Contents-armel 38 | f134666bc09535cbc917f63022ea31613da15ec3c0ce1c664981ace325acdd6a 28039 contrib/Contents-armel.gz` 39 | 40 | r := strings.NewReader(txt) 41 | release, err := ParseRelease(r) 42 | if err != nil { 43 | t.Errorf("error received: %s", err) 44 | } 45 | if len(release.MD5Sum) != 8 { 46 | t.Errorf("wrong num of MD5Sum") 47 | } 48 | if len(release.SHA256) != 8 { 49 | t.Errorf("wrong num of SHA256") 50 | } 51 | if len(release.PackagePaths) != 8 { 52 | t.Errorf("wrong num of paths") 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /utils/cookie.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "net/http" 7 | "os" 8 | "time" 9 | ) 10 | 11 | // Cookies maps the json file exported from Chrome containing the cookies 12 | type Cookies []struct { 13 | Domain string `json:"domain"` 14 | ExpirationDate float64 `json:"expirationDate,omitempty"` 15 | HostOnly bool `json:"hostOnly"` 16 | HTTPOnly bool `json:"httpOnly"` 17 | Name string `json:"name"` 18 | Path string `json:"path"` 19 | SameSite string `json:"sameSite"` 20 | Secure bool `json:"secure"` 21 | Session bool `json:"session"` 22 | StoreID string `json:"storeId"` 23 | Value string `json:"value"` 24 | ID int `json:"id"` 25 | } 26 | 27 | // ParseCookieFile transforms JSON file in slice of http.Cookie 28 | func ParseCookieFile(filename string) ([]*http.Cookie, error) { 29 | var cookies Cookies 30 | jsonFile, err := os.Open(filename) 31 | if err != nil { 32 | return nil, err 33 | } 34 | defer jsonFile.Close() 35 | 36 | byteValue, _ := ioutil.ReadAll(jsonFile) 37 | err = json.Unmarshal(byteValue, &cookies) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | httpCookies := make([]*http.Cookie, len(cookies)) 43 | for i, c := range cookies { 44 | secs := int64(c.ExpirationDate) 45 | nsecs := int64((c.ExpirationDate - float64(secs)) * 1e9) 46 | 47 | httpCookies[i] = &http.Cookie{ 48 | Name: c.Name, 49 | Value: c.Value, 50 | Path: c.Path, 51 | Domain: c.Domain, 52 | Expires: time.Unix(secs, nsecs), 53 | Secure: c.Secure, 54 | HttpOnly: c.HTTPOnly, 55 | } 56 | } 57 | 58 | return httpCookies, nil 59 | } -------------------------------------------------------------------------------- /utils/cookie_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "path" 7 | "reflect" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestParseCookieFile(t *testing.T) { 13 | type Test struct { 14 | Name string 15 | Filename string 16 | CreateFile bool 17 | FileContent string 18 | ExpectedCookies []*http.Cookie 19 | ExpectedError string 20 | } 21 | tests := []Test{ 22 | { "FileNotExist", path.Join(os.TempDir(), "notexist"), false, "", []*http.Cookie(nil), "open /tmp/notexist: no such file or directory"}, 23 | { "WrongJson",path.Join(os.TempDir(), "wrongjson"), true, "wrong", []*http.Cookie(nil), "invalid character 'w' looking for beginning of value"}, 24 | { "ExportedJson", path.Join(os.TempDir(), "cookie_test_file"), true, "[\n {\n \"domain\": \"test.com\",\n \"expirationDate\": 1627486224,\n \"hostOnly\": true,\n \"httpOnly\": false,\n \"name\": \"testCookie\",\n \"path\": \"/\",\n \"sameSite\": \"unspecified\",\n \"secure\": false,\n \"session\": false,\n \"storeId\": \"0\",\n \"value\": \"testValue\",\n \"id\": 1\n }\n]", []*http.Cookie{&http.Cookie{ 25 | Name: "testCookie", 26 | Value: "testValue", 27 | Path: "/", 28 | Domain: "test.com", 29 | Expires: time.Unix(1627486224, 0), 30 | RawExpires: "", 31 | MaxAge: 0, 32 | Secure: false, 33 | HttpOnly: false, 34 | SameSite: 0, 35 | Raw: "", 36 | Unparsed: nil, 37 | }}, 38 | "open /tmp/notexist: no such file or directory"}, 39 | } 40 | 41 | for _, v := range tests { 42 | if v.CreateFile { 43 | file, err := os.Create(v.Filename) 44 | if err != nil { 45 | t.Errorf("%s: cannot create a temporary file", v.Name) 46 | } 47 | defer os.Remove(v.Filename) 48 | 49 | if _, err = file.Write([]byte(v.FileContent)); err != nil { 50 | t.Errorf("%s: can't write on file", v.Name) 51 | } 52 | } 53 | 54 | had, err := ParseCookieFile(v.Filename) 55 | if v.ExpectedError == "" && err != nil { 56 | t.Errorf("%s: wrong error: expected=nil had=%#v", v.Name, err) 57 | } else if err != nil && err.Error() != v.ExpectedError { 58 | t.Errorf("%s: wrong error: expected=%#v had=%#v", v.Name, v.ExpectedError, err.Error()) 59 | } 60 | if reflect.DeepEqual(had, v.ExpectedCookies) == false { 61 | t.Errorf("%s: wrong cookie: expected=%#v had=%#v", v.Name, v.ExpectedCookies, had) 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /utils/global_ttl_map.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | // GlobalTTLMap is a cache shared between all the rules 9 | type GlobalTTLMap struct { 10 | Caches map[string]*TTLMap 11 | } 12 | 13 | var ( 14 | instance *GlobalTTLMap 15 | once sync.Once 16 | ) 17 | 18 | // GetGlobalTTLMapInstance returns the unique GlobalTTLMap (singleton) 19 | func GetGlobalTTLMapInstance(gcdelay time.Duration) *GlobalTTLMap { 20 | once.Do(func() { 21 | instance = &GlobalTTLMap{ 22 | Caches: make(map[string]*TTLMap), 23 | } 24 | instance.Caches["global"] = NewTTLMap(gcdelay) 25 | }) 26 | return instance 27 | } 28 | 29 | // GetNamedTTLMap return a Cache stored on the globalTTLMap with a name 30 | func GetNamedTTLMap(name string, gcdelay time.Duration) *TTLMap { 31 | i := GetGlobalTTLMapInstance(gcdelay) 32 | if v, ok := i.Caches[name]; ok { 33 | return v 34 | } 35 | i.Caches[name] = NewTTLMap(gcdelay) 36 | return i.Caches[name] 37 | } 38 | -------------------------------------------------------------------------------- /utils/global_ttl_map_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestGetGlobalTTLMapInstance(t *testing.T) { 9 | m1 := GetGlobalTTLMapInstance(1 * time.Second) 10 | m2 := GetGlobalTTLMapInstance(1 * time.Second) 11 | if m1 != m2 { 12 | t.Errorf("the instances returned are different") 13 | } 14 | } 15 | 16 | func TestGetNamedTTLMap(t *testing.T) { 17 | m1 := GetNamedTTLMap("test", 1*time.Second) 18 | m2 := GetNamedTTLMap("test", 1*time.Second) 19 | m3 := GetNamedTTLMap("test2", 1*time.Second) 20 | 21 | if m1 != m2 { 22 | t.Errorf("the instances returned are different") 23 | } 24 | if m1 == m3 { 25 | t.Errorf("the instances returned should be different") 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /utils/html.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "golang.org/x/net/html" 6 | "strings" 7 | ) 8 | 9 | // HTMLMeta contains information from the HTML page 10 | type HTMLMeta struct { 11 | Title string `json:"title"` 12 | Description string `json:"description"` 13 | Image string `json:"image"` 14 | SiteName string `json:"site_name"` 15 | } 16 | 17 | // GetMetaFromHTML extracts info from an HTML page and store them on a HTMLMeta struct 18 | func GetMetaFromHTML(s string) *HTMLMeta { 19 | z := html.NewTokenizer(strings.NewReader(s)) 20 | titleFound := false 21 | hm := &HTMLMeta{} 22 | 23 | for { 24 | tt := z.Next() 25 | switch tt { 26 | case html.StartTagToken, html.SelfClosingTagToken: 27 | 28 | t := z.Token() 29 | if t.Data == `body` { 30 | return hm 31 | } 32 | if t.Data == "title" { 33 | titleFound = true 34 | } 35 | if t.Data == "meta" { 36 | desc, ok := extractMetaProperty(t, "description") 37 | if ok { 38 | hm.Description = desc 39 | } 40 | 41 | ogTitle, ok := extractMetaProperty(t, "og:title") 42 | if ok { 43 | hm.Title = ogTitle 44 | } 45 | 46 | ogDesc, ok := extractMetaProperty(t, "og:description") 47 | if ok { 48 | hm.Description = ogDesc 49 | } 50 | 51 | ogImage, ok := extractMetaProperty(t, "og:image") 52 | if ok { 53 | hm.Image = ogImage 54 | } 55 | 56 | ogSiteName, ok := extractMetaProperty(t, "og:site_name") 57 | if ok { 58 | hm.SiteName = ogSiteName 59 | } 60 | } 61 | case html.TextToken: 62 | if titleFound { 63 | t := z.Token() 64 | hm.Title = t.Data 65 | titleFound = false 66 | } 67 | case html.ErrorToken: 68 | return hm 69 | } 70 | } 71 | } 72 | 73 | func extractMetaProperty(t html.Token, prop string) (content string, ok bool) { 74 | for _, attr := range t.Attr { 75 | if attr.Key == "name" && attr.Val == prop { 76 | ok = true 77 | } 78 | 79 | if attr.Key == "content" { 80 | content = attr.Val 81 | } 82 | } 83 | 84 | return 85 | } 86 | 87 | // ExtractTextFromHTML returns a string with only the text version of the web page 88 | func ExtractTextFromHTML(s string) string { 89 | ret := "" 90 | domDocTest := html.NewTokenizer(strings.NewReader(s)) 91 | previousStartTokenTest := domDocTest.Token() 92 | for { 93 | tt := domDocTest.Next() 94 | switch { 95 | case tt == html.ErrorToken: 96 | return ret // End of the document, done 97 | case tt == html.StartTagToken: 98 | previousStartTokenTest = domDocTest.Token() 99 | case tt == html.TextToken: 100 | if previousStartTokenTest.Data == "script" || 101 | previousStartTokenTest.Data == "noscript" || 102 | previousStartTokenTest.Data == "style" { 103 | continue 104 | } 105 | TxtContent := strings.TrimSpace(html.UnescapeString(string(domDocTest.Text()))) 106 | if len(TxtContent) > 0 { 107 | ret = fmt.Sprintf("%s %s", ret, TxtContent) 108 | } 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /utils/html_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestGetMetaFromHTML(t *testing.T) { 9 | html := "test" 10 | type Test struct { 11 | Name string 12 | HTML string 13 | Expected *HTMLMeta 14 | } 15 | tests := []Test{ 16 | {"CorrectHtml", html, &HTMLMeta{Title: "og_title", Description: "og_description", Image: "og_image", SiteName: "og_site_name"}}, 17 | {"WrongHtml", "", &HTMLMeta{}}, 18 | {"EmptyHtml", "", &HTMLMeta{}}, 19 | } 20 | 21 | for _, v := range tests { 22 | had := GetMetaFromHTML(v.HTML) 23 | if reflect.DeepEqual(had, v.Expected) == false { 24 | t.Errorf("%s: wrong parsing: expected=%#v had=%#v", v.Name, v.Expected, had) 25 | } 26 | } 27 | } 28 | 29 | func TestExtractTextFromHTML(t *testing.T) { 30 | html := "test

title

this is a text!" 31 | type Test struct { 32 | Name string 33 | HTML string 34 | Expected string 35 | } 36 | tests := []Test{ 37 | {"CorrectHtml", html, " test title this is a text!"}, 38 | {"NoTextHtml", "", ""}, 39 | {"EmptyHtml", "", ""}, 40 | } 41 | 42 | for _, v := range tests { 43 | had := ExtractTextFromHTML(v.HTML) 44 | if had != v.Expected { 45 | t.Errorf("%s: wrong parsing: expected=%#v had=%#v", v.Name, v.Expected, had) 46 | } 47 | } 48 | } 49 | --------------------------------------------------------------------------------