├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .goreleaser.yml ├── .octocov.yml ├── .tbls.yml ├── CHANGELOG.md ├── CREDITS ├── LICENSE ├── Makefile ├── README.md ├── client ├── client.go ├── file.go ├── k8s.go ├── k8s │ └── k8s.go └── ssh.go ├── cmd └── hrv │ ├── cmd │ ├── cat.go │ ├── completion.go │ ├── configtest.go │ ├── count.go │ ├── cp.go │ ├── fetch.go │ ├── generateK8sConfig.go │ ├── info.go │ ├── logs.go │ ├── root.go │ ├── stream.go │ ├── tags.go │ ├── targets.go │ └── version.go │ └── main.go ├── collector └── collector.go ├── config ├── config.go └── config_test.go ├── db └── db.go ├── doc ├── fetch.png ├── schema │ ├── README.md │ ├── logs.md │ ├── logs.svg │ ├── metas.md │ ├── metas.svg │ ├── schema.svg │ ├── tags.md │ ├── tags.svg │ ├── targets.md │ ├── targets.svg │ ├── targets_tags.md │ └── targets_tags.svg ├── screencast.svg └── stream.png ├── go.mod ├── go.sum ├── logger └── logger.go ├── parser ├── combined_log.go ├── none.go ├── parser.go ├── parser_test.go ├── regexp.go └── syslog.go ├── stdout └── stdout.go ├── testdata ├── apache.log ├── test.yml.template └── test_config.yml └── version └── version.go /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | job-test: 11 | name: Test 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | go_version: [1.17] 16 | steps: 17 | - name: Set up Go ${{ matrix.go_version }} 18 | uses: actions/setup-go@v2 19 | with: 20 | go-version: ${{ matrix.go_version }} 21 | 22 | - name: Check out source code 23 | uses: actions/checkout@v2 24 | 25 | - name: Run test 26 | run: make ci 27 | 28 | - name: Run octocov 29 | uses: k1LoW/octocov-action@v0 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | hrv 2 | dist/ 3 | coverage.out 4 | *.db 5 | *.log 6 | *.yml 7 | .envrc 8 | .go-version 9 | !cmd/hrv 10 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | - go mod tidy 5 | builds: 6 | - 7 | id: hrv-darwin 8 | binary: hrv 9 | main: ./cmd/hrv/main.go 10 | ldflags: 11 | - -s -w -X github.com/k1LoW/harvest.version={{.Version}} -X github.com/k1LoW/harvest.commit={{.FullCommit}} -X github.com/k1LoW/harvest.date={{.Date}} -X github.com/k1LoW/harvest/version.Version={{.Version}} 12 | env: 13 | - CGO_ENABLED=1 14 | goos: 15 | - darwin 16 | goarch: 17 | - amd64 18 | - 19 | id: hrv-linux 20 | binary: hrv 21 | main: ./cmd/hrv/main.go 22 | ldflags: 23 | - -s -w -X github.com/k1LoW/harvest.version={{.Version}} -X github.com/k1LoW/harvest.commit={{.FullCommit}} -X github.com/k1LoW/harvest.date={{.Date}} -X github.com/k1LoW/harvest/version.Version={{.Version}} 24 | - -linkmode external 25 | - -extldflags "-static" 26 | env: 27 | - CGO_ENABLED=1 28 | - CC=/usr/local/bin/x86_64-linux-musl-cc 29 | goos: 30 | - linux 31 | goarch: 32 | - amd64 33 | archives: 34 | - 35 | id: harvest-archive 36 | name_template: '{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' 37 | format_overrides: 38 | - goos: darwin 39 | format: zip 40 | files: 41 | - CREDITS 42 | - README.md 43 | - CHANGELOG.md 44 | checksum: 45 | name_template: 'checksums.txt' 46 | snapshot: 47 | name_template: "{{ .Version }}-next" 48 | changelog: 49 | skip: true 50 | sort: asc 51 | filters: 52 | exclude: 53 | - '^docs:' 54 | - '^test:' 55 | brews: 56 | - 57 | name: harvest 58 | github: 59 | owner: k1LoW 60 | name: homebrew-tap 61 | commit_author: 62 | name: k1LoW 63 | email: k1lowxb@gmail.com 64 | homepage: https://github.com/k1LoW/harvest 65 | description: Portable log aggregation tool for middle-scale system operation/observation. 66 | install: | 67 | system './hrv', 'completion', 'bash', '--out', 'hrv.bash' 68 | system './hrv', 'completion', 'zsh', '--out', 'hrv.zsh' 69 | bin.install 'hrv' 70 | bash_completion.install 'hrv.bash' => 'hrv' 71 | zsh_completion.install 'hrv.zsh' => '_hrv' 72 | nfpms: 73 | - 74 | id: harvest-nfpms 75 | file_name_template: "{{ .ProjectName }}_{{ .Version }}-1_{{ .Arch }}" 76 | builds: 77 | - hrv-linux 78 | homepage: https://github.com/k1LoW/harvest 79 | maintainer: Ken'ichiro Oyama 80 | description: Portable log aggregation tool for middle-scale system operation/observation. 81 | license: MIT 82 | formats: 83 | - deb 84 | - rpm 85 | bindir: /usr/bin 86 | epoch: 1 87 | -------------------------------------------------------------------------------- /.octocov.yml: -------------------------------------------------------------------------------- 1 | # generated by octocov init 2 | coverage: 3 | if: true 4 | codeToTestRatio: 5 | code: 6 | - '**/*.go' 7 | - '!**/*_test.go' 8 | test: 9 | - '**/*_test.go' 10 | testExecutionTime: 11 | if: true 12 | diff: 13 | datastores: 14 | - artifact://${GITHUB_REPOSITORY} 15 | comment: 16 | if: is_pull_request 17 | report: 18 | if: is_default_branch 19 | datastores: 20 | - artifact://${GITHUB_REPOSITORY} 21 | -------------------------------------------------------------------------------- /.tbls.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dsn: sqlite://${PWD}/harvest.db 3 | docPath: doc/schema 4 | format: 5 | adjust: true 6 | er: 7 | format: svg 8 | relations: 9 | - 10 | table: logs 11 | columns: 12 | - target_id 13 | parentTable: targets 14 | parentColumns: 15 | - id 16 | def: logs -> targets 17 | - 18 | table: targets_tags 19 | columns: 20 | - target_id 21 | parentTable: targets 22 | parentColumns: 23 | - id 24 | def: targets_tags -> targets 25 | - 26 | table: targets_tags 27 | columns: 28 | - tag_id 29 | parentTable: tags 30 | parentColumns: 31 | - id 32 | def: targets_tags -> tags 33 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [v0.17.3](https://github.com/k1LoW/harvest/compare/v0.17.2...v0.17.3) (2020-07-23) 4 | 5 | * Update completion [#64](https://github.com/k1LoW/harvest/pull/64) ([k1LoW](https://github.com/k1LoW)) 6 | 7 | ## [v0.17.2](https://github.com/k1LoW/harvest/compare/v0.17.1...v0.17.2) (2020-07-07) 8 | 9 | * Fix log collect command for syslog (2 space like `Jun 6 19`) [#63](https://github.com/k1LoW/harvest/pull/63) ([k1LoW](https://github.com/k1LoW)) 10 | 11 | ## [v0.17.1](https://github.com/k1LoW/harvest/compare/v0.17.0...v0.17.1) (2020-03-05) 12 | 13 | * Fix zsh completion [#62](https://github.com/k1LoW/harvest/pull/62) ([k1LoW](https://github.com/k1LoW)) 14 | 15 | ## [v0.17.0](https://github.com/k1LoW/harvest/compare/v0.16.2...v0.17.0) (2020-01-11) 16 | 17 | * Add `hrv completion` [#61](https://github.com/k1LoW/harvest/pull/61) ([k1LoW](https://github.com/k1LoW)) 18 | 19 | ## [v0.16.2](https://github.com/k1LoW/harvest/compare/v0.16.1...v0.16.2) (2019-11-27) 20 | 21 | * Fix `hrv count` timestamp ordering [#60](https://github.com/k1LoW/harvest/pull/60) ([k1LoW](https://github.com/k1LoW)) 22 | * Use GitHub Actions [#59](https://github.com/k1LoW/harvest/pull/59) ([k1LoW](https://github.com/k1LoW)) 23 | 24 | ## [v0.16.1](https://github.com/k1LoW/harvest/compare/v0.16.0...v0.16.1) (2019-10-18) 25 | 26 | * Fix panic: runtime error: invalid memory address or nil pointer dereference [#58](https://github.com/k1LoW/harvest/pull/58) ([k1LoW](https://github.com/k1LoW)) 27 | 28 | ## [v0.16.0](https://github.com/k1LoW/harvest/compare/v0.15.5...v0.16.0) (2019-10-18) 29 | 30 | * Add `hrv info [DB_FILE]` [#57](https://github.com/k1LoW/harvest/pull/57) ([k1LoW](https://github.com/k1LoW)) 31 | * Fix timestamp grouping [#56](https://github.com/k1LoW/harvest/pull/56) ([k1LoW](https://github.com/k1LoW)) 32 | * Add table `metas` for saving `hrv fetch` info [#55](https://github.com/k1LoW/harvest/pull/55) ([k1LoW](https://github.com/k1LoW)) 33 | 34 | ## [v0.15.5](https://github.com/k1LoW/harvest/compare/v0.15.4...v0.15.5) (2019-10-16) 35 | 36 | * k8s client use --start-time and --end-time [#54](https://github.com/k1LoW/harvest/pull/54) ([k1LoW](https://github.com/k1LoW)) 37 | * Refactor RegexpParser [#53](https://github.com/k1LoW/harvest/pull/53) ([k1LoW](https://github.com/k1LoW)) 38 | 39 | ## [v0.15.4](https://github.com/k1LoW/harvest/compare/v0.15.3...v0.15.4) (2019-10-15) 40 | 41 | * Fix k8s timestamp parse [#52](https://github.com/k1LoW/harvest/pull/52) ([k1LoW](https://github.com/k1LoW)) 42 | 43 | ## [v0.15.3](https://github.com/k1LoW/harvest/compare/v0.15.2...v0.15.3) (2019-10-10) 44 | 45 | * SSH Client should fetch last line when SSH session closed. [#51](https://github.com/k1LoW/harvest/pull/51) ([k1LoW](https://github.com/k1LoW)) 46 | * Support parse 'unixtime' [#50](https://github.com/k1LoW/harvest/pull/50) ([k1LoW](https://github.com/k1LoW)) 47 | 48 | ## [v0.15.2](https://github.com/k1LoW/harvest/compare/v0.15.1...v0.15.2) (2019-09-28) 49 | 50 | * Fix: grep error Binary file (standard input) matches [#49](https://github.com/k1LoW/harvest/pull/49) ([k1LoW](https://github.com/k1LoW)) 51 | 52 | ## [v0.15.1](https://github.com/k1LoW/harvest/compare/v0.15.0...v0.15.1) (2019-09-26) 53 | 54 | * Fix configtest [#48](https://github.com/k1LoW/harvest/pull/48) ([k1LoW](https://github.com/k1LoW)) 55 | 56 | ## [v0.15.0](https://github.com/k1LoW/harvest/compare/v0.14.2...v0.15.0) (2019-09-26) 57 | 58 | * Add `hrv count` [#46](https://github.com/k1LoW/harvest/pull/46) ([k1LoW](https://github.com/k1LoW)) 59 | 60 | ## [v0.14.2](https://github.com/k1LoW/harvest/compare/v0.14.1...v0.14.2) (2019-09-25) 61 | 62 | * Fix fetch error handling [#47](https://github.com/k1LoW/harvest/pull/47) ([k1LoW](https://github.com/k1LoW)) 63 | 64 | ## [v0.14.1](https://github.com/k1LoW/harvest/compare/v0.14.0...v0.14.1) (2019-09-21) 65 | 66 | * Fix chan close timing (panic: send on closed channel) [#45](https://github.com/k1LoW/harvest/pull/45) ([k1LoW](https://github.com/k1LoW)) 67 | 68 | ## [v0.14.0](https://github.com/k1LoW/harvest/compare/v0.13.1...v0.14.0) (2019-09-18) 69 | 70 | * Do faster `hrv fetch` via ssh/file using grep timestamp [#44](https://github.com/k1LoW/harvest/pull/44) ([k1LoW](https://github.com/k1LoW)) 71 | 72 | ## [v0.13.1](https://github.com/k1LoW/harvest/compare/v0.13.0...v0.13.1) (2019-09-17) 73 | 74 | * Fix endtime (time.Time) is nil when no --end-time [#43](https://github.com/k1LoW/harvest/pull/43) ([k1LoW](https://github.com/k1LoW)) 75 | 76 | ## [v0.13.0](https://github.com/k1LoW/harvest/compare/v0.12.0...v0.13.0) (2019-09-15) 77 | 78 | * Add araddon/dataparse to parse `--*-time` option [#42](https://github.com/k1LoW/harvest/pull/42) ([k1LoW](https://github.com/k1LoW)) 79 | * Add `--duration` option [#41](https://github.com/k1LoW/harvest/pull/41) ([k1LoW](https://github.com/k1LoW)) 80 | * Fix `hrv cat` ts condition [#40](https://github.com/k1LoW/harvest/pull/40) ([k1LoW](https://github.com/k1LoW)) 81 | * Separate timestamp columns for grouping [#39](https://github.com/k1LoW/harvest/pull/39) ([k1LoW](https://github.com/k1LoW)) 82 | * Increase maxScanTokenSize [#38](https://github.com/k1LoW/harvest/pull/38) ([k1LoW](https://github.com/k1LoW)) 83 | * Refactor parser.Log struct [#37](https://github.com/k1LoW/harvest/pull/37) ([k1LoW](https://github.com/k1LoW)) 84 | * Refactor log pipeline [#36](https://github.com/k1LoW/harvest/pull/36) ([k1LoW](https://github.com/k1LoW)) 85 | * io.EOF is successful completion [#35](https://github.com/k1LoW/harvest/pull/35) ([k1LoW](https://github.com/k1LoW)) 86 | 87 | ## [v0.12.0](https://github.com/k1LoW/harvest/compare/v0.11.0...v0.12.0) (2019-08-19) 88 | 89 | * Use STDERR instead of STDOUT [#34](https://github.com/k1LoW/harvest/pull/34) ([k1LoW](https://github.com/k1LoW)) 90 | * Add gosec [#33](https://github.com/k1LoW/harvest/pull/33) ([k1LoW](https://github.com/k1LoW)) 91 | * Add --verbose option [#32](https://github.com/k1LoW/harvest/pull/32) ([k1LoW](https://github.com/k1LoW)) 92 | 93 | ## [v0.11.0](https://github.com/k1LoW/harvest/compare/v0.10.0...v0.11.0) (2019-06-24) 94 | 95 | * Add `hrv generate-k8s-config` [#31](https://github.com/k1LoW/harvest/pull/31) ([k1LoW](https://github.com/k1LoW)) 96 | 97 | ## [v0.10.0](https://github.com/k1LoW/harvest/compare/v0.9.0...v0.10.0) (2019-06-23) 98 | 99 | * Support Kubernetes logs [#30](https://github.com/k1LoW/harvest/pull/30) ([k1LoW](https://github.com/k1LoW)) 100 | 101 | ## [v0.9.0](https://github.com/k1LoW/harvest/compare/v0.8.0...v0.9.0) (2019-06-18) 102 | 103 | * [BREAKING] Update schema [#29](https://github.com/k1LoW/harvest/pull/29) ([k1LoW](https://github.com/k1LoW)) 104 | 105 | ## [v0.8.0](https://github.com/k1LoW/harvest/compare/v0.7.0...v0.8.0) (2019-06-16) 106 | 107 | * Change `--url-regexp` -> `--source` [#28](https://github.com/k1LoW/harvest/pull/28) ([k1LoW](https://github.com/k1LoW)) 108 | * Fix `hrv targets` output ( when source is `file://` ) [#27](https://github.com/k1LoW/harvest/pull/27) ([k1LoW](https://github.com/k1LoW)) 109 | 110 | ## [v0.7.0](https://github.com/k1LoW/harvest/compare/v0.6.3...v0.7.0) (2019-06-16) 111 | 112 | * Add `hrv tags` [#26](https://github.com/k1LoW/harvest/pull/26) ([k1LoW](https://github.com/k1LoW)) 113 | * [BREAKING CHANGE] Change filter logic of `--tag` option [#25](https://github.com/k1LoW/harvest/pull/25) ([k1LoW](https://github.com/k1LoW)) 114 | * [BREAKING CHANGE] Change command [#24](https://github.com/k1LoW/harvest/pull/24) ([k1LoW](https://github.com/k1LoW)) 115 | * [BREAKING] Change config format `urls:` -> `sources:` [#23](https://github.com/k1LoW/harvest/pull/23) ([k1LoW](https://github.com/k1LoW)) 116 | 117 | ## [v0.6.3](https://github.com/k1LoW/harvest/compare/v0.6.2...v0.6.3) (2019-05-17) 118 | 119 | * Fix target filter [#22](https://github.com/k1LoW/harvest/pull/22) ([k1LoW](https://github.com/k1LoW)) 120 | * Skip configtest when target type = 'none' [#21](https://github.com/k1LoW/harvest/pull/21) ([k1LoW](https://github.com/k1LoW)) 121 | 122 | ## [v0.6.2](https://github.com/k1LoW/harvest/compare/v0.6.1...v0.6.2) (2019-03-16) 123 | 124 | * Fix goreleaser.yml for for CGO_ENABLED=1 [#20](https://github.com/k1LoW/harvest/pull/20) ([k1LoW](https://github.com/k1LoW)) 125 | 126 | ## [v0.6.1](https://github.com/k1LoW/harvest/compare/v0.6.0...v0.6.1) (2019-03-16) 127 | 128 | * Fix `file://` log aggregation [#19](https://github.com/k1LoW/harvest/pull/19) ([k1LoW](https://github.com/k1LoW)) 129 | * Fix CGO_ENABLED [#18](https://github.com/k1LoW/harvest/pull/18) ([k1LoW](https://github.com/k1LoW)) 130 | 131 | ## [v0.6.0](https://github.com/k1LoW/harvest/compare/v0.5.0...v0.6.0) (2019-03-16) 132 | 133 | * Use goreleaser [#17](https://github.com/k1LoW/harvest/pull/17) ([k1LoW](https://github.com/k1LoW)) 134 | * Fix maxContentStash logic [#16](https://github.com/k1LoW/harvest/pull/16) ([k1LoW](https://github.com/k1LoW)) 135 | * Add parser type `none` [#15](https://github.com/k1LoW/harvest/pull/15) ([k1LoW](https://github.com/k1LoW)) 136 | * Add `--without-mark` option [#14](https://github.com/k1LoW/harvest/pull/14) ([k1LoW](https://github.com/k1LoW)) 137 | 138 | ## [v0.5.0](https://github.com/k1LoW/harvest/compare/v0.4.0...v0.5.0) (2019-02-27) 139 | 140 | * Preset default passphrase for all targets [#13](https://github.com/k1LoW/harvest/pull/13) ([k1LoW](https://github.com/k1LoW)) 141 | 142 | ## [v0.4.0](https://github.com/k1LoW/harvest/compare/v0.3.0...v0.4.0) (2019-02-21) 143 | 144 | * Add `hrv ls-logs` [#12](https://github.com/k1LoW/harvest/pull/12) ([k1LoW](https://github.com/k1LoW)) 145 | * Change config.yml format ( `logs:` -> `targetSets:` ) [#11](https://github.com/k1LoW/harvest/pull/11) ([k1LoW](https://github.com/k1LoW)) 146 | * Add `hrv cp` [#10](https://github.com/k1LoW/harvest/pull/10) ([k1LoW](https://github.com/k1LoW)) 147 | * Fix configtest targets order [#9](https://github.com/k1LoW/harvest/pull/9) ([k1LoW](https://github.com/k1LoW)) 148 | * Add `--preset-ssh-key-passphrase` option [#8](https://github.com/k1LoW/harvest/pull/8) ([k1LoW](https://github.com/k1LoW)) 149 | 150 | ## [v0.3.0](https://github.com/k1LoW/harvest/compare/v0.2.3...v0.3.0) (2019-02-19) 151 | 152 | * Add `hrv ls-targets` [#7](https://github.com/k1LoW/harvest/pull/7) ([k1LoW](https://github.com/k1LoW)) 153 | 154 | ## [v0.2.3](https://github.com/k1LoW/harvest/compare/v0.2.2...v0.2.3) (2019-02-15) 155 | 156 | * Fix build*Command ( append more `sudo` ) [#6](https://github.com/k1LoW/harvest/pull/6) ([k1LoW](https://github.com/k1LoW)) 157 | 158 | ## [v0.2.2](https://github.com/k1LoW/harvest/compare/v0.2.1...v0.2.2) (2019-02-15) 159 | 160 | * Show MultiLine when configtest error [#5](https://github.com/k1LoW/harvest/pull/5) ([k1LoW](https://github.com/k1LoW)) 161 | * Show error message when log read error [#4](https://github.com/k1LoW/harvest/pull/4) ([k1LoW](https://github.com/k1LoW)) 162 | * Fix build command [#3](https://github.com/k1LoW/harvest/pull/3) ([k1LoW](https://github.com/k1LoW)) 163 | 164 | ## [v0.2.1](https://github.com/k1LoW/harvest/compare/v0.2.0...v0.2.1) (2019-02-14) 165 | 166 | * Change directories [#2](https://github.com/k1LoW/harvest/pull/2) ([k1LoW](https://github.com/k1LoW)) 167 | 168 | ## [v0.2.0](https://github.com/k1LoW/harvest/compare/51449d0b6a46...v0.2.0) (2019-02-14) 169 | 170 | * Add `hrv configtest` [#1](https://github.com/k1LoW/harvest/pull/1) ([k1LoW](https://github.com/k1LoW)) 171 | 172 | ## [v0.1.0](https://github.com/k1LoW/harvest/compare/51449d0b6a46...v0.1.0) (2019-02-13) 173 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2019 Ken'ichiro Oyama 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PKG = github.com/k1LoW/harvest 2 | COMMIT = $$(git describe --tags --always) 3 | OSNAME=${shell uname -s} 4 | ifeq ($(OSNAME),Darwin) 5 | DATE = $$(gdate --utc '+%Y-%m-%d_%H:%M:%S') 6 | else 7 | DATE = $$(date --utc '+%Y-%m-%d_%H:%M:%S') 8 | endif 9 | 10 | export GO111MODULE=on 11 | 12 | BUILD_LDFLAGS = -X $(PKG).commit=$(COMMIT) -X $(PKG).date=$(DATE) 13 | 14 | default: test 15 | 16 | ci: depsdev test build integration sec 17 | 18 | test: 19 | go test ./... -coverprofile=coverage.out -covermode=count 20 | 21 | sec: 22 | gosec ./... 23 | 24 | integration: 25 | @cat testdata/test.yml.template | sed -e "s|__PWD__|${PWD}|" > testdata/test.yml 26 | @./hrv fetch -c testdata/test.yml -o test.db --start-time='2019-01-01 00:00:00' -v 27 | test `./hrv cat test.db | grep -c ''` -gt 0 || exit 1 28 | @rm test.db 29 | 30 | build: 31 | go build -ldflags="$(BUILD_LDFLAGS)" ./cmd/hrv 32 | 33 | depsdev: 34 | go install github.com/Songmu/ghch/cmd/ghch@v0.10.2 35 | go install github.com/Songmu/gocredits/cmd/gocredits@v0.2.0 36 | go install github.com/securego/gosec/v2/cmd/gosec@v2.8.1 37 | 38 | dbdoc: build 39 | @cat testdata/test.yml.template | sed -e "s|__PWD__|${PWD}|" > testdata/test.yml 40 | @./hrv fetch -c testdata/test.yml -o harvest.db --start-time='2019-01-01 00:00:00' 41 | @tbls doc -f 42 | @rm harvest.db 43 | 44 | prerelease: 45 | ghch -w -N ${VER} 46 | git add CHANGELOG.md 47 | git commit -m'Bump up version number' 48 | git tag ${VER} 49 | 50 | release: 51 | goreleaser --rm-dist 52 | 53 | .PHONY: default test 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Harvest [![Build Status](https://github.com/k1LoW/filt/workflows/build/badge.svg)](https://github.com/k1LoW/filt/actions) [![GitHub release](https://img.shields.io/github/release/k1LoW/harvest.svg)](https://github.com/k1LoW/harvest/releases) [![Go Report Card](https://goreportcard.com/badge/github.com/k1LoW/harvest)](https://goreportcard.com/report/github.com/k1LoW/harvest) 2 | 3 | > Portable log aggregation tool for middle-scale system operation/troubleshooting. 4 | 5 | ![screencast](doc/screencast.svg) 6 | 7 | Harvest provides the `hrv` command with the following features. 8 | 9 | - Agentless. 10 | - Portable. 11 | - Only 1 config file. 12 | - Fetch various remote/local log data via SSH/exec/Kubernetes API. ( `hrv fetch` ) 13 | - Output all fetched logs in the order of timestamp. ( `hrv cat` ) 14 | - Stream various remote/local logs via SSH/exec/Kubernetes API. ( `hrv stream` ) 15 | - Copy remote/local raw logs via SSH/exec. ( `hrv cp` ) 16 | 17 | ## Quick Start ( for Kubernetes ) 18 | 19 | ``` console 20 | $ hrv generate-k8s-config > cluster.yml 21 | $ hrv stream -c cluster.yml --tag='kube_apiserver or coredns' --with-path --with-timestamp 22 | ``` 23 | 24 | ## Usage 25 | 26 | ### :beetle: Fetch and output remote/local log data 27 | 28 | #### 1. Set log sources (and log type) in config.yml 29 | 30 | ``` yaml 31 | --- 32 | targetSets: 33 | - 34 | description: webproxy syslog 35 | type: syslog 36 | sources: 37 | - 'ssh://webproxy.example.com/var/log/syslog*' 38 | tags: 39 | - webproxy 40 | - syslog 41 | - 42 | description: webproxy NGINX access log 43 | type: combinedLog 44 | sources: 45 | - 'ssh://webproxy.example.com/var/log/nginx/access_log*' 46 | tags: 47 | - webproxy 48 | - nginx 49 | - 50 | description: app log 51 | type: regexp 52 | regexp: 'time:([^\t]+)' 53 | timeFormat: 'Jan 02 15:04:05' # Golang time format and 'unixtime' 54 | timeZone: '+0900' 55 | sources: 56 | - 'ssh://app-1.example.com/var/log/ltsv.log*' 57 | - 'ssh://app-2.example.com/var/log/ltsv.log*' 58 | - 'ssh://app-3.example.com/var/log/ltsv.log*' 59 | tags: 60 | - app 61 | - 62 | description: db dump log 63 | type: regexp 64 | regexp: '"ts":"([^"]+)"' 65 | timeFormat: '2006-01-02T15:04:05.999-0700' 66 | sources: 67 | - 'ssh://db.example.com/var/log/tcpdp/eth0/dump*' 68 | tags: 69 | - db 70 | - query 71 | - 72 | description: PostgreSQL log 73 | type: regexp 74 | regexp: '^\[?(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \w{3})' 75 | timeFormat: '2006-01-02 15:04:05 MST' 76 | multiLine: true 77 | sources: 78 | - 'ssh://db.example.com/var/log/postgresql/postgresql*' 79 | tags: 80 | - db 81 | - postgresql 82 | - 83 | description: local Apache access log 84 | type: combinedLog 85 | sources: 86 | - 'file:///path/to/httpd/access.log' 87 | tags: 88 | - httpd 89 | - 90 | description: api on Kubernetes 91 | type: k8s 92 | sources: 93 | - 'k8s://context-name/namespace/pod-name*' 94 | tags: 95 | - api 96 | - k8s 97 | ``` 98 | 99 | You can use `hrv configtest` for config test. 100 | 101 | ``` console 102 | $ hrv configtest -c config.yml 103 | ``` 104 | 105 | #### 2. Fetch target log data via SSH/exec/Kubernetes API ( `hrv fecth` ) 106 | 107 | ``` console 108 | $ hrv fetch -c config.yml --tag=webproxy,db 109 | ``` 110 | 111 | #### 3. Output log data ( `hrv cat` ) 112 | 113 | ``` console 114 | $ hrv cat harvest-20181215T2338+900.db --with-timestamp --with-host --with-path | less -R 115 | ``` 116 | 117 | #### 4. Count log data ( `hrv count` ) 118 | 119 | ``` console 120 | $ hrv count harvest-20191015T2338+900.db -g minute -g webproxy -b db 121 | ts webproxy db 122 | 2019-09-24 08:01:00 9618 5910 123 | 2019-09-24 08:02:00 9767 5672 124 | 2019-09-24 08:03:00 10815 7394 125 | 2019-09-24 08:04:00 11782 7109 126 | 2019-09-24 08:05:00 9896 6346 127 | [...] 128 | 2019-09-24 08:24:00 11619 5646 129 | 2019-09-24 08:25:00 10541 6097 130 | 2019-09-24 08:26:00 11336 5264 131 | 2019-09-24 08:27:00 1102 5261 132 | 2019-09-24 08:28:00 1318 6660 133 | 2019-09-24 08:29:00 10362 5663 134 | 2019-09-24 08:30:00 11136 5373 135 | 2019-09-24 08:31:00 1748 1340 136 | ``` 137 | 138 | ### :beetle: Stream remote/local logs 139 | 140 | #### 1. [Set config.yml](#1-set-log-sources-and-log-type-in-configyml) 141 | 142 | #### 2. Stream target logs via SSH/exec/Kubernetes API ( `hrv stream` ) 143 | 144 | ``` console 145 | $ hrv stream -c config.yml --with-timestamp --with-host --with-path --with-tag 146 | ``` 147 | 148 | ### :beetle: Copy remote/local raw logs 149 | 150 | #### 1. [Set config.yml](#1-set-log-sources-and-log-type-in-configyml) 151 | 152 | #### 2. Copy remote/local raw logs to local directory via SSH/exec ( `hrv cp` ) 153 | 154 | ``` console 155 | $ hrv cp -c config.yml 156 | ``` 157 | 158 | ### --tag filter operators 159 | 160 | The following operators can be used to filter targets 161 | 162 | `not`, `and`, `or`, `!`, `&&`, `||` 163 | 164 | ``` console 165 | $ hrv stream -c config.yml --tag='webproxy or db' --with-timestamp --with-host --with-path 166 | ``` 167 | 168 | #### `,` is converted to ` or ` 169 | 170 | ``` console 171 | $ hrv stream -c config.yml --tag='webproxy,db' 172 | ``` 173 | 174 | is converted to 175 | 176 | ``` console 177 | $ hrv stream -c config.yml --tag='webproxy or db' 178 | ``` 179 | 180 | ### --source filter 181 | 182 | filter targets using source regexp 183 | 184 | ``` console 185 | $ hrv fetch -c config.yml --source='app-[0-9].example' 186 | ``` 187 | 188 | ## Architecture 189 | 190 | ### `hrv fetch` and `hrv cat` 191 | 192 | ![img](doc/fetch.png) 193 | 194 | ### `hrv stream` 195 | 196 | ![img](doc/stream.png) 197 | 198 | ## Installation 199 | 200 | ```console 201 | $ brew install k1LoW/tap/harvest 202 | ``` 203 | 204 | or 205 | 206 | ```console 207 | $ go get github.com/k1LoW/harvest/cmd/hrv 208 | ``` 209 | 210 | ## What is "middle-scale system"? 211 | 212 | - < 50 instances 213 | - < 1 million logs per `hrv fetch` 214 | 215 | ### What if you are operating a large-scale/super-large-scale/hyper-large-scale system? 216 | 217 | Let's consider agent-base log collector/platform, service mesh and distributed tracing platform! 218 | 219 | ## Internal 220 | 221 | - [harvest-*.db database schema](doc/schema) 222 | 223 | ## Requirements 224 | 225 | - UNIX commands 226 | - date 227 | - find 228 | - grep 229 | - head 230 | - ls 231 | - tail 232 | - xargs 233 | - zcat 234 | - sudo 235 | - SQLite 236 | 237 | ## WANT 238 | 239 | - tag DAG 240 | - Viewer / Visualizer 241 | 242 | ## References 243 | 244 | - [Hayabusa](https://github.com/hirolovesbeer/hayabusa): A Simple and Fast Full-Text Search Engine for Massive System Log Data 245 | - Make simple with a combination of commands. 246 | - Full-Text Search Engine using SQLite FTS. 247 | - [stern](https://github.com/wercker/stern): ⎈ Multi pod and container log tailing for Kubernetes 248 | - Multiple Kubernetes log streaming architecture. 249 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "math/rand" 9 | "path/filepath" 10 | "regexp" 11 | "time" 12 | 13 | "go.uber.org/zap" 14 | ) 15 | 16 | const ( 17 | initialScanTokenSize = 4096 18 | maxScanTokenSize = 1024 * 1024 19 | ) 20 | 21 | // Client ... 22 | type Client interface { 23 | Read(ctx context.Context, st, et *time.Time, timeFormat, timeZone string) error 24 | Tailf(ctx context.Context) error 25 | RandomOne(ctx context.Context) error 26 | Ls(ctx context.Context, st *time.Time, et *time.Time) error 27 | Copy(ctx context.Context, filePath string, dstDir string) error 28 | Out() <-chan Line 29 | } 30 | 31 | // Line ... 32 | type Line struct { 33 | Host string 34 | Path string 35 | Content string 36 | TimeZone string 37 | TimestampViaClient *time.Time 38 | } 39 | 40 | var syslogTimestampAMRe = regexp.MustCompile(`^([a-zA-Z]{3}) ([0-9] .+)$`) 41 | 42 | // buildReadCommand ... 43 | func buildReadCommand(path string, st, et *time.Time, timeFormat, timeZone string) string { 44 | dir := filepath.Dir(path) 45 | base := filepath.Base(path) 46 | 47 | stRunes := []rune(st.Format(fmt.Sprintf("%s %s", timeFormat, timeZone))) 48 | etRunes := []rune(et.Format(fmt.Sprintf("%s %s", timeFormat, timeZone))) 49 | 50 | matches := []rune{} 51 | for idx, r := range stRunes { 52 | if r != etRunes[idx] { 53 | break 54 | } 55 | matches = append(matches, r) 56 | } 57 | 58 | grepStr := string(matches) 59 | // for syslog timestamp 60 | if syslogTimestampAMRe.MatchString(grepStr) { 61 | grepStr = syslogTimestampAMRe.ReplaceAllString(string(matches), "$1 $2") 62 | } 63 | 64 | findStart := st.Format("2006-01-02 15:04:05 MST") 65 | 66 | cmd := fmt.Sprintf("sudo find %s/ -type f -name '%s' -newermt '%s' | xargs sudo ls -tr | xargs sudo zcat -f | grep -a '%s'", dir, base, findStart, grepStr) 67 | if timeFormat == "unixtime" { 68 | cmd = fmt.Sprintf("sudo find %s/ -type f -name '%s' -newermt '%s' | xargs sudo ls -tr | xargs sudo zcat -f", dir, base, findStart) 69 | } 70 | 71 | return cmd 72 | } 73 | 74 | // buildTailfCommand ... 75 | func buildTailfCommand(path string) string { 76 | dir := filepath.Dir(path) 77 | base := filepath.Base(path) 78 | 79 | cmd := fmt.Sprintf("sudo find %s/ -type f -name '%s' | xargs sudo ls -tr | tail -1 | xargs sudo tail -F", dir, base) 80 | 81 | return cmd 82 | } 83 | 84 | // buildLsCommand ... 85 | func buildLsCommand(path string, st *time.Time) string { 86 | dir := filepath.Dir(path) 87 | base := filepath.Base(path) 88 | 89 | stStr := st.Format("2006-01-02 15:04:05 MST") 90 | 91 | cmd := fmt.Sprintf("sudo find %s/ -type f -name '%s' -newermt '%s' | xargs sudo ls -tr", dir, base, stStr) 92 | 93 | return cmd 94 | } 95 | 96 | // buildRandomOneCommand ... 97 | func buildRandomOneCommand(path string) string { 98 | dir := filepath.Dir(path) 99 | base := filepath.Base(path) 100 | rand.Seed(time.Now().UnixNano()) 101 | 102 | // why tail -2 -> for 0 line log 103 | cmd := fmt.Sprintf("sudo find %s/ -type f -name '%s' | xargs sudo ls -tr | tail -2 | xargs sudo zcat -f | head -%d | tail -1", dir, base, rand.Intn(100)) // #nosec 104 | 105 | return cmd 106 | } 107 | 108 | func bindReaderAndChan(ctx context.Context, l *zap.Logger, r *io.Reader, lineChan chan Line, host string, path string, tz string) { 109 | defer func() { 110 | l.Debug("Close chan client.Line") 111 | close(lineChan) 112 | }() 113 | scanner := bufio.NewScanner(*r) 114 | buf := make([]byte, initialScanTokenSize) 115 | scanner.Buffer(buf, maxScanTokenSize) 116 | L: 117 | for scanner.Scan() { 118 | select { 119 | case <-ctx.Done(): 120 | break L 121 | default: 122 | lineChan <- Line{ 123 | Host: host, 124 | Path: path, 125 | Content: scanner.Text(), 126 | TimeZone: tz, 127 | } 128 | } 129 | } 130 | if scanner.Err() != nil { 131 | l.Error("Fetch error", zap.Error(scanner.Err())) 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /client/file.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "runtime" 11 | "strings" 12 | "time" 13 | 14 | "go.uber.org/zap" 15 | ) 16 | 17 | // FileClient ... 18 | type FileClient struct { 19 | path string 20 | lineChan chan Line 21 | logger *zap.Logger 22 | } 23 | 24 | // NewFileClient ... 25 | func NewFileClient(l *zap.Logger, path string) (Client, error) { 26 | return &FileClient{ 27 | path: path, 28 | lineChan: make(chan Line), 29 | logger: l, 30 | }, nil 31 | } 32 | 33 | // Read ... 34 | func (c *FileClient) Read(ctx context.Context, st, et *time.Time, timeFormat, timeZone string) error { 35 | cmd := buildReadCommand(c.path, st, et, timeFormat, timeZone) 36 | if runtime.GOOS == "darwin" { 37 | cmd = strings.Replace(cmd, "zcat", "gzcat", -1) 38 | } 39 | 40 | return c.Exec(ctx, cmd) 41 | } 42 | 43 | // Tailf ... 44 | func (c *FileClient) Tailf(ctx context.Context) error { 45 | cmd := buildTailfCommand(c.path) 46 | return c.Exec(ctx, cmd) 47 | } 48 | 49 | // Ls ... 50 | func (c *FileClient) Ls(ctx context.Context, st *time.Time, et *time.Time) error { 51 | cmd := buildLsCommand(c.path, st) 52 | return c.Exec(ctx, cmd) 53 | } 54 | 55 | // Copy ... 56 | func (c *FileClient) Copy(ctx context.Context, filePath string, dstDir string) error { 57 | dstLogFilePath := filepath.Join(dstDir, filePath) 58 | dstLogDir := filepath.Dir(dstLogFilePath) 59 | err := os.MkdirAll(dstLogDir, 0755) // #nosec 60 | if err != nil { 61 | return err 62 | } 63 | catCmd := fmt.Sprintf("sudo cat %s > %s", filePath, dstLogFilePath) 64 | cmd := exec.CommandContext(ctx, "sh", "-c", catCmd) // #nosec 65 | err = cmd.Run() 66 | if err != nil { 67 | return err 68 | } 69 | return nil 70 | } 71 | 72 | // RandomOne ... 73 | func (c *FileClient) RandomOne(ctx context.Context) error { 74 | cmd := buildRandomOneCommand(c.path) 75 | if runtime.GOOS == "darwin" { 76 | cmd = strings.Replace(cmd, "zcat", "gzcat", -1) 77 | } 78 | return c.Exec(ctx, cmd) 79 | } 80 | 81 | // Exec ... 82 | func (c *FileClient) Exec(ctx context.Context, cmdStr string) error { 83 | c.logger.Info("Create new local exec session") 84 | tzCmd := exec.Command("date", `+%z`) // #nosec 85 | tzOut, err := tzCmd.Output() 86 | if err != nil { 87 | return err 88 | } 89 | 90 | innerCtx, cancel := context.WithCancel(ctx) 91 | defer cancel() 92 | cmd := exec.CommandContext(innerCtx, "sh", "-c", cmdStr) // #nosec 93 | 94 | stdout, err := cmd.StdoutPipe() 95 | if err != nil { 96 | return err 97 | } 98 | // FIXME 99 | // _, err = cmd.StderrPipe() 100 | // if err != nil { 101 | // return err 102 | // } 103 | 104 | r := stdout.(io.Reader) 105 | 106 | err = cmd.Start() 107 | if err != nil { 108 | return err 109 | } 110 | 111 | bindReaderAndChan(ctx, c.logger, &r, c.lineChan, "localhost", c.path, strings.TrimRight(string(tzOut), "\n")) 112 | cancel() 113 | 114 | err = cmd.Wait() 115 | if err != nil { 116 | return err 117 | } 118 | 119 | c.logger.Info("Close local exec session") 120 | return nil 121 | } 122 | 123 | // Out ... 124 | func (c *FileClient) Out() <-chan Line { 125 | return c.lineChan 126 | } 127 | -------------------------------------------------------------------------------- /client/k8s.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "regexp" 9 | "strings" 10 | "time" 11 | 12 | "github.com/k1LoW/harvest/client/k8s" 13 | "github.com/pkg/errors" 14 | "go.uber.org/zap" 15 | corev1 "k8s.io/api/core/v1" 16 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 17 | "k8s.io/apimachinery/pkg/watch" 18 | "k8s.io/client-go/kubernetes" 19 | v1 "k8s.io/client-go/kubernetes/typed/core/v1" 20 | ) 21 | 22 | type K8sClient struct { 23 | contextName string 24 | namespace string 25 | pod string 26 | podFilter *regexp.Regexp 27 | clientset *kubernetes.Clientset 28 | lineChan chan Line 29 | logger *zap.Logger 30 | } 31 | 32 | // NewK8sClient ... 33 | func NewK8sClient(l *zap.Logger, host, path string) (Client, error) { 34 | contextName := host 35 | splited := strings.Split(path, "/") 36 | ns := splited[1] 37 | p := splited[2] 38 | pRegexp := regexp.MustCompile(strings.Replace(strings.Replace(p, ".*", "*", -1), "*", ".*", -1)) 39 | 40 | clientset, err := k8s.NewKubeClientSet(contextName) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return &K8sClient{ 46 | contextName: contextName, 47 | namespace: ns, 48 | pod: p, 49 | podFilter: pRegexp, 50 | clientset: clientset, 51 | lineChan: make(chan Line), 52 | logger: l, 53 | }, nil 54 | } 55 | 56 | // Read ... 57 | func (c *K8sClient) Read(ctx context.Context, st, et *time.Time, timeFormat, timeZone string) error { 58 | sinceSeconds := time.Now().Unix() - st.Unix() 59 | return c.Stream(ctx, false, &sinceSeconds, nil) 60 | } 61 | 62 | // Tailf ... 63 | func (c *K8sClient) Tailf(ctx context.Context) error { 64 | sinceSeconds := int64(1) 65 | return c.Stream(ctx, true, &sinceSeconds, nil) 66 | } 67 | 68 | // Ls ... 69 | func (c *K8sClient) Ls(ctx context.Context, st *time.Time, et *time.Time) error { 70 | defer func() { 71 | c.logger.Debug("Close chan client.Line") 72 | close(c.lineChan) 73 | }() 74 | list, err := c.clientset.CoreV1().Pods(c.namespace).List(metav1.ListOptions{}) 75 | if err != nil { 76 | return err 77 | } 78 | for _, i := range list.Items { 79 | for _, container := range i.Spec.Containers { 80 | l := strings.Join([]string{"", i.GetNamespace(), i.GetName(), container.Name}, "/") 81 | c.lineChan <- Line{ 82 | Host: c.contextName, 83 | Path: l, 84 | Content: fmt.Sprintf("%s STDOUT/STDERR", l), 85 | TimeZone: "", 86 | } 87 | } 88 | } 89 | return nil 90 | } 91 | 92 | // Copy ... 93 | func (c *K8sClient) Copy(ctx context.Context, filePath string, dstDir string) error { 94 | c.logger.Error("not implemented: cp k8s sources") 95 | return nil 96 | } 97 | 98 | // RandomOne ... 99 | func (c *K8sClient) RandomOne(ctx context.Context) error { 100 | tailLines := int64(1) 101 | return c.Stream(ctx, false, nil, &tailLines) 102 | } 103 | 104 | // Out ... 105 | func (c *K8sClient) Out() <-chan Line { 106 | return c.lineChan 107 | } 108 | 109 | // Reference code: 110 | // https://github.com/wercker/stern/blob/473d1b605673d8f4bfe5f86b3748d02c87d339d7/stern/main.go 111 | // https://github.com/wercker/stern/blob/473d1b605673d8f4bfe5f86b3748d02c87d339d7/stern/tail.go 112 | 113 | // Copyright 2016 Wercker Holding BV 114 | // 115 | // Licensed under the Apache License, Version 2.0 (the "License"); 116 | // you may not use this file except in compliance with the License. 117 | // You may obtain a copy of the License at 118 | // 119 | // http://www.apache.org/licenses/LICENSE-2.0 120 | // 121 | // Unless required by applicable law or agreed to in writing, software 122 | // distributed under the License is distributed on an "AS IS" BASIS, 123 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 124 | // See the License for the specific language governing permissions and 125 | // limitations under the License. 126 | 127 | // targetContainer is a target to watch 128 | type targetContainer struct { 129 | namespace string 130 | pod string 131 | container string 132 | } 133 | 134 | func (tc *targetContainer) getID() string { 135 | return fmt.Sprintf("%s-%s-%s", tc.namespace, tc.pod, tc.container) 136 | } 137 | 138 | // Stream ... 139 | func (c *K8sClient) Stream(ctx context.Context, follow bool, sinceSeconds, tailLines *int64) error { 140 | defer func() { 141 | c.logger.Debug("Close chan client.Line") 142 | close(c.lineChan) 143 | }() 144 | innerCtx, cancel := context.WithCancel(ctx) 145 | defer cancel() 146 | added, removed, err := watchContainers(innerCtx, c.clientset.CoreV1().Pods(c.namespace), c.podFilter) 147 | if err != nil { 148 | return err 149 | } 150 | 151 | tails := make(map[string]*Tail) 152 | 153 | go func() { 154 | for tc := range added { 155 | id := tc.getID() 156 | if tails[id] != nil { 157 | continue 158 | } 159 | 160 | tail := NewTail(c.logger, c.lineChan, c.contextName, tc.namespace, tc.pod, tc.container) 161 | tails[id] = tail 162 | 163 | tail.Start(innerCtx, c.clientset.CoreV1().Pods(tc.namespace), follow, sinceSeconds, tailLines) 164 | } 165 | }() 166 | 167 | go func() { 168 | for tc := range removed { 169 | id := tc.getID() 170 | if tails[id] == nil { 171 | delete(tails, id) 172 | continue 173 | } 174 | tails[id].Close() 175 | delete(tails, id) 176 | } 177 | }() 178 | 179 | go func() { 180 | ticker := time.NewTicker(1 * time.Second) 181 | L: 182 | for { 183 | select { 184 | case <-ticker.C: 185 | for id, t := range tails { 186 | if t.Closed { 187 | delete(tails, id) 188 | } 189 | } 190 | if len(tails) == 0 { 191 | cancel() 192 | } 193 | case <-innerCtx.Done(): 194 | break L 195 | } 196 | } 197 | }() 198 | 199 | <-innerCtx.Done() 200 | 201 | return nil 202 | } 203 | 204 | func watchContainers(ctx context.Context, i v1.PodInterface, podFilter *regexp.Regexp) (chan *targetContainer, chan *targetContainer, error) { 205 | watcher, err := i.Watch(metav1.ListOptions{Watch: true}) 206 | if err != nil { 207 | return nil, nil, errors.Wrap(err, "failed to set up watch") 208 | } 209 | 210 | added := make(chan *targetContainer) 211 | removed := make(chan *targetContainer) 212 | 213 | go func() { 214 | for { 215 | select { 216 | case e := <-watcher.ResultChan(): 217 | if e.Object == nil { 218 | // Closed because of error 219 | return 220 | } 221 | 222 | pod := e.Object.(*corev1.Pod) 223 | 224 | if !podFilter.MatchString(pod.Name) { 225 | continue 226 | } 227 | 228 | switch e.Type { 229 | case watch.Added, watch.Modified: 230 | var statuses []corev1.ContainerStatus 231 | statuses = append(statuses, pod.Status.InitContainerStatuses...) 232 | statuses = append(statuses, pod.Status.ContainerStatuses...) 233 | 234 | for _, c := range statuses { 235 | added <- &targetContainer{ 236 | namespace: pod.Namespace, 237 | pod: pod.Name, 238 | container: c.Name, 239 | } 240 | } 241 | case watch.Deleted: 242 | var containers []corev1.Container 243 | containers = append(containers, pod.Spec.Containers...) 244 | containers = append(containers, pod.Spec.InitContainers...) 245 | 246 | for _, c := range containers { 247 | removed <- &targetContainer{ 248 | namespace: pod.Namespace, 249 | pod: pod.Name, 250 | container: c.Name, 251 | } 252 | } 253 | } 254 | case <-ctx.Done(): 255 | watcher.Stop() 256 | close(added) 257 | close(removed) 258 | return 259 | } 260 | } 261 | }() 262 | 263 | return added, removed, nil 264 | } 265 | 266 | type Tail struct { 267 | ContextName string 268 | Namespace string 269 | PodName string 270 | ContainerName string 271 | Closed bool 272 | lineChan chan Line 273 | closed chan struct{} 274 | logger *zap.Logger 275 | } 276 | 277 | // NewTail returns a new tail for a Kubernetes container inside a pod 278 | func NewTail(l *zap.Logger, lineChan chan Line, contextName, namespace, podName, containerName string) *Tail { 279 | return &Tail{ 280 | ContextName: contextName, 281 | Namespace: namespace, 282 | PodName: podName, 283 | ContainerName: containerName, 284 | lineChan: lineChan, 285 | closed: make(chan struct{}), 286 | logger: l, 287 | } 288 | } 289 | 290 | // Start starts tailing 291 | func (t *Tail) Start(ctx context.Context, i v1.PodInterface, follow bool, sinceSeconds, tailLines *int64) { 292 | go func() { 293 | t.logger.Debug(fmt.Sprintf("Open stream: /%s/%s/%s", t.Namespace, t.PodName, t.ContainerName)) 294 | req := i.GetLogs(t.PodName, &corev1.PodLogOptions{ 295 | Follow: follow, 296 | Timestamps: true, 297 | Container: t.ContainerName, 298 | SinceSeconds: sinceSeconds, 299 | TailLines: tailLines, 300 | }) 301 | 302 | stream, err := req.Stream() 303 | if err != nil { 304 | t.logger.Error(fmt.Sprintf("Error opening stream to /%s/%s/%s", t.Namespace, t.PodName, t.ContainerName)) 305 | return 306 | } 307 | 308 | reader := bufio.NewReader(stream) 309 | L: 310 | for { 311 | line, err := reader.ReadBytes('\n') 312 | if err != nil { 313 | if err != io.EOF { 314 | t.logger.Error(fmt.Sprintf("%s", err)) 315 | } 316 | break L 317 | } 318 | 319 | lineWithTs := strings.TrimSuffix(string(line), "\n") 320 | splitted := strings.Split(lineWithTs, " ") 321 | ts, err := time.Parse(time.RFC3339Nano, splitted[0]) 322 | if err != nil { 323 | t.logger.Error(fmt.Sprintf("%s", err)) 324 | } 325 | 326 | select { 327 | case <-ctx.Done(): 328 | break L 329 | default: 330 | t.lineChan <- Line{ 331 | Host: t.ContextName, 332 | Path: strings.Join([]string{"", t.Namespace, t.PodName, t.ContainerName}, "/"), 333 | Content: strings.Join(splitted[1:], " "), 334 | TimeZone: "", 335 | TimestampViaClient: &ts, 336 | } 337 | } 338 | } 339 | t.Close() 340 | }() 341 | } 342 | 343 | // Close stops tailing 344 | func (t *Tail) Close() { 345 | t.logger.Debug(fmt.Sprintf("Close stream to /%s/%s/%s", t.Namespace, t.PodName, t.ContainerName)) 346 | t.Closed = true 347 | close(t.closed) 348 | } 349 | -------------------------------------------------------------------------------- /client/k8s/k8s.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/mitchellh/go-homedir" 10 | "github.com/pkg/errors" 11 | "k8s.io/client-go/kubernetes" 12 | "k8s.io/client-go/tools/clientcmd" 13 | 14 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 15 | _ "k8s.io/client-go/plugin/pkg/client/auth/azure" 16 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 17 | _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" 18 | ) 19 | 20 | func NewKubeClientSet(contextName string) (*kubernetes.Clientset, error) { 21 | kubeconfig, err := getKubeConfig() 22 | if err != nil { 23 | return nil, err 24 | } 25 | clientConfig := newClientConfig(kubeconfig, contextName) 26 | c, err := clientConfig.ClientConfig() 27 | if err != nil { 28 | return nil, err 29 | } 30 | clientset, err := kubernetes.NewForConfig(c) 31 | if err != nil { 32 | return nil, err 33 | } 34 | return clientset, err 35 | } 36 | 37 | // GetCurrentContext ... 38 | func GetCurrentContext() (string, error) { 39 | kubeconfig, err := getKubeConfig() 40 | if err != nil { 41 | return "", err 42 | } 43 | clientConfig := newClientConfig(kubeconfig, "") 44 | rc, err := clientConfig.RawConfig() 45 | if err != nil { 46 | return "", err 47 | } 48 | return rc.CurrentContext, nil 49 | } 50 | 51 | func GetContainers(contextName string, namespace string, podFilter *regexp.Regexp) ([]string, error) { 52 | clientset, err := NewKubeClientSet(contextName) 53 | if err != nil { 54 | return nil, err 55 | } 56 | list, err := clientset.CoreV1().Pods(namespace).List(metav1.ListOptions{}) 57 | if err != nil { 58 | return nil, err 59 | } 60 | containers := []string{} 61 | for _, i := range list.Items { 62 | if !podFilter.MatchString(i.GetName()) { 63 | continue 64 | } 65 | for _, c := range i.Spec.Containers { 66 | containers = append(containers, strings.Join([]string{"", i.GetNamespace(), i.GetName(), c.Name}, "/")) 67 | } 68 | } 69 | return containers, nil 70 | } 71 | 72 | func newClientConfig(configPath string, contextName string) clientcmd.ClientConfig { 73 | configPathList := filepath.SplitList(configPath) 74 | configLoadingRules := &clientcmd.ClientConfigLoadingRules{} 75 | if len(configPathList) <= 1 { 76 | configLoadingRules.ExplicitPath = configPath 77 | } else { 78 | configLoadingRules.Precedence = configPathList 79 | } 80 | return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 81 | configLoadingRules, 82 | &clientcmd.ConfigOverrides{ 83 | CurrentContext: contextName, 84 | }, 85 | ) 86 | } 87 | 88 | func getKubeConfig() (string, error) { 89 | var kubeconfig string 90 | 91 | if kubeconfig = os.Getenv("KUBECONFIG"); kubeconfig != "" { 92 | return kubeconfig, nil 93 | } 94 | 95 | home, err := homedir.Dir() 96 | if err != nil { 97 | return "", errors.Wrap(err, "failed to get user home directory") 98 | } 99 | 100 | kubeconfig = filepath.Join(home, ".kube/config") 101 | 102 | return kubeconfig, nil 103 | } 104 | -------------------------------------------------------------------------------- /client/ssh.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "strings" 11 | "time" 12 | 13 | "github.com/k1LoW/sshc" 14 | "go.uber.org/zap" 15 | "golang.org/x/crypto/ssh" 16 | ) 17 | 18 | // SSHClient ... 19 | type SSHClient struct { 20 | host string 21 | path string 22 | client *ssh.Client 23 | lineChan chan Line 24 | logger *zap.Logger 25 | } 26 | 27 | // NewSSHClient ... 28 | func NewSSHClient(l *zap.Logger, host string, user string, port int, path string, passphrase []byte) (Client, error) { 29 | options := []sshc.Option{} 30 | if user != "" { 31 | options = append(options, sshc.User(user)) 32 | } 33 | if port > 0 { 34 | options = append(options, sshc.Port(port)) 35 | } 36 | options = append(options, sshc.Passphrase(passphrase)) 37 | 38 | client, err := sshc.NewClient(host, options...) 39 | if err != nil { 40 | return nil, err 41 | } 42 | return &SSHClient{ 43 | client: client, 44 | host: host, 45 | path: path, 46 | lineChan: make(chan Line), 47 | logger: l, 48 | }, nil 49 | } 50 | 51 | // Read ... 52 | func (c *SSHClient) Read(ctx context.Context, st, et *time.Time, timeFormat, timeZone string) error { 53 | cmd := buildReadCommand(c.path, st, et, timeFormat, timeZone) 54 | return c.Exec(ctx, cmd) 55 | } 56 | 57 | // Tailf ... 58 | func (c *SSHClient) Tailf(ctx context.Context) error { 59 | cmd := buildTailfCommand(c.path) 60 | return c.Exec(ctx, cmd) 61 | } 62 | 63 | // Ls ... 64 | func (c *SSHClient) Ls(ctx context.Context, st *time.Time, et *time.Time) error { 65 | cmd := buildLsCommand(c.path, st) 66 | return c.Exec(ctx, cmd) 67 | } 68 | 69 | // Copy ... 70 | func (c *SSHClient) Copy(ctx context.Context, filePath string, dstDir string) error { 71 | dstLogFilePath := filepath.Join(dstDir, c.host, filePath) 72 | dstLogDir := filepath.Dir(dstLogFilePath) 73 | err := os.MkdirAll(dstLogDir, 0755) // #nosec 74 | if err != nil { 75 | return err 76 | } 77 | catCmd := fmt.Sprintf("ssh %s sudo cat %s > %s", c.host, filePath, dstLogFilePath) 78 | cmd := exec.CommandContext(ctx, "sh", "-c", catCmd) // #nosec 79 | err = cmd.Run() 80 | if err != nil { 81 | return err 82 | } 83 | return nil 84 | } 85 | 86 | // RandomOne ... 87 | func (c *SSHClient) RandomOne(ctx context.Context) error { 88 | cmd := buildRandomOneCommand(c.path) 89 | 90 | return c.Exec(ctx, cmd) 91 | } 92 | 93 | // Exec ... 94 | func (c *SSHClient) Exec(ctx context.Context, cmd string) error { 95 | session, err := c.client.NewSession() 96 | if err != nil { 97 | return err 98 | } 99 | c.logger.Debug("Create new SSH session") 100 | defer session.Close() 101 | 102 | var tzOut []byte 103 | err = func() error { 104 | session, err := c.client.NewSession() 105 | if err != nil { 106 | return err 107 | } 108 | defer session.Close() 109 | tzCmd := `date +"%z"` 110 | tzOut, err = session.Output(tzCmd) 111 | if err != nil { 112 | return err 113 | } 114 | return nil 115 | }() 116 | if err != nil { 117 | return err 118 | } 119 | 120 | stdout, err := session.StdoutPipe() 121 | if err != nil { 122 | return err 123 | } 124 | // FIXME 125 | // _, err = session.StderrPipe() 126 | // if err != nil { 127 | // return err 128 | // } 129 | 130 | go bindReaderAndChan(ctx, c.logger, &stdout, c.lineChan, c.host, c.path, strings.TrimRight(string(tzOut), "\n")) 131 | 132 | err = session.Start(cmd) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | go func() { 138 | <-ctx.Done() 139 | err = session.Close() 140 | if err != nil && err != io.EOF { 141 | c.logger.Error(fmt.Sprintf("%s", err)) 142 | } 143 | }() 144 | 145 | // TODO: use session.Signal() 146 | // https://github.com/golang/go/issues/16597 147 | _ = session.Wait() 148 | 149 | c.logger.Debug("Close SSH session") 150 | 151 | return nil 152 | } 153 | 154 | // Out ... 155 | func (c *SSHClient) Out() <-chan Line { 156 | return c.lineChan 157 | } 158 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/cat.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "os" 27 | "strconv" 28 | "strings" 29 | 30 | "github.com/antonmedv/expr" 31 | "github.com/k1LoW/harvest/db" 32 | "github.com/k1LoW/harvest/logger" 33 | "github.com/k1LoW/harvest/stdout" 34 | "github.com/spf13/cobra" 35 | "go.uber.org/zap" 36 | ) 37 | 38 | var ( 39 | match string 40 | ) 41 | 42 | // catCmd represents the cat command 43 | var catCmd = &cobra.Command{ 44 | Use: "cat [DB_FILE]", 45 | Short: "cat logs from harvest-*.db", 46 | Long: `cat logs from harvest-*.db`, 47 | Args: cobra.ExactArgs(1), 48 | Run: func(cmd *cobra.Command, args []string) { 49 | l := logger.NewLogger(verbose) 50 | dbPath := args[0] 51 | 52 | if _, err := os.Lstat(dbPath); err != nil { 53 | l.Error(fmt.Sprintf("%s not exists", dbPath), zap.String("error", err.Error())) 54 | os.Exit(1) 55 | } 56 | 57 | ctx, cancel := context.WithCancel(context.Background()) 58 | defer cancel() 59 | 60 | d, err := db.AttachDB(ctx, l, dbPath) 61 | if err != nil { 62 | l.Error("DB attach error", zap.String("error", err.Error())) 63 | os.Exit(1) 64 | } 65 | 66 | cond, err := buildCondition(d) 67 | if err != nil { 68 | l.Error("option error", zap.String("error", err.Error())) 69 | os.Exit(1) 70 | } 71 | 72 | hLen, tLen, err := getCatStdoutLengthes(d, withHost, withPath, withTag) 73 | if err != nil { 74 | l.Error("option error", zap.String("error", err.Error())) 75 | os.Exit(1) 76 | } 77 | 78 | hosts, err := d.GetHosts() 79 | if err != nil { 80 | l.Error("DB query error", zap.String("error", err.Error())) 81 | os.Exit(1) 82 | } 83 | 84 | sout, err := stdout.NewStdout( 85 | withTimestamp, 86 | withTimestampNano, 87 | withHost, 88 | withPath, 89 | withTag, 90 | withoutMark, 91 | hLen, 92 | tLen, 93 | noColor, 94 | ) 95 | if err != nil { 96 | l.Error("cat error", zap.String("error", err.Error())) 97 | os.Exit(1) 98 | } 99 | 100 | sout.Out(d.Cat(cond), hosts) 101 | }, 102 | } 103 | 104 | func buildCondition(db *db.DB) (string, error) { 105 | matchCond := []string{} 106 | cond := []string{} 107 | if match != "" { 108 | matchCond = append(matchCond, match) 109 | } 110 | 111 | if stStr != "" || etStr != "" || duStr != "" { 112 | st, et, err := parseTimes(stStr, etStr, duStr) 113 | if err != nil { 114 | return "", err 115 | } 116 | cond = append(cond, fmt.Sprintf("ts_unixnano >= %d", st.UnixNano())) 117 | cond = append(cond, fmt.Sprintf("ts_unixnano <= %d", et.UnixNano())) 118 | } 119 | 120 | if tag != "" { 121 | tagExpr := strings.Replace(tag, ",", " or ", -1) 122 | allTags, err := db.GetTags() 123 | if err != nil { 124 | return "", err 125 | } 126 | tt, err := db.GetTargetIdAndTags() 127 | if err != nil { 128 | return "", err 129 | } 130 | targetIds := []string{} 131 | for targetId, targetTags := range tt { 132 | targetExpr := []string{"true"} 133 | targetExpr = append(targetExpr, fmt.Sprintf("(%s)", tagExpr)) 134 | tags := map[string]interface{}{} 135 | for _, tag := range allTags { 136 | if contains(targetTags, tag) { 137 | tags[tag] = true 138 | } else { 139 | tags[tag] = false 140 | } 141 | } 142 | out, err := expr.Eval(strings.Join(targetExpr, " and "), tags) 143 | if err != nil { 144 | return "", err 145 | } 146 | if out.(bool) { 147 | targetIds = append(targetIds, strconv.FormatInt(targetId, 10)) 148 | } 149 | } 150 | cond = append(cond, fmt.Sprintf("( target_id IN (%s) )", strings.Join(targetIds, ", "))) 151 | } 152 | 153 | if len(matchCond) > 0 { 154 | cond = append(cond, fmt.Sprintf("content MATCH '%s'", strings.Join(matchCond, " AND "))) 155 | } 156 | 157 | if len(cond) == 0 { 158 | return "", nil 159 | } 160 | 161 | return fmt.Sprintf(" WHERE %s", strings.Join(cond, " AND ")), nil // #nosec 162 | } 163 | 164 | func getCatStdoutLengthes(d *db.DB, withHost, withPath, withTag bool) (int, int, error) { 165 | var ( 166 | hLen int 167 | tLen int 168 | err error 169 | ) 170 | if withHost && withPath { 171 | hLen, err = d.GetColumnMaxLength("host", "path") 172 | if err != nil { 173 | return 0, 0, err 174 | } 175 | } else if withHost { 176 | hLen, err = d.GetColumnMaxLength("host") 177 | if err != nil { 178 | return 0, 0, err 179 | } 180 | } else if withPath { 181 | hLen, err = d.GetColumnMaxLength("path") 182 | if err != nil { 183 | return 0, 0, err 184 | } 185 | } 186 | if withTag { 187 | tLen, err = d.GetTagMaxLength() 188 | if err != nil { 189 | return 0, 0, err 190 | } 191 | } 192 | return hLen, tLen, nil 193 | } 194 | 195 | func contains(ss []string, t string) bool { 196 | for _, s := range ss { 197 | if s == t { 198 | return true 199 | } 200 | } 201 | return false 202 | } 203 | 204 | func init() { 205 | rootCmd.AddCommand(catCmd) 206 | catCmd.Flags().BoolVarP(&withTimestamp, "with-timestamp", "", false, "output with timestamp") 207 | catCmd.Flags().BoolVarP(&withTimestampNano, "with-timestamp-nano", "", false, "output with timestamp nano sec") 208 | catCmd.Flags().BoolVarP(&withHost, "with-host", "", false, "output with host") 209 | catCmd.Flags().BoolVarP(&withPath, "with-path", "", false, "output with path") 210 | catCmd.Flags().BoolVarP(&withTag, "with-tag", "", false, "output with tag") 211 | catCmd.Flags().BoolVarP(&withoutMark, "without-mark", "", false, "output without prefix mark") 212 | catCmd.Flags().StringVarP(&match, "match", "", "", "filter logs using SQLite FTS `MATCH` query") 213 | catCmd.Flags().StringVarP(&tag, "tag", "", "", "filter logs using tag") 214 | catCmd.Flags().StringVarP(&stStr, "start-time", "", "", "log start time (format: 2006-01-02 15:04:05)") 215 | catCmd.Flags().StringVarP(&etStr, "end-time", "", "", "log end time (format: 2006-01-02 15:04:05)") 216 | catCmd.Flags().StringVarP(&duStr, "duration", "", "", "log duration") 217 | catCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print debugging messages.") 218 | catCmd.Flags().BoolVarP(&noColor, "no-color", "", false, "disable colorize output") 219 | err := catCmd.MarkZshCompPositionalArgumentFile(1) 220 | if err != nil { 221 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 222 | os.Exit(1) 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/completion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Ken'ichiro Oyama 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | package cmd 23 | 24 | import ( 25 | "fmt" 26 | "os" 27 | 28 | "github.com/spf13/cobra" 29 | ) 30 | 31 | var out string 32 | 33 | // completionCmd represents the completion command 34 | var completionCmd = &cobra.Command{ 35 | Use: "completion", 36 | Short: "output shell completion code", 37 | Long: `output shell completion code. 38 | To configure your shell to load completions for each session 39 | 40 | # bash 41 | echo '. <(hrv completion bash)' > ~/.bashrc 42 | 43 | # zsh 44 | hrv completion zsh > $fpath[1]/_hrv 45 | 46 | # fish 47 | hrv completion fish ~/.config/fish/completions/hrv.fish 48 | `, 49 | ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, 50 | Args: func(cmd *cobra.Command, args []string) error { 51 | if len(args) != 1 { 52 | return fmt.Errorf("accepts 1 arg, received %d", len(args)) 53 | } 54 | if err := cobra.OnlyValidArgs(cmd, args); err != nil { 55 | return err 56 | } 57 | return nil 58 | }, 59 | Run: func(cmd *cobra.Command, args []string) { 60 | var ( 61 | o *os.File 62 | err error 63 | ) 64 | sh := args[0] 65 | if out == "" { 66 | o = os.Stdout 67 | } else { 68 | o, err = os.Create(out) 69 | if err != nil { 70 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 71 | os.Exit(1) 72 | } 73 | defer func() { 74 | err := o.Close() 75 | if err != nil { 76 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 77 | os.Exit(1) 78 | } 79 | }() 80 | } 81 | 82 | switch sh { 83 | case "bash": 84 | if err := cmd.Root().GenBashCompletion(o); err != nil { 85 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 86 | os.Exit(1) 87 | } 88 | case "zsh": 89 | if err := cmd.Root().GenZshCompletion(o); err != nil { 90 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 91 | os.Exit(1) 92 | } 93 | case "fish": 94 | if err := cmd.Root().GenFishCompletion(o, true); err != nil { 95 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 96 | os.Exit(1) 97 | } 98 | case "powershell": 99 | if err := cmd.Root().GenPowerShellCompletion(o); err != nil { 100 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 101 | os.Exit(1) 102 | } 103 | } 104 | }, 105 | } 106 | 107 | func init() { 108 | rootCmd.AddCommand(completionCmd) 109 | completionCmd.Flags().StringVarP(&out, "out", "o", "", "output file path") 110 | } 111 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/configtest.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "os" 27 | "sync" 28 | 29 | "github.com/k1LoW/harvest/collector" 30 | "github.com/k1LoW/harvest/config" 31 | "github.com/k1LoW/harvest/logger" 32 | "github.com/k1LoW/harvest/parser" 33 | "github.com/labstack/gommon/color" 34 | "github.com/spf13/cobra" 35 | "go.uber.org/zap" 36 | ) 37 | 38 | // configtestCmd represents the configtest command 39 | var configtestCmd = &cobra.Command{ 40 | Use: "configtest", 41 | Short: "configtest", 42 | Long: `configtest.`, 43 | Run: func(cmd *cobra.Command, args []string) { 44 | l := logger.NewLogger(verbose) 45 | 46 | cfg, err := config.NewConfig() 47 | if err != nil { 48 | l.Error("Config error", zap.String("error", err.Error())) 49 | os.Exit(1) 50 | } 51 | err = cfg.LoadConfigFile(configPath) 52 | if err != nil { 53 | l.Error("Config error", zap.String("error", err.Error())) 54 | os.Exit(1) 55 | } 56 | 57 | ctx, cancel := context.WithCancel(context.Background()) 58 | defer cancel() 59 | 60 | targets, err := cfg.FilterTargets(tag, sourceRe) 61 | if err != nil { 62 | l.Error("tag option error", zap.String("error", err.Error())) 63 | os.Exit(1) 64 | } 65 | if len(targets) == 0 { 66 | l.Error("No targets") 67 | os.Exit(1) 68 | } 69 | l.Info(fmt.Sprintf("Target count: %d", len(targets))) 70 | 71 | if presetSSHKeyPassphrase { 72 | err = presetSSHKeyPassphraseToTargets(targets) 73 | if err != nil { 74 | l.Error("option error", zap.String("error", err.Error())) 75 | os.Exit(1) 76 | } 77 | } 78 | 79 | l.Info("Test timestamp parsing") 80 | fmt.Println("") 81 | 82 | cChan := make(chan struct{}, 1) 83 | var wg sync.WaitGroup 84 | 85 | failure := 0 86 | for _, t := range targets { 87 | wg.Add(1) 88 | cChan <- struct{}{} 89 | c, err := collector.NewCollector(ctx, t, l) 90 | if err != nil { 91 | failure++ 92 | <-cChan 93 | wg.Done() 94 | l.Error("ConfigTest error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 95 | continue 96 | } 97 | logChan := make(chan parser.Log) 98 | go func(t *config.Target, logChan chan parser.Log) { 99 | defer wg.Done() 100 | fmt.Printf("%s: ", t.Source) 101 | logRead := false 102 | for log := range logChan { 103 | if logRead { 104 | continue 105 | } 106 | if log.Timestamp != nil { 107 | fmt.Printf("%s\n", color.Green("OK", color.B)) 108 | } else if t.Type == "none" { 109 | fmt.Printf("%s\n", color.Yellow("Skip (because type=none)", color.B)) 110 | } else { 111 | fmt.Printf("%s\n", color.Red("Timestamp parse error", color.B)) 112 | fmt.Printf(" %s %s\n", color.Red(" Type:"), color.Red(t.Type)) 113 | fmt.Printf(" %s %s\n", color.Red(" Regexp:"), color.Red(t.Regexp)) 114 | fmt.Printf(" %s %s\n", color.Red("TimeFormat:"), color.Red(t.TimeFormat)) 115 | fmt.Printf(" %s %s\n", color.Red(" MultiLine:"), color.Red(t.MultiLine)) 116 | fmt.Printf(" %s %s\n", color.Red(" Log:"), color.Red(log.Content)) 117 | fmt.Println("") 118 | failure++ 119 | } 120 | logRead = true 121 | } 122 | if !logRead { 123 | fmt.Printf("%s\n", color.Red("Log read error", color.B)) 124 | failure++ 125 | } 126 | }(t, logChan) 127 | err = c.ConfigTest(logChan, t.MultiLine) 128 | if err != nil { 129 | failure++ 130 | l.Error("ConfigTest error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 131 | } 132 | <-cChan 133 | } 134 | 135 | wg.Wait() 136 | 137 | fmt.Println("") 138 | if failure > 0 { 139 | fmt.Println(color.Red(fmt.Sprintf("%d targets, %d failure\n", len(targets), failure), color.B)) 140 | } else { 141 | fmt.Println(color.Green(fmt.Sprintf("%d targets, %d failure\n", len(targets), failure), color.B)) 142 | } 143 | }, 144 | } 145 | 146 | func init() { 147 | rootCmd.AddCommand(configtestCmd) 148 | configtestCmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path") 149 | _ = configtestCmd.MarkFlagFilename("config", "yaml", "yml") 150 | configtestCmd.Flags().StringVarP(&tag, "tag", "", "", "filter targets using tag") 151 | configtestCmd.Flags().StringVarP(&sourceRe, "source", "", "", "filter targets using source regexp") 152 | configtestCmd.Flags().BoolVarP(&presetSSHKeyPassphrase, "preset-ssh-key-passphrase", "", false, "preset SSH key passphrase") 153 | configtestCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print debugging messages.") 154 | } 155 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/count.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2019 Ken'ichiro Oyama 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | package cmd 23 | 24 | import ( 25 | "context" 26 | "fmt" 27 | "os" 28 | "strings" 29 | 30 | "github.com/k1LoW/harvest/db" 31 | "github.com/k1LoW/harvest/logger" 32 | "github.com/spf13/cobra" 33 | "go.uber.org/zap" 34 | ) 35 | 36 | var ( 37 | groups []string 38 | matches []string 39 | delimiter string 40 | ) 41 | 42 | // countCmd represents the count command 43 | var countCmd = &cobra.Command{ 44 | Use: "count [DB_FILE]", 45 | Short: "count logs from harvest-*.db", 46 | Long: `count logs from harvest-*.db.`, 47 | Args: cobra.ExactArgs(1), 48 | Run: func(cmd *cobra.Command, args []string) { 49 | os.Exit(runCount(args, groups, matches)) 50 | }, 51 | } 52 | 53 | // runCount ... 54 | func runCount(args, groups, matches []string) int { 55 | l := logger.NewLogger(verbose) 56 | dbPath := args[0] 57 | 58 | if _, err := os.Lstat(dbPath); err != nil { 59 | l.Error(fmt.Sprintf("%s not exists", dbPath), zap.String("error", err.Error())) 60 | return 1 61 | } 62 | 63 | ctx, cancel := context.WithCancel(context.Background()) 64 | defer cancel() 65 | 66 | d, err := db.AttachDB(ctx, l, dbPath) 67 | if err != nil { 68 | l.Error("DB attach error", zap.String("error", err.Error())) 69 | return 1 70 | } 71 | 72 | results, err := d.Count(groups, matches) 73 | if err != nil { 74 | l.Error("DB attach error", zap.String("error", err.Error())) 75 | return 1 76 | } 77 | 78 | for _, line := range results { 79 | fmt.Println(strings.Join(line, delimiter)) 80 | } 81 | 82 | return 0 83 | } 84 | 85 | func init() { 86 | countCmd.Flags().StringSliceVarP(&groups, "group-by", "g", []string{}, "group logs using time, host, desctiption, and tag") 87 | countCmd.Flags().StringSliceVarP(&matches, "match", "m", []string{}, "group logs using SQLite `%LIKE%` query") 88 | countCmd.Flags().StringVarP(&delimiter, "delimiter", "d", "\t", "delmiter") 89 | countCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print debugging messages.") 90 | err := countCmd.MarkZshCompPositionalArgumentFile(1) 91 | if err != nil { 92 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 93 | os.Exit(1) 94 | } 95 | rootCmd.AddCommand(countCmd) 96 | } 97 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/cp.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "os" 27 | "path/filepath" 28 | "sync" 29 | "time" 30 | 31 | "github.com/k1LoW/harvest/collector" 32 | "github.com/k1LoW/harvest/config" 33 | "github.com/k1LoW/harvest/logger" 34 | "github.com/k1LoW/harvest/parser" 35 | "github.com/k1LoW/harvest/stdout" 36 | "github.com/spf13/cobra" 37 | "go.uber.org/zap" 38 | ) 39 | 40 | var ( 41 | dstDir string 42 | ) 43 | 44 | // cpCmd represents the dl command 45 | var cpCmd = &cobra.Command{ 46 | Use: "cp", 47 | Short: "copy raw logs from targets", 48 | Long: `copy raw logs from targets.`, 49 | Run: func(cmd *cobra.Command, args []string) { 50 | l := logger.NewLogger(verbose) 51 | 52 | cfg, err := config.NewConfig() 53 | if err != nil { 54 | l.Error("Config error", zap.String("error", err.Error())) 55 | os.Exit(1) 56 | } 57 | err = cfg.LoadConfigFile(configPath) 58 | if err != nil { 59 | l.Error("Config error", zap.String("error", err.Error())) 60 | os.Exit(1) 61 | } 62 | 63 | ctx, cancel := context.WithCancel(context.Background()) 64 | defer cancel() 65 | 66 | if dstDir == "" { 67 | dstDir = fmt.Sprintf("harvest-%s", time.Now().Format("20060102T150405-0700")) 68 | } 69 | dstDir, err = filepath.Abs(dstDir) 70 | if err != nil { 71 | l.Error("option error", zap.String("error", err.Error())) 72 | os.Exit(1) 73 | } 74 | l.Info(fmt.Sprintf("Mkdir %s", dstDir)) 75 | l.Info("Directory initialized") 76 | 77 | if _, err := os.Lstat(dstDir); err == nil { 78 | l.Error(fmt.Sprintf("%s already exists", dstDir), zap.String("error", err.Error())) 79 | os.Exit(1) 80 | } 81 | 82 | targets, err := cfg.FilterTargets(tag, sourceRe) 83 | if err != nil { 84 | l.Error("tag option error", zap.String("error", err.Error())) 85 | os.Exit(1) 86 | } 87 | if len(targets) == 0 { 88 | l.Error("No targets") 89 | os.Exit(1) 90 | } 91 | l.Info(fmt.Sprintf("Target count: %d", len(targets))) 92 | 93 | if presetSSHKeyPassphrase { 94 | err = presetSSHKeyPassphraseToTargets(targets) 95 | if err != nil { 96 | l.Error("option error", zap.String("error", err.Error())) 97 | os.Exit(1) 98 | } 99 | } 100 | 101 | st, et, err := parseTimes(stStr, etStr, duStr) 102 | if err != nil { 103 | l.Error("option error", zap.String("error", err.Error())) 104 | os.Exit(1) 105 | } 106 | 107 | l.Debug(fmt.Sprintf("Client concurrency: %d", concurrency)) 108 | l.Info(fmt.Sprintf("Log timestamp: %s - %s", st.Format("2006-01-02 15:04:05-0700"), et.Format("2006-01-02 15:04:05-0700"))) 109 | 110 | l.Debug("Start copying logs from targets") 111 | 112 | sout, err := stdout.NewStdout( 113 | false, 114 | false, 115 | false, 116 | false, 117 | false, 118 | false, 119 | 0, 120 | 0, 121 | noColor, 122 | ) 123 | if err != nil { 124 | l.Error("fetch error", zap.String("error", err.Error())) 125 | os.Exit(1) 126 | } 127 | 128 | hosts := getHosts(targets) 129 | logChan := make(chan parser.Log) 130 | 131 | go sout.Out(logChan, hosts) 132 | 133 | cChan := make(chan struct{}, concurrency) 134 | var wg sync.WaitGroup 135 | 136 | for _, t := range targets { 137 | wg.Add(1) 138 | go func(t *config.Target) { 139 | cChan <- struct{}{} 140 | defer wg.Done() 141 | c, err := collector.NewCollector(ctx, t, l) 142 | if err != nil { 143 | l.Error("Copy error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 144 | } 145 | err = c.Copy(logChan, st, et, dstDir) 146 | if err != nil { 147 | l.Error("Copy error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 148 | } 149 | <-cChan 150 | }(t) 151 | } 152 | 153 | wg.Wait() 154 | 155 | l.Debug("Copy finished") 156 | }, 157 | } 158 | 159 | func init() { 160 | rootCmd.AddCommand(cpCmd) 161 | cpCmd.Flags().StringVarP(&dstDir, "out", "o", "", "dst dir") 162 | cpCmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path") 163 | _ = cpCmd.MarkFlagFilename("config", "yaml", "yml") 164 | cpCmd.Flags().IntVarP(&concurrency, "concurrency", "C", defaultConcurrency, "concurrency") 165 | cpCmd.Flags().StringVarP(&tag, "tag", "", "", "filter targets using tag") 166 | cpCmd.Flags().StringVarP(&sourceRe, "source", "", "", "filter targets using source regexp") 167 | cpCmd.Flags().StringVarP(&stStr, "start-time", "", "", "log start time (format: 2006-01-02 15:04:05)") 168 | cpCmd.Flags().StringVarP(&etStr, "end-time", "", "", "log end time (default: latest) (format: 2006-01-02 15:04:05)") 169 | cpCmd.Flags().StringVarP(&duStr, "duration", "", "", "log duration") 170 | cpCmd.Flags().BoolVarP(&presetSSHKeyPassphrase, "preset-ssh-key-passphrase", "", false, "preset SSH key passphrase") 171 | cpCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print debugging messages.") 172 | } 173 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/fetch.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "os" 27 | "strconv" 28 | "sync" 29 | "time" 30 | 31 | "github.com/k1LoW/harvest/collector" 32 | "github.com/k1LoW/harvest/config" 33 | "github.com/k1LoW/harvest/db" 34 | "github.com/k1LoW/harvest/logger" 35 | "github.com/spf13/cobra" 36 | "go.uber.org/zap" 37 | ) 38 | 39 | var ( 40 | stStr string 41 | etStr string 42 | duStr string 43 | dbPath string 44 | concurrency int 45 | ) 46 | 47 | const ( 48 | defaultConcurrency = 10 49 | ) 50 | 51 | // fetchCmd represents the fetch command 52 | var fetchCmd = &cobra.Command{ 53 | Use: "fetch", 54 | Short: "fetch from targets", 55 | Long: `fetch from targets.`, 56 | Run: func(cmd *cobra.Command, args []string) { 57 | l := logger.NewLogger(verbose) 58 | 59 | cfg, err := config.NewConfig() 60 | if err != nil { 61 | l.Error("Config error", zap.String("error", err.Error())) 62 | os.Exit(1) 63 | } 64 | err = cfg.LoadConfigFile(configPath) 65 | if err != nil { 66 | l.Error("Config error", zap.String("error", err.Error())) 67 | os.Exit(1) 68 | } 69 | 70 | ctx, cancel := context.WithCancel(context.Background()) 71 | defer cancel() 72 | 73 | if dbPath == "" { 74 | dbPath = fmt.Sprintf("harvest-%s.db", time.Now().Format("20060102T150405-0700")) 75 | } 76 | if _, err := os.Lstat(dbPath); err == nil { 77 | l.Error(fmt.Sprintf("%s already exists", dbPath)) 78 | os.Exit(1) 79 | } 80 | d, err := db.NewDB(ctx, l, cfg, dbPath) 81 | if err != nil { 82 | l.Error("DB initialize error", zap.String("error", err.Error())) 83 | os.Exit(1) 84 | } 85 | 86 | _ = d.SetMeta("option.tag", tag) 87 | _ = d.SetMeta("option.source", sourceRe) 88 | _ = d.SetMeta("option.start-time", stStr) 89 | _ = d.SetMeta("option.end-time", etStr) 90 | _ = d.SetMeta("option.duration", duStr) 91 | _ = d.SetMeta("option.concurrency", strconv.Itoa(concurrency)) 92 | 93 | targets, err := cfg.FilterTargets(tag, sourceRe) 94 | if err != nil { 95 | l.Error("tag option error", zap.String("error", err.Error())) 96 | os.Exit(1) 97 | } 98 | if len(targets) == 0 { 99 | l.Error("No targets") 100 | os.Exit(1) 101 | } 102 | l.Info(fmt.Sprintf("Target count: %d", len(targets))) 103 | 104 | _ = d.SetMeta("fetch.target-count", strconv.Itoa(len(targets))) 105 | 106 | if presetSSHKeyPassphrase { 107 | err = presetSSHKeyPassphraseToTargets(targets) 108 | if err != nil { 109 | l.Error("option error", zap.String("error", err.Error())) 110 | os.Exit(1) 111 | } 112 | } 113 | 114 | st, et, err := parseTimes(stStr, etStr, duStr) 115 | if err != nil { 116 | l.Error("option error", zap.String("error", err.Error())) 117 | os.Exit(1) 118 | } 119 | 120 | l.Debug(fmt.Sprintf("Client concurrency: %d", concurrency)) 121 | l.Info(fmt.Sprintf("Log timestamp: %s - %s", st.Format(time.RFC3339), et.Format(time.RFC3339))) 122 | l.Debug("Start fetching from targets") 123 | 124 | _ = d.SetMeta("fetch.started_at", time.Now().Format(time.RFC3339)) 125 | 126 | go d.StartInsert() 127 | 128 | cChan := make(chan struct{}, concurrency) 129 | var wg sync.WaitGroup 130 | 131 | finished := 0 132 | for _, t := range targets { 133 | wg.Add(1) 134 | go func(t *config.Target) { 135 | cChan <- struct{}{} 136 | defer func() { 137 | finished = finished + 1 138 | l.Info(fmt.Sprintf("Fetching progress: %d/%d", finished, len(targets))) 139 | wg.Done() 140 | }() 141 | c, err := collector.NewCollector(ctx, t, l) 142 | if err != nil { 143 | l.Error("Fetch error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 144 | return 145 | } 146 | err = c.Fetch(d.In(), st, et, t.MultiLine) 147 | if err != nil { 148 | l.Error("Fetch error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 149 | } 150 | <-cChan 151 | }(t) 152 | } 153 | 154 | wg.Wait() 155 | 156 | l.Info("Fetch finished") 157 | _ = d.SetMeta("fetch.finished_at", time.Now().Format(time.RFC3339)) 158 | }, 159 | } 160 | 161 | func init() { 162 | rootCmd.AddCommand(fetchCmd) 163 | fetchCmd.Flags().StringVarP(&dbPath, "out", "o", "", "db path") 164 | fetchCmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path") 165 | _ = fetchCmd.MarkFlagFilename("config", "yaml", "yml") 166 | fetchCmd.Flags().IntVarP(&concurrency, "concurrency", "C", defaultConcurrency, "concurrency") 167 | fetchCmd.Flags().StringVarP(&tag, "tag", "", "", "filter targets using tag") 168 | fetchCmd.Flags().StringVarP(&sourceRe, "source", "", "", "filter targets using source regexp") 169 | fetchCmd.Flags().StringVarP(&stStr, "start-time", "", "", "log start time (format: 2006-01-02 15:04:05)") 170 | fetchCmd.Flags().StringVarP(&etStr, "end-time", "", "", "log end time (default: latest) (format: 2006-01-02 15:04:05)") 171 | fetchCmd.Flags().StringVarP(&duStr, "duration", "", "", "log duration") 172 | fetchCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print debugging messages.") 173 | fetchCmd.Flags().BoolVarP(&presetSSHKeyPassphrase, "preset-ssh-key-passphrase", "", false, "preset SSH key passphrase") 174 | } 175 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/generateK8sConfig.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2019 Ken'ichiro Oyama 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | package cmd 23 | 24 | import ( 25 | "fmt" 26 | "os" 27 | "regexp" 28 | "strings" 29 | 30 | "github.com/k1LoW/harvest/client/k8s" 31 | "github.com/k1LoW/harvest/config" 32 | "github.com/k1LoW/harvest/logger" 33 | "github.com/spf13/cobra" 34 | "go.uber.org/zap" 35 | "gopkg.in/yaml.v2" 36 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 37 | ) 38 | 39 | var ( 40 | contextName string 41 | namespace string 42 | ) 43 | 44 | // generateK8sConfigCmd represents the generateK8sConfig command 45 | var generateK8sConfigCmd = &cobra.Command{ 46 | Use: "generate-k8s-config", 47 | Short: "generate harvest config.yml via Kubernetes cluster", 48 | Long: `generate harvest config.yml via Kubernetes cluster.`, 49 | Run: func(cmd *cobra.Command, args []string) { 50 | l := logger.NewLogger(verbose) 51 | 52 | if contextName == "" { 53 | cc, err := k8s.GetCurrentContext() 54 | if err != nil { 55 | l.Error("kube config error", zap.String("error", err.Error())) 56 | os.Exit(1) 57 | } 58 | contextName = cc 59 | } 60 | 61 | clientset, err := k8s.NewKubeClientSet(contextName) 62 | if err != nil { 63 | l.Error("kube config error", zap.String("error", err.Error())) 64 | os.Exit(1) 65 | } 66 | 67 | list, err := clientset.CoreV1().Pods(namespace).List(metav1.ListOptions{}) 68 | if err != nil { 69 | l.Error("error", zap.String("error", err.Error())) 70 | os.Exit(1) 71 | } 72 | 73 | c, err := config.NewConfig() 74 | if err != nil { 75 | l.Error("error", zap.String("error", err.Error())) 76 | os.Exit(1) 77 | } 78 | 79 | re := regexp.MustCompile(`[+\-*\/%.]`) 80 | reNumber := regexp.MustCompile(`^\d+$`) 81 | tagetSetMap := map[string]struct{}{} 82 | 83 | for _, i := range list.Items { 84 | source := strings.Join([]string{"k8s:/", contextName, i.ObjectMeta.Namespace, fmt.Sprintf("%s*", i.ObjectMeta.GenerateName)}, "/") 85 | if _, ok := tagetSetMap[source]; ok { 86 | continue 87 | } else { 88 | tagetSetMap[source] = struct{}{} 89 | } 90 | tags := []string{re.ReplaceAllString(contextName, "_"), re.ReplaceAllString(i.ObjectMeta.Namespace, "_")} 91 | for _, v := range i.ObjectMeta.Labels { 92 | if reNumber.MatchString(v) { 93 | continue 94 | } 95 | switch v { 96 | case "true", "false": 97 | continue 98 | default: 99 | tags = append(tags, re.ReplaceAllString(v, "_")) 100 | } 101 | } 102 | c.TargetSets = append(c.TargetSets, &config.TargetSet{ 103 | Sources: []string{source}, 104 | Description: "Generated by `hrv generate-k8s-config`", 105 | Type: "k8s", 106 | MultiLine: false, 107 | Tags: uniqueTags(tags), 108 | }) 109 | } 110 | y, err := yaml.Marshal(&c) 111 | if err != nil { 112 | l.Error("generate error", zap.String("error", err.Error())) 113 | os.Exit(1) 114 | } 115 | fmt.Printf("%s\n", string(y)) 116 | }, 117 | } 118 | 119 | func init() { 120 | rootCmd.AddCommand(generateK8sConfigCmd) 121 | generateK8sConfigCmd.Flags().StringVarP(&contextName, "context", "c", "", "kubernetes context. default:current context") 122 | generateK8sConfigCmd.Flags().StringVarP(&namespace, "namespace", "n", "", "kubernetes namespace") 123 | } 124 | 125 | func uniqueTags(tags []string) []string { 126 | keys := make(map[string]bool) 127 | list := []string{} 128 | for _, t := range tags { 129 | if _, value := keys[t]; !value { 130 | keys[t] = true 131 | list = append(list, t) 132 | } 133 | } 134 | return list 135 | } 136 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/info.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2019 Ken'ichiro Oyama 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | package cmd 23 | 24 | import ( 25 | "context" 26 | "fmt" 27 | "os" 28 | 29 | "github.com/k1LoW/harvest/db" 30 | "github.com/k1LoW/harvest/logger" 31 | "github.com/spf13/cobra" 32 | "go.uber.org/zap" 33 | ) 34 | 35 | // infoCmd represents the info command 36 | var infoCmd = &cobra.Command{ 37 | Use: "info [DB_FILE]", 38 | Short: "print information of harvest-*.db", 39 | Long: `print information of harvest-*.db.`, 40 | Args: cobra.ExactArgs(1), 41 | Run: func(cmd *cobra.Command, args []string) { 42 | l := logger.NewLogger(verbose) 43 | dbPath := args[0] 44 | 45 | if _, err := os.Lstat(dbPath); err != nil { 46 | l.Error(fmt.Sprintf("%s not exists", dbPath), zap.String("error", err.Error())) 47 | os.Exit(1) 48 | } 49 | 50 | ctx, cancel := context.WithCancel(context.Background()) 51 | defer cancel() 52 | 53 | d, err := db.AttachDB(ctx, l, dbPath) 54 | if err != nil { 55 | l.Error("DB attach error", zap.String("error", err.Error())) 56 | os.Exit(1) 57 | } 58 | 59 | metas, err := d.GetMetaAll() 60 | if err != nil { 61 | l.Error("info error", zap.String("error", err.Error())) 62 | os.Exit(1) 63 | } 64 | for _, m := range metas { 65 | fmt.Printf("%s=%s\n", m.Key, m.Value) 66 | } 67 | }, 68 | } 69 | 70 | func init() { 71 | infoCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print debugging messages.") 72 | err := infoCmd.MarkZshCompPositionalArgumentFile(1) 73 | if err != nil { 74 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 75 | os.Exit(1) 76 | } 77 | rootCmd.AddCommand(infoCmd) 78 | } 79 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/logs.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "os" 27 | "sync" 28 | 29 | "github.com/k1LoW/harvest/collector" 30 | "github.com/k1LoW/harvest/config" 31 | "github.com/k1LoW/harvest/logger" 32 | "github.com/k1LoW/harvest/parser" 33 | "github.com/spf13/cobra" 34 | "go.uber.org/zap" 35 | ) 36 | 37 | // logsCmd represents the logs command 38 | var logsCmd = &cobra.Command{ 39 | Use: "logs", 40 | Short: "list target logs", 41 | Long: `list target logs.`, 42 | Run: func(cmd *cobra.Command, args []string) { 43 | l := logger.NewLogger(verbose) 44 | 45 | cfg, err := config.NewConfig() 46 | if err != nil { 47 | l.Error("Config error", zap.String("error", err.Error())) 48 | os.Exit(1) 49 | } 50 | err = cfg.LoadConfigFile(configPath) 51 | if err != nil { 52 | l.Error("Config error", zap.String("error", err.Error())) 53 | os.Exit(1) 54 | } 55 | 56 | ctx, cancel := context.WithCancel(context.Background()) 57 | defer cancel() 58 | 59 | targets, err := cfg.FilterTargets(tag, sourceRe) 60 | if err != nil { 61 | l.Error("tag option error", zap.String("error", err.Error())) 62 | os.Exit(1) 63 | } 64 | if len(targets) == 0 { 65 | l.Error("No targets") 66 | os.Exit(1) 67 | } 68 | 69 | if presetSSHKeyPassphrase { 70 | err = presetSSHKeyPassphraseToTargets(targets) 71 | if err != nil { 72 | l.Error("option error", zap.String("error", err.Error())) 73 | os.Exit(1) 74 | } 75 | } 76 | 77 | st, et, err := parseTimes(stStr, etStr, duStr) 78 | if err != nil { 79 | l.Error("option error", zap.String("error", err.Error())) 80 | os.Exit(1) 81 | } 82 | 83 | l.Info(fmt.Sprintf("Log timestamp: %s - %s", st.Format("2006-01-02 15:04:05-0700"), et.Format("2006-01-02 15:04:05-0700"))) 84 | 85 | logChan := make(chan parser.Log) 86 | 87 | waiter := make(chan struct{}) 88 | 89 | go func() { 90 | defer func() { 91 | waiter <- struct{}{} 92 | }() 93 | for log := range logChan { 94 | fmt.Printf("%s:%s\n", log.Host, log.Content) 95 | } 96 | }() 97 | 98 | cChan := make(chan struct{}, concurrency) 99 | var wg sync.WaitGroup 100 | 101 | for _, t := range targets { 102 | wg.Add(1) 103 | go func(t *config.Target) { 104 | cChan <- struct{}{} 105 | defer wg.Done() 106 | c, err := collector.NewCollector(ctx, t, l) 107 | if err != nil { 108 | l.Error("Ls error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 109 | } 110 | err = c.LsLogs(logChan, st, et) 111 | if err != nil { 112 | l.Error("Ls error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 113 | } 114 | <-cChan 115 | }(t) 116 | } 117 | 118 | wg.Wait() 119 | close(logChan) 120 | <-waiter 121 | }, 122 | } 123 | 124 | func init() { 125 | rootCmd.AddCommand(logsCmd) 126 | logsCmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path") 127 | _ = logsCmd.MarkFlagFilename("config", "yaml", "yml") 128 | logsCmd.Flags().IntVarP(&concurrency, "concurrency", "C", defaultConcurrency, "concurrency") 129 | logsCmd.Flags().StringVarP(&tag, "tag", "", "", "filter targets using tag") 130 | logsCmd.Flags().StringVarP(&sourceRe, "source", "", "", "filter targets using source regexp") 131 | logsCmd.Flags().StringVarP(&stStr, "start-time", "", "", "log start time (format: 2006-01-02 15:04:05)") 132 | logsCmd.Flags().StringVarP(&etStr, "end-time", "", "", "log end time (default: latest) (format: 2006-01-02 15:04:05)") 133 | logsCmd.Flags().StringVarP(&duStr, "duration", "", "", "log duration") 134 | logsCmd.Flags().BoolVarP(&presetSSHKeyPassphrase, "preset-ssh-key-passphrase", "", false, "preset SSH key passphrase") 135 | logsCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print debugging messages.") 136 | } 137 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/root.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | "time" 27 | 28 | "github.com/Songmu/prompter" 29 | "github.com/araddon/dateparse" 30 | "github.com/k1LoW/duration" 31 | "github.com/k1LoW/harvest/config" 32 | "github.com/spf13/cobra" 33 | ) 34 | 35 | const ( 36 | defaultDuration = "1 hour" 37 | ) 38 | 39 | var ( 40 | tag string 41 | configPath string 42 | sourceRe string 43 | withTimestamp bool 44 | withTimestampNano bool 45 | withHost bool 46 | withPath bool 47 | withTag bool 48 | withoutMark bool 49 | noColor bool 50 | presetSSHKeyPassphrase bool 51 | verbose bool 52 | ) 53 | 54 | // rootCmd represents the base command when called without any subcommands 55 | var rootCmd = &cobra.Command{ 56 | Use: "hrv", 57 | Short: "Portable log aggregation tool for middle-scale system operation/troubleshooting", 58 | Long: `Portable log aggregation tool for middle-scale system operation/troubleshooting.`, 59 | } 60 | 61 | func Execute() { 62 | if err := rootCmd.Execute(); err != nil { 63 | fmt.Println(err) 64 | os.Exit(1) 65 | } 66 | } 67 | 68 | func init() {} 69 | 70 | type hostPassphrase struct { 71 | host string 72 | passphrase []byte 73 | } 74 | 75 | func presetSSHKeyPassphraseToTargets(targets []*config.Target) error { 76 | hpMap := map[string]hostPassphrase{} 77 | var defaultPassohrase []byte 78 | 79 | yn := prompter.YN("Do you preset default passphrase for all targets?", true) 80 | if yn { 81 | fmt.Println("Preset default passphrase") 82 | defaultPassohrase = []byte(prompter.Password("Enter default passphrase")) 83 | } else { 84 | fmt.Println("Preset passphrase for each target") 85 | } 86 | 87 | for i, target := range targets { 88 | if target.Scheme != "ssh" { 89 | continue 90 | } 91 | if yn { 92 | targets[i].SSHKeyPassphrase = defaultPassohrase 93 | continue 94 | } 95 | if hp, ok := hpMap[target.Host]; ok { 96 | targets[i].SSHKeyPassphrase = hp.passphrase 97 | continue 98 | } 99 | passphrase := []byte(prompter.Password(fmt.Sprintf("Enter passphrase for host '%s'", target.Host))) 100 | targets[i].SSHKeyPassphrase = passphrase 101 | hpMap[target.Host] = hostPassphrase{ 102 | host: target.Host, 103 | passphrase: passphrase, 104 | } 105 | } 106 | return nil 107 | } 108 | 109 | func parseTimes(stStr, etStr, duStr string) (*time.Time, *time.Time, error) { 110 | var ( 111 | stt time.Time 112 | ett time.Time 113 | ) 114 | loc, err := time.LoadLocation("Local") 115 | if err != nil { 116 | return nil, nil, err 117 | } 118 | if stStr != "" { 119 | layout, err := dateparse.ParseFormat(stStr) 120 | if err != nil { 121 | return nil, nil, err 122 | } 123 | stt, err = time.ParseInLocation(layout, stStr, loc) 124 | if err != nil { 125 | return nil, nil, err 126 | } 127 | } 128 | if etStr != "" { 129 | layout, err := dateparse.ParseFormat(etStr) 130 | if err != nil { 131 | return nil, nil, err 132 | } 133 | ett, err = time.ParseInLocation(layout, etStr, loc) 134 | if err != nil { 135 | return nil, nil, err 136 | } 137 | } 138 | 139 | switch { 140 | case stStr != "" && etStr != "": 141 | case stStr != "" && etStr == "": 142 | if duStr == "" { 143 | ett = time.Now() 144 | } else { 145 | du, err := duration.Parse(duStr) 146 | if err != nil { 147 | return nil, nil, err 148 | } 149 | ett = stt.Add(du) 150 | } 151 | case stStr == "" && etStr != "": 152 | du, err := duration.Parse(duStr) 153 | if err != nil { 154 | return nil, nil, err 155 | } 156 | stt = ett.Add(-du) 157 | case stStr == "" && etStr == "": 158 | ett = time.Now() 159 | if duStr == "" { 160 | duStr = defaultDuration 161 | } 162 | du, err := duration.Parse(duStr) 163 | if err != nil { 164 | return nil, nil, err 165 | } 166 | stt = ett.Add(-du) 167 | } 168 | return &stt, &ett, nil 169 | } 170 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/stream.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "os" 27 | "strings" 28 | "sync" 29 | "time" 30 | 31 | "github.com/k1LoW/harvest/collector" 32 | "github.com/k1LoW/harvest/config" 33 | "github.com/k1LoW/harvest/logger" 34 | "github.com/k1LoW/harvest/parser" 35 | "github.com/k1LoW/harvest/stdout" 36 | "github.com/spf13/cobra" 37 | "go.uber.org/zap" 38 | ) 39 | 40 | // streamCmd represents the stream command 41 | var streamCmd = &cobra.Command{ 42 | Use: "stream", 43 | Short: "output stream from targets", 44 | Long: `output stream from targets.`, 45 | Run: func(cmd *cobra.Command, args []string) { 46 | l := logger.NewLogger(verbose) 47 | 48 | cfg, err := config.NewConfig() 49 | if err != nil { 50 | l.Error("Config error", zap.String("error", err.Error())) 51 | os.Exit(1) 52 | } 53 | err = cfg.LoadConfigFile(configPath) 54 | if err != nil { 55 | l.Error("Config error", zap.String("error", err.Error())) 56 | os.Exit(1) 57 | } 58 | 59 | ctx, cancel := context.WithCancel(context.Background()) 60 | defer cancel() 61 | 62 | targets, err := cfg.FilterTargets(tag, sourceRe) 63 | if err != nil { 64 | l.Error("tag option error", zap.String("error", err.Error())) 65 | os.Exit(1) 66 | } 67 | if len(targets) == 0 { 68 | l.Error("No targets") 69 | os.Exit(1) 70 | } 71 | l.Info(fmt.Sprintf("Target count: %d", len(targets))) 72 | 73 | if presetSSHKeyPassphrase { 74 | err = presetSSHKeyPassphraseToTargets(targets) 75 | if err != nil { 76 | l.Error("option error", zap.String("error", err.Error())) 77 | os.Exit(1) 78 | } 79 | } 80 | 81 | hLen, tLen, err := getStreamStdoutLengthes(targets, withHost, withPath, withTag) 82 | if err != nil { 83 | l.Error("option error", zap.String("error", err.Error())) 84 | os.Exit(1) 85 | } 86 | 87 | sout, err := stdout.NewStdout( 88 | withTimestamp, 89 | withTimestampNano, 90 | withHost, 91 | withPath, 92 | withTag, 93 | withoutMark, 94 | hLen, 95 | tLen, 96 | noColor, 97 | ) 98 | if err != nil { 99 | l.Error("fetch error", zap.String("error", err.Error())) 100 | os.Exit(1) 101 | } 102 | 103 | hosts := getHosts(targets) 104 | logChan := make(chan parser.Log) 105 | 106 | go sout.Out(logChan, hosts) 107 | 108 | var wg sync.WaitGroup 109 | 110 | for _, t := range targets { 111 | wg.Add(1) 112 | go func(t *config.Target) { 113 | defer wg.Done() 114 | c, err := collector.NewCollector(ctx, t, l) 115 | if err != nil { 116 | l.Error("Stream error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 117 | return 118 | } 119 | err = c.Stream(logChan, t.MultiLine) 120 | if err != nil { 121 | l.Error("Stream error", zap.String("host", t.Host), zap.String("path", t.Path), zap.String("error", err.Error())) 122 | } 123 | }(t) 124 | time.Sleep(100 * time.Millisecond) 125 | } 126 | 127 | wg.Wait() 128 | }, 129 | } 130 | 131 | func getHosts(targets []*config.Target) []string { 132 | hosts := []string{} 133 | for _, target := range targets { 134 | hosts = append(hosts, target.Host) 135 | } 136 | return hosts 137 | } 138 | 139 | func getStreamStdoutLengthes(targets []*config.Target, withHost, withPath, withTag bool) (int, int, error) { 140 | var ( 141 | hLen int 142 | tLen int 143 | err error 144 | ) 145 | if withHost && withPath { 146 | hLen, err = getMaxLength(targets, "hostpath") 147 | if err != nil { 148 | return 0, 0, err 149 | } 150 | } else if withHost { 151 | hLen, err = getMaxLength(targets, "host") 152 | if err != nil { 153 | return 0, 0, err 154 | } 155 | } else if withPath { 156 | hLen, err = getMaxLength(targets, "path") 157 | if err != nil { 158 | return 0, 0, err 159 | } 160 | } 161 | if withTag { 162 | tLen, err = getMaxLength(targets, "tags") 163 | if err != nil { 164 | return 0, 0, err 165 | } 166 | } 167 | return hLen, tLen, nil 168 | } 169 | 170 | func getMaxLength(targets []*config.Target, key string) (int, error) { 171 | var ( 172 | length int 173 | err error 174 | ) 175 | for _, target := range targets { 176 | var c int 177 | switch key { 178 | case "host": 179 | c = target.GetHostLength() 180 | case "path": 181 | c, err = target.GetPathLength() 182 | if err != nil { 183 | return 0, err 184 | } 185 | case "hostpath": 186 | pLen, err := target.GetPathLength() 187 | if err != nil { 188 | return 0, err 189 | } 190 | c = target.GetHostLength() + pLen 191 | case "tags": 192 | c = len(fmt.Sprintf("[%s]", strings.Join(target.Tags, "]["))) 193 | } 194 | if length < c { 195 | length = c 196 | } 197 | } 198 | return length, nil 199 | } 200 | 201 | func init() { 202 | rootCmd.AddCommand(streamCmd) 203 | streamCmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path") 204 | _ = streamCmd.MarkFlagFilename("config", "yaml", "yml") 205 | streamCmd.Flags().BoolVarP(&withTimestamp, "with-timestamp", "", false, "output with timestamp") 206 | streamCmd.Flags().BoolVarP(&withTimestampNano, "with-timestamp-nano", "", false, "output with timestamp nano sec") 207 | streamCmd.Flags().BoolVarP(&withHost, "with-host", "", false, "output with host") 208 | streamCmd.Flags().BoolVarP(&withPath, "with-path", "", false, "output with path") 209 | streamCmd.Flags().BoolVarP(&withTag, "with-tag", "", false, "output with tag") 210 | streamCmd.Flags().BoolVarP(&withoutMark, "without-mark", "", false, "output without prefix mark") 211 | streamCmd.Flags().StringVarP(&tag, "tag", "", "", "filter targets using tag (format: foo,bar)") 212 | streamCmd.Flags().StringVarP(&sourceRe, "source", "", "", "filter targets using source regexp") 213 | streamCmd.Flags().BoolVarP(&noColor, "no-color", "", false, "disable colorize output") 214 | streamCmd.Flags().BoolVarP(&presetSSHKeyPassphrase, "preset-ssh-key-passphrase", "", false, "preset SSH key passphrase") 215 | streamCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print debugging messages.") 216 | } 217 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/tags.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | 27 | "github.com/k1LoW/harvest/config" 28 | "github.com/k1LoW/harvest/logger" 29 | "github.com/spf13/cobra" 30 | "go.uber.org/zap" 31 | ) 32 | 33 | // tagsCmd represents the targets command 34 | var tagsCmd = &cobra.Command{ 35 | Use: "tags", 36 | Short: "list tags", 37 | Long: `list tags.`, 38 | Run: func(cmd *cobra.Command, args []string) { 39 | l := logger.NewLogger(verbose) 40 | 41 | cfg, err := config.NewConfig() 42 | if err != nil { 43 | l.Error("Config error", zap.String("error", err.Error())) 44 | os.Exit(1) 45 | } 46 | err = cfg.LoadConfigFile(configPath) 47 | if err != nil { 48 | l.Error("Config error", zap.String("error", err.Error())) 49 | os.Exit(1) 50 | } 51 | tags := cfg.Tags() 52 | for tag, count := range tags { 53 | fmt.Printf("%s:%d\n", tag, count) 54 | } 55 | }, 56 | } 57 | 58 | func init() { 59 | rootCmd.AddCommand(tagsCmd) 60 | tagsCmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path") 61 | _ = tagsCmd.MarkFlagFilename("config", "yaml", "yml") 62 | } 63 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/targets.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | 27 | "github.com/k1LoW/harvest/config" 28 | "github.com/k1LoW/harvest/logger" 29 | "github.com/spf13/cobra" 30 | "go.uber.org/zap" 31 | ) 32 | 33 | // targetsCmd represents the targets command 34 | var targetsCmd = &cobra.Command{ 35 | Use: "targets", 36 | Short: "list targets", 37 | Long: `list targets.`, 38 | Run: func(cmd *cobra.Command, args []string) { 39 | l := logger.NewLogger(verbose) 40 | 41 | cfg, err := config.NewConfig() 42 | if err != nil { 43 | l.Error("Config error", zap.String("error", err.Error())) 44 | os.Exit(1) 45 | } 46 | err = cfg.LoadConfigFile(configPath) 47 | if err != nil { 48 | l.Error("Config error", zap.String("error", err.Error())) 49 | os.Exit(1) 50 | } 51 | targets, err := cfg.FilterTargets(tag, sourceRe) 52 | if err != nil { 53 | l.Error("tag option error", zap.String("error", err.Error())) 54 | os.Exit(1) 55 | } 56 | if len(targets) == 0 { 57 | l.Error("No targets") 58 | os.Exit(1) 59 | } 60 | for _, t := range targets { 61 | host := t.Host 62 | if host == "" { 63 | host = "localhost" 64 | } 65 | fmt.Printf("%s:%s\n", host, t.Path) 66 | } 67 | }, 68 | } 69 | 70 | func init() { 71 | rootCmd.AddCommand(targetsCmd) 72 | targetsCmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path") 73 | _ = targetsCmd.MarkFlagFilename("config", "yaml", "yml") 74 | targetsCmd.Flags().StringVarP(&tag, "tag", "", "", "filter targets using tag (format: foo,bar)") 75 | targetsCmd.Flags().StringVarP(&sourceRe, "source", "", "", "filter targets using source regexp") 76 | } 77 | -------------------------------------------------------------------------------- /cmd/hrv/cmd/version.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | 26 | "github.com/k1LoW/harvest/version" 27 | "github.com/spf13/cobra" 28 | ) 29 | 30 | // versionCmd represents the version command 31 | var versionCmd = &cobra.Command{ 32 | Use: "version", 33 | Short: "print hrv version", 34 | Long: `print hrv version.`, 35 | Run: func(cmd *cobra.Command, args []string) { 36 | fmt.Println(version.Version) 37 | }, 38 | } 39 | 40 | func init() { 41 | rootCmd.AddCommand(versionCmd) 42 | } 43 | -------------------------------------------------------------------------------- /cmd/hrv/main.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package main 22 | 23 | import ( 24 | "github.com/k1LoW/harvest/cmd/hrv/cmd" 25 | _ "github.com/mattn/go-sqlite3" 26 | ) 27 | 28 | func main() { 29 | cmd.Execute() 30 | } 31 | -------------------------------------------------------------------------------- /collector/collector.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/k1LoW/harvest/client" 9 | "github.com/k1LoW/harvest/config" 10 | "github.com/k1LoW/harvest/parser" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | // Collector ... 15 | type Collector struct { 16 | client client.Client 17 | parser parser.Parser 18 | target *config.Target 19 | ctx context.Context 20 | logger *zap.Logger 21 | } 22 | 23 | // NewCollector ... 24 | func NewCollector(ctx context.Context, t *config.Target, l *zap.Logger) (*Collector, error) { 25 | var ( 26 | err error 27 | c client.Client 28 | p parser.Parser 29 | ) 30 | 31 | l = l.With(zap.String("host", t.Host), zap.String("path", t.Path)) 32 | 33 | // Set client 34 | switch t.Scheme { 35 | case "ssh": 36 | sshc, err := client.NewSSHClient(l, t.Host, t.User, t.Port, t.Path, t.SSHKeyPassphrase) 37 | if err != nil { 38 | return nil, err 39 | } 40 | c = sshc 41 | case "file": 42 | filec, err := client.NewFileClient(l, t.Path) 43 | if err != nil { 44 | return nil, err 45 | } 46 | c = filec 47 | case "k8s": 48 | k8sc, err := client.NewK8sClient(l, t.Host, t.Path) 49 | if err != nil { 50 | return nil, err 51 | } 52 | c = k8sc 53 | default: 54 | return nil, fmt.Errorf("unsupport scheme: %s", t.Scheme) 55 | } 56 | 57 | // Set parser 58 | switch t.Type { 59 | case "syslog": 60 | p, err = parser.NewSyslogParser(t, l) 61 | if err != nil { 62 | return nil, err 63 | } 64 | case "combinedLog": 65 | p, err = parser.NewCombinedLogParser(t, l) 66 | if err != nil { 67 | return nil, err 68 | } 69 | case "none", "k8s": 70 | p, err = parser.NewNoneParser(t, l) 71 | if err != nil { 72 | return nil, err 73 | } 74 | default: // regexp 75 | p, err = parser.NewRegexpParser(t, l) 76 | if err != nil { 77 | return nil, err 78 | } 79 | } 80 | 81 | return &Collector{ 82 | client: c, 83 | parser: p, 84 | target: t, 85 | ctx: ctx, 86 | logger: l, 87 | }, nil 88 | } 89 | 90 | // Fetch ... 91 | func (c *Collector) Fetch(dbChan chan parser.Log, st *time.Time, et *time.Time, multiLine bool) error { 92 | waiter := make(chan struct{}) 93 | innerCtx, cancel := context.WithCancel(c.ctx) 94 | defer cancel() 95 | 96 | go func() { 97 | defer func() { 98 | cancel() 99 | waiter <- struct{}{} 100 | }() 101 | for log := range c.parser.Parse(innerCtx, cancel, c.client.Out(), c.target.TimeZone, st, et) { 102 | dbChan <- log 103 | } 104 | }() 105 | 106 | err := c.client.Read(innerCtx, st, et, c.target.TimeFormat, c.target.TimeZone) 107 | if err != nil { 108 | return err 109 | } 110 | 111 | <-waiter 112 | return nil 113 | } 114 | 115 | // Stream ... 116 | func (c *Collector) Stream(logChan chan parser.Log, multiLine bool) error { 117 | waiter := make(chan struct{}) 118 | innerCtx, cancel := context.WithCancel(c.ctx) 119 | defer cancel() 120 | 121 | go func() { 122 | defer func() { 123 | cancel() 124 | waiter <- struct{}{} 125 | }() 126 | for log := range c.parser.Parse(innerCtx, cancel, c.client.Out(), c.target.TimeZone, nil, nil) { 127 | logChan <- log 128 | } 129 | }() 130 | 131 | err := c.client.Tailf(innerCtx) 132 | if err != nil { 133 | return err 134 | } 135 | 136 | <-waiter 137 | return nil 138 | } 139 | 140 | // LsLogs ... 141 | func (c *Collector) LsLogs(logChan chan parser.Log, st *time.Time, et *time.Time) error { 142 | waiter := make(chan struct{}) 143 | innerCtx, cancel := context.WithCancel(c.ctx) 144 | defer cancel() 145 | 146 | go func() { 147 | defer func() { 148 | cancel() 149 | waiter <- struct{}{} 150 | }() 151 | for line := range c.client.Out() { 152 | logChan <- parser.Log{ 153 | Host: line.Host, 154 | Path: line.Path, 155 | Content: line.Content, 156 | Target: c.target, 157 | } 158 | } 159 | }() 160 | 161 | err := c.client.Ls(innerCtx, st, et) 162 | if err != nil { 163 | return err 164 | } 165 | 166 | <-waiter 167 | return nil 168 | } 169 | 170 | // Copy ... 171 | func (c *Collector) Copy(logChan chan parser.Log, st *time.Time, et *time.Time, dstDir string) error { 172 | waiter := make(chan struct{}) 173 | innerCtx, cancel := context.WithCancel(c.ctx) 174 | defer cancel() 175 | fileChan := make(chan parser.Log) 176 | 177 | go func() { 178 | defer func() { 179 | cancel() 180 | waiter <- struct{}{} 181 | }() 182 | files := []parser.Log{} 183 | for file := range fileChan { 184 | files = append(files, file) 185 | } 186 | for _, file := range files { 187 | filePath := file.Content 188 | c.logger.Debug(fmt.Sprintf("Start copying %s", filePath), zap.String("host", c.target.Host), zap.String("path", c.target.Path)) 189 | err := c.client.Copy(innerCtx, filePath, dstDir) 190 | if err != nil { 191 | c.logger.Error("Copy error", zap.String("host", c.target.Host), zap.String("path", c.target.Path), zap.String("error", err.Error())) 192 | } else { 193 | c.logger.Debug(fmt.Sprintf("Copy %s finished", filePath), zap.String("host", c.target.Host), zap.String("path", c.target.Path)) 194 | } 195 | } 196 | }() 197 | 198 | err := c.LsLogs(fileChan, st, et) 199 | if err != nil { 200 | return err 201 | } 202 | close(fileChan) 203 | 204 | <-waiter 205 | return nil 206 | } 207 | 208 | // ConfigTest ... 209 | func (c *Collector) ConfigTest(logChan chan parser.Log, multiLine bool) error { 210 | waiter := make(chan struct{}) 211 | innerCtx, cancel := context.WithCancel(c.ctx) 212 | defer cancel() 213 | 214 | go func() { 215 | defer func() { 216 | cancel() 217 | waiter <- struct{}{} 218 | }() 219 | for log := range c.parser.Parse(innerCtx, cancel, c.client.Out(), c.target.TimeZone, nil, nil) { 220 | logChan <- log 221 | } 222 | }() 223 | 224 | err := c.client.RandomOne(innerCtx) 225 | if err != nil { 226 | return err 227 | } 228 | 229 | <-waiter 230 | close(logChan) 231 | return nil 232 | } 233 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/url" 7 | "path/filepath" 8 | "regexp" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/antonmedv/expr" 13 | "github.com/k1LoW/harvest/client/k8s" 14 | "github.com/pkg/errors" 15 | "gopkg.in/yaml.v2" 16 | ) 17 | 18 | // TargetSet ... 19 | type TargetSet struct { 20 | Sources []string `yaml:"sources"` 21 | Description string `yaml:"description,omitempty"` 22 | Type string `yaml:"type"` 23 | Regexp string `yaml:"regexp,omitempty"` 24 | MultiLine bool `yaml:"multiLine,omitempty"` 25 | TimeFormat string `yaml:"timeFormat,omitempty"` 26 | TimeZone string `yaml:"timeZone,omitempty"` 27 | Tags []string `yaml:"tags"` 28 | } 29 | 30 | // Target ... 31 | type Target struct { 32 | Source string `db:"source"` 33 | Description string `db:"description"` 34 | Type string `db:"type"` 35 | Regexp string `db:"regexp"` 36 | MultiLine bool `db:"multi_line"` 37 | TimeFormat string `db:"time_format"` 38 | TimeZone string `db:"time_zone"` 39 | Tags []string 40 | Scheme string `db:"scheme"` 41 | Host string `db:"host"` 42 | User string `db:"user"` 43 | Port int `db:"port"` 44 | Path string `db:"path"` 45 | SSHKeyPassphrase []byte 46 | Id int64 `db:"id"` 47 | } 48 | 49 | func (t *Target) GetHostLength() int { 50 | return len(t.Host) 51 | } 52 | 53 | func (t *Target) GetPathLength() (int, error) { 54 | if t.Scheme == "k8s" { 55 | contextName := t.Host 56 | splited := strings.Split(t.Path, "/") 57 | namespace := splited[1] 58 | podFilter := regexp.MustCompile(strings.Replace(strings.Replace(splited[2], ".*", "*", -1), "*", ".*", -1)) 59 | containers, err := k8s.GetContainers(contextName, namespace, podFilter) 60 | if err != nil { 61 | return 0, err 62 | } 63 | length := 0 64 | for _, c := range containers { 65 | if length < len(c) { 66 | length = len(c) 67 | } 68 | } 69 | return length, nil 70 | } 71 | return len(t.Path), nil 72 | } 73 | 74 | type Tags map[string]int 75 | 76 | // Config ... 77 | type Config struct { 78 | Targets []*Target `yaml:"-"` 79 | TargetSets []*TargetSet `yaml:"targetSets"` 80 | } 81 | 82 | // NewConfig ... 83 | func NewConfig() (*Config, error) { 84 | return &Config{ 85 | Targets: []*Target{}, 86 | TargetSets: []*TargetSet{}, 87 | }, nil 88 | } 89 | 90 | // LoadConfigFile ... 91 | func (c *Config) LoadConfigFile(path string) error { 92 | if path == "" { 93 | return errors.New("failed to load config file") 94 | } 95 | fullPath, err := filepath.Abs(path) 96 | if err != nil { 97 | return errors.Wrap(errors.WithStack(err), "failed to load config file") 98 | } 99 | buf, err := ioutil.ReadFile(filepath.Clean(fullPath)) 100 | if err != nil { 101 | return errors.Wrap(errors.WithStack(err), "failed to load config file") 102 | } 103 | err = yaml.Unmarshal(buf, c) 104 | if err != nil { 105 | return errors.Wrap(errors.WithStack(err), "failed to load config file") 106 | } 107 | for _, t := range c.TargetSets { 108 | for _, src := range t.Sources { 109 | target := Target{} 110 | target.Source = src 111 | target.Description = t.Description 112 | target.Type = t.Type 113 | target.Regexp = t.Regexp 114 | target.MultiLine = t.MultiLine 115 | target.TimeFormat = t.TimeFormat 116 | target.TimeZone = t.TimeZone 117 | target.Tags = t.Tags 118 | 119 | u, err := url.Parse(src) 120 | if err != nil { 121 | return err 122 | } 123 | target.Scheme = u.Scheme 124 | target.Path = u.Path 125 | target.User = u.User.Username() 126 | if strings.Contains(u.Host, ":") { 127 | splited := strings.Split(u.Host, ":") 128 | target.Host = splited[0] 129 | target.Port, _ = strconv.Atoi(splited[1]) 130 | } else { 131 | target.Host = u.Host 132 | target.Port = 0 133 | } 134 | if target.Host == "" { 135 | target.Host = "localhost" 136 | } 137 | 138 | c.Targets = append(c.Targets, &target) 139 | } 140 | } 141 | return nil 142 | } 143 | 144 | func (c *Config) Tags() Tags { 145 | tags := map[string]int{} 146 | for _, t := range c.TargetSets { 147 | for _, tag := range t.Tags { 148 | if _, ok := tags[tag]; !ok { 149 | tags[tag] = 0 150 | } 151 | tags[tag] = tags[tag] + 1 152 | } 153 | } 154 | return tags 155 | } 156 | 157 | func (c *Config) FilterTargets(tagExpr, sourceRe string) ([]*Target, error) { 158 | allTags := c.Tags() 159 | targets := []*Target{} 160 | tagExpr = strings.Replace(tagExpr, ",", " or ", -1) 161 | for _, target := range c.Targets { 162 | tags := map[string]interface{}{ 163 | "hrv_source": target.Source, 164 | } 165 | for tag := range allTags { 166 | if contains(target.Tags, tag) { 167 | tags[tag] = true 168 | } else { 169 | tags[tag] = false 170 | } 171 | } 172 | targetExpr := []string{"true"} 173 | if tagExpr != "" { 174 | targetExpr = append(targetExpr, fmt.Sprintf("(%s)", tagExpr)) 175 | } 176 | if sourceRe != "" { 177 | targetExpr = append(targetExpr, fmt.Sprintf(`(hrv_source matches "%s")`, sourceRe)) 178 | } 179 | out, err := expr.Eval(strings.Join(targetExpr, " and "), tags) 180 | if err != nil { 181 | return targets, err 182 | } 183 | if out.(bool) { 184 | targets = append(targets, target) 185 | } 186 | } 187 | return targets, nil 188 | } 189 | 190 | func contains(ss []string, t string) bool { 191 | for _, s := range ss { 192 | if s == t { 193 | return true 194 | } 195 | } 196 | return false 197 | } 198 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | ) 8 | 9 | func TestFilterTargets(t *testing.T) { 10 | var tests = []struct { 11 | tagExpr string 12 | sourceRe string 13 | want int 14 | }{ 15 | {"", "", 8}, 16 | {"webproxy", "", 2}, 17 | {"webproxy and syslog", "", 1}, 18 | {"webproxy or app", "", 5}, 19 | {"!app", "", 5}, 20 | {"webproxy,app", "", 5}, 21 | {"app", ".*app-1.*", 1}, 22 | {"", "app-1", 1}, 23 | {"db", ".*app-1.*", 0}, 24 | } 25 | 26 | for _, tt := range tests { 27 | c, err := NewConfig() 28 | if err != nil { 29 | t.Fatalf("%v", err) 30 | } 31 | err = c.LoadConfigFile(filepath.Join(testdataDir(), "test_config.yml")) 32 | if err != nil { 33 | t.Fatalf("%v", err) 34 | } 35 | filtered, err := c.FilterTargets(tt.tagExpr, tt.sourceRe) 36 | if err != nil { 37 | t.Fatalf("%v", err) 38 | } 39 | got := len(filtered) 40 | if got != tt.want { 41 | t.Errorf("\ngot %v\nwant %v", got, tt.want) 42 | } 43 | } 44 | } 45 | 46 | func testdataDir() string { 47 | wd, _ := os.Getwd() 48 | dir, _ := filepath.Abs(filepath.Join(filepath.Dir(wd), "testdata")) 49 | return dir 50 | } 51 | -------------------------------------------------------------------------------- /doc/fetch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/k1LoW/harvest/dc5cbf8bb64110506e11cb7709d9541ee1062f15/doc/fetch.png -------------------------------------------------------------------------------- /doc/schema/README.md: -------------------------------------------------------------------------------- 1 | # harvest.db 2 | 3 | ## Tables 4 | 5 | | Name | Columns | Comment | Type | 6 | | ------------------------------- | ------- | ------- | ------------- | 7 | | [targets](targets.md) | 13 | | table | 8 | | [tags](tags.md) | 2 | | table | 9 | | [targets_tags](targets_tags.md) | 3 | | table | 10 | | [logs](logs.md) | 14 | | virtual table | 11 | | [metas](metas.md) | 3 | | table | 12 | 13 | ## Relations 14 | 15 | ![er](schema.svg) 16 | 17 | --- 18 | 19 | > Generated by [tbls](https://github.com/k1LoW/tbls) 20 | -------------------------------------------------------------------------------- /doc/schema/logs.md: -------------------------------------------------------------------------------- 1 | # logs 2 | 3 | ## Description 4 | 5 |
6 | Table Definition 7 | 8 | ```sql 9 | CREATE VIRTUAL TABLE logs USING FTS4( 10 | host, 11 | path, 12 | target_id INTEGER, 13 | ts, 14 | ts_unixnano INTEGER, 15 | ts_year INTEGER, 16 | ts_month INTEGER, 17 | ts_day INTEGER, 18 | ts_hour INTEGER, 19 | ts_minute INTEGER, 20 | ts_second INTEGER, 21 | ts_time_zone, 22 | filled_by_prev_ts INTEGER, 23 | content 24 | ) 25 | ``` 26 | 27 |
28 | 29 | ## Columns 30 | 31 | | Name | Type | Default | Nullable | Children | Parents | Comment | 32 | | ----------------- | ---- | ------- | -------- | -------- | --------------------- | ------- | 33 | | host | | | true | | | | 34 | | path | | | true | | | | 35 | | target_id | | | true | | [targets](targets.md) | | 36 | | ts | | | true | | | | 37 | | ts_unixnano | | | true | | | | 38 | | ts_year | | | true | | | | 39 | | ts_month | | | true | | | | 40 | | ts_day | | | true | | | | 41 | | ts_hour | | | true | | | | 42 | | ts_minute | | | true | | | | 43 | | ts_second | | | true | | | | 44 | | ts_time_zone | | | true | | | | 45 | | filled_by_prev_ts | | | true | | | | 46 | | content | | | true | | | | 47 | 48 | ## Relations 49 | 50 | ![er](logs.svg) 51 | 52 | --- 53 | 54 | > Generated by [tbls](https://github.com/k1LoW/tbls) 55 | -------------------------------------------------------------------------------- /doc/schema/logs.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | logs 11 | 12 | 13 | 14 | logs 15 | 16 | 17 | logs 18 | 19 | [virtual table] 20 | 21 | host 22 | [] 23 | 24 | path 25 | [] 26 | 27 | target_id 28 | [] 29 | 30 | ts 31 | [] 32 | 33 | ts_unixnano 34 | [] 35 | 36 | ts_year 37 | [] 38 | 39 | ts_month 40 | [] 41 | 42 | ts_day 43 | [] 44 | 45 | ts_hour 46 | [] 47 | 48 | ts_minute 49 | [] 50 | 51 | ts_second 52 | [] 53 | 54 | ts_time_zone 55 | [] 56 | 57 | filled_by_prev_ts 58 | [] 59 | 60 | content 61 | [] 62 | 63 | 64 | 65 | 66 | targets 67 | 68 | 69 | targets 70 | 71 | [table] 72 | 73 | id 74 | [INTEGER] 75 | 76 | source 77 | [TEXT] 78 | 79 | description 80 | [TEXT] 81 | 82 | type 83 | [TEXT] 84 | 85 | regexp 86 | [TEXT] 87 | 88 | multi_line 89 | [INTEGER] 90 | 91 | time_format 92 | [TEXT] 93 | 94 | time_zone 95 | [TEXT] 96 | 97 | scheme 98 | [TEXT] 99 | 100 | host 101 | [TEXT] 102 | 103 | user 104 | [TEXT] 105 | 106 | port 107 | [INTEGER] 108 | 109 | path 110 | [TEXT] 111 | 112 | 113 | 114 | logs:target_id->targets:id 115 | 116 | 117 | logs -> targets 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /doc/schema/metas.md: -------------------------------------------------------------------------------- 1 | # metas 2 | 3 | ## Description 4 | 5 |
6 | Table Definition 7 | 8 | ```sql 9 | CREATE TABLE metas ( 10 | id INTEGER PRIMARY KEY AUTOINCREMENT, 11 | key TEXT NOT NULL, 12 | value TEXT NOT NULL, 13 | UNIQUE(key) 14 | ) 15 | ``` 16 | 17 |
18 | 19 | ## Columns 20 | 21 | | Name | Type | Default | Nullable | Children | Parents | Comment | 22 | | ----- | ------- | ------- | -------- | -------- | ------- | ------- | 23 | | id | INTEGER | | true | | | | 24 | | key | TEXT | | false | | | | 25 | | value | TEXT | | false | | | | 26 | 27 | ## Constraints 28 | 29 | | Name | Type | Definition | 30 | | ------------------------ | ----------- | ---------------- | 31 | | id | PRIMARY KEY | PRIMARY KEY (id) | 32 | | sqlite_autoindex_metas_1 | UNIQUE | UNIQUE (key) | 33 | 34 | ## Indexes 35 | 36 | | Name | Definition | 37 | | ------------------------ | ------------ | 38 | | sqlite_autoindex_metas_1 | UNIQUE (key) | 39 | 40 | ## Relations 41 | 42 | ![er](metas.svg) 43 | 44 | --- 45 | 46 | > Generated by [tbls](https://github.com/k1LoW/tbls) 47 | -------------------------------------------------------------------------------- /doc/schema/metas.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | metas 11 | 12 | 13 | 14 | metas 15 | 16 | 17 | metas 18 | 19 | [table] 20 | 21 | id 22 | [INTEGER] 23 | 24 | key 25 | [TEXT] 26 | 27 | value 28 | [TEXT] 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /doc/schema/schema.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | harvest.db 11 | 12 | 13 | 14 | targets 15 | 16 | 17 | targets 18 | 19 | [table] 20 | 21 | id 22 | [INTEGER] 23 | 24 | source 25 | [TEXT] 26 | 27 | description 28 | [TEXT] 29 | 30 | type 31 | [TEXT] 32 | 33 | regexp 34 | [TEXT] 35 | 36 | multi_line 37 | [INTEGER] 38 | 39 | time_format 40 | [TEXT] 41 | 42 | time_zone 43 | [TEXT] 44 | 45 | scheme 46 | [TEXT] 47 | 48 | host 49 | [TEXT] 50 | 51 | user 52 | [TEXT] 53 | 54 | port 55 | [INTEGER] 56 | 57 | path 58 | [TEXT] 59 | 60 | 61 | 62 | tags 63 | 64 | 65 | tags 66 | 67 | [table] 68 | 69 | id 70 | [INTEGER] 71 | 72 | name 73 | [TEXT] 74 | 75 | 76 | 77 | targets_tags 78 | 79 | 80 | targets_tags 81 | 82 | [table] 83 | 84 | id 85 | [INTEGER] 86 | 87 | target_id 88 | [INTEGER] 89 | 90 | tag_id 91 | [INTEGER] 92 | 93 | 94 | 95 | targets_tags:target_id->targets:id 96 | 97 | 98 | targets_tags -> targets 99 | 100 | 101 | 102 | targets_tags:tag_id->tags:id 103 | 104 | 105 | targets_tags -> tags 106 | 107 | 108 | 109 | logs 110 | 111 | 112 | logs 113 | 114 | [virtual table] 115 | 116 | host 117 | [] 118 | 119 | path 120 | [] 121 | 122 | target_id 123 | [] 124 | 125 | ts 126 | [] 127 | 128 | ts_unixnano 129 | [] 130 | 131 | ts_year 132 | [] 133 | 134 | ts_month 135 | [] 136 | 137 | ts_day 138 | [] 139 | 140 | ts_hour 141 | [] 142 | 143 | ts_minute 144 | [] 145 | 146 | ts_second 147 | [] 148 | 149 | ts_time_zone 150 | [] 151 | 152 | filled_by_prev_ts 153 | [] 154 | 155 | content 156 | [] 157 | 158 | 159 | 160 | logs:target_id->targets:id 161 | 162 | 163 | logs -> targets 164 | 165 | 166 | 167 | metas 168 | 169 | 170 | metas 171 | 172 | [table] 173 | 174 | id 175 | [INTEGER] 176 | 177 | key 178 | [TEXT] 179 | 180 | value 181 | [TEXT] 182 | 183 | 184 | 185 | -------------------------------------------------------------------------------- /doc/schema/tags.md: -------------------------------------------------------------------------------- 1 | # tags 2 | 3 | ## Description 4 | 5 |
6 | Table Definition 7 | 8 | ```sql 9 | CREATE TABLE tags ( 10 | id INTEGER PRIMARY KEY AUTOINCREMENT, 11 | name TEXT NOT NULL, 12 | UNIQUE(name) 13 | ) 14 | ``` 15 | 16 |
17 | 18 | ## Columns 19 | 20 | | Name | Type | Default | Nullable | Children | Parents | Comment | 21 | | ---- | ------- | ------- | -------- | ------------------------------- | ------- | ------- | 22 | | id | INTEGER | | true | [targets_tags](targets_tags.md) | | | 23 | | name | TEXT | | false | | | | 24 | 25 | ## Constraints 26 | 27 | | Name | Type | Definition | 28 | | ----------------------- | ----------- | ---------------- | 29 | | id | PRIMARY KEY | PRIMARY KEY (id) | 30 | | sqlite_autoindex_tags_1 | UNIQUE | UNIQUE (name) | 31 | 32 | ## Indexes 33 | 34 | | Name | Definition | 35 | | ----------------------- | ------------- | 36 | | sqlite_autoindex_tags_1 | UNIQUE (name) | 37 | 38 | ## Relations 39 | 40 | ![er](tags.svg) 41 | 42 | --- 43 | 44 | > Generated by [tbls](https://github.com/k1LoW/tbls) 45 | -------------------------------------------------------------------------------- /doc/schema/tags.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | tags 11 | 12 | 13 | 14 | tags 15 | 16 | 17 | tags 18 | 19 | [table] 20 | 21 | id 22 | [INTEGER] 23 | 24 | name 25 | [TEXT] 26 | 27 | 28 | 29 | 30 | targets_tags 31 | 32 | 33 | targets_tags 34 | 35 | [table] 36 | 37 | id 38 | [INTEGER] 39 | 40 | target_id 41 | [INTEGER] 42 | 43 | tag_id 44 | [INTEGER] 45 | 46 | 47 | 48 | targets_tags:tag_id->tags:id 49 | 50 | 51 | targets_tags -> tags 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /doc/schema/targets.md: -------------------------------------------------------------------------------- 1 | # targets 2 | 3 | ## Description 4 | 5 |
6 | Table Definition 7 | 8 | ```sql 9 | CREATE TABLE targets ( 10 | id INTEGER PRIMARY KEY AUTOINCREMENT, 11 | source TEXT NOT NULL, 12 | description TEXT, 13 | type TEXT NOT NULL, 14 | regexp TEXT, 15 | multi_line INTEGER, 16 | time_format TEXT, 17 | time_zone TEXT, 18 | scheme TEXT NOT NULL, 19 | host TEXT, 20 | user TEXT, 21 | port INTEGER, 22 | path TEXT NOT NULL 23 | ) 24 | ``` 25 | 26 |
27 | 28 | ## Columns 29 | 30 | | Name | Type | Default | Nullable | Children | Parents | Comment | 31 | | ----------- | ------- | ------- | -------- | ----------------------------------------------- | ------- | ------- | 32 | | id | INTEGER | | true | [logs](logs.md) [targets_tags](targets_tags.md) | | | 33 | | source | TEXT | | false | | | | 34 | | description | TEXT | | true | | | | 35 | | type | TEXT | | false | | | | 36 | | regexp | TEXT | | true | | | | 37 | | multi_line | INTEGER | | true | | | | 38 | | time_format | TEXT | | true | | | | 39 | | time_zone | TEXT | | true | | | | 40 | | scheme | TEXT | | false | | | | 41 | | host | TEXT | | true | | | | 42 | | user | TEXT | | true | | | | 43 | | port | INTEGER | | true | | | | 44 | | path | TEXT | | false | | | | 45 | 46 | ## Constraints 47 | 48 | | Name | Type | Definition | 49 | | ---- | ----------- | ---------------- | 50 | | id | PRIMARY KEY | PRIMARY KEY (id) | 51 | 52 | ## Relations 53 | 54 | ![er](targets.svg) 55 | 56 | --- 57 | 58 | > Generated by [tbls](https://github.com/k1LoW/tbls) 59 | -------------------------------------------------------------------------------- /doc/schema/targets.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | targets 11 | 12 | 13 | 14 | targets 15 | 16 | 17 | targets 18 | 19 | [table] 20 | 21 | id 22 | [INTEGER] 23 | 24 | source 25 | [TEXT] 26 | 27 | description 28 | [TEXT] 29 | 30 | type 31 | [TEXT] 32 | 33 | regexp 34 | [TEXT] 35 | 36 | multi_line 37 | [INTEGER] 38 | 39 | time_format 40 | [TEXT] 41 | 42 | time_zone 43 | [TEXT] 44 | 45 | scheme 46 | [TEXT] 47 | 48 | host 49 | [TEXT] 50 | 51 | user 52 | [TEXT] 53 | 54 | port 55 | [INTEGER] 56 | 57 | path 58 | [TEXT] 59 | 60 | 61 | 62 | 63 | logs 64 | 65 | 66 | logs 67 | 68 | [virtual table] 69 | 70 | host 71 | [] 72 | 73 | path 74 | [] 75 | 76 | target_id 77 | [] 78 | 79 | ts 80 | [] 81 | 82 | ts_unixnano 83 | [] 84 | 85 | ts_year 86 | [] 87 | 88 | ts_month 89 | [] 90 | 91 | ts_day 92 | [] 93 | 94 | ts_hour 95 | [] 96 | 97 | ts_minute 98 | [] 99 | 100 | ts_second 101 | [] 102 | 103 | ts_time_zone 104 | [] 105 | 106 | filled_by_prev_ts 107 | [] 108 | 109 | content 110 | [] 111 | 112 | 113 | 114 | logs:target_id->targets:id 115 | 116 | 117 | logs -> targets 118 | 119 | 120 | 121 | targets_tags 122 | 123 | 124 | targets_tags 125 | 126 | [table] 127 | 128 | id 129 | [INTEGER] 130 | 131 | target_id 132 | [INTEGER] 133 | 134 | tag_id 135 | [INTEGER] 136 | 137 | 138 | 139 | targets_tags:target_id->targets:id 140 | 141 | 142 | targets_tags -> targets 143 | 144 | 145 | 146 | -------------------------------------------------------------------------------- /doc/schema/targets_tags.md: -------------------------------------------------------------------------------- 1 | # targets_tags 2 | 3 | ## Description 4 | 5 |
6 | Table Definition 7 | 8 | ```sql 9 | CREATE TABLE targets_tags ( 10 | id INTEGER PRIMARY KEY AUTOINCREMENT, 11 | target_id INTEGER NOT NULL, 12 | tag_id INTEGER NOT NULL, 13 | UNIQUE(target_id, tag_id) 14 | ) 15 | ``` 16 | 17 |
18 | 19 | ## Columns 20 | 21 | | Name | Type | Default | Nullable | Children | Parents | Comment | 22 | | --------- | ------- | ------- | -------- | -------- | --------------------- | ------- | 23 | | id | INTEGER | | true | | | | 24 | | target_id | INTEGER | | false | | [targets](targets.md) | | 25 | | tag_id | INTEGER | | false | | [tags](tags.md) | | 26 | 27 | ## Constraints 28 | 29 | | Name | Type | Definition | 30 | | ------------------------------- | ----------- | -------------------------- | 31 | | id | PRIMARY KEY | PRIMARY KEY (id) | 32 | | sqlite_autoindex_targets_tags_1 | UNIQUE | UNIQUE (target_id, tag_id) | 33 | 34 | ## Indexes 35 | 36 | | Name | Definition | 37 | | ------------------------------- | -------------------------- | 38 | | sqlite_autoindex_targets_tags_1 | UNIQUE (target_id, tag_id) | 39 | 40 | ## Relations 41 | 42 | ![er](targets_tags.svg) 43 | 44 | --- 45 | 46 | > Generated by [tbls](https://github.com/k1LoW/tbls) 47 | -------------------------------------------------------------------------------- /doc/schema/targets_tags.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | targets_tags 11 | 12 | 13 | 14 | targets_tags 15 | 16 | 17 | targets_tags 18 | 19 | [table] 20 | 21 | id 22 | [INTEGER] 23 | 24 | target_id 25 | [INTEGER] 26 | 27 | tag_id 28 | [INTEGER] 29 | 30 | 31 | 32 | 33 | targets 34 | 35 | 36 | targets 37 | 38 | [table] 39 | 40 | id 41 | [INTEGER] 42 | 43 | source 44 | [TEXT] 45 | 46 | description 47 | [TEXT] 48 | 49 | type 50 | [TEXT] 51 | 52 | regexp 53 | [TEXT] 54 | 55 | multi_line 56 | [INTEGER] 57 | 58 | time_format 59 | [TEXT] 60 | 61 | time_zone 62 | [TEXT] 63 | 64 | scheme 65 | [TEXT] 66 | 67 | host 68 | [TEXT] 69 | 70 | user 71 | [TEXT] 72 | 73 | port 74 | [INTEGER] 75 | 76 | path 77 | [TEXT] 78 | 79 | 80 | 81 | targets_tags:target_id->targets:id 82 | 83 | 84 | targets_tags -> targets 85 | 86 | 87 | 88 | tags 89 | 90 | 91 | tags 92 | 93 | [table] 94 | 95 | id 96 | [INTEGER] 97 | 98 | name 99 | [TEXT] 100 | 101 | 102 | 103 | targets_tags:tag_id->tags:id 104 | 105 | 106 | targets_tags -> tags 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /doc/stream.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/k1LoW/harvest/dc5cbf8bb64110506e11cb7709d9541ee1062f15/doc/stream.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/k1LoW/harvest 2 | 3 | require ( 4 | github.com/Azure/go-autorest/autorest v0.2.0 // indirect 5 | github.com/Songmu/prompter v0.2.0 6 | github.com/antonmedv/expr v1.1.4 7 | github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 8 | github.com/go-sql-driver/mysql v1.4.1 // indirect 9 | github.com/google/gofuzz v1.0.0 // indirect 10 | github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d // indirect 11 | github.com/grpc-ecosystem/grpc-gateway v1.9.2 // indirect 12 | github.com/imdario/mergo v0.3.7 // indirect 13 | github.com/jmoiron/sqlx v1.2.0 14 | github.com/k1LoW/duration v1.1.0 15 | github.com/k1LoW/sshc v0.0.0-20190816012013-f54f6b90bfef 16 | github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect 17 | github.com/labstack/gommon v0.2.8 18 | github.com/lib/pq v1.2.0 // indirect 19 | github.com/mattn/go-colorable v0.1.2 // indirect 20 | github.com/mattn/go-sqlite3 v1.10.0 21 | github.com/mitchellh/go-homedir v1.1.0 22 | github.com/pkg/errors v0.8.1 23 | github.com/spf13/cobra v1.0.1-0.20200719220246-c6fe2d4df810 24 | github.com/stretchr/testify v1.4.0 // indirect 25 | go.uber.org/zap v1.10.0 26 | golang.org/x/crypto v0.17.0 27 | golang.org/x/net v0.17.0 // indirect 28 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 29 | gopkg.in/inf.v0 v0.9.0 // indirect 30 | gopkg.in/yaml.v2 v2.3.0 31 | k8s.io/api v0.0.0-20190313235455-40a48860b5ab 32 | k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1 33 | k8s.io/client-go v11.0.0+incompatible 34 | k8s.io/klog v0.3.3 // indirect 35 | k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a // indirect 36 | sigs.k8s.io/yaml v1.1.0 // indirect 37 | ) 38 | 39 | go 1.13 40 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "os" 5 | 6 | "go.uber.org/zap" 7 | "go.uber.org/zap/zapcore" 8 | ) 9 | 10 | // NewLogger ... 11 | func NewLogger(verbose bool) *zap.Logger { 12 | encoderConfig := zapcore.EncoderConfig{ 13 | TimeKey: "ts", 14 | LevelKey: "level", 15 | NameKey: "logger", 16 | CallerKey: "caller", 17 | MessageKey: "msg", 18 | StacktraceKey: "stacktrace", 19 | EncodeLevel: zapcore.CapitalLevelEncoder, 20 | EncodeTime: zapcore.ISO8601TimeEncoder, 21 | EncodeDuration: zapcore.StringDurationEncoder, 22 | EncodeCaller: zapcore.ShortCallerEncoder, 23 | } 24 | 25 | var logLevel zapcore.Level 26 | 27 | if verbose { 28 | logLevel = zapcore.DebugLevel 29 | } else { 30 | logLevel = zapcore.InfoLevel 31 | } 32 | 33 | stderrCore := zapcore.NewCore( 34 | zapcore.NewConsoleEncoder(encoderConfig), 35 | zapcore.AddSync(os.Stderr), 36 | logLevel, 37 | ) 38 | 39 | logger := zap.New(stderrCore) 40 | 41 | return logger 42 | } 43 | -------------------------------------------------------------------------------- /parser/combined_log.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/k1LoW/harvest/config" 5 | "go.uber.org/zap" 6 | ) 7 | 8 | // NewCombinedLogParser ... 9 | func NewCombinedLogParser(t *config.Target, l *zap.Logger) (Parser, error) { 10 | t.Regexp = `^[\d\.]+ - [^ ]+ \[(.+)\] .+$` 11 | t.TimeFormat = "02/Jan/2006:15:04:05 -0700" 12 | t.MultiLine = false 13 | return NewRegexpParser(t, l) 14 | } 15 | -------------------------------------------------------------------------------- /parser/none.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "time" 7 | 8 | "github.com/k1LoW/harvest/client" 9 | "github.com/k1LoW/harvest/config" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | // NoneParser ... 14 | type NoneParser struct { 15 | target *config.Target 16 | logger *zap.Logger 17 | } 18 | 19 | // NewNoneParser ... 20 | func NewNoneParser(t *config.Target, l *zap.Logger) (Parser, error) { 21 | return &NoneParser{ 22 | target: t, 23 | logger: l, 24 | }, nil 25 | } 26 | 27 | // Parse ... 28 | func (p *NoneParser) Parse(ctx context.Context, cancel context.CancelFunc, lineChan <-chan client.Line, tz string, st *time.Time, et *time.Time) <-chan Log { 29 | if p.target.MultiLine { 30 | return p.parseMultipleLine(ctx, cancel, lineChan, tz, st, et) 31 | } 32 | return p.parseSingleLine(ctx, cancel, lineChan, tz, st, et) 33 | } 34 | 35 | func (p *NoneParser) parseSingleLine(ctx context.Context, cancel context.CancelFunc, lineChan <-chan client.Line, tz string, st *time.Time, et *time.Time) <-chan Log { 36 | logChan := make(chan Log) 37 | logStarted := false 38 | logEnded := false 39 | 40 | var prevTs *time.Time 41 | 42 | if st == nil { 43 | logStarted = true 44 | } 45 | 46 | go func() { 47 | defer func() { 48 | p.logger.Debug("Close chan parser.Log") 49 | close(logChan) 50 | }() 51 | 52 | for line := range lineChan { 53 | if logEnded { 54 | continue 55 | } 56 | var ( 57 | ts *time.Time 58 | filledByPrevTs bool 59 | ) 60 | 61 | if line.TimestampViaClient != nil { 62 | ts = line.TimestampViaClient 63 | prevTs = ts 64 | } else { 65 | ts = prevTs 66 | if ts != nil { 67 | filledByPrevTs = true 68 | } 69 | } 70 | if ts == nil { 71 | logStarted = true 72 | } 73 | 74 | if !logStarted && ts != nil && ts.UnixNano() > st.UnixNano() { 75 | logStarted = true 76 | } 77 | 78 | if !logStarted { 79 | continue 80 | } 81 | 82 | if ts != nil && et != nil && ts.UnixNano() > et.UnixNano() { 83 | p.logger.Debug("Cancel parse, because timestamp period out") 84 | logEnded = true 85 | cancel() 86 | continue 87 | } 88 | 89 | logChan <- Log{ 90 | Host: line.Host, 91 | Path: line.Path, 92 | Timestamp: ts, 93 | FilledByPrevTs: filledByPrevTs, 94 | Content: line.Content, 95 | Target: p.target, 96 | } 97 | } 98 | }() 99 | 100 | return logChan 101 | } 102 | 103 | func (p *NoneParser) parseMultipleLine(ctx context.Context, cancel context.CancelFunc, lineChan <-chan client.Line, tz string, st *time.Time, et *time.Time) <-chan Log { 104 | logChan := make(chan Log) 105 | logStarted := false 106 | logEnded := false 107 | contentStash := []string{} 108 | 109 | var ( 110 | hostStash string 111 | pathStash string 112 | prevTs *time.Time 113 | ) 114 | 115 | if st == nil { 116 | logStarted = true 117 | } 118 | 119 | go func() { 120 | defer func() { 121 | logChan <- Log{ 122 | Host: hostStash, 123 | Path: pathStash, 124 | Timestamp: prevTs, 125 | FilledByPrevTs: false, 126 | Content: strings.Join(contentStash, "\n"), 127 | Target: p.target, 128 | } 129 | close(logChan) 130 | }() 131 | 132 | for line := range lineChan { 133 | if logEnded { 134 | continue 135 | } 136 | hostStash = line.Host 137 | pathStash = line.Path 138 | var ts *time.Time 139 | 140 | if line.TimestampViaClient != nil { 141 | ts = line.TimestampViaClient 142 | } else { 143 | logStarted = true 144 | } 145 | 146 | if !logStarted && ts != nil && ts.UnixNano() > st.UnixNano() { 147 | logStarted = true 148 | } 149 | 150 | if !logStarted { 151 | continue 152 | } 153 | 154 | if ts != nil && et != nil && ts.UnixNano() > et.UnixNano() { 155 | p.logger.Debug("Cancel parse, because timestamp period out") 156 | logEnded = true 157 | cancel() 158 | continue 159 | } 160 | 161 | if ts == nil && (strings.HasPrefix(line.Content, " ") || strings.HasPrefix(line.Content, "\t")) { 162 | contentStash = append(contentStash, line.Content) 163 | if len(contentStash) > maxContentStash { 164 | logChan <- Log{ 165 | Host: line.Host, 166 | Path: line.Path, 167 | Timestamp: prevTs, 168 | FilledByPrevTs: false, 169 | Content: strings.Join(contentStash, "\n"), 170 | Target: p.target, 171 | } 172 | logChan <- Log{ 173 | Host: line.Host, 174 | Path: line.Path, 175 | Timestamp: prevTs, 176 | FilledByPrevTs: false, 177 | Content: "Harvest parse error: too many rows", 178 | Target: p.target, 179 | } 180 | contentStash = nil 181 | } 182 | continue 183 | } 184 | 185 | // ts > 0 or ^.+ 186 | 187 | if len(contentStash) > 0 { 188 | logChan <- Log{ 189 | Host: line.Host, 190 | Path: line.Path, 191 | Timestamp: prevTs, 192 | FilledByPrevTs: false, 193 | Content: strings.Join(contentStash, "\n"), 194 | Target: p.target, 195 | } 196 | } 197 | 198 | contentStash = nil 199 | contentStash = append(contentStash, line.Content) 200 | prevTs = ts 201 | } 202 | }() 203 | 204 | return logChan 205 | } 206 | -------------------------------------------------------------------------------- /parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/k1LoW/harvest/client" 10 | "github.com/k1LoW/harvest/config" 11 | ) 12 | 13 | const maxContentStash = 1000 14 | 15 | // Log ... 16 | type Log struct { 17 | Host string `db:"host"` 18 | Path string `db:"path"` 19 | Timestamp *time.Time 20 | TimestampUnixNano int64 `db:"ts_unixnano"` 21 | FilledByPrevTs bool `db:"filled_by_prev_ts"` 22 | Content string `db:"content"` 23 | Target *config.Target `db:"target"` 24 | } 25 | 26 | // Parser ... 27 | type Parser interface { 28 | Parse(ctx context.Context, cancel context.CancelFunc, lineChan <-chan client.Line, tz string, st *time.Time, et *time.Time) <-chan Log 29 | } 30 | 31 | func parseTime(tf string, tz string, content string) (*time.Time, error) { 32 | if tf == "unixtime" { 33 | ui, _ := strconv.ParseInt(content, 10, 64) 34 | ut := time.Unix(ui, 0) 35 | return &ut, nil 36 | } 37 | if tz == "" { 38 | t, err := time.Parse(fmt.Sprintf("2006-01-02 %s", tf), fmt.Sprintf("%s %s", time.Now().Format("2006-01-02"), content)) 39 | return &t, err 40 | } 41 | t, err := time.Parse(fmt.Sprintf("2006-01-02 -0700 %s", tf), fmt.Sprintf("%s %s %s", time.Now().Format("2006-01-02"), tz, content)) 42 | return &t, err 43 | } 44 | -------------------------------------------------------------------------------- /parser/parser_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | var parseTimeTests = []struct { 10 | tf string 11 | tz string 12 | content string 13 | want string 14 | }{ 15 | { 16 | "02/Jan/2006:15:04:05 -0700", 17 | "", 18 | "04/Feb/2019:00:13:49 +0900", 19 | "2019-02-04T00:13:49.000000000 +09:00", 20 | }, 21 | { 22 | "02/Jan/2006:15:04:05 -0700", 23 | "+0500", 24 | "04/Feb/2019:00:13:49 +0900", 25 | "2019-02-04T00:13:49.000000000 +09:00", 26 | }, 27 | } 28 | 29 | func TestParseTime(t *testing.T) { 30 | for _, tt := range parseTimeTests { 31 | got, err := parseTime(tt.tf, tt.tz, tt.content) 32 | if err != nil { 33 | t.Errorf("%v", err) 34 | } 35 | want, err := time.Parse("2006-01-02T15:04:05.999999999 -07:00", tt.want) 36 | if err != nil { 37 | t.Errorf("%v", err) 38 | } 39 | if got.UnixNano() != want.UnixNano() { 40 | t.Errorf("\ngot %s\nwant %s", got, want) 41 | } 42 | } 43 | } 44 | 45 | var parseTimeDetectionTests = []struct { 46 | tf string 47 | tz string 48 | content string 49 | want string 50 | }{ 51 | { 52 | "Jan 02 15:04:05", 53 | "+0900", 54 | "Mar 05 23:59:59", 55 | fmt.Sprintf("%d-%s", time.Now().Year(), "03-05T23:59:59.000000000 +09:00"), 56 | }, 57 | { 58 | "Jan 02 15:04:05", 59 | "+0000", 60 | "Mar 05 23:59:59", 61 | fmt.Sprintf("%d-%s", time.Now().Year(), "03-05T23:59:59.000000000 +00:00"), 62 | }, 63 | { 64 | "Jan 02 15:04:05", 65 | "", 66 | "Mar 05 23:59:59", 67 | fmt.Sprintf("%d-%s", time.Now().Year(), "03-05T23:59:59.000000000 +00:00"), 68 | }, 69 | } 70 | 71 | func TestParseTimeDetection(t *testing.T) { 72 | for _, tt := range parseTimeDetectionTests { 73 | got, err := parseTime(tt.tf, tt.tz, tt.content) 74 | if err != nil { 75 | t.Errorf("%v", err) 76 | } 77 | want, err := time.Parse("2006-01-02T15:04:05.999999999 -07:00", tt.want) 78 | if err != nil { 79 | t.Errorf("%v", err) 80 | } 81 | if got.UnixNano() != want.UnixNano() { 82 | t.Errorf("\ngot %s\nwant %s", got, want) 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /parser/regexp.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "strings" 7 | "time" 8 | 9 | "github.com/k1LoW/harvest/client" 10 | "github.com/k1LoW/harvest/config" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | // RegexpParser ... 15 | type RegexpParser struct { 16 | target *config.Target 17 | logger *zap.Logger 18 | } 19 | 20 | // NewRegexpParser ... 21 | func NewRegexpParser(t *config.Target, l *zap.Logger) (Parser, error) { 22 | return &RegexpParser{ 23 | target: t, 24 | logger: l, 25 | }, nil 26 | } 27 | 28 | // Parse ... 29 | func (p *RegexpParser) Parse(ctx context.Context, cancel context.CancelFunc, lineChan <-chan client.Line, tz string, st *time.Time, et *time.Time) <-chan Log { 30 | if p.target.MultiLine { 31 | return p.parseMultipleLine(ctx, cancel, lineChan, tz, st, et) 32 | } 33 | return p.parseSingleLine(ctx, cancel, lineChan, tz, st, et) 34 | } 35 | 36 | func (p *RegexpParser) parseSingleLine(ctx context.Context, cancel context.CancelFunc, lineChan <-chan client.Line, tz string, st *time.Time, et *time.Time) <-chan Log { 37 | logChan := make(chan Log) 38 | logStarted := false 39 | logEnded := false 40 | re := regexp.MustCompile(p.target.Regexp) 41 | 42 | var prevTs *time.Time 43 | 44 | if st == nil { 45 | logStarted = true 46 | } 47 | 48 | go func() { 49 | defer func() { 50 | p.logger.Debug("Close chan parser.Log") 51 | close(logChan) 52 | }() 53 | 54 | lineTZ := tz 55 | 56 | for line := range lineChan { 57 | if logEnded { 58 | continue 59 | } 60 | 61 | var ( 62 | ts *time.Time 63 | err error 64 | filledByPrevTs bool 65 | ) 66 | 67 | if tz == "" { 68 | lineTZ = line.TimeZone 69 | } 70 | 71 | if p.target.TimeFormat != "" { 72 | m := re.FindStringSubmatch(line.Content) 73 | if len(m) > 1 { 74 | ts, err = parseTime(p.target.TimeFormat, lineTZ, m[1]) 75 | if err == nil { 76 | prevTs = ts 77 | } 78 | } 79 | } 80 | if ts == nil { 81 | if line.TimestampViaClient != nil { 82 | ts = line.TimestampViaClient 83 | prevTs = ts 84 | } else { 85 | ts = prevTs 86 | if ts != nil { 87 | filledByPrevTs = true 88 | } 89 | } 90 | } 91 | 92 | if !logStarted && ts != nil && ts.UnixNano() > st.UnixNano() { 93 | logStarted = true 94 | } 95 | 96 | if !logStarted { 97 | continue 98 | } 99 | 100 | if ts != nil && et != nil && ts.UnixNano() > et.UnixNano() { 101 | p.logger.Debug("Cancel parse, because timestamp period out") 102 | logEnded = true 103 | cancel() 104 | continue 105 | } 106 | 107 | logChan <- Log{ 108 | Host: line.Host, 109 | Path: line.Path, 110 | Timestamp: ts, 111 | FilledByPrevTs: filledByPrevTs, 112 | Content: line.Content, 113 | Target: p.target, 114 | } 115 | } 116 | }() 117 | 118 | return logChan 119 | } 120 | 121 | func (p *RegexpParser) parseMultipleLine(ctx context.Context, cancel context.CancelFunc, lineChan <-chan client.Line, tz string, st *time.Time, et *time.Time) <-chan Log { 122 | logChan := make(chan Log) 123 | logStarted := false 124 | logEnded := false 125 | re := regexp.MustCompile(p.target.Regexp) 126 | contentStash := []string{} 127 | var ( 128 | prevTs *time.Time 129 | hostStash string 130 | pathStash string 131 | ) 132 | 133 | if st == nil { 134 | logStarted = true 135 | } 136 | 137 | go func() { 138 | defer func() { 139 | logChan <- Log{ 140 | Host: hostStash, 141 | Path: pathStash, 142 | Timestamp: prevTs, 143 | FilledByPrevTs: false, 144 | Content: strings.Join(contentStash, "\n"), 145 | Target: p.target, 146 | } 147 | p.logger.Debug("Close chan parser.Log") 148 | close(logChan) 149 | }() 150 | 151 | lineTZ := tz 152 | for line := range lineChan { 153 | if logEnded { 154 | continue 155 | } 156 | var ts *time.Time 157 | 158 | hostStash = line.Host 159 | pathStash = line.Path 160 | 161 | if tz == "" { 162 | lineTZ = line.TimeZone 163 | } 164 | if p.target.TimeFormat != "" { 165 | m := re.FindStringSubmatch(line.Content) 166 | if len(m) > 1 { 167 | ts, _ = parseTime(p.target.TimeFormat, lineTZ, m[1]) 168 | } 169 | } 170 | if ts == nil { 171 | if line.TimestampViaClient != nil { 172 | ts = line.TimestampViaClient 173 | } 174 | } 175 | 176 | if !logStarted && ts != nil && ts.UnixNano() > st.UnixNano() { 177 | logStarted = true 178 | } 179 | 180 | if !logStarted { 181 | continue 182 | } 183 | 184 | if ts != nil && et != nil && ts.UnixNano() > et.UnixNano() { 185 | p.logger.Debug("Cancel parse, because timestamp period out") 186 | logEnded = true 187 | cancel() 188 | continue 189 | } 190 | 191 | if ts == nil { 192 | contentStash = append(contentStash, line.Content) 193 | if len(contentStash) > maxContentStash { 194 | logChan <- Log{ 195 | Host: line.Host, 196 | Path: line.Path, 197 | Timestamp: prevTs, 198 | FilledByPrevTs: false, 199 | Content: strings.Join(contentStash, "\n"), 200 | Target: p.target, 201 | } 202 | logChan <- Log{ 203 | Host: line.Host, 204 | Path: line.Path, 205 | Timestamp: prevTs, 206 | FilledByPrevTs: false, 207 | Content: "Harvest parse error: too many rows", 208 | Target: p.target, 209 | } 210 | contentStash = nil 211 | } 212 | continue 213 | } 214 | 215 | // ts > 0 216 | 217 | if len(contentStash) > 0 { 218 | logChan <- Log{ 219 | Host: line.Host, 220 | Path: line.Path, 221 | Timestamp: prevTs, 222 | FilledByPrevTs: false, 223 | Content: strings.Join(contentStash, "\n"), 224 | Target: p.target, 225 | } 226 | } 227 | 228 | contentStash = nil 229 | contentStash = append(contentStash, line.Content) 230 | prevTs = ts 231 | } 232 | }() 233 | 234 | return logChan 235 | } 236 | -------------------------------------------------------------------------------- /parser/syslog.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/k1LoW/harvest/config" 5 | "go.uber.org/zap" 6 | ) 7 | 8 | // NewSyslogParser ... 9 | func NewSyslogParser(t *config.Target, l *zap.Logger) (Parser, error) { 10 | t.Regexp = `^(\w{3} ?\d{1,2} \d{2}:\d{2}:\d{2}) .+$` 11 | t.TimeFormat = "Jan 2 15:04:05" 12 | t.MultiLine = false 13 | return NewRegexpParser(t, l) 14 | } 15 | -------------------------------------------------------------------------------- /stdout/stdout.go: -------------------------------------------------------------------------------- 1 | package stdout 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/k1LoW/harvest/parser" 8 | "github.com/labstack/gommon/color" 9 | ) 10 | 11 | const ( 12 | tsParseFmt = "2006-01-02T15:04:05-07:00" 13 | tsNanoParseFmt = "2006-01-02T15:04:05.000000000-07:00" 14 | ) 15 | 16 | var colorizeMap = []struct { 17 | colorFunc func(interface{}, ...string) string 18 | bar string 19 | }{ 20 | {color.Yellow, "█ "}, 21 | {color.Magenta, "█ "}, 22 | {color.Green, "█ "}, 23 | {color.Cyan, "█ "}, 24 | {color.Yellow, "▚ "}, 25 | {color.Magenta, "▚ "}, 26 | {color.Green, "▚ "}, 27 | {color.Cyan, "▚ "}, 28 | {color.Yellow, "║ "}, 29 | {color.Magenta, "║ "}, 30 | {color.Green, "║ "}, 31 | {color.Cyan, "║ "}, 32 | {color.Yellow, "▒ "}, 33 | {color.Magenta, "▒ "}, 34 | {color.Green, "▒ "}, 35 | {color.Cyan, "▒ "}, 36 | {color.Yellow, "▓ "}, 37 | {color.Magenta, "▓ "}, 38 | {color.Green, "▓ "}, 39 | {color.Cyan, "▓ "}, 40 | } 41 | 42 | // Stdout ... 43 | type Stdout struct { 44 | withTimestamp bool 45 | withTimestampNano bool 46 | withHost bool 47 | withPath bool 48 | withTag bool 49 | withoutMark bool 50 | hFmt string 51 | tFmt string 52 | noColor bool 53 | } 54 | 55 | // NewStdout ... 56 | func NewStdout(withTimestamp bool, 57 | withTimestampNano bool, 58 | withHost bool, 59 | withPath bool, 60 | withTag bool, 61 | withoutMark bool, 62 | hLen int, 63 | tLen int, 64 | noColor bool, 65 | ) (*Stdout, error) { 66 | return &Stdout{ 67 | withTimestamp: withTimestamp, 68 | withTimestampNano: withTimestampNano, 69 | withHost: withHost, 70 | withPath: withPath, 71 | withTag: withTag, 72 | withoutMark: withoutMark, 73 | hFmt: fmt.Sprintf("%%-%ds ", hLen), 74 | tFmt: fmt.Sprintf("%%-%ds ", tLen), 75 | noColor: noColor, 76 | }, nil 77 | } 78 | 79 | // Out ... 80 | func (s *Stdout) Out(logChan chan parser.Log, hosts []string) { 81 | if s.noColor { 82 | color.Disable() 83 | } else { 84 | color.Enable() 85 | } 86 | 87 | for log := range logChan { 88 | var ( 89 | bar string 90 | ts string 91 | filledByPrevTs string 92 | host string 93 | tag string 94 | ) 95 | 96 | colorFunc := func(msg interface{}, styles ...string) string { 97 | return msg.(string) 98 | } 99 | 100 | if s.withTimestamp { 101 | if log.Timestamp == nil { 102 | ts = fmt.Sprintf(fmt.Sprintf("%%-%ds", len(tsParseFmt)), "-") 103 | } else { 104 | ts = log.Timestamp.Format(tsParseFmt) 105 | } 106 | } 107 | if s.withTimestampNano { 108 | if log.Timestamp == nil { 109 | ts = fmt.Sprintf(fmt.Sprintf("%%-%ds", len(tsNanoParseFmt)), "-") 110 | } else { 111 | ts = log.Timestamp.Format(tsNanoParseFmt) 112 | } 113 | } 114 | if s.withTimestamp || s.withTimestampNano { 115 | if log.FilledByPrevTs { 116 | filledByPrevTs = "* " 117 | } else { 118 | filledByPrevTs = " " 119 | } 120 | } 121 | 122 | if s.withHost && s.withPath { 123 | host = fmt.Sprintf(s.hFmt, fmt.Sprintf("%s:%s", log.Host, log.Path)) 124 | } else if s.withHost { 125 | host = fmt.Sprintf(s.hFmt, log.Host) 126 | } else if s.withPath { 127 | host = fmt.Sprintf(s.hFmt, log.Path) 128 | } 129 | if s.withTag { 130 | tag = fmt.Sprintf(s.tFmt, fmt.Sprintf("%v", log.Target.Tags)) 131 | } 132 | 133 | if s.withTimestamp || s.withTimestampNano || s.withHost || s.withPath { 134 | for i, h := range hosts { 135 | if h == log.Host { 136 | colorFunc = colorizeMap[i%len(colorizeMap)].colorFunc 137 | if s.withoutMark { 138 | bar = "" 139 | } else { 140 | bar = colorFunc(colorizeMap[i%len(colorizeMap)].bar) 141 | } 142 | } 143 | } 144 | } 145 | 146 | fmt.Printf("%s%s%s%s%s%s\n", bar, colorFunc(ts), color.White(filledByPrevTs, color.B), colorizeTag(colorFunc, tag), color.Grey(host), log.Content) 147 | } 148 | } 149 | 150 | func colorizeTag(colorFunc func(interface{}, ...string) string, tag string) string { 151 | colorized := []string{} 152 | tags := strings.Split(tag, " ") 153 | for _, t := range tags { 154 | colorized = append(colorized, colorFunc(t, color.B)) 155 | } 156 | return strings.Join(colorized, " ") 157 | } 158 | -------------------------------------------------------------------------------- /testdata/test.yml.template: -------------------------------------------------------------------------------- 1 | --- 2 | targetSets: 3 | - 4 | description: dummy access.log 5 | type: combinedLog 6 | sources: 7 | - 'file://__PWD__/testdata/apache.log*' 8 | tags: 9 | - httpd 10 | -------------------------------------------------------------------------------- /testdata/test_config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | targetSets: 3 | - 4 | description: webproxy syslog 5 | type: syslog 6 | sources: 7 | - 'ssh://webproxy.example.com/var/log/syslog*' 8 | tags: 9 | - webproxy 10 | - syslog 11 | - 12 | description: webproxy NGINX access log 13 | type: combinedLog 14 | sources: 15 | - 'ssh://webproxy.example.com/var/log/nginx/access_log*' 16 | tags: 17 | - webproxy 18 | - nginx 19 | - 20 | description: app log 21 | type: regexp 22 | regexp: 'time:([^\t]+)' 23 | timeFormat: 'Jan 02 15:04:05' 24 | timeZone: '+0900' 25 | sources: 26 | - 'ssh://app-1.example.com/var/log/ltsv.log*' 27 | - 'ssh://app-2.example.com/var/log/ltsv.log*' 28 | - 'ssh://app-3.example.com/var/log/ltsv.log*' 29 | tags: 30 | - app 31 | - 32 | description: db dump log 33 | type: regexp 34 | regexp: '"ts":"([^"]+)"' 35 | timeFormat: '2006-01-02T15:04:05.999-0700' 36 | sources: 37 | - 'ssh://db.example.com/var/log/tcpdp/eth0/dump*' 38 | tags: 39 | - db 40 | - query 41 | - 42 | description: PostgreSQL log 43 | type: regexp 44 | regexp: '^\[?(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \w{3})' 45 | timeFormat: '2006-01-02 15:04:05 MST' 46 | multiLine: true 47 | sources: 48 | - 'ssh://db.example.com/var/log/postgresql/postgresql*' 49 | tags: 50 | - db 51 | - postgresql 52 | - 53 | description: local Apache access log 54 | type: combinedLog 55 | sources: 56 | - 'file:///path/to/httpd/access.log' 57 | tags: 58 | - httpd 59 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | // Name for this 4 | const Name string = "hrv" 5 | 6 | // Version for this 7 | var Version = "dev" 8 | --------------------------------------------------------------------------------