├── .github ├── release.yml └── workflows │ ├── ci.yml │ └── tagpr.yml ├── .gitignore ├── .goreleaser ├── deb_amd64.yml ├── deb_arm64.yml ├── deb_arm64_static.yml ├── rpm_amd64.yml └── rpm_arm64.yml ├── .octocov.yml ├── .tagpr ├── CHANGELOG.md ├── CREDITS ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── config.go ├── probe.go ├── proxy.go ├── read.go ├── root.go └── version.go ├── docker-compose.yml ├── dumper ├── conn │ └── conn.go ├── dumper.go ├── dumper_test.go ├── hex │ ├── hex.go │ └── hex_test.go ├── mysql │ ├── const.go │ ├── encoding.go │ ├── encoding_test.go │ ├── mysql.go │ └── mysql_test.go └── pg │ ├── pg.go │ └── pg_test.go ├── go.mod ├── go.sum ├── integration_test.go ├── logger └── logger.go ├── main.go ├── misc └── pcap │ ├── mysql │ └── prepare │ │ └── main.go │ └── pg │ └── prepare │ └── main.go ├── reader ├── payload_buffer.go ├── proxy_protocol.go ├── proxy_protocol_test.go ├── reader.go └── reader_test.go ├── server ├── probe_server.go ├── proxy.go └── server.go ├── template ├── control.template └── tcpdp.spec.template ├── testdata ├── haproxy │ └── haproxy.cfg ├── mariadb.conf.d │ └── custom.cnf ├── mysql.conf.d │ └── custom.cnf ├── pcap │ ├── mysql_prepare.pcap │ └── pg_prepare.pcap ├── query │ └── long.sql └── terraform │ └── benchmark.tf └── version └── version.go /.github/release.yml: -------------------------------------------------------------------------------- 1 | changelog: 2 | exclude: 3 | labels: 4 | - tagpr 5 | categories: 6 | - title: Breaking Changes 🛠 7 | labels: 8 | - breaking-change 9 | - title: New Features 🎉 10 | labels: 11 | - enhancement 12 | - title: Fix bug 🐛 13 | labels: 14 | - bug 15 | - title: Other Changes 16 | labels: 17 | - "*" 18 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | job-test: 11 | name: Test 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Check out source code 15 | uses: actions/checkout@v4 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v5 19 | with: 20 | go-version-file: go.mod 21 | 22 | - name: Setup packages 23 | run: | 24 | sudo apt-get update 25 | sudo apt-get install -y libpcap-dev netcat 26 | 27 | - name: Run Docker containers 28 | run: | 29 | docker --version 30 | docker-compose --version 31 | docker-compose up -d postgres mysql57 proxy-protocol-proxy-linux proxy-protocol-mariadb 32 | while ! nc -w 1 127.0.0.1 33066 > /dev/null 2>&1; do sleep 1; echo 'sleeping'; done; 33 | while ! nc -w 1 127.0.0.1 54322 > /dev/null 2>&1; do sleep 1; echo 'sleeping'; done; 34 | while ! nc -w 1 127.0.0.1 33081 > /dev/null 2>&1; do sleep 1; echo 'sleeping'; done; 35 | while ! nc -w 1 127.0.0.1 33068 > /dev/null 2>&1; do sleep 1; echo 'sleeping'; done; 36 | while ! nc -w 1 127.0.0.1 33069 > /dev/null 2>&1; do sleep 1; echo 'sleeping'; done; 37 | while ! nc -w 1 127.0.0.1 33070 > /dev/null 2>&1; do sleep 1; echo 'sleeping'; done; 38 | while ! nc -w 1 127.0.0.1 33071 > /dev/null 2>&1; do sleep 1; echo 'sleeping'; done; 39 | 40 | - name: Test 41 | run: make ci 42 | 43 | - name: Run octocov 44 | uses: k1LoW/octocov-action@v1 45 | -------------------------------------------------------------------------------- /.github/workflows/tagpr.yml: -------------------------------------------------------------------------------- 1 | name: tagpr 2 | on: 3 | push: 4 | branches: 5 | - main 6 | 7 | jobs: 8 | tagpr: 9 | runs-on: ubuntu-latest 10 | outputs: 11 | tagpr-tag: ${{ steps.run-tagpr.outputs.tag }} 12 | go-version: ${{ steps.setup-go.outputs.go-version }} 13 | env: 14 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 15 | steps: 16 | - name: Check out source code 17 | uses: actions/checkout@v4 18 | 19 | - name: Set up Go 20 | id: setup-go 21 | uses: actions/setup-go@v5 22 | with: 23 | go-version-file: go.mod 24 | cache: true 25 | 26 | - name: Run tagpr 27 | id: run-tagpr 28 | uses: Songmu/tagpr@v1 29 | 30 | ubuntu-amd64: 31 | needs: tagpr 32 | if: needs.tagpr.outputs.tagpr-tag != '' 33 | name: Build packages on Ubuntu amd64 34 | runs-on: ubuntu-latest 35 | steps: 36 | - name: Check out source code 37 | uses: actions/checkout@v4 38 | with: 39 | fetch-depth: 0 40 | 41 | - name: Build 42 | uses: k1LoW/run-on-container@v1 43 | with: 44 | run: | 45 | uname -a 46 | 47 | apt-get -qq update 48 | apt-get install -qq gcc g++ make debhelper dh-make clang git curl devscripts fakeroot byacc bison flex libpcap-dev 49 | git config --global --add safe.directory $WORKSPACE 50 | 51 | export LIBPCAP_FILE=libpcap-$LIBPCAP_VERSION.tar.gz 52 | export LIBPCAP_URL=https://www.tcpdump.org/release/$LIBPCAP_FILE 53 | export GO_FILE=go$GO_VERSION.linux-amd64.tar.gz 54 | export GO_URL=https://storage.googleapis.com/golang/$GO_FILE 55 | export GOPATH=/go 56 | export PATH=$GOPATH/bin:/usr/local/go/bin:$PATH 57 | curl -OL $GO_URL 58 | tar -C /usr/local -xzf $GO_FILE 59 | rm $GO_FILE 60 | curl -OL $LIBPCAP_URL 61 | tar -C /usr/local/src -xzf $LIBPCAP_FILE 62 | rm $LIBPCAP_FILE 63 | 64 | mkdir -p $GOPATH/src $GOPATH/bin 65 | chmod -R 777 $GOPATH 66 | go version 67 | 68 | echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list 69 | apt-get -qq update 70 | apt-get install -qq goreleaser 71 | goreleaser -v 72 | goreleaser release --config .goreleaser/deb_amd64.yml --clean --skip=publish 73 | image: ubuntu:latest 74 | platform: linux/amd64 75 | args: '--env LIBPCAP_VERSION --env GO_VERSION' 76 | env: 77 | GO_VERSION: ${{ needs.tagpr.outputs.go-version }} 78 | LIBPCAP_VERSION: 1.10.4 79 | 80 | - name: Check dist/ 81 | run: ls dist/ 82 | 83 | - name: Install test 84 | uses: k1LoW/run-on-container@v1 85 | with: 86 | run: | 87 | apt-get -qq update 88 | apt-get install -qq libpcap0.8 89 | dpkg -i dist/tcpdp*.deb 90 | tcpdp version 91 | image: ubuntu:latest 92 | platform: linux/amd64 93 | 94 | - name: Upload dist/ 95 | uses: actions/upload-artifact@v4 96 | with: 97 | name: dist-deb-amd64 98 | path: | 99 | dist/*.deb 100 | dist/*.tar.gz 101 | dist/checksums* 102 | 103 | ubuntu-arm64: 104 | needs: tagpr 105 | if: needs.tagpr.outputs.tagpr-tag != '' 106 | name: Build packages on Ubuntu arm64 107 | runs-on: ubuntu-latest 108 | steps: 109 | - name: Check out source code 110 | uses: actions/checkout@v4 111 | with: 112 | fetch-depth: 0 113 | 114 | - name: Build 115 | uses: k1LoW/run-on-container@v1 116 | with: 117 | run: | 118 | uname -a 119 | 120 | apt-get -qq update 121 | apt-get install -qq gcc g++ make debhelper dh-make clang git curl devscripts fakeroot byacc bison flex libpcap-dev 122 | git config --global --add safe.directory $WORKSPACE 123 | 124 | export LIBPCAP_FILE=libpcap-$LIBPCAP_VERSION.tar.gz 125 | export LIBPCAP_URL=https://www.tcpdump.org/release/$LIBPCAP_FILE 126 | export GO_FILE=go$GO_VERSION.linux-amd64.tar.gz 127 | export GO_URL=https://storage.googleapis.com/golang/$GO_FILE 128 | export GOPATH=/go 129 | export PATH=$GOPATH/bin:/usr/local/go/bin:$PATH 130 | curl -OL $GO_URL 131 | tar -C /usr/local -xzf $GO_FILE 132 | rm $GO_FILE 133 | curl -OL $LIBPCAP_URL 134 | tar -C /usr/local/src -xzf $LIBPCAP_FILE 135 | rm $LIBPCAP_FILE 136 | 137 | mkdir -p $GOPATH/src $GOPATH/bin 138 | chmod -R 777 $GOPATH 139 | go version 140 | 141 | echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list 142 | apt-get -qq update 143 | apt-get install -qq goreleaser 144 | goreleaser -v 145 | goreleaser release --config .goreleaser/deb_arm64.yml --clean --skip=publish 146 | image: ubuntu:latest 147 | platform: linux/arm64 148 | args: '--env LIBPCAP_VERSION --env GO_VERSION' 149 | env: 150 | GO_VERSION: ${{ needs.tagpr.outputs.go-version }} 151 | LIBPCAP_VERSION: 1.10.4 152 | 153 | - name: Check dist/ 154 | run: ls dist/ 155 | 156 | - name: Install test 157 | uses: k1LoW/run-on-container@v1 158 | with: 159 | run: | 160 | apt-get -qq update 161 | apt-get install -qq libpcap0.8 162 | dpkg -i dist/tcpdp*.deb 163 | tcpdp version 164 | image: ubuntu:latest 165 | platform: linux/arm64 166 | 167 | - name: Upload dist/ 168 | uses: actions/upload-artifact@v4 169 | with: 170 | name: dist-deb-arm64 171 | path: | 172 | dist/*.deb 173 | dist/*.tar.gz 174 | dist/checksums* 175 | 176 | centos-amd64: 177 | needs: tagpr 178 | if: needs.tagpr.outputs.tagpr-tag != '' 179 | name: Build packages on CentOS amd64 180 | runs-on: ubuntu-latest 181 | steps: 182 | - name: Check out source code 183 | uses: actions/checkout@v4 184 | with: 185 | fetch-depth: 0 186 | 187 | - name: Build 188 | uses: k1LoW/run-on-container@v1 189 | with: 190 | run: | 191 | uname -a 192 | 193 | yum install -y epel-release make clang glibc glibc-static gcc byacc flex libpcap-devel 194 | yum remove git* 195 | yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm 196 | yum install -y git 197 | 198 | git config --global --add safe.directory $WORKSPACE 199 | 200 | export LIBPCAP_FILE=libpcap-$LIBPCAP_VERSION.tar.gz 201 | export LIBPCAP_URL=https://www.tcpdump.org/release/$LIBPCAP_FILE 202 | export GO_FILE=go$GO_VERSION.linux-amd64.tar.gz 203 | export GO_URL=https://storage.googleapis.com/golang/$GO_FILE 204 | export GOPATH=/go 205 | export PATH=$GOPATH/bin:/usr/local/go/bin:$PATH 206 | curl -OL $GO_URL 207 | tar -C /usr/local -xzf $GO_FILE 208 | rm $GO_FILE 209 | curl -OL $LIBPCAP_URL 210 | tar -C /usr/local/src -xzf $LIBPCAP_FILE 211 | rm $LIBPCAP_FILE 212 | 213 | mkdir -p $GOPATH/src $GOPATH/bin 214 | chmod -R 777 $GOPATH 215 | go version 216 | 217 | echo '[goreleaser] 218 | name=GoReleaser 219 | baseurl=https://repo.goreleaser.com/yum/ 220 | enabled=1 221 | gpgcheck=0' | tee /etc/yum.repos.d/goreleaser.repo 222 | yum install -y goreleaser 223 | goreleaser -v 224 | goreleaser release --config .goreleaser/rpm_amd64.yml --clean --skip=publish 225 | image: centos:7 226 | platform: linux/amd64 227 | args: '--env LIBPCAP_VERSION --env GO_VERSION' 228 | env: 229 | GO_VERSION: ${{ needs.tagpr.outputs.go-version }} 230 | LIBPCAP_VERSION: 1.10.4 231 | 232 | - name: Check dist/ 233 | run: ls dist/ 234 | 235 | - name: Install test 236 | uses: k1LoW/run-on-container@v1 237 | with: 238 | run: | 239 | yum install -y dist/tcpdp*.rpm 240 | tcpdp version 241 | image: centos:7 242 | platform: linux/amd64 243 | 244 | - name: Upload dist/ 245 | uses: actions/upload-artifact@v4 246 | with: 247 | name: dist-rpm-amd64 248 | path: | 249 | dist/*.rpm 250 | dist/*.tar.gz 251 | dist/checksums* 252 | 253 | centos-arm64: 254 | needs: tagpr 255 | if: needs.tagpr.outputs.tagpr-tag != '' 256 | name: Build packages on CentOS arm64 257 | runs-on: ubuntu-latest 258 | steps: 259 | - name: Check out source code 260 | uses: actions/checkout@v4 261 | with: 262 | fetch-depth: 0 263 | 264 | - name: Build 265 | uses: k1LoW/run-on-container@v1 266 | with: 267 | run: | 268 | uname -a 269 | 270 | yum install -y epel-release make clang glibc glibc-static gcc byacc flex libpcap-devel 271 | yum remove git* 272 | yum -y install epel-release centos-release-scl 273 | yum -y groupinstall "Development Tools" 274 | yum -y install wget perl-CPAN gettext-devel perl-devel openssl-devel zlib-devel curl curl-devel expat-devel getopt asciidoc xmlto docbook2X devtoolset-10 275 | ln -s /usr/bin/db2x_docbook2texi /usr/bin/docbook2x-texi 276 | export GIT_VER="v2.44.0" 277 | wget https://github.com/git/git/archive/${GIT_VER}.tar.gz 278 | tar -xvf ${GIT_VER}.tar.gz 279 | rm -f ${GIT_VER}.tar.gz 280 | cd git-* 281 | scl enable devtoolset-10 'make configure && ./configure --prefix=/usr && make && make install' 282 | cd $WORKSPACE 283 | rm -rf git-* 284 | git config --global --add safe.directory $WORKSPACE 285 | 286 | export LIBPCAP_FILE=libpcap-$LIBPCAP_VERSION.tar.gz 287 | export LIBPCAP_URL=https://www.tcpdump.org/release/$LIBPCAP_FILE 288 | export GO_FILE=go$GO_VERSION.linux-arm64.tar.gz 289 | export GO_URL=https://storage.googleapis.com/golang/$GO_FILE 290 | export GOPATH=/go 291 | export PATH=$GOPATH/bin:/usr/local/go/bin:$PATH 292 | curl -OL $GO_URL 293 | tar -C /usr/local -xzf $GO_FILE 294 | rm $GO_FILE 295 | curl -OL $LIBPCAP_URL 296 | tar -C /usr/local/src -xzf $LIBPCAP_FILE 297 | rm $LIBPCAP_FILE 298 | 299 | mkdir -p $GOPATH/src $GOPATH/bin 300 | chmod -R 777 $GOPATH 301 | go version 302 | 303 | echo '[goreleaser] 304 | name=GoReleaser 305 | baseurl=https://repo.goreleaser.com/yum/ 306 | enabled=1 307 | gpgcheck=0' | tee /etc/yum.repos.d/goreleaser.repo 308 | yum install -y goreleaser 309 | goreleaser -v 310 | goreleaser release --config .goreleaser/rpm_arm64.yml --clean --skip=publish 311 | image: centos:7 312 | platform: linux/arm64 313 | args: '--env LIBPCAP_VERSION --env GO_VERSION' 314 | env: 315 | GO_VERSION: ${{ needs.tagpr.outputs.go-version }} 316 | LIBPCAP_VERSION: 1.10.4 317 | 318 | - name: Check dist/ 319 | run: ls dist/ 320 | 321 | - name: Install test 322 | uses: k1LoW/run-on-container@v1 323 | with: 324 | run: | 325 | yum install -y dist/tcpdp*.rpm 326 | tcpdp version 327 | image: centos:7 328 | platform: linux/arm64 329 | 330 | - name: Upload dist/ 331 | uses: actions/upload-artifact@v4 332 | with: 333 | name: dist-rpm-rpm64 334 | path: | 335 | dist/*.rpm 336 | dist/*.tar.gz 337 | dist/checksums* 338 | release: 339 | runs-on: ubuntu-latest 340 | needs: 341 | - tagpr 342 | - ubuntu-amd64 343 | - ubuntu-arm64 344 | - centos-amd64 345 | - centos-arm64 346 | steps: 347 | - name: Merge Artifacts 348 | uses: actions/upload-artifact/merge@v4 349 | with: 350 | name: merged-artifacts 351 | pattern: dist-* 352 | delete-merged: true 353 | - name: Download Artifacts 354 | uses: actions/download-artifact@v4 355 | with: 356 | name: merged-artifacts 357 | path: dist/ 358 | - name: Check dist/ 359 | run: ls dist/ 360 | - name: Setup ghr 361 | uses: k1LoW/gh-setup@v1 362 | with: 363 | repo: tcnksm/ghr 364 | github-token: ${{ secrets.GITHUB_TOKEN }} 365 | bin-match: ghr 366 | force: true 367 | - name: Release 368 | run: | 369 | ghr -u k1LoW -r tcpdp -t ${{ secrets.GITHUB_TOKEN }} -replace ${{ needs.tagpr.outputs.tagpr-tag }} dist/ 370 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | .terraform 3 | *.tfstate 4 | *.tfstate.backup 5 | coverage*out 6 | tcpdp 7 | -------------------------------------------------------------------------------- /.goreleaser/deb_amd64.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | - go mod tidy 5 | builds: 6 | - 7 | id: tcpdp-linux 8 | flags: 9 | - -buildvcs=false 10 | ldflags: 11 | - -s -w -X github.com/k1LoW/tcpdp.version={{.Version}} -X github.com/k1LoW/tcpdp.commit={{.FullCommit}} -X github.com/k1LoW/tcpdp.date={{.Date}} -X github.com/k1LoW/tcpdp/version.Version={{.Version}} 12 | env: 13 | - CGO_ENABLED=1 14 | goos: 15 | - linux 16 | goarch: 17 | - amd64 18 | archives: 19 | - 20 | id: tcpdp-archive 21 | name_template: '{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' 22 | builds: 23 | - tcpdp-linux 24 | files: 25 | - LICENSE 26 | - CREDITS 27 | - README.md 28 | - CHANGELOG.md 29 | checksum: 30 | name_template: 'checksums-deb_amd64.txt' 31 | snapshot: 32 | name_template: "{{ .Version }}-next" 33 | changelog: 34 | skip: true 35 | sort: asc 36 | filters: 37 | exclude: 38 | - '^docs:' 39 | - '^test:' 40 | nfpms: 41 | - 42 | id: tcpdp-nfpms 43 | file_name_template: "{{ .ProjectName }}_{{ .Version }}-1_{{ .Arch }}" 44 | builds: 45 | - tcpdp-linux 46 | homepage: https://github.com/k1LoW/tcpdp 47 | maintainer: Ken'ichiro Oyama 48 | description: tcpdp is TCP dump tool with custom dumper and structured logger written in Go. 49 | license: MIT 50 | formats: 51 | - deb 52 | bindir: /usr/bin 53 | dependencies: 54 | - libpcap0.8 55 | epoch: 1 56 | release: 57 | draft: true 58 | skip_upload: true 59 | -------------------------------------------------------------------------------- /.goreleaser/deb_arm64.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | - go mod tidy 5 | builds: 6 | - 7 | id: tcpdp-linux 8 | flags: 9 | - -buildvcs=false 10 | ldflags: 11 | - -s -w -X github.com/k1LoW/tcpdp.version={{.Version}} -X github.com/k1LoW/tcpdp.commit={{.FullCommit}} -X github.com/k1LoW/tcpdp.date={{.Date}} -X github.com/k1LoW/tcpdp/version.Version={{.Version}} 12 | env: 13 | - CGO_ENABLED=1 14 | goos: 15 | - linux 16 | goarch: 17 | - arm64 18 | archives: 19 | - 20 | id: tcpdp-archive 21 | name_template: '{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' 22 | builds: 23 | - tcpdp-linux 24 | files: 25 | - LICENSE 26 | - CREDITS 27 | - README.md 28 | - CHANGELOG.md 29 | checksum: 30 | name_template: 'checksums-deb_arm64.txt' 31 | snapshot: 32 | name_template: "{{ .Version }}-next" 33 | changelog: 34 | skip: true 35 | sort: asc 36 | filters: 37 | exclude: 38 | - '^docs:' 39 | - '^test:' 40 | nfpms: 41 | - 42 | id: tcpdp-nfpms 43 | file_name_template: "{{ .ProjectName }}_{{ .Version }}-1_{{ .Arch }}" 44 | builds: 45 | - tcpdp-linux 46 | homepage: https://github.com/k1LoW/tcpdp 47 | maintainer: Ken'ichiro Oyama 48 | description: tcpdp is TCP dump tool with custom dumper and structured logger written in Go. 49 | license: MIT 50 | formats: 51 | - deb 52 | bindir: /usr/bin 53 | dependencies: 54 | - libpcap0.8 55 | epoch: 1 56 | release: 57 | draft: true 58 | skip_upload: true 59 | -------------------------------------------------------------------------------- /.goreleaser/deb_arm64_static.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | - go mod tidy 5 | builds: 6 | - 7 | id: tcpdp-linux-static 8 | flags: 9 | - -a 10 | - -tags 11 | - netgo 12 | - -installsuffix 13 | - netgo 14 | - -buildvcs=false 15 | ldflags: 16 | - -s -w -X github.com/k1LoW/tcpdp.version={{.Version}} -X github.com/k1LoW/tcpdp.commit={{.FullCommit}} -X github.com/k1LoW/tcpdp.date={{.Date}} -X github.com/k1LoW/tcpdp/version.Version={{.Version}} 17 | - -linkmode external 18 | - -extldflags '-static' 19 | env: 20 | - CGO_ENABLED=1 21 | goos: 22 | - linux 23 | goarch: 24 | - arm64 25 | archives: 26 | - 27 | id: tcpdp-archive-static 28 | name_template: '{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_static_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' 29 | builds: 30 | - tcpdp-linux-static 31 | files: 32 | - LICENSE 33 | - CREDITS 34 | - README.md 35 | - CHANGELOG.md 36 | checksum: 37 | name_template: 'checksums-deb_arm64_static.txt' 38 | snapshot: 39 | name_template: "{{ .Version }}-next" 40 | changelog: 41 | skip: true 42 | sort: asc 43 | filters: 44 | exclude: 45 | - '^docs:' 46 | - '^test:' 47 | nfpms: 48 | - 49 | id: tcpdp-nfpms-static 50 | file_name_template: "{{ .ProjectName }}_{{ .Version }}-1_{{ .Arch }}_static" 51 | builds: 52 | - tcpdp-linux-static 53 | homepage: https://github.com/k1LoW/tcpdp 54 | maintainer: Ken'ichiro Oyama 55 | description: tcpdp is TCP dump tool with custom dumper and structured logger written in Go. 56 | license: MIT 57 | formats: 58 | - deb 59 | bindir: /usr/bin 60 | epoch: 1 61 | release: 62 | draft: true 63 | skip_upload: true 64 | -------------------------------------------------------------------------------- /.goreleaser/rpm_amd64.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | - go mod tidy 5 | builds: 6 | - 7 | id: tcpdp-linux 8 | flags: 9 | - -buildvcs=false 10 | ldflags: 11 | - -s -w -X github.com/k1LoW/tcpdp.version={{.Version}} -X github.com/k1LoW/tcpdp.commit={{.FullCommit}} -X github.com/k1LoW/tcpdp.date={{.Date}} -X github.com/k1LoW/tcpdp/version.Version={{.Version}} 12 | env: 13 | - CGO_ENABLED=1 14 | goos: 15 | - linux 16 | goarch: 17 | - amd64 18 | archives: 19 | - 20 | id: tcpdp-archive 21 | name_template: '{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}.el7' 22 | builds: 23 | - tcpdp-linux 24 | files: 25 | - LICENSE 26 | - CREDITS 27 | - README.md 28 | - CHANGELOG.md 29 | checksum: 30 | name_template: 'checksums-rpm_amd64.txt' 31 | snapshot: 32 | name_template: "{{ .Version }}-next" 33 | changelog: 34 | skip: true 35 | sort: asc 36 | filters: 37 | exclude: 38 | - '^docs:' 39 | - '^test:' 40 | nfpms: 41 | - 42 | id: tcpdp-nfpms 43 | file_name_template: "{{ .ProjectName }}_{{ .Version }}-1_{{ .Arch }}" 44 | builds: 45 | - tcpdp-linux 46 | homepage: https://github.com/k1LoW/tcpdp 47 | maintainer: Ken'ichiro Oyama 48 | description: tcpdp is TCP dump tool with custom dumper and structured logger written in Go. 49 | license: MIT 50 | formats: 51 | - rpm 52 | bindir: /usr/bin 53 | dependencies: 54 | - libpcap-devel 55 | epoch: 1 56 | release: 57 | draft: true 58 | skip_upload: true 59 | -------------------------------------------------------------------------------- /.goreleaser/rpm_arm64.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | - go mod tidy 5 | builds: 6 | - 7 | id: tcpdp-linux 8 | flags: 9 | - -buildvcs=false 10 | ldflags: 11 | - -s -w -X github.com/k1LoW/tcpdp.version={{.Version}} -X github.com/k1LoW/tcpdp.commit={{.FullCommit}} -X github.com/k1LoW/tcpdp.date={{.Date}} -X github.com/k1LoW/tcpdp/version.Version={{.Version}} 12 | env: 13 | - CGO_ENABLED=1 14 | goos: 15 | - linux 16 | goarch: 17 | - arm64 18 | archives: 19 | - 20 | id: tcpdp-archive 21 | name_template: '{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}.el7' 22 | builds: 23 | - tcpdp-linux 24 | files: 25 | - LICENSE 26 | - CREDITS 27 | - README.md 28 | - CHANGELOG.md 29 | checksum: 30 | name_template: 'checksums-rpm_arm64.txt' 31 | snapshot: 32 | name_template: "{{ .Version }}-next" 33 | changelog: 34 | skip: true 35 | sort: asc 36 | filters: 37 | exclude: 38 | - '^docs:' 39 | - '^test:' 40 | nfpms: 41 | - 42 | id: tcpdp-nfpms 43 | file_name_template: "{{ .ProjectName }}_{{ .Version }}-1_{{ .Arch }}" 44 | builds: 45 | - tcpdp-linux 46 | homepage: https://github.com/k1LoW/tcpdp 47 | maintainer: Ken'ichiro Oyama 48 | description: tcpdp is TCP dump tool with custom dumper and structured logger written in Go. 49 | license: MIT 50 | formats: 51 | - rpm 52 | bindir: /usr/bin 53 | dependencies: 54 | - libpcap-devel 55 | epoch: 1 56 | release: 57 | draft: true 58 | skip_upload: true 59 | -------------------------------------------------------------------------------- /.octocov.yml: -------------------------------------------------------------------------------- 1 | # generated by octocov init 2 | coverage: 3 | paths: 4 | - coverage.out 5 | - coverage-integration.out 6 | codeToTestRatio: 7 | code: 8 | - '**/*.go' 9 | - '!**/*_test.go' 10 | test: 11 | - '**/*_test.go' 12 | testExecutionTime: 13 | if: true 14 | diff: 15 | datastores: 16 | - artifact://${GITHUB_REPOSITORY} 17 | comment: 18 | if: is_pull_request 19 | report: 20 | if: is_default_branch 21 | datastores: 22 | - artifact://${GITHUB_REPOSITORY} 23 | -------------------------------------------------------------------------------- /.tagpr: -------------------------------------------------------------------------------- 1 | # config file for the tagpr in git config format 2 | # The tagpr generates the initial configuration, which you can rewrite to suit your environment. 3 | # CONFIGURATIONS: 4 | # tagpr.releaseBranch 5 | # Generally, it is "main." It is the branch for releases. The pcpr tracks this branch, 6 | # creates or updates a pull request as a release candidate, or tags when they are merged. 7 | # 8 | # tagpr.versionFile 9 | # Versioning file containing the semantic version needed to be updated at release. 10 | # It will be synchronized with the "git tag". 11 | # Often this is a meta-information file such as gemspec, setup.cfg, package.json, etc. 12 | # Sometimes the source code file, such as version.go or Bar.pm, is used. 13 | # If you do not want to use versioning files but only git tags, specify the "-" string here. 14 | # You can specify multiple version files by comma separated strings. 15 | # 16 | # tagpr.vPrefix 17 | # Flag whether or not v-prefix is added to semver when git tagging. (e.g. v1.2.3 if true) 18 | # This is only a tagging convention, not how it is described in the version file. 19 | # 20 | # tagpr.changelog (Optional) 21 | # Flag whether or not changelog is added or changed during the release. 22 | # 23 | # tagpr.command (Optional) 24 | # Command to change files just before release. 25 | # 26 | # tagpr.tmplate (Optional) 27 | # Pull request template in go template format 28 | # 29 | # tagpr.release (Optional) 30 | # GitHub Release creation behavior after tagging [true, draft, false] 31 | # If this value is not set, the release is to be created. 32 | [tagpr] 33 | vPrefix = true 34 | releaseBranch = main 35 | release = draft 36 | versionFile = version/version.go 37 | command = "make prerelease_for_tagpr" 38 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [v0.23.9](https://github.com/k1LoW/tcpdp/compare/v0.23.8...v0.23.9) - 2024-03-28 4 | 5 | ## [v0.23.8](https://github.com/k1LoW/tcpdp/compare/v0.23.7...v0.23.8) - 2024-03-28 6 | 7 | ## [v0.23.7](https://github.com/k1LoW/tcpdp/compare/v0.23.6...v0.23.7) - 2024-03-28 8 | 9 | ## [v0.23.6](https://github.com/k1LoW/tcpdp/compare/v0.23.5...v0.23.6) - 2024-03-28 10 | 11 | ## [v0.23.5](https://github.com/k1LoW/tcpdp/compare/v0.23.4...v0.23.5) - 2024-03-28 12 | 13 | ## [v0.23.4](https://github.com/k1LoW/tcpdp/compare/v0.23.3...v0.23.4) - 2024-03-28 14 | 15 | ## [v0.23.3](https://github.com/k1LoW/tcpdp/compare/v0.23.2...v0.23.3) - 2024-03-28 16 | 17 | ## [v0.23.2](https://github.com/k1LoW/tcpdp/compare/v0.23.1...v0.23.2) - 2024-03-28 18 | 19 | ## [v0.23.1](https://github.com/k1LoW/tcpdp/compare/v0.23.0...v0.23.1) - 2024-03-28 20 | 21 | ## [v0.23.0](https://github.com/k1LoW/tcpdp/compare/v0.22.2...v0.23.0) - 2024-03-28 22 | ### Breaking Changes 🛠 23 | - Update go version by @k1LoW in https://github.com/k1LoW/tcpdp/pull/106 24 | ### Other Changes 25 | - Bump golang.org/x/text from 0.3.7 to 0.3.8 by @dependabot in https://github.com/k1LoW/tcpdp/pull/102 26 | - Bump gopkg.in/yaml.v3 from 3.0.0-20210107192922-496545a6307b to 3.0.0 by @dependabot in https://github.com/k1LoW/tcpdp/pull/103 27 | - Add Go 1.22 by @k1LoW in https://github.com/k1LoW/tcpdp/pull/104 28 | - Fix build pipeline by @k1LoW in https://github.com/k1LoW/tcpdp/pull/105 29 | - Setup tagpr by @k1LoW in https://github.com/k1LoW/tcpdp/pull/107 30 | - Cleanup unnecessary code/files by @k1LoW in https://github.com/k1LoW/tcpdp/pull/109 31 | 32 | ## [v0.22.2](https://github.com/k1LoW/tcpdp/compare/v0.22.1...v0.22.2) (2022-09-08) 33 | 34 | * Fix build and release pipeline [#100](https://github.com/k1LoW/tcpdp/pull/100) ([k1LoW](https://github.com/k1LoW)) 35 | * Revert "Update go and pkgs" [#101](https://github.com/k1LoW/tcpdp/pull/101) ([k1LoW](https://github.com/k1LoW)) 36 | * Update go and pkgs [#99](https://github.com/k1LoW/tcpdp/pull/99) ([k1LoW](https://github.com/k1LoW)) 37 | * Update the GoText package [#98](https://github.com/k1LoW/tcpdp/pull/98) ([rnakamine](https://github.com/rnakamine)) 38 | * Use octocov [#97](https://github.com/k1LoW/tcpdp/pull/97) ([k1LoW](https://github.com/k1LoW)) 39 | 40 | ## [v0.22.1](https://github.com/k1LoW/tcpdp/compare/v0.22.0...v0.22.1) (2021-09-07) 41 | 42 | * Fix null pointer dereference [#96](https://github.com/k1LoW/tcpdp/pull/96) ([k1LoW](https://github.com/k1LoW)) 43 | 44 | ## [v0.22.0](https://github.com/k1LoW/tcpdp/compare/v0.21.1...v0.22.0) (2020-11-25) 45 | 46 | * Fix payloadBuffer invalid nil pointer dereference [#95](https://github.com/k1LoW/tcpdp/pull/95) ([k1LoW](https://github.com/k1LoW)) 47 | * Use GitHub Actions [#94](https://github.com/k1LoW/tcpdp/pull/94) ([k1LoW](https://github.com/k1LoW)) 48 | * Bump up go version [#93](https://github.com/k1LoW/tcpdp/pull/93) ([k1LoW](https://github.com/k1LoW)) 49 | * fix a typo in doc string [#92](https://github.com/k1LoW/tcpdp/pull/92) ([kentaro](https://github.com/kentaro)) 50 | * Fix payloadBuffer.Get ( when buffer deleted, return nil ) [#91](https://github.com/k1LoW/tcpdp/pull/91) ([k1LoW](https://github.com/k1LoW)) 51 | * Add make sec [#89](https://github.com/k1LoW/tcpdp/pull/89) ([k1LoW](https://github.com/k1LoW)) 52 | 53 | ## [v0.21.2](https://github.com/k1LoW/tcpdp/compare/v0.21.1...v0.21.2) (2019-08-09) 54 | 55 | * Fix payloadBuffer.Get ( when buffer deleted, return nil ) [#91](https://github.com/k1LoW/tcpdp/pull/91) ([k1LoW](https://github.com/k1LoW)) 56 | * Add make sec [#89](https://github.com/k1LoW/tcpdp/pull/89) ([k1LoW](https://github.com/k1LoW)) 57 | 58 | ## [v0.21.0](https://github.com/k1LoW/tcpdp/compare/v0.20.3...v0.21.0) (2019-06-19) 59 | 60 | * Add lock/unlock to the place to operate pMap.buffers [#88](https://github.com/k1LoW/tcpdp/pull/88) ([k1LoW](https://github.com/k1LoW)) 61 | * Log *addr when dumper.Read error ( ex. SSL connection ) [#86](https://github.com/k1LoW/tcpdp/pull/86) ([k1LoW](https://github.com/k1LoW)) 62 | 63 | ## [v0.20.3](https://github.com/k1LoW/tcpdp/compare/v0.20.2...v0.20.3) (2019-06-11) 64 | 65 | * Add more memStats metrics [#85](https://github.com/k1LoW/tcpdp/pull/85) ([k1LoW](https://github.com/k1LoW)) 66 | * handle FIN / RST to delete unused buffer [#84](https://github.com/k1LoW/tcpdp/pull/84) ([k1LoW](https://github.com/k1LoW)) 67 | * Add mutex to payloadBufferManager [#83](https://github.com/k1LoW/tcpdp/pull/83) ([k1LoW](https://github.com/k1LoW)) 68 | * hundle dumper.Read error and delete unnecessary buffer [#82](https://github.com/k1LoW/tcpdp/pull/82) ([k1LoW](https://github.com/k1LoW)) 69 | 70 | ## [v0.20.2](https://github.com/k1LoW/tcpdp/compare/v0.20.1...v0.20.2) (2019-06-09) 71 | 72 | * Fix payload buffer leak [#81](https://github.com/k1LoW/tcpdp/pull/81) ([k1LoW](https://github.com/k1LoW)) 73 | 74 | ## [v0.20.1](https://github.com/k1LoW/tcpdp/compare/v0.20.0...v0.20.1) (2019-06-06) 75 | 76 | * Fix output `tcpdp version` 77 | 78 | ## [v0.20.0](https://github.com/k1LoW/tcpdp/compare/v0.19.0...v0.20.0) (2019-06-04) 79 | 80 | * Add `log.enableInternal` to log tcpdp internal stats [#80](https://github.com/k1LoW/tcpdp/pull/80) ([k1LoW](https://github.com/k1LoW)) 81 | 82 | ## [v0.19.0](https://github.com/k1LoW/tcpdp/compare/v0.18.0...v0.19.0) (2019-05-15) 83 | 84 | * Add dumper `conn` for tracking only connection [#79](https://github.com/k1LoW/tcpdp/pull/79) ([k1LoW](https://github.com/k1LoW)) 85 | 86 | ## [v0.18.0](https://github.com/k1LoW/tcpdp/compare/v0.17.0...v0.18.0) (2019-05-14) 87 | 88 | * Build using Go 1.12.x [#78](https://github.com/k1LoW/tcpdp/pull/78) ([k1LoW](https://github.com/k1LoW)) 89 | * Support `-i any` (Linux only) [#77](https://github.com/k1LoW/tcpdp/pull/77) ([k1LoW](https://github.com/k1LoW)) 90 | 91 | ## [v0.17.0](https://github.com/k1LoW/tcpdp/compare/v0.16.1...v0.17.0) (2019-02-17) 92 | 93 | * Gonize integration test [#76](https://github.com/k1LoW/tcpdp/pull/76) ([k1LoW](https://github.com/k1LoW)) 94 | * Support proxy protocol [#75](https://github.com/k1LoW/tcpdp/pull/75) ([k1LoW](https://github.com/k1LoW)) 95 | 96 | ## [v0.16.1](https://github.com/k1LoW/tcpdp/compare/v0.16.0...v0.16.1) (2018-12-17) 97 | 98 | * Refactor NewProbeServer and logging info [#74](https://github.com/k1LoW/tcpdp/pull/74) ([k1LoW](https://github.com/k1LoW)) 99 | 100 | ## [v0.16.0](https://github.com/k1LoW/tcpdp/compare/v0.15.0...v0.16.0) (2018-12-16) 101 | 102 | * Add `--filter` option for override BPF [#73](https://github.com/k1LoW/tcpdp/pull/73) ([k1LoW](https://github.com/k1LoW)) 103 | * Support multi target [#72](https://github.com/k1LoW/tcpdp/pull/72) ([k1LoW](https://github.com/k1LoW)) 104 | 105 | ## [v0.15.0](https://github.com/k1LoW/tcpdp/compare/v0.14.1...v0.15.0) (2018-12-12) 106 | 107 | * Add `--stdout` option [#71](https://github.com/k1LoW/tcpdp/pull/71) ([k1LoW](https://github.com/k1LoW)) 108 | 109 | ## [v0.14.1](https://github.com/k1LoW/tcpdp/compare/v0.14.0...v0.14.1) (2018-12-07) 110 | 111 | * Fix COM_STMT_EXECUTE with zero stmt_execute_values [#70](https://github.com/k1LoW/tcpdp/pull/70) ([k1LoW](https://github.com/k1LoW)) 112 | * Fix break packets for parsing HandshakeResponse320 [#69](https://github.com/k1LoW/tcpdp/pull/69) ([k1LoW](https://github.com/k1LoW)) 113 | 114 | ## [v0.14.0](https://github.com/k1LoW/tcpdp/compare/v0.13.1...v0.14.0) (2018-11-28) 115 | 116 | * Add Unsupport SSL warning when Protocol::HandshakeResponse320 [#68](https://github.com/k1LoW/tcpdp/pull/68) ([k1LoW](https://github.com/k1LoW)) 117 | * Analyze username and database name via Protocol::HandshakeResponse320 [#67](https://github.com/k1LoW/tcpdp/pull/67) ([k1LoW](https://github.com/k1LoW)) 118 | * Check SSLRequest when parsing Handshake and write warning to dump.log [#66](https://github.com/k1LoW/tcpdp/pull/66) ([k1LoW](https://github.com/k1LoW)) 119 | 120 | ## [v0.13.1](https://github.com/k1LoW/tcpdp/compare/v0.13.0...v0.13.1) (2018-10-23) 121 | 122 | * Write `mtu` `mss` to log [#65](https://github.com/k1LoW/tcpdp/pull/65) ([k1LoW](https://github.com/k1LoW)) 123 | * Fix default snapshotLength and Fix auto detect snapshotLength [#64](https://github.com/k1LoW/tcpdp/pull/64) ([k1LoW](https://github.com/k1LoW)) 124 | 125 | ## [v0.13.0](https://github.com/k1LoW/tcpdp/compare/v0.12.0...v0.13.0) (2018-10-22) 126 | 127 | * Add `--snapshot-length` option to `tcpdp probe` [#63](https://github.com/k1LoW/tcpdp/pull/63) ([k1LoW](https://github.com/k1LoW)) 128 | * Check clientSSL when parsing HandshakeResponse and write warning to dump.log [#62](https://github.com/k1LoW/tcpdp/pull/62) ([k1LoW](https://github.com/k1LoW)) 129 | * Add *log.fileName config option [#61](https://github.com/k1LoW/tcpdp/pull/61) ([k1LoW](https://github.com/k1LoW)) 130 | 131 | ## [v0.12.0](https://github.com/k1LoW/tcpdp/compare/v0.11.0...v0.12.0) (2018-10-14) 132 | 133 | * Make internal packet buffer [#57](https://github.com/k1LoW/tcpdp/pull/57) ([k1LoW](https://github.com/k1LoW)) 134 | * Add `make lint` [#60](https://github.com/k1LoW/tcpdp/pull/60) ([k1LoW](https://github.com/k1LoW)) 135 | * Support PostgreSQL long query [#59](https://github.com/k1LoW/tcpdp/pull/59) ([k1LoW](https://github.com/k1LoW)) 136 | * Fix probe / proxy starting log format [#56](https://github.com/k1LoW/tcpdp/pull/56) ([k1LoW](https://github.com/k1LoW)) 137 | * Fix `--use-server-starter` option ( replace miss ) [#55](https://github.com/k1LoW/tcpdp/pull/55) ([k1LoW](https://github.com/k1LoW)) 138 | * Disable promiscuous mode [#54](https://github.com/k1LoW/tcpdp/pull/54) ([k1LoW](https://github.com/k1LoW)) 139 | * Add rotationTime `secondly` [#51](https://github.com/k1LoW/tcpdp/pull/51) [#53](https://github.com/k1LoW/tcpdp/pull/53) ([k1LoW](https://github.com/k1LoW)) 140 | 141 | ## [v0.11.0](https://github.com/k1LoW/tcpdp/compare/v0.10.0...v0.11.0) (2018-10-10) 142 | 143 | * Add `--immediate-mode` option to `tcpdp probe` [#50](https://github.com/k1LoW/tcpdp/pull/50) ([k1LoW](https://github.com/k1LoW)) 144 | * Add `--buffer-size (-B)` option to `tcpdp probe` [#49](https://github.com/k1LoW/tcpdp/pull/49) ([k1LoW](https://github.com/k1LoW)) 145 | * Log pcap_stats when shutdown probe server or packets dropped [#48](https://github.com/k1LoW/tcpdp/pull/48) ([k1LoW](https://github.com/k1LoW)) 146 | * Increase snaplen so as not to lost packets [#47](https://github.com/k1LoW/tcpdp/pull/47) ([k1LoW](https://github.com/k1LoW)) 147 | * Add logger to PacketReader [#46](https://github.com/k1LoW/tcpdp/pull/46) ([k1LoW](https://github.com/k1LoW)) 148 | * bMap should be per direction ( not per connection ) [#45](https://github.com/k1LoW/tcpdp/pull/45) ([k1LoW](https://github.com/k1LoW)) 149 | * Support MySQL long query with payload_length [#44](https://github.com/k1LoW/tcpdp/pull/44) ([k1LoW](https://github.com/k1LoW)) 150 | * Support long packet [#43](https://github.com/k1LoW/tcpdp/pull/43) ([k1LoW](https://github.com/k1LoW)) 151 | * update ghch [#42](https://github.com/k1LoW/tcpdp/pull/42) ([pyama86](https://github.com/pyama86)) 152 | 153 | ## [v0.10.0](https://github.com/k1LoW/tcpdp/compare/v0.9.1...v0.10.0) (2018-09-30) 154 | 155 | * Build deb package [#39](https://github.com/k1LoW/tcpdp/pull/39) ([k1LoW](https://github.com/k1LoW)) 156 | * Build RPM package [#38](https://github.com/k1LoW/tcpdp/pull/38) ([k1LoW](https://github.com/k1LoW)) 157 | * Support MySQL client default-character-set [#37](https://github.com/k1LoW/tcpdp/pull/37) ([k1LoW](https://github.com/k1LoW)) 158 | * Separate tcpdp/dumper to tcpdp/dumper and tcpdp/dumper/* [#36](https://github.com/k1LoW/tcpdp/pull/36) ([k1LoW](https://github.com/k1LoW)) 159 | * Let's stop vendor [#35](https://github.com/k1LoW/tcpdp/pull/35) ([pyama86](https://github.com/pyama86)) 160 | * Fix busy loop [#34](https://github.com/k1LoW/tcpdp/pull/34) ([k1LoW](https://github.com/k1LoW)) 161 | * Support parse MySQL compressed packet [#33](https://github.com/k1LoW/tcpdp/pull/33) ([k1LoW](https://github.com/k1LoW)) 162 | * Use LABEL instead of MAINTAINER (for deprecation) [#32](https://github.com/k1LoW/tcpdp/pull/32) ([hfm](https://github.com/hfm)) 163 | 164 | ## [v0.9.1](https://github.com/k1LoW/tcpdp/compare/v0.9.0...v0.9.1) (2018-09-27) 165 | 166 | * Fix parsing `--target` and generating BPF Filter [#31](https://github.com/k1LoW/tcpdp/pull/31) ([k1LoW](https://github.com/k1LoW)) 167 | * Remove `-X $(PKG).date=` from -ldflags and add `-X $(PKG).version=` [#30](https://github.com/k1LoW/tcpdp/pull/30) ([k1LoW](https://github.com/k1LoW)) 168 | * Fix `make crossbuild` [#29](https://github.com/k1LoW/tcpdp/pull/29) ([k1LoW](https://github.com/k1LoW)) 169 | 170 | ## [v0.9.0](https://github.com/k1LoW/tcpdp/compare/v0.8.0...v0.9.0) (2018-09-25) 171 | 172 | * I want to do arbitrary processing after rotate [#28](https://github.com/k1LoW/tcpdp/pull/28) ([pyama86](https://github.com/pyama86)) 173 | * support any ip sniffing [#27](https://github.com/k1LoW/tcpdp/pull/27) ([pyama86](https://github.com/pyama86)) 174 | 175 | ## [v0.8.0](https://github.com/k1LoW/tcpdp/compare/v0.7.0...v0.8.0) (2018-09-24) 176 | 177 | * [BREAKING]Fix HexDumper output [#26](https://github.com/k1LoW/tcpdp/pull/26) ([k1LoW](https://github.com/k1LoW)) 178 | * Add more values to error log when `probe` [#25](https://github.com/k1LoW/tcpdp/pull/25) ([k1LoW](https://github.com/k1LoW)) 179 | * [BREAKING]Fix *Dumber Read when direction = Unknown [#24](https://github.com/k1LoW/tcpdp/pull/24) ([k1LoW](https://github.com/k1LoW)) 180 | * Remove dumper.ReadInitialDumpValues [#23](https://github.com/k1LoW/tcpdp/pull/23) ([k1LoW](https://github.com/k1LoW)) 181 | 182 | ## [v0.7.0](https://github.com/k1LoW/tcpdp/compare/v0.6.1...v0.7.0) (2018-09-23) 183 | 184 | * [BREAKING]Parse PostgreSQL MessageParse / MessageBind / ( MessageExecute ) [#22](https://github.com/k1LoW/tcpdp/pull/22) ([k1LoW](https://github.com/k1LoW)) 185 | * Fix read StartupMessage [#21](https://github.com/k1LoW/tcpdp/pull/21) ([k1LoW](https://github.com/k1LoW)) 186 | * [BREAKING]Parse MySQL COM_STMT_PREPARE / COM_STMT_EXECUTE [#20](https://github.com/k1LoW/tcpdp/pull/20) ([k1LoW](https://github.com/k1LoW)) 187 | * Fix logic that read packet [#19](https://github.com/k1LoW/tcpdp/pull/19) ([k1LoW](https://github.com/k1LoW)) 188 | 189 | ## [v0.6.1](https://github.com/k1LoW/tcpdp/compare/v0.6.0...v0.6.1) (2018-09-19) 190 | 191 | * Remove pprof [#18](https://github.com/k1LoW/tcpdp/pull/18) ([k1LoW](https://github.com/k1LoW)) 192 | * Disable dump.log when execute `tcpdp read` [#17](https://github.com/k1LoW/tcpdp/pull/17) ([k1LoW](https://github.com/k1LoW)) 193 | 194 | ## [v0.6.0](https://github.com/k1LoW/tcpdp/compare/v0.5.0...v0.6.0) (2018-09-19) 195 | 196 | * Fix panic when exec root command with invalid option. [#16](https://github.com/k1LoW/tcpdp/pull/16) ([k1LoW](https://github.com/k1LoW)) 197 | * Add `read` command for read pcap file. [#15](https://github.com/k1LoW/tcpdp/pull/15) ([k1LoW](https://github.com/k1LoW)) 198 | 199 | ## [v0.5.0](https://github.com/k1LoW/tcpdp/compare/v0.4.1...v0.5.0) (2018-09-14) 200 | 201 | * `--target` can set port only [#13](https://github.com/k1LoW/tcpdp/pull/13) ([k1LoW](https://github.com/k1LoW)) 202 | 203 | ## [v0.4.1](https://github.com/k1LoW/tcpdp/compare/v0.4.0...v0.4.1) (2018-09-14) 204 | 205 | * Add `conn_id` to `probe` dump.log [#12](https://github.com/k1LoW/tcpdp/pull/12) ([k1LoW](https://github.com/k1LoW)) 206 | * Fix -d parse logic [#11](https://github.com/k1LoW/tcpdp/pull/11) ([k1LoW](https://github.com/k1LoW)) 207 | 208 | ## [v0.4.0](https://github.com/k1LoW/tcpdp/compare/v0.3.0...v0.4.0) (2018-09-14) 209 | 210 | * [BREAKING] Rename package `tcpdp` -> `tcpdp` [#10](https://github.com/k1LoW/tcpdp/pull/10) ([k1LoW](https://github.com/k1LoW)) 211 | * [BREAKING] Rename command `server` -> `proxy` [#9](https://github.com/k1LoW/tcpdp/pull/9) ([k1LoW](https://github.com/k1LoW)) 212 | * Add `probe` command like tcpdump [#8](https://github.com/k1LoW/tcpdp/pull/8) ([k1LoW](https://github.com/k1LoW)) 213 | * Refactor Dumper struct [#7](https://github.com/k1LoW/tcpdp/pull/7) ([k1LoW](https://github.com/k1LoW)) 214 | 215 | ## [v0.3.0](https://github.com/k1LoW/tcprxy/compare/v0.2.1...v0.3.0) (2018-09-08) 216 | 217 | * Analyze database name via Protocol::HandshakeResponse41 [#6](https://github.com/k1LoW/tcprxy/pull/6) ([k1LoW](https://github.com/k1LoW)) 218 | 219 | ## [v0.2.1](https://github.com/k1LoW/tcprxy/compare/v0.2.0...v0.2.1) (2018-09-06) 220 | 221 | * Fix `tcprxy config` output [#5](https://github.com/k1LoW/tcprxy/pull/5) ([k1LoW](https://github.com/k1LoW)) 222 | 223 | ## [v0.2.0](https://github.com/k1LoW/tcprxy/compare/v0.1.0...v0.2.0) (2018-08-30) 224 | 225 | * Add pidfile config [#4](https://github.com/k1LoW/tcprxy/pull/4) ([k1LoW](https://github.com/k1LoW)) 226 | * Add log config [#3](https://github.com/k1LoW/tcprxy/pull/3) ([k1LoW](https://github.com/k1LoW)) 227 | * Fix hex dump log config [#2](https://github.com/k1LoW/tcprxy/pull/2) ([k1LoW](https://github.com/k1LoW)) 228 | 229 | ## [v0.1.0](https://github.com/k1LoW/tcprxy/compare/33d46026c86c...v0.1.0) (2018-08-29) 230 | 231 | * Add dumper for MySQL query [#1](https://github.com/k1LoW/tcprxy/pull/1) ([k1LoW](https://github.com/k1LoW)) 232 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2018 Ken'ichiro Oyama 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PKG = github.com/k1LoW/tcpdp 2 | COMMIT = $$(git describe --tags --always) 3 | OSNAME=${shell uname -s} 4 | ifeq ($(OSNAME),Darwin) 5 | export LO = lo0 6 | export MYSQL_DISABLE_SSL = --ssl-mode=DISABLED 7 | export GOMPLATE_OS=darwin 8 | else 9 | export LO = lo 10 | export MYSQL_DISABLE_SSL = --ssl-mode=DISABLED 11 | export GOMPLATE_OS=linux 12 | endif 13 | 14 | export GO111MODULE=on 15 | 16 | BUILD_LDFLAGS = -X $(PKG).commit=$(COMMIT) 17 | RELEASE_BUILD_LDFLAGS = -s -w $(BUILD_LDFLAGS) 18 | 19 | SOURCES=Makefile CHANGELOG.md README.md LICENSE go.mod go.sum dumper logger reader server cmd version main.go 20 | 21 | export POSTGRES_PORT=54322 22 | export POSTGRES_USER=postgres 23 | export POSTGRES_PASSWORD=pgpass 24 | export POSTGRES_DB=testdb 25 | 26 | export MYSQL_PORT=33066 27 | export MYSQL_DATABASE=testdb 28 | export MYSQL_ROOT_PASSWORD=mypass 29 | 30 | default: build 31 | ci: depsdev test_race test_with_integration sec 32 | 33 | test: 34 | go test -v $(shell go list ./... | grep -v misc) -coverprofile=coverage.out -covermode=count 35 | 36 | sec: 37 | gosec ./... 38 | 39 | test_race: 40 | go test $(shell go list ./... | grep -v misc) -race 41 | 42 | test_with_integration: build 43 | go test -v $(shell go list ./... | grep -v misc) -tags integration -coverprofile=coverage-integration.out -covermode=count 44 | 45 | build: 46 | go build -ldflags="$(BUILD_LDFLAGS)" 47 | 48 | depsdev: 49 | go install github.com/Songmu/ghch/cmd/ghch@latest 50 | go install github.com/Songmu/gocredits/cmd/gocredits@latest 51 | go install github.com/securego/gosec/v2/cmd/gosec@latest 52 | go install github.com/tcnksm/ghr@latest 53 | go install github.com/hairyhenderson/gomplate/v3/cmd/gomplate@v3.9.0 54 | go install github.com/x-motemen/gobump/cmd/gobump@master 55 | 56 | prerelease_for_tagpr: 57 | gocredits -w . 58 | git add CHANGELOG.md CREDITS go.mod go.sum 59 | 60 | release: 61 | ghr -username k1LoW -replace ${ver} dist/ 62 | 63 | .PHONY: default test 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tcpdp [![build](https://github.com/k1LoW/tcpdp/workflows/build/badge.svg)](https://github.com/k1LoW/tcpdp/actions) [![GitHub release](https://img.shields.io/github/release/k1LoW/tcpdp.svg)](https://github.com/k1LoW/tcpdp/releases) ![Coverage](https://raw.githubusercontent.com/k1LoW/octocovs/main/badges/k1LoW/tcpdp/coverage.svg) ![Code to Test Ratio](https://raw.githubusercontent.com/k1LoW/octocovs/main/badges/k1LoW/tcpdp/ratio.svg) ![Test Execution Time](https://raw.githubusercontent.com/k1LoW/octocovs/main/badges/k1LoW/tcpdp/time.svg) 2 | 3 | tcpdp is TCP dump tool with custom dumper and structured logger written in Go. 4 | 5 | `tcpdp` has 3 modes: 6 | 7 | - TCP **Proxy** server mode 8 | - **Probe** mode ( using libpcap ) 9 | - **Read** pcap file mode 10 | 11 | ## Usage 12 | 13 | ### `tcpdp proxy` : TCP proxy server mode 14 | 15 | ``` console 16 | $ tcpdp proxy -l localhost:12345 -r localhost:1234 -d hex # hex.Dump() 17 | ``` 18 | 19 | ``` console 20 | $ tcpdp proxy -l localhost:55432 -r db.internal.example.com:5432 -d pg # Dump query of PostgreSQL 21 | ``` 22 | 23 | ``` console 24 | $ tcpdp proxy -l localhost:33306 -r db.example.com:3306 -d mysql # Dump query of MySQL 25 | ``` 26 | 27 | #### With server-starter 28 | 29 | https://github.com/lestrrat-go/server-starter 30 | 31 | ``` console 32 | $ start_server --port 33306 -- tcpdp proxy -s -r db.example.com:3306 -d mysql 33 | ``` 34 | 35 | #### With config file 36 | 37 | ``` console 38 | $ tcpdp proxy -c config.toml 39 | ``` 40 | 41 | ### `tcpdp probe` : Probe mode (like tcpdump) 42 | 43 | ``` console 44 | $ tcpdp probe -i lo0 -t localhost:3306 -d mysql # is almost the same setting as 'tcpdump -i lo0 host 127.0.0.1 and tcp port 3306' 45 | ``` 46 | 47 | ``` console 48 | $ tcpdp probe -i eth0 -t 3306 -d hex # is almost the same setting as 'tcpdump -i eth0 tcp port 3306' 49 | ``` 50 | 51 | ### `tcpdp read` : Read pcap file mode 52 | 53 | ``` console 54 | $ tcpdump -i eth0 host 127.0.0.1 and tcp port 3306 -w mysql.pcap 55 | $ tcpdp read mysql.pcap -d mysql -t 3306 -f ltsv 56 | ``` 57 | 58 | ### `tcpdp config` Create config 59 | 60 | ``` console 61 | $ tcpdp config > myconfig.toml 62 | ``` 63 | 64 | #### Show current config 65 | 66 | ``` console 67 | $ tcpdp config 68 | ``` 69 | 70 | #### config format 71 | 72 | ``` toml 73 | [tcpdp] 74 | pidfile = "/var/run/tcpdp.pid" 75 | dumper = "mysql" 76 | 77 | [probe] 78 | target = "db.example.com:3306" 79 | interface = "en0" 80 | bufferSize = "2MB" 81 | immediateMode = false 82 | snapshotLength = "auto" 83 | internalBufferLength = 10000 84 | filter = "" 85 | 86 | [proxy] 87 | useServerStarter = false 88 | listenAddr = "localhost:3306" 89 | remoteAddr = "db.example.com:3306" 90 | 91 | [log] 92 | dir = "/var/log/tcpdp" 93 | enable = true 94 | enableInternal = true 95 | stdout = true 96 | format = "ltsv" 97 | rotateEnable = true 98 | rotationTime = "daily" 99 | rotationCount = 7 100 | # You can execute arbitrary commands after rotate 101 | # $1 = prev filename 102 | # $2 = current filename 103 | rotationHook = "/path/to/after_rotate.sh" 104 | fileName = "tcpdp.log" 105 | 106 | [dumpLog] 107 | dir = "/var/log/dump" 108 | enable = true 109 | stdout = false 110 | format = "json" 111 | rotateEnable = true 112 | rotationTime = "hourly" 113 | rotationCount = 24 114 | fileName = "dump.log" 115 | ``` 116 | 117 | ## Installation 118 | 119 | ```console 120 | $ go get github.com/k1LoW/tcpdp 121 | ``` 122 | 123 | ## Architecture 124 | 125 | ### tcpdp proxy connection diagram 126 | 127 | ``` 128 | client_addr 129 | ^ 130 | | tcpdp 131 | +----------|---------------+ 132 | | v | 133 | | proxy_listen_addr | 134 | | + ^ | 135 | | | | +--------+ | 136 | | |<----+ dumper | | 137 | | | |<--+ | | 138 | | | | +--------+ | 139 | | v + | 140 | | proxy_client_addr | 141 | | ^ | 142 | +----------|---------------+ 143 | | 144 | v 145 | remote_addr 146 | ``` 147 | 148 | ### tcpdp probe connection diagram 149 | 150 | ``` 151 | server 152 | +--------------------------+ 153 | | | 154 | | +---+---+ 155 | | <--------------| eth0 |-----------> 156 | | interface +---+---+ 157 | | /target ^ | 158 | | | | 159 | | tcpdp | | 160 | | +--------+ | | 161 | | | dumper +------+ | 162 | | +--------+ | 163 | +--------------------------+ 164 | ``` 165 | 166 | ### tcpdp read diagram 167 | 168 | ``` 169 | tcpdp 170 | +--------+ STDIN +--------+ STDOUT 171 | | *.pcap +------>+ dumper +--------> 172 | +--------+ +--------+ 173 | ``` 174 | 175 | ## tcpdp.log ( `tcpdp proxy` or `tcpdp probe` ) 176 | 177 | | key | description | mode | 178 | | --- | ----------- | ---- | 179 | | ts | timestamp | proxy / probe / read | 180 | | level | log level | proxy / probe | 181 | | msg | log message | proxy / probe | 182 | | error | error info | proxy / probe | 183 | | caller | error caller | proxy / probe | 184 | | conn_id | TCP connection ID by tcpdp | proxy / probe | 185 | | target | probe target | proxy / probe | 186 | | dumper | dumper type | proxy / probe | 187 | | use_server_starter | use server_starter | proxy | 188 | | conn_seq_num | TCP comunication sequence number by tcpdp | proxy | 189 | | client_addr | client address | tcpdp.log, hex, mysql, pg | proxy | 190 | | remote_addr | remote address | proxy | 191 | | proxy_listen_addr | listen address| proxy | 192 | | direction | client to remote: `->` / remote to client: `<-` | proxy | 193 | | interface | probe target interface | probe | 194 | | mtu | interface MTU (Maximum Transmission Unit) | probe | 195 | | mss | TCP connection MSS (Max Segment Size) | probe | 196 | | probe_target_addr | probe target address | probe | 197 | | filter | BPF (Berkeley Packet Filter) | probe | 198 | | buffer_size | libpcap buffer_size | probe | 199 | | immediate_mode | libpcap immediate_mode | probe | 200 | | snapshot_length | libpcap snapshot length | probe | 201 | | internal_buffer_length | tcpdp internal packet buffer length | probe | 202 | 203 | ## Dumper 204 | 205 | ### mysql 206 | 207 | MySQL query dumper 208 | 209 | **NOTICE: MySQL query dumper require `--target` option when `tcpdp proxy` `tcpdp probe`** 210 | 211 | | key | description | mode | 212 | | --- | ----------- | ---- | 213 | | ts | timestamp | proxy / probe / read | 214 | | conn_id | TCP connection ID by tcpdp | proxy / probe / read | 215 | | conn_seq_num | TCP comunication sequence number by tcpdp | proxy | 216 | | client_addr | client address | proxy | 217 | | proxy_listen_addr | listen address| proxy | 218 | | proxy_client_addr | proxy client address | proxy | 219 | | remote_addr | remote address | proxy | 220 | | direction | client to remote: `->` / remote to client: `<-` | proxy | 221 | | interface | probe target interface | probe | 222 | | src_addr | src address | probe / read | 223 | | dst_addr | dst address | probe / read | 224 | | probe_target_addr | probe target address | probe | 225 | | proxy_protocol_src_addr | proxy protocol src address | probe / proxy /read | 226 | | proxy_protocol_dst_addr | proxy protocol dst address | probe / proxy /read | 227 | | query | SQL query | proxy / probe / read | 228 | | stmt_id | statement id | proxy / probe / read | 229 | | stmt_prepare_query | prepared statement query | proxy / probe / read | 230 | | stmt_execute_values | prepared statement execute values | proxy / probe / read | 231 | | character_set | [character set](https://dev.mysql.com/doc/internals/en/character-set.html) | proxy / probe / read | 232 | | username | username | proxy / probe / read | 233 | | database | database | proxy / probe / read | 234 | | seq_num | sequence number by MySQL | proxy / probe / read | 235 | | command_id | [command_id](https://dev.mysql.com/doc/internals/en/com-query.html) for MySQL | proxy / probe / read | 236 | 237 | ### pg 238 | 239 | PostgreSQL query dumper 240 | 241 | **NOTICE: PostgreSQL query dumper require `--target` option `tcpdp proxy` `tcpdp probe`** 242 | 243 | | key | description | mode | 244 | | --- | ----------- | ---- | 245 | | ts | timestamp | proxy / probe / read | 246 | | conn_id | TCP connection ID by tcpdp | proxy / probe / read | 247 | | conn_seq_num | TCP comunication sequence number by tcpdp | proxy | 248 | | client_addr | client address | proxy | 249 | | proxy_listen_addr | listen address| proxy | 250 | | proxy_client_addr | proxy client address | proxy | 251 | | remote_addr | remote address | proxy | 252 | | direction | client to remote: `->` / remote to client: `<-` | proxy | 253 | | interface | probe target interface | probe | 254 | | src_addr | src address | probe / read | 255 | | dst_addr | dst address | probe / read | 256 | | probe_target_addr | probe target address | probe | 257 | | proxy_protocol_src_addr | proxy protocol src address | probe / proxy /read | 258 | | proxy_protocol_dst_addr | proxy protocol dst address | probe / proxy /read | 259 | | query | SQL query | proxy / probe / read | 260 | | portal_name | portal Name | proxy / probe / read | 261 | | stmt_name | prepared statement name | proxy / probe / read | 262 | | parse_query | prepared statement query | proxy / probe / read | 263 | | bind_values | prepared statement bind(execute) values | proxy / probe / read | 264 | | username | username | proxy / probe / read | 265 | | database | database | proxy / probe / read | 266 | | message_type | [message type](https://www.postgresql.org/docs/current/static/protocol-overview.html#PROTOCOL-MESSAGE-CONCEPTS) for PostgreSQL | proxy / probe / read | 267 | 268 | ### hex 269 | 270 | | key | description | mode | 271 | | --- | ----------- | ---- | 272 | | ts | timestamp | proxy / probe / read | 273 | | conn_id | TCP connection ID by tcpdp | proxy / probe / read | 274 | | conn_seq_num | TCP comunication sequence number by tcpdp | proxy | 275 | | client_addr | client address | proxy | 276 | | proxy_listen_addr | listen address| proxy | 277 | | proxy_client_addr | proxy client address | proxy | 278 | | remote_addr | remote address | proxy | 279 | | direction | client to remote: `->` / remote to client: `<-` | proxy | 280 | | interface | probe target interface | probe | 281 | | src_addr | src address | probe / read | 282 | | dst_addr | dst address | probe / read | 283 | | probe_target_addr | probe target address | probe | 284 | | proxy_protocol_src_addr | proxy protocol src address | probe / proxy /read | 285 | | proxy_protocol_dst_addr | proxy protocol dst address | probe / proxy /read | 286 | | bytes | bytes string by hex.Dump | proxy / probe / read | 287 | | ascii | ascii string by hex.Dump | proxy / probe / read | 288 | 289 | ## References 290 | 291 | - https://github.com/jpillora/go-tcp-proxy 292 | - https://github.com/dmmlabo/tcpserver_go 293 | -------------------------------------------------------------------------------- /cmd/config.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | "text/template" 27 | 28 | "github.com/spf13/cobra" 29 | "github.com/spf13/viper" 30 | ) 31 | 32 | // configCmd represents the config command 33 | var configCmd = &cobra.Command{ 34 | Use: "config", 35 | Short: "Show currnt config", 36 | Long: `Show currnt config.`, 37 | Run: func(cmd *cobra.Command, args []string) { 38 | const cfgTemplate = `[tcpdp] 39 | pidfile = "{{ .tcpdp.pidfile }}" 40 | dumper = "{{ .tcpdp.dumper }}" 41 | proxyProtocol = {{ .tcpdp.proxyprotocol }} 42 | 43 | [probe] 44 | interface = "{{ .probe.interface }}" 45 | target = "{{ .probe.target }}" 46 | bufferSize = "{{ .probe.buffersize }}" 47 | immediateMode = {{ .probe.immediatemode }} 48 | snapshotLength = "{{ .probe.snapshotlength }}" 49 | internalBufferLength = {{ .probe.internalbufferlength }} 50 | filter = "{{ .probe.filter }}" 51 | 52 | [proxy] 53 | useServerStarter = {{ .proxy.useserverstarter }} 54 | listenAddr = "{{ .proxy.listenaddr }}" 55 | remoteAddr = "{{ .proxy.remoteaddr }}" 56 | 57 | [log] 58 | dir = "{{ .log.dir }}" 59 | enable = {{ .log.enable }} 60 | enableInternal = {{ .log.enableinternal }} 61 | stdout = {{ .log.stdout }} 62 | format = "{{ .log.format }}" 63 | rotateEnable = {{ .log.rotateenable }} 64 | rotationTime = "{{ .log.rotationtime }}" 65 | rotationCount = {{ .log.rotationcount }} 66 | {{ if (ne .log.rotationhook "") -}} 67 | rotationHook = "{{ .log.rotationhook }}" 68 | fileName = "{{ .log.filename }}" 69 | {{ else -}} 70 | fileName = "{{ .log.filename }}" 71 | {{- end }} 72 | 73 | [dumpLog] 74 | dir = "{{ .dumplog.dir }}" 75 | enable = {{ .dumplog.enable }} 76 | stdout = {{ .dumplog.stdout }} 77 | format = "{{ .dumplog.format }}" 78 | rotateEnable = {{ .dumplog.rotateenable }} 79 | rotationTime = "{{ .dumplog.rotationtime }}" 80 | rotationCount = {{ .dumplog.rotationcount }} 81 | {{ if (ne .dumplog.rotationhook "") -}} 82 | rotationHook = "{{ .dumplog.rotationhook }}" 83 | fileName = "{{ .dumplog.filename }}" 84 | {{ else -}} 85 | fileName = "{{ .dumplog.filename }}" 86 | {{- end -}} 87 | ` 88 | tpl, err := template.New("config").Parse(cfgTemplate) 89 | if err != nil { 90 | fmt.Println(err) 91 | os.Exit(1) 92 | } 93 | 94 | if err := tpl.Execute(os.Stdout, viper.AllSettings()); err != nil { 95 | fmt.Println(err) 96 | os.Exit(1) 97 | } 98 | }, 99 | } 100 | 101 | func init() { 102 | configCmd.Flags().StringVarP(&cfgFile, "config", "c", "", "config file path") 103 | rootCmd.AddCommand(configCmd) 104 | } 105 | -------------------------------------------------------------------------------- /cmd/probe.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "net" 27 | "os" 28 | "os/signal" 29 | "runtime" 30 | "syscall" 31 | 32 | "github.com/k1LoW/tcpdp/server" 33 | "github.com/spf13/cobra" 34 | "github.com/spf13/viper" 35 | "go.uber.org/zap" 36 | ) 37 | 38 | var ( 39 | probeDumper string 40 | probeProxyProtocol bool 41 | ) 42 | 43 | const snaplenAuto = "auto" 44 | const snaplenDefault = 0xFFFF 45 | 46 | // probeCmd represents the probe command 47 | var probeCmd = &cobra.Command{ 48 | Use: "probe", 49 | Short: "Probe mode", 50 | Long: "`tcp probe` dump packets like tcpdump.", 51 | Run: func(cmd *cobra.Command, args []string) { 52 | err := viper.ReadInConfig() 53 | if err != nil { 54 | logger.Warn("Config file not found.", zap.Error(err)) 55 | } 56 | if cfgFile == "" { 57 | viper.Set("tcpdp.dumper", probeDumper) // because share with `proxy` 58 | viper.Set("tcpdp.proxyProtocol", probeProxyProtocol) // because share with `proxy` 59 | } 60 | if logToStdout { 61 | viper.Set("log.enable", true) 62 | viper.Set("log.stdout", true) 63 | viper.Set("dumpLog.enable", true) 64 | viper.Set("dumpLog.stdout", true) 65 | } 66 | 67 | dumper := viper.GetString("tcpdp.dumper") 68 | target := viper.GetString("probe.target") 69 | device := viper.GetString("probe.interface") 70 | snapshotLength := viper.GetString("probe.snapshotLength") 71 | ifi, err := net.InterfaceByName(device) 72 | if err != nil && !(device == "any" && runtime.GOOS == "linux") { 73 | logger.Fatal("interface error.", zap.Error(err)) 74 | } 75 | mtu := 1500 76 | if device != "any" { 77 | mtu = ifi.MTU 78 | } 79 | if snapshotLength == snaplenAuto { 80 | snapshotLength = fmt.Sprintf("%dB (auto)", mtu+14+4) // 14:Ethernet header 4:FCS 81 | viper.Set("probe.snapshotLength", fmt.Sprintf("%dB", mtu+14+4)) 82 | } 83 | internalBufferLength := viper.GetInt("probe.internalBufferLength") 84 | 85 | defer logger.Sync() 86 | 87 | signalChan := make(chan os.Signal, 1) 88 | signal.Ignore() 89 | signal.Notify(signalChan, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) 90 | 91 | s, err := server.NewProbeServer(context.Background(), logger) 92 | if err != nil { 93 | logger.Fatal("NewProbeServer error.", zap.Error(err)) 94 | } 95 | 96 | pcapConfig := s.PcapConfig() 97 | 98 | logger.Info("Starting probe.", 99 | zap.String("dumper", dumper), 100 | zap.String("interface", pcapConfig.Device), 101 | zap.String("mtu", fmt.Sprintf("%d", mtu)), 102 | zap.String("probe_target_addr", target), 103 | zap.String("filter", pcapConfig.Filter), 104 | zap.String("buffer_size", pcapConfig.BufferSize), 105 | zap.Bool("immediate_mode", pcapConfig.ImmediateMode), 106 | zap.String("snapshot_length", pcapConfig.SnapshotLength), 107 | zap.Int("internal_buffer_length", internalBufferLength), 108 | ) 109 | 110 | go s.Start() 111 | 112 | sc := <-signalChan 113 | 114 | switch sc { 115 | case syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM: 116 | logger.Info("Shutting down probe...") 117 | s.Shutdown() 118 | <-s.ClosedChan 119 | default: 120 | logger.Info("Unexpected signal") 121 | os.Exit(1) 122 | } 123 | }, 124 | } 125 | 126 | func init() { 127 | probeCmd.Flags().StringVarP(&cfgFile, "config", "c", "", "config file path") 128 | probeCmd.Flags().StringP("target", "t", "", "target addr. (ex. \"localhost:80\", \"3306\")") 129 | probeCmd.Flags().StringP("interface", "i", "", "interface") 130 | probeCmd.Flags().StringP("buffer-size", "B", "2MB", "buffer size (pcap_buffer_size)") 131 | probeCmd.Flags().BoolP("immediate-mode", "", false, "immediate mode") 132 | probeCmd.Flags().StringP("snapshot-length", "s", fmt.Sprintf("%dB", snaplenDefault), "snapshot length") 133 | probeCmd.Flags().StringVarP(&probeDumper, "dumper", "d", "hex", "dumper") 134 | probeCmd.Flags().BoolVarP(&logToStdout, "stdout", "", false, "output all log to STDOUT") 135 | probeCmd.Flags().StringP("filter", "", "", "override Berkekey Packet Filter") 136 | probeCmd.Flags().BoolVarP(&probeProxyProtocol, "proxy-protocol", "", false, "accept proxy protocol") 137 | 138 | if err := viper.BindPFlag("probe.target", probeCmd.Flags().Lookup("target")); err != nil { 139 | fmt.Println(err) 140 | os.Exit(1) 141 | } 142 | if err := viper.BindPFlag("probe.interface", probeCmd.Flags().Lookup("interface")); err != nil { 143 | fmt.Println(err) 144 | os.Exit(1) 145 | } 146 | if err := viper.BindPFlag("probe.bufferSize", probeCmd.Flags().Lookup("buffer-size")); err != nil { 147 | fmt.Println(err) 148 | os.Exit(1) 149 | } 150 | if err := viper.BindPFlag("probe.immediateMode", probeCmd.Flags().Lookup("immediate-mode")); err != nil { 151 | fmt.Println(err) 152 | os.Exit(1) 153 | } 154 | if err := viper.BindPFlag("probe.snapshotLength", probeCmd.Flags().Lookup("snapshot-length")); err != nil { 155 | fmt.Println(err) 156 | os.Exit(1) 157 | } 158 | if err := viper.BindPFlag("probe.filter", probeCmd.Flags().Lookup("filter")); err != nil { 159 | fmt.Println(err) 160 | os.Exit(1) 161 | } 162 | 163 | rootCmd.AddCommand(probeCmd) 164 | } 165 | -------------------------------------------------------------------------------- /cmd/proxy.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "net" 27 | "os" 28 | "os/signal" 29 | "syscall" 30 | 31 | "github.com/k1LoW/tcpdp/server" 32 | "github.com/spf13/cobra" 33 | "github.com/spf13/viper" 34 | "go.uber.org/zap" 35 | ) 36 | 37 | var ( 38 | proxyDumper string 39 | proxyProxyProtocol bool 40 | ) 41 | 42 | // proxyCmd represents the proxy command 43 | var proxyCmd = &cobra.Command{ 44 | Use: "proxy", 45 | Short: "TCP proxy server mode", 46 | Long: "`tcpdp proxy` run TCP proxy server.", 47 | Run: func(cmd *cobra.Command, args []string) { 48 | err := viper.ReadInConfig() 49 | if err != nil { 50 | logger.Warn("Config file not found.", zap.Error(err)) 51 | } 52 | if cfgFile == "" { 53 | viper.Set("tcpdp.dumper", proxyDumper) // because share with `probe` 54 | viper.Set("tcpdp.proxyProtocol", proxyProxyProtocol) // because share with `probe` 55 | } 56 | if logToStdout { 57 | viper.Set("log.enable", true) 58 | viper.Set("log.stdout", true) 59 | viper.Set("dumpLog.enable", true) 60 | viper.Set("dumpLog.stdout", true) 61 | } 62 | 63 | dumper := viper.GetString("tcpdp.dumper") 64 | listenAddr := viper.GetString("proxy.listenAddr") 65 | remoteAddr := viper.GetString("proxy.remoteAddr") 66 | useServerStarter := viper.GetBool("proxy.useServerStarter") 67 | 68 | defer logger.Sync() 69 | 70 | lAddr, err := net.ResolveTCPAddr("tcp", listenAddr) 71 | if err != nil { 72 | logger.Fatal("error", zap.Error(err)) 73 | os.Exit(1) 74 | } 75 | rAddr, err := net.ResolveTCPAddr("tcp", remoteAddr) 76 | if err != nil { 77 | logger.Fatal("error", zap.Error(err)) 78 | os.Exit(1) 79 | } 80 | 81 | signalChan := make(chan os.Signal, 1) 82 | signal.Ignore() 83 | signal.Notify(signalChan, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) 84 | 85 | s := server.NewServer(context.Background(), lAddr, rAddr, logger) 86 | 87 | if useServerStarter { 88 | logger.Info(fmt.Sprintf("Starting proxy. [server_starter] <-> %s:%d", rAddr.IP, rAddr.Port), 89 | zap.String("dumper", dumper), 90 | zap.String("remote_addr", remoteAddr), 91 | zap.Bool("use_server_starter", useServerStarter), 92 | zap.Bool("proxy_protocol", proxyProxyProtocol), 93 | ) 94 | } else { 95 | logger.Info(fmt.Sprintf("Starting proxy. %s:%d <-> %s:%d", lAddr.IP, lAddr.Port, rAddr.IP, rAddr.Port), 96 | zap.String("dumper", dumper), 97 | zap.String("proxy_listen_addr", listenAddr), 98 | zap.String("remote_addr", remoteAddr), 99 | zap.Bool("use_server_starter", useServerStarter), 100 | zap.Bool("proxy_protocol", proxyProxyProtocol), 101 | ) 102 | } 103 | 104 | go s.Start() 105 | 106 | sc := <-signalChan 107 | 108 | switch sc { 109 | case syscall.SIGINT: 110 | logger.Info("Shutting down proxy...") 111 | s.Shutdown() 112 | s.Wg.Wait() 113 | <-s.ClosedChan 114 | case syscall.SIGQUIT, syscall.SIGTERM: 115 | logger.Info("Graceful Shutting down proxy...") 116 | s.GracefulShutdown() 117 | s.Wg.Wait() 118 | <-s.ClosedChan 119 | default: 120 | logger.Info("Unexpected signal") 121 | os.Exit(1) 122 | } 123 | }, 124 | } 125 | 126 | func init() { 127 | proxyCmd.Flags().StringVarP(&cfgFile, "config", "c", "", "config file path") 128 | proxyCmd.Flags().StringP("listen", "l", "localhost:8080", "listen address") 129 | proxyCmd.Flags().StringP("remote", "r", "localhost:80", "remote address") 130 | proxyCmd.Flags().StringVarP(&proxyDumper, "dumper", "d", "hex", "dumper") 131 | proxyCmd.Flags().BoolP("use-server-starter", "s", false, "use server_starter") 132 | proxyCmd.Flags().BoolVarP(&logToStdout, "stdout", "", false, "output all log to STDOUT") 133 | proxyCmd.Flags().BoolVarP(&proxyProxyProtocol, "proxy-protocol", "", false, "accept proxy protocol") 134 | 135 | if err := viper.BindPFlag("proxy.listenAddr", proxyCmd.Flags().Lookup("listen")); err != nil { 136 | fmt.Println(err) 137 | os.Exit(1) 138 | } 139 | if err := viper.BindPFlag("proxy.remoteAddr", proxyCmd.Flags().Lookup("remote")); err != nil { 140 | fmt.Println(err) 141 | os.Exit(1) 142 | } 143 | if err := viper.BindPFlag("proxy.useServerStarter", proxyCmd.Flags().Lookup("use-server-starter")); err != nil { 144 | fmt.Println(err) 145 | os.Exit(1) 146 | } 147 | 148 | rootCmd.AddCommand(proxyCmd) 149 | } 150 | -------------------------------------------------------------------------------- /cmd/read.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "io/ioutil" 27 | "os" 28 | 29 | "github.com/google/gopacket" 30 | "github.com/google/gopacket/pcap" 31 | "github.com/k1LoW/tcpdp/dumper" 32 | "github.com/k1LoW/tcpdp/dumper/conn" 33 | "github.com/k1LoW/tcpdp/dumper/hex" 34 | "github.com/k1LoW/tcpdp/dumper/mysql" 35 | "github.com/k1LoW/tcpdp/dumper/pg" 36 | "github.com/k1LoW/tcpdp/reader" 37 | "github.com/spf13/cobra" 38 | "github.com/spf13/viper" 39 | ) 40 | 41 | var ( 42 | readDumper string 43 | readTarget string 44 | ) 45 | 46 | const readIternalBufferLength = 10000 47 | 48 | // readCmd represents the read command 49 | var readCmd = &cobra.Command{ 50 | Use: "read [PCAP]", 51 | Short: "Read pcap file mode", 52 | Long: "Read pcap format file and dump.", 53 | Args: func(cmd *cobra.Command, args []string) error { 54 | fi, _ := os.Stdin.Stat() 55 | if (fi.Mode() & os.ModeCharDevice) != 0 { 56 | if len(args) != 1 { 57 | return fmt.Errorf("Error: %s", "requires pcap file path") 58 | } 59 | } 60 | return nil 61 | }, 62 | Run: func(cmd *cobra.Command, args []string) { 63 | viper.Set("tcpdp.dumper", readDumper) // because share with `server` 64 | viper.Set("log.enable", false) 65 | viper.Set("log.stdout", false) 66 | viper.Set("dumpLog.enable", false) 67 | viper.Set("dumpLog.stdout", true) 68 | 69 | defer logger.Sync() 70 | 71 | var pcapFile string 72 | 73 | fi, _ := os.Stdin.Stat() 74 | 75 | if (fi.Mode() & os.ModeCharDevice) != 0 { 76 | pcapFile = args[0] 77 | } else { 78 | pcap, _ := ioutil.ReadAll(os.Stdin) 79 | tmpfile, _ := ioutil.TempFile("", "tcpdptmp") 80 | defer func() { 81 | if err := tmpfile.Close(); err != nil { 82 | fmt.Println(err) 83 | os.Exit(1) 84 | } 85 | if err := os.Remove(tmpfile.Name()); err != nil { 86 | fmt.Println(err) 87 | os.Exit(1) 88 | } 89 | }() 90 | pcapFile = tmpfile.Name() 91 | if _, err := tmpfile.Write(pcap); err != nil { 92 | fmt.Println(err) 93 | os.Exit(1) 94 | } 95 | } 96 | 97 | handle, err := pcap.OpenOffline(pcapFile) 98 | if err != nil { 99 | fmt.Println(err) 100 | os.Exit(1) 101 | } 102 | defer handle.Close() 103 | 104 | var d dumper.Dumper 105 | switch readDumper { 106 | case "hex": 107 | d = hex.NewDumper() 108 | case "pg": 109 | d = pg.NewDumper() 110 | case "mysql": 111 | d = mysql.NewDumper() 112 | case "conn": 113 | d = conn.NewDumper() 114 | default: 115 | d = hex.NewDumper() 116 | } 117 | 118 | packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) 119 | 120 | ctx, cancel := context.WithCancel(context.Background()) 121 | 122 | proxyProtocol := viper.GetBool("tcpdp.proxyProtocol") 123 | enableInternal := viper.GetBool("log.enableInternal") 124 | 125 | r := reader.NewPacketReader( 126 | ctx, 127 | cancel, 128 | packetSource, 129 | d, 130 | []dumper.DumpValue{}, 131 | logger, 132 | readIternalBufferLength, 133 | proxyProtocol, 134 | enableInternal, 135 | ) 136 | 137 | t, err := reader.ParseTarget(readTarget) 138 | if err != nil { 139 | fmt.Println(err) 140 | os.Exit(1) 141 | } 142 | 143 | if err := r.ReadAndDump(t); err != nil { 144 | fmt.Println(err) 145 | os.Exit(1) 146 | } 147 | }, 148 | } 149 | 150 | func init() { 151 | readCmd.Flags().StringVarP(&readTarget, "target", "t", "", "target addr. (ex. \"localhost:80\", \"3306\")") 152 | readCmd.Flags().StringP("format", "f", "json", "STDOUT format. (\"console\", \"json\" , \"ltsv\") ") 153 | readCmd.Flags().StringVarP(&readDumper, "dumper", "d", "hex", "dumper") 154 | 155 | if err := viper.BindPFlag("dumpLog.stdoutFormat", readCmd.Flags().Lookup("format")); err != nil { 156 | fmt.Println(err) 157 | os.Exit(1) 158 | } 159 | 160 | rootCmd.AddCommand(readCmd) 161 | } 162 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | 27 | l "github.com/k1LoW/tcpdp/logger" 28 | "github.com/spf13/cobra" 29 | "github.com/spf13/viper" 30 | "go.uber.org/zap" 31 | ) 32 | 33 | var ( 34 | cfgFile string 35 | logger *zap.Logger 36 | logToStdout bool 37 | ) 38 | 39 | // rootCmd represents the base command when called without any subcommands 40 | var rootCmd = &cobra.Command{ 41 | Use: "tcpdp", 42 | Short: "tcpdp is TCP dump tool with custom dumper and structured logger written in Go.", 43 | Long: `tcpdp is TCP dump tool with custom dumper and structured logger written in Go.`, 44 | } 45 | 46 | // Execute adds all child commands to the root command and sets flags appropriately. 47 | // This is called by main.main(). It only needs to happen once to the rootCmd. 48 | func Execute() { 49 | if err := rootCmd.Execute(); err != nil { 50 | os.Exit(1) 51 | } 52 | } 53 | 54 | func init() { 55 | cobra.OnInitialize(initConfig) 56 | } 57 | 58 | func initConfig() { 59 | viper.SetDefault("tcpdp.pidfile", "./tcpdp.pid") 60 | viper.SetDefault("tcpdp.dumper", "hex") 61 | viper.SetDefault("tcpdp.proxyProtocol", false) 62 | 63 | viper.SetDefault("proxy.useServerStarter", false) 64 | viper.SetDefault("proxy.listenAddr", "localhost:8080") 65 | viper.SetDefault("proxy.remoteAddr", "localhost:80") 66 | 67 | viper.SetDefault("probe.target", "localhost:80") 68 | viper.SetDefault("probe.interface", "") 69 | viper.SetDefault("probe.bufferSize", "2MB") 70 | viper.SetDefault("probe.immediateMode", false) 71 | viper.SetDefault("probe.internalBufferLength", 10000) 72 | viper.SetDefault("probe.snapshotLength", fmt.Sprintf("%dB", snaplenDefault)) 73 | viper.SetDefault("probe.filter", "") 74 | 75 | viper.SetDefault("log.dir", ".") 76 | viper.SetDefault("log.enable", true) 77 | viper.SetDefault("log.enableInternal", false) 78 | viper.SetDefault("log.stdout", true) 79 | viper.SetDefault("log.format", "json") 80 | viper.SetDefault("log.rotateEnable", true) 81 | viper.SetDefault("log.rotationTime", "daily") 82 | viper.SetDefault("log.rotationCount", 7) 83 | viper.SetDefault("log.rotationHook", "") 84 | viper.SetDefault("log.fileName", "tcpdp.log") 85 | 86 | viper.SetDefault("dumpLog.dir", ".") 87 | viper.SetDefault("dumpLog.enable", true) 88 | viper.SetDefault("dumpLog.stdout", false) 89 | viper.SetDefault("dumpLog.format", "json") 90 | viper.SetDefault("dumpLog.rotateEnable", true) 91 | viper.SetDefault("dumpLog.rotationTime", "daily") 92 | viper.SetDefault("dumpLog.rotationCount", 7) 93 | viper.SetDefault("dumpLog.rotationHook", "") 94 | viper.SetDefault("dumpLog.fileName", "dump.log") 95 | 96 | if cfgFile != "" { 97 | viper.SetConfigFile(cfgFile) 98 | } else { 99 | viper.SetConfigName("tcpdp") 100 | viper.AddConfigPath("/etc/tcpdp/") 101 | viper.AddConfigPath("$HOME/.tcpdp") 102 | viper.AddConfigPath(".") 103 | } 104 | _ = viper.ReadInConfig() 105 | logger = l.NewLogger() 106 | } 107 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | 26 | "github.com/k1LoW/tcpdp/version" 27 | "github.com/spf13/cobra" 28 | ) 29 | 30 | // versionCmd represents the version command 31 | var versionCmd = &cobra.Command{ 32 | Use: "version", 33 | Short: "Print tcpdp version", 34 | Long: `Print tcpdp version.`, 35 | Run: func(cmd *cobra.Command, args []string) { 36 | fmt.Println(version.Version) 37 | }, 38 | } 39 | 40 | func init() { 41 | rootCmd.AddCommand(versionCmd) 42 | } 43 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | services: 4 | postgres: 5 | image: postgres:10 6 | restart: always 7 | ports: 8 | - "54322:5432" 9 | environment: 10 | - POSTGRES_USER=postgres 11 | - POSTGRES_PASSWORD=pgpass 12 | - POSTGRES_DB=testdb 13 | mysql57: 14 | image: mysql:5.7 15 | restart: always 16 | ports: 17 | - "33066:3306" 18 | volumes: 19 | - ./testdata/mysql.conf.d:/etc/mysql/conf.d 20 | environment: 21 | - MYSQL_DATABASE=testdb 22 | - MYSQL_ROOT_PASSWORD=mypass 23 | proxy-protocol-proxy-mac: 24 | image: mminks/haproxy-docker-logging 25 | restart: always 26 | ports: 27 | - "33068:33068" 28 | - "33069:33069" 29 | - "33070:33070" 30 | - "33071:33071" 31 | volumes: 32 | - ./testdata/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg 33 | proxy-protocol-proxy-linux: 34 | image: mminks/haproxy-docker-logging 35 | restart: always 36 | network_mode: host 37 | volumes: 38 | - ./testdata/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg 39 | extra_hosts: 40 | - "host.docker.internal:127.0.0.1" 41 | expose: 42 | - "33068" 43 | - "33069" 44 | - "33070" 45 | - "33071" 46 | proxy-protocol-mariadb: 47 | image: mariadb:10.4 48 | restart: always 49 | ports: 50 | - "33081:3306" 51 | volumes: 52 | - ./testdata/mariadb.conf.d:/etc/mysql/conf.d 53 | environment: 54 | - MYSQL_DATABASE=testdb 55 | - MYSQL_ROOT_PASSWORD=mypass 56 | -------------------------------------------------------------------------------- /dumper/conn/conn.go: -------------------------------------------------------------------------------- 1 | package conn 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/k1LoW/tcpdp/dumper" 7 | "github.com/k1LoW/tcpdp/logger" 8 | "go.uber.org/zap" 9 | "go.uber.org/zap/zapcore" 10 | ) 11 | 12 | type Dumper struct { 13 | name string 14 | logger *zap.Logger 15 | } 16 | 17 | // NewDumper returns a Dumper 18 | func NewDumper() *Dumper { 19 | dumper := &Dumper{ 20 | name: "conn", 21 | logger: logger.NewHexLogger(), 22 | } 23 | return dumper 24 | } 25 | 26 | // Name return dumper name 27 | func (h *Dumper) Name() string { 28 | return h.name 29 | } 30 | 31 | // Dump TCP 32 | func (h *Dumper) Dump(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata, additional []dumper.DumpValue) error { 33 | values := []dumper.DumpValue{} 34 | values = append(values, connMetadata.DumpValues...) 35 | values = append(values, additional...) 36 | values = append(values, dumper.DumpValue{ 37 | Key: "ts", 38 | Value: time.Now(), 39 | }) 40 | 41 | h.Log(values) 42 | return nil 43 | } 44 | 45 | // Read return byte to analyzed string 46 | func (h *Dumper) Read(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata) ([]dumper.DumpValue, error) { 47 | return []dumper.DumpValue{ 48 | dumper.DumpValue{ 49 | Key: "dummy", 50 | Value: "dummy", 51 | }, 52 | }, nil 53 | } 54 | 55 | // Log values 56 | func (h *Dumper) Log(values []dumper.DumpValue) { 57 | fields := []zapcore.Field{} 58 | for _, kv := range values { 59 | if kv.Key == "dummy" { 60 | continue 61 | } 62 | fields = append(fields, zap.Any(kv.Key, kv.Value)) 63 | } 64 | h.logger.Info("-", fields...) 65 | } 66 | 67 | // NewConnMetadata ... 68 | func (h *Dumper) NewConnMetadata() *dumper.ConnMetadata { 69 | return &dumper.ConnMetadata{ 70 | DumpValues: []dumper.DumpValue{}, 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /dumper/dumper.go: -------------------------------------------------------------------------------- 1 | package dumper 2 | 3 | // Direction of TCP commnication 4 | type Direction int 5 | 6 | const ( 7 | // ClientToRemote is client->proxy->remote 8 | ClientToRemote Direction = iota 9 | // RemoteToClient is client<-proxy<-remote 10 | RemoteToClient 11 | // SrcToDst is src->dst 12 | SrcToDst 13 | // DstToSrc is dst->src 14 | DstToSrc 15 | // Unknown direction 16 | Unknown Direction = 9 17 | ) 18 | 19 | func (d Direction) String() string { 20 | switch d { 21 | case ClientToRemote, SrcToDst: 22 | return "->" 23 | case RemoteToClient, DstToSrc: 24 | return "<-" 25 | default: 26 | return "?" 27 | } 28 | } 29 | 30 | // DumpValue ... 31 | type DumpValue struct { 32 | Key string 33 | Value interface{} 34 | } 35 | 36 | // ConnMetadata is metadada per TCP connection 37 | type ConnMetadata struct { 38 | DumpValues []DumpValue 39 | Internal interface{} // internal metadata for dumper 40 | Fin bool 41 | } 42 | 43 | // Dumper interface 44 | type Dumper interface { 45 | Name() string 46 | Dump(in []byte, direction Direction, connMetadata *ConnMetadata, additional []DumpValue) error 47 | Read(in []byte, direction Direction, connMetadata *ConnMetadata) ([]DumpValue, error) 48 | Log(values []DumpValue) 49 | NewConnMetadata() *ConnMetadata 50 | } 51 | -------------------------------------------------------------------------------- /dumper/dumper_test.go: -------------------------------------------------------------------------------- 1 | package dumper 2 | 3 | import ( 4 | "io" 5 | 6 | "go.uber.org/zap" 7 | "go.uber.org/zap/zapcore" 8 | ) 9 | 10 | // newTestLogger return zap.Logger for test 11 | func newTestLogger(out io.Writer) *zap.Logger { 12 | encoderConfig := zapcore.EncoderConfig{ 13 | TimeKey: "ts", 14 | LevelKey: "level", 15 | NameKey: "logger", 16 | CallerKey: "caller", 17 | MessageKey: "msg", 18 | StacktraceKey: "stacktrace", 19 | EncodeLevel: zapcore.LowercaseLevelEncoder, 20 | EncodeTime: zapcore.ISO8601TimeEncoder, 21 | EncodeDuration: zapcore.StringDurationEncoder, 22 | EncodeCaller: zapcore.ShortCallerEncoder, 23 | } 24 | 25 | logger := zap.New(zapcore.NewCore( 26 | zapcore.NewJSONEncoder(encoderConfig), 27 | zapcore.AddSync(out), 28 | zapcore.DebugLevel, 29 | )) 30 | 31 | return logger 32 | } 33 | -------------------------------------------------------------------------------- /dumper/hex/hex.go: -------------------------------------------------------------------------------- 1 | package hex 2 | 3 | import ( 4 | "encoding/hex" 5 | "strings" 6 | "time" 7 | 8 | "github.com/k1LoW/tcpdp/dumper" 9 | "github.com/k1LoW/tcpdp/logger" 10 | "go.uber.org/zap" 11 | "go.uber.org/zap/zapcore" 12 | ) 13 | 14 | // Dumper ... 15 | type Dumper struct { 16 | name string 17 | logger *zap.Logger 18 | } 19 | 20 | // NewDumper returns a Dumper 21 | func NewDumper() *Dumper { 22 | dumper := &Dumper{ 23 | name: "hex", 24 | logger: logger.NewHexLogger(), 25 | } 26 | return dumper 27 | } 28 | 29 | // Name return dumper name 30 | func (h *Dumper) Name() string { 31 | return h.name 32 | } 33 | 34 | // Dump TCP 35 | func (h *Dumper) Dump(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata, additional []dumper.DumpValue) error { 36 | values := []dumper.DumpValue{} 37 | read, err := h.Read(in, direction, connMetadata) 38 | if err != nil { 39 | return err 40 | } 41 | values = append(values, read...) 42 | values = append(values, connMetadata.DumpValues...) 43 | values = append(values, additional...) 44 | values = append(values, dumper.DumpValue{ 45 | Key: "ts", 46 | Value: time.Now(), 47 | }) 48 | 49 | h.Log(values) 50 | return nil 51 | } 52 | 53 | // Read return byte to analyzed string 54 | func (h *Dumper) Read(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata) ([]dumper.DumpValue, error) { 55 | hexdump := strings.Split(hex.Dump(in), "\n") 56 | byteString := []string{} 57 | ascii := []string{} 58 | for _, hd := range hexdump { 59 | if hd == "" { 60 | continue 61 | } 62 | byteString = append(byteString, strings.TrimRight(strings.Replace(hd[10:58], " ", " ", 1), " ")) 63 | ascii = append(ascii, hd[61:len(hd)-1]) 64 | } 65 | 66 | return []dumper.DumpValue{ 67 | dumper.DumpValue{ 68 | Key: "bytes", 69 | Value: strings.Join(byteString, " "), 70 | }, 71 | dumper.DumpValue{ 72 | Key: "ascii", 73 | Value: strings.Join(ascii, ""), 74 | }, 75 | }, nil 76 | } 77 | 78 | // Log values 79 | func (h *Dumper) Log(values []dumper.DumpValue) { 80 | fields := []zapcore.Field{} 81 | for _, kv := range values { 82 | fields = append(fields, zap.Any(kv.Key, kv.Value)) 83 | } 84 | h.logger.Info("-", fields...) 85 | } 86 | 87 | // NewConnMetadata ... 88 | func (h *Dumper) NewConnMetadata() *dumper.ConnMetadata { 89 | return &dumper.ConnMetadata{ 90 | DumpValues: []dumper.DumpValue{}, 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /dumper/hex/hex_test.go: -------------------------------------------------------------------------------- 1 | package hex 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | 8 | "github.com/k1LoW/tcpdp/dumper" 9 | "go.uber.org/zap" 10 | "go.uber.org/zap/zapcore" 11 | ) 12 | 13 | var hexReadTests = []struct { 14 | description string 15 | in []byte 16 | direction dumper.Direction 17 | expected []dumper.DumpValue 18 | }{ 19 | { 20 | "MySQL HandshakeResponse41 packet (https://dev.mysql.com/doc/internals/en/connection-phase-packets.html)", 21 | // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html 22 | []byte{ 23 | 0x54, 0x00, 0x00, 0x01, 0x8d, 0xa6, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 24 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 25 | 0x00, 0x00, 0x00, 0x00, 0x70, 0x61, 0x6d, 0x00, 0x14, 0xab, 0x09, 0xee, 0xf6, 0xbc, 0xb1, 0x32, 26 | 0x3e, 0x61, 0x14, 0x38, 0x65, 0xc0, 0x99, 0x1d, 0x95, 0x7d, 0x75, 0xd4, 0x47, 0x74, 0x65, 0x73, 27 | 0x74, 0x00, 0x6d, 0x79, 0x73, 0x71, 0x6c, 0x5f, 0x6e, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 28 | 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x00, 29 | }, 30 | dumper.SrcToDst, 31 | []dumper.DumpValue{ 32 | dumper.DumpValue{ 33 | Key: "bytes", 34 | Value: "54 00 00 01 8d a6 0f 00 00 00 00 01 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 70 61 6d 00 14 ab 09 ee f6 bc b1 32 3e 61 14 38 65 c0 99 1d 95 7d 75 d4 47 74 65 73 74 00 6d 79 73 71 6c 5f 6e 61 74 69 76 65 5f 70 61 73 73 77 6f 72 64 00", 35 | }, 36 | dumper.DumpValue{ 37 | Key: "ascii", 38 | Value: "T...................................pam........2>a.8e....}u.Gtest.mysql_native_password.", 39 | }, 40 | }, 41 | }, 42 | { 43 | "MySQL COM_QUERY packet", 44 | []byte{ 45 | 0x14, 0x00, 0x00, 0x00, 0x03, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x20, 0x2a, 0x20, 0x66, 0x72, 46 | 0x6f, 0x6d, 0x20, 0x70, 0x6f, 0x73, 0x74, 0x73, 47 | }, 48 | dumper.SrcToDst, 49 | []dumper.DumpValue{ 50 | dumper.DumpValue{ 51 | Key: "bytes", 52 | Value: "14 00 00 00 03 73 65 6c 65 63 74 20 2a 20 66 72 6f 6d 20 70 6f 73 74 73", 53 | }, 54 | dumper.DumpValue{ 55 | Key: "ascii", 56 | Value: ".....select * from posts", 57 | }, 58 | }, 59 | }, 60 | } 61 | 62 | func TestHexRead(t *testing.T) { 63 | for _, tt := range hexReadTests { 64 | out := new(bytes.Buffer) 65 | dumper := &Dumper{ 66 | logger: newTestLogger(out), 67 | } 68 | in := tt.in 69 | direction := tt.direction 70 | connMetadata := dumper.NewConnMetadata() 71 | 72 | actual, err := dumper.Read(in, direction, connMetadata) 73 | if err != nil { 74 | t.Errorf("%v", err) 75 | } 76 | 77 | expected := tt.expected 78 | 79 | if len(actual) != len(expected) { 80 | t.Errorf("actual %v\nwant %v", actual, expected) 81 | } 82 | for i := 0; i < len(actual); i++ { 83 | v := actual[i].Value 84 | ev := expected[i].Value 85 | switch v.(type) { 86 | case []interface{}: 87 | for j := 0; j < len(v.([]interface{})); j++ { 88 | if v.([]interface{})[j] != ev.([]interface{})[j] { 89 | t.Errorf("actual %#v\nwant %#v", v.([]interface{})[j], ev.([]interface{})[j]) 90 | } 91 | } 92 | default: 93 | if actual[i] != expected[i] { 94 | t.Errorf("actual %#v\nwant %#v", actual[i], expected[i]) 95 | } 96 | } 97 | } 98 | } 99 | } 100 | 101 | // newTestLogger return zap.Logger for test 102 | func newTestLogger(out io.Writer) *zap.Logger { 103 | encoderConfig := zapcore.EncoderConfig{ 104 | TimeKey: "ts", 105 | LevelKey: "level", 106 | NameKey: "logger", 107 | CallerKey: "caller", 108 | MessageKey: "msg", 109 | StacktraceKey: "stacktrace", 110 | EncodeLevel: zapcore.LowercaseLevelEncoder, 111 | EncodeTime: zapcore.ISO8601TimeEncoder, 112 | EncodeDuration: zapcore.StringDurationEncoder, 113 | EncodeCaller: zapcore.ShortCallerEncoder, 114 | } 115 | 116 | logger := zap.New(zapcore.NewCore( 117 | zapcore.NewJSONEncoder(encoderConfig), 118 | zapcore.AddSync(out), 119 | zapcore.DebugLevel, 120 | )) 121 | 122 | return logger 123 | } 124 | -------------------------------------------------------------------------------- /dumper/mysql/const.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | const ( 4 | comQuery = 0x03 5 | comStmtPrepare = 0x16 6 | comStmtExecute = 0x17 7 | 8 | comStmtPrepareOK = 0x00 9 | ) 10 | 11 | type dataType byte 12 | 13 | const ( 14 | typeDecimal dataType = 0x00 15 | typeTiny = 0x01 16 | typeShort = 0x02 17 | typeLong = 0x03 18 | typeFloat = 0x04 19 | typeDouble = 0x05 20 | typeNull = 0x06 21 | typeTimestamp = 0x07 22 | typeLonglong = 0x08 23 | typeInt24 = 0x09 24 | typeDate = 0x0a 25 | typeTime = 0x0b 26 | typeDatetime = 0x0c 27 | typeYear = 0x0d 28 | typeNewdate = 0x0e 29 | typeVarchar = 0x0f 30 | typeBit = 0x10 31 | typeNewdecimal = 0xf6 32 | typeEnum = 0xf7 33 | typeSet = 0xf8 34 | typeTinyBlob = 0xf9 35 | typeMediumblob = 0xfa 36 | typeLongblob = 0xfb 37 | typeBlob = 0xfc 38 | typeVarString = 0xfd 39 | typeString = 0xfe 40 | typeGeometry = 0xff 41 | ) 42 | 43 | type clientCapability uint32 44 | 45 | const ( 46 | clientLongPassword clientCapability = 1 << iota 47 | clientFoundRows 48 | clientLongFlag 49 | clientConnectWithDB 50 | clientNoSchema 51 | clientCompress 52 | clientODBC 53 | clientLocalFiles 54 | clientIgnoreSpace 55 | clientProtocol41 56 | clientInteractive 57 | clientSSL 58 | clientIgnoreSIGPIPE 59 | clientTransactions 60 | clientReserved 61 | clientSecureConnection 62 | clientMultiStatements 63 | clientMultiResults 64 | clientPSMultiResults 65 | clientPluginAuth 66 | clientConnectAttrs 67 | clientPluginAuthLenEncClientData 68 | clientCanHandleExpiredPasswords 69 | clientSessionTrack 70 | clientDeprecateEOF 71 | ) 72 | 73 | type charSet uint32 74 | 75 | const ( 76 | charSetUnknown charSet = 0 77 | charSetBig5 = 1 78 | charSetDec8 = 3 79 | charSetCp850 = 4 80 | charSetHp8 = 6 81 | charSetKoi8r = 7 82 | charSetLatin1 = 8 83 | charSetLatin2 = 9 84 | charSetSwe7 = 10 85 | charSetASCII = 11 86 | charSetUjis = 12 87 | charSetSjis = 13 88 | charSetHebrew = 16 89 | charSetTis620 = 18 90 | charSetEuckr = 19 91 | charSetKoi8u = 22 92 | charSetGb2312 = 24 93 | charSetGreek = 25 94 | charSetCp1250 = 26 95 | charSetGbk = 28 96 | charSetLatin5 = 30 97 | charSetArmscii8 = 32 98 | charSetUtf8 = 33 99 | charSetUcs2 = 35 100 | charSetCp866 = 36 101 | charSetKeybcs2 = 37 102 | charSetMacce = 38 103 | charSetMacroman = 39 104 | charSetCp852 = 40 105 | charSetLatin7 = 41 106 | charSetCp1251 = 51 107 | charSetUtf16 = 54 108 | charSetUtf16le = 56 109 | charSetCp1256 = 57 110 | charSetCp1257 = 59 111 | charSetUtf32 = 60 112 | charSetBinary = 63 113 | charSetGeostd8 = 92 114 | charSetCp932 = 95 115 | charSetEucjpms = 97 116 | charSetGb18030 = 248 117 | charSetUtf8mb4 = 255 118 | ) 119 | 120 | func (c charSet) String() string { 121 | switch c { 122 | case charSetBig5: 123 | return "big5" 124 | case charSetDec8: 125 | return "dec8" 126 | case charSetCp850: 127 | return "cp850" 128 | case charSetHp8: 129 | return "hp8" 130 | case charSetKoi8r: 131 | return "koi8r" 132 | case charSetLatin1: 133 | return "latin1" 134 | case charSetLatin2: 135 | return "latin2" 136 | case charSetSwe7: 137 | return "swe7" 138 | case charSetASCII: 139 | return "ascii" 140 | case charSetUjis: 141 | return "ujis" 142 | case charSetSjis: 143 | return "sjis" 144 | case charSetHebrew: 145 | return "hebrew" 146 | case charSetTis620: 147 | return "tis620" 148 | case charSetEuckr: 149 | return "euckr" 150 | case charSetKoi8u: 151 | return "koi8u" 152 | case charSetGb2312: 153 | return "gb2312" 154 | case charSetGreek: 155 | return "greek" 156 | case charSetCp1250: 157 | return "cp1250" 158 | case charSetGbk: 159 | return "gbk" 160 | case charSetLatin5: 161 | return "latin5" 162 | case charSetArmscii8: 163 | return "armscii8" 164 | case charSetUtf8: 165 | return "utf8" 166 | case charSetUcs2: 167 | return "ucs2" 168 | case charSetCp866: 169 | return "cp866" 170 | case charSetKeybcs2: 171 | return "keybcs2" 172 | case charSetMacce: 173 | return "macce" 174 | case charSetMacroman: 175 | return "macroman" 176 | case charSetCp852: 177 | return "cp852" 178 | case charSetLatin7: 179 | return "latin7" 180 | case charSetCp1251: 181 | return "cp1251" 182 | case charSetUtf16: 183 | return "utf16" 184 | case charSetUtf16le: 185 | return "utf16le" 186 | case charSetCp1256: 187 | return "cp1256" 188 | case charSetCp1257: 189 | return "cp1257" 190 | case charSetUtf32: 191 | return "utf32" 192 | case charSetBinary: 193 | return "binary" 194 | case charSetGeostd8: 195 | return "geostd8" 196 | case charSetCp932: 197 | return "cp932" 198 | case charSetEucjpms: 199 | return "eucjpms" 200 | case charSetGb18030: 201 | return "gb18030" 202 | case charSetUtf8mb4: 203 | return "utf8mb4" 204 | default: 205 | return "" 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /dumper/mysql/encoding.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "strings" 7 | 8 | "golang.org/x/text/encoding/japanese" 9 | "golang.org/x/text/transform" 10 | ) 11 | 12 | func readString(src []byte, srcCharSet charSet) string { 13 | switch srcCharSet { 14 | case charSetUjis, charSetEucjpms: 15 | buff := bytes.NewBuffer(src) 16 | dst, err := ioutil.ReadAll(transform.NewReader(buff, japanese.EUCJP.NewDecoder())) 17 | if err != nil { 18 | return strings.TrimRight(string(src), "\x00") 19 | } 20 | return strings.TrimRight(string(dst), "\x00") 21 | case charSetSjis, charSetCp932: 22 | buff := bytes.NewBuffer(src) 23 | dst, err := ioutil.ReadAll(transform.NewReader(buff, japanese.ShiftJIS.NewDecoder())) 24 | if err != nil { 25 | return strings.TrimRight(string(src), "\x00") 26 | } 27 | return strings.TrimRight(string(dst), "\x00") 28 | default: 29 | return strings.TrimRight(string(src), "\x00") 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /dumper/mysql/encoding_test.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var readStringTests = []struct { 8 | in []byte 9 | charSet charSet 10 | expected string 11 | }{ 12 | { 13 | []byte{ 14 | 0x14, 0x00, 0x00, 0x00, 0x03, 0x53, 0x45, 0x4c, 0x45, 0x43, 0x54, 0x20, 0x27, 0x82, 0xa0, 0x82, 15 | 0xa2, 0x82, 0xa4, 0x82, 0xa6, 0x82, 0xa8, 0x27, 16 | }, 17 | charSetSjis, 18 | "SELECT 'あいうえお'", 19 | }, 20 | { 21 | []byte{ 22 | 0x14, 0x00, 0x00, 0x00, 0x03, 0x53, 0x45, 0x4c, 0x45, 0x43, 0x54, 0x20, 0x27, 0xa4, 0xa2, 0xa4, 23 | 0xa4, 0xa4, 0xa6, 0xa4, 0xa8, 0xa4, 0xaa, 0x27, 24 | }, 25 | charSetUjis, 26 | "SELECT 'あいうえお'", 27 | }, 28 | { 29 | []byte{ 30 | 0x14, 0x00, 0x00, 0x00, 0x03, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x20, 0x2a, 0x20, 0x66, 0x72, 31 | 0x6f, 0x6d, 0x20, 0x70, 0x6f, 0x73, 0x74, 0x73, 32 | }, 33 | charSetUnknown, 34 | "select * from posts", 35 | }, 36 | } 37 | 38 | func TestReadString(t *testing.T) { 39 | for _, tt := range readStringTests { 40 | actual := readString(tt.in[5:], tt.charSet) 41 | if actual != tt.expected { 42 | t.Errorf("actual %#v\nwant %#v", actual, tt.expected) 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /dumper/mysql/mysql.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | "math" 10 | "time" 11 | 12 | "github.com/k1LoW/tcpdp/dumper" 13 | "github.com/k1LoW/tcpdp/logger" 14 | "github.com/pkg/errors" 15 | "go.uber.org/zap" 16 | "go.uber.org/zap/zapcore" 17 | ) 18 | 19 | // Dumper struct 20 | type Dumper struct { 21 | name string 22 | logger *zap.Logger 23 | } 24 | 25 | type clientCapabilities map[clientCapability]bool 26 | 27 | type stmtNumParams map[int]int // statement_id:num_params 28 | 29 | type connMetadataInternal struct { 30 | clientCapabilities clientCapabilities 31 | stmtNumParams stmtNumParams 32 | charSet charSet 33 | payloadLength uint32 34 | longPacketCache []byte 35 | } 36 | 37 | // NewDumper returns a Dumper 38 | func NewDumper() *Dumper { 39 | dumper := &Dumper{ 40 | name: "mysql", 41 | logger: logger.NewQueryLogger(), 42 | } 43 | return dumper 44 | } 45 | 46 | // Name return dumper name 47 | func (m *Dumper) Name() string { 48 | return m.name 49 | } 50 | 51 | // Dump query of MySQL 52 | func (m *Dumper) Dump(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata, additional []dumper.DumpValue) error { 53 | read, _ := m.Read(in, direction, connMetadata) 54 | if len(read) == 0 { 55 | return nil 56 | } 57 | 58 | values := []dumper.DumpValue{} 59 | values = append(values, read...) 60 | values = append(values, connMetadata.DumpValues...) 61 | values = append(values, additional...) 62 | 63 | m.Log(values) 64 | return nil 65 | } 66 | 67 | // Read return byte to analyzed string 68 | func (m *Dumper) Read(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata) ([]dumper.DumpValue, error) { 69 | values, handshakeErr := m.readHandshakeResponse(in, direction, connMetadata) 70 | 71 | connMetadata.DumpValues = append(connMetadata.DumpValues, values...) 72 | cSet := connMetadata.Internal.(connMetadataInternal).charSet 73 | 74 | if len(connMetadata.Internal.(connMetadataInternal).longPacketCache) > 0 { 75 | internal := connMetadata.Internal.(connMetadataInternal) 76 | in = append(internal.longPacketCache, in...) 77 | internal.longPacketCache = nil 78 | connMetadata.Internal = internal 79 | } 80 | 81 | if handshakeErr != nil { 82 | return values, handshakeErr 83 | } 84 | 85 | // Client Compress 86 | compressed, ok := connMetadata.Internal.(connMetadataInternal).clientCapabilities[clientCompress] 87 | if ok && compressed { 88 | // https://dev.mysql.com/doc/internals/en/compressed-packet-header.html 89 | buff := bytes.NewBuffer(in) 90 | lenCompressed := int(bytesToUint64(readBytes(buff, 3))) // 3:length of compressed payload 91 | _ = readBytes(buff, 1) // 1:compressed sequence id 92 | lenUncompressed := bytesToUint64(readBytes(buff, 3)) // 3:length of payload before compression 93 | if buff.Len() == lenCompressed { 94 | if lenUncompressed > 0 { 95 | // https://dev.mysql.com/doc/internals/en/compressed-payload.html 96 | r, err := zlib.NewReader(buff) 97 | if err != nil { 98 | return values, err 99 | } 100 | newBuff := new(bytes.Buffer) 101 | _, err = io.Copy(newBuff, r) // #nosec 102 | if err != nil { 103 | return values, err 104 | } 105 | in = newBuff.Bytes() 106 | } else { 107 | // https://dev.mysql.com/doc/internals/en/uncompressed-payload.html 108 | in = buff.Bytes() 109 | } 110 | } 111 | } 112 | 113 | if direction == dumper.RemoteToClient || direction == dumper.DstToSrc || direction == dumper.Unknown { 114 | // COM_STMT_PREPARE Response https://dev.mysql.com/doc/internals/en/com-stmt-prepare-response.html 115 | if len(in) >= 16 && in[4] == comStmtPrepareOK && in[13] == 0x00 { 116 | buff := bytes.NewBuffer(in[5:]) 117 | stmtID := readBytes(buff, 4) 118 | stmtIDNum := int(bytesToUint64(stmtID)) 119 | _ = readBytes(buff, 2) 120 | numParams := readBytes(buff, 2) 121 | numParamsNum := int(bytesToUint64(numParams)) 122 | connMetadata.Internal.(connMetadataInternal).stmtNumParams[stmtIDNum] = numParamsNum 123 | } 124 | return []dumper.DumpValue{}, nil 125 | } 126 | 127 | if len(in) < 6 { 128 | return []dumper.DumpValue{}, nil 129 | } 130 | 131 | var payloadLength uint32 132 | internal := connMetadata.Internal.(connMetadataInternal) 133 | if internal.payloadLength > 0 { 134 | payloadLength = internal.payloadLength 135 | } else { 136 | pl := make([]byte, 3) 137 | copy(pl, in[0:3]) 138 | payloadLength = bytesToUint32(pl) // 3:payload_length 139 | } 140 | if uint32(len(in[4:])) < payloadLength { 141 | internal.payloadLength = payloadLength 142 | internal.longPacketCache = append(internal.longPacketCache, in...) 143 | connMetadata.Internal = internal 144 | return []dumper.DumpValue{}, nil 145 | } 146 | internal.payloadLength = uint32(0) 147 | connMetadata.Internal = internal 148 | 149 | seqNum := int64(in[3]) 150 | commandID := in[4] 151 | 152 | var dumps = []dumper.DumpValue{} 153 | switch commandID { 154 | case comQuery: 155 | query := readString(in[5:], cSet) 156 | dumps = []dumper.DumpValue{ 157 | dumper.DumpValue{ 158 | Key: "query", 159 | Value: query, 160 | }, 161 | } 162 | case comStmtPrepare: 163 | stmtPrepare := readString(in[5:], cSet) 164 | dumps = []dumper.DumpValue{ 165 | dumper.DumpValue{ 166 | Key: "stmt_prepare_query", 167 | Value: stmtPrepare, 168 | }, 169 | } 170 | case comStmtExecute: 171 | // https://dev.mysql.com/doc/internals/en/com-stmt-execute.html 172 | buff := bytes.NewBuffer(in[5:]) 173 | stmtID := readBytes(buff, 4) // 4:stmt-id 174 | stmtIDNum := int(bytesToUint64(stmtID)) 175 | numParamsNum, ok := connMetadata.Internal.(connMetadataInternal).stmtNumParams[stmtIDNum] 176 | if ok && numParamsNum > 0 { 177 | _ = readBytes(buff, 5) // 1:flags 4:iteration-count 178 | _ = readBytes(buff, (numParamsNum+7)/8) // NULL-bitmap, length: (num-params+7)/8 179 | newParamsBoundFlag, _ := buff.ReadByte() 180 | if newParamsBoundFlag == 0x01 { 181 | // type of each parameter, length: num-params * 2 182 | dataTypes := []dataType{} 183 | for i := 0; i < numParamsNum; i++ { 184 | t := readMysqlType(buff) 185 | dataTypes = append(dataTypes, t) 186 | _, _ = buff.ReadByte() 187 | } 188 | // value of each parameter 189 | values := []interface{}{} 190 | for i := 0; i < numParamsNum; i++ { 191 | // https://dev.mysql.com/doc/internals/en/binary-protocol-value.html 192 | v := readBinaryProtocolValue(buff, dataTypes[i], cSet) 193 | values = append(values, v) 194 | } 195 | dumps = []dumper.DumpValue{ 196 | dumper.DumpValue{ 197 | Key: "stmt_id", 198 | Value: stmtIDNum, 199 | }, 200 | dumper.DumpValue{ 201 | Key: "stmt_execute_values", 202 | Value: values, 203 | }, 204 | } 205 | } else { 206 | dumps = []dumper.DumpValue{ 207 | dumper.DumpValue{ 208 | Key: "stmt_id", 209 | Value: stmtIDNum, 210 | }, 211 | dumper.DumpValue{ 212 | Key: "stmt_execute_values", 213 | Value: []interface{}{}, 214 | }, 215 | } 216 | } 217 | } else if ok && numParamsNum == 0 { 218 | dumps = []dumper.DumpValue{ 219 | dumper.DumpValue{ 220 | Key: "stmt_id", 221 | Value: stmtIDNum, 222 | }, 223 | dumper.DumpValue{ 224 | Key: "stmt_execute_values", 225 | Value: []interface{}{}, 226 | }, 227 | } 228 | } else { 229 | values := readString(in[5:], cSet) 230 | dumps = []dumper.DumpValue{ 231 | dumper.DumpValue{ 232 | Key: "stmt_id", 233 | Value: stmtIDNum, 234 | }, 235 | dumper.DumpValue{ 236 | Key: "stmt_execute_values", 237 | Value: []string{values}, 238 | }, 239 | } 240 | } 241 | default: 242 | return []dumper.DumpValue{}, nil 243 | } 244 | 245 | return append(dumps, []dumper.DumpValue{ 246 | dumper.DumpValue{ 247 | Key: "seq_num", 248 | Value: seqNum, 249 | }, 250 | dumper.DumpValue{ 251 | Key: "command_id", 252 | Value: commandID, 253 | }, 254 | }...), nil 255 | } 256 | 257 | // Log values 258 | func (m *Dumper) Log(values []dumper.DumpValue) { 259 | fields := []zapcore.Field{} 260 | for _, kv := range values { 261 | fields = append(fields, zap.Any(kv.Key, kv.Value)) 262 | } 263 | m.logger.Info("-", fields...) 264 | } 265 | 266 | // NewConnMetadata return metadata per TCP connection 267 | func (m *Dumper) NewConnMetadata() *dumper.ConnMetadata { 268 | return &dumper.ConnMetadata{ 269 | DumpValues: []dumper.DumpValue{}, 270 | Internal: connMetadataInternal{ 271 | stmtNumParams: stmtNumParams{}, 272 | clientCapabilities: clientCapabilities{}, 273 | charSet: charSetUnknown, 274 | payloadLength: uint32(0), 275 | }, 276 | } 277 | } 278 | 279 | func (m *Dumper) readHandshakeResponse(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata) ([]dumper.DumpValue, error) { 280 | values := []dumper.DumpValue{} 281 | if direction == dumper.RemoteToClient || direction == dumper.DstToSrc { 282 | return values, nil 283 | } 284 | 285 | if len(connMetadata.Internal.(connMetadataInternal).clientCapabilities) > 0 { 286 | return values, nil 287 | } 288 | 289 | if len(in) < 9 { 290 | return values, nil 291 | } 292 | 293 | clientCapabilities := binary.LittleEndian.Uint32(in[4:8]) 294 | 295 | // parse Protocol::HandshakeResponse41 296 | if len(in) > 35 && clientCapabilities&uint32(clientProtocol41) > 0 && bytes.Compare(in[13:36], []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}) == 0 { 297 | internal := connMetadata.Internal.(connMetadataInternal) 298 | 299 | cSet := charSet(uint32(in[12])) 300 | values = append(values, dumper.DumpValue{ 301 | Key: "character_set", 302 | Value: cSet.String(), 303 | }) 304 | internal.charSet = cSet 305 | connMetadata.Internal = internal 306 | connMetadata.Internal.(connMetadataInternal).clientCapabilities[clientProtocol41] = true 307 | 308 | if clientCapabilities&uint32(clientSSL) > 0 { 309 | // tcpdp mysql dumper not support SSL connection. 310 | err := errors.New("client is trying to connect using SSL. tcpdp mysql dumper not support SSL connection") 311 | fields := []zapcore.Field{ 312 | zap.Error(err), 313 | } 314 | for _, kv := range connMetadata.DumpValues { 315 | fields = append(fields, zap.Any(kv.Key, kv.Value)) 316 | } 317 | for _, kv := range values { 318 | fields = append(fields, zap.Any(kv.Key, kv.Value)) 319 | } 320 | m.logger.Warn("-", fields...) 321 | return values, err 322 | } 323 | 324 | buff := bytes.NewBuffer(in[36:]) 325 | readed, _ := buff.ReadBytes(0x00) 326 | username := readString(readed, cSet) 327 | values = append(values, dumper.DumpValue{ 328 | Key: "username", 329 | Value: username, 330 | }) 331 | if clientCapabilities&uint32(clientPluginAuthLenEncClientData) > 0 { 332 | connMetadata.Internal.(connMetadataInternal).clientCapabilities[clientPluginAuthLenEncClientData] = true 333 | n := readLengthEncodedInteger(buff) 334 | _, _ = buff.Read(make([]byte, n)) 335 | } else if clientCapabilities&uint32(clientSecureConnection) > 0 { 336 | connMetadata.Internal.(connMetadataInternal).clientCapabilities[clientSecureConnection] = true 337 | l, _ := buff.ReadByte() 338 | _, _ = buff.Read(make([]byte, l)) 339 | } else { 340 | _, _ = buff.ReadString(0x00) 341 | } 342 | if clientCapabilities&uint32(clientConnectWithDB) > 0 { 343 | connMetadata.Internal.(connMetadataInternal).clientCapabilities[clientConnectWithDB] = true 344 | readed, _ := buff.ReadBytes(0x00) 345 | database := readString(readed, cSet) 346 | values = append(values, dumper.DumpValue{ 347 | Key: "database", 348 | Value: database, 349 | }) 350 | } 351 | connMetadata.Internal.(connMetadataInternal).clientCapabilities[clientCompress] = (clientCapabilities&uint32(clientCompress) > 0) 352 | return values, nil 353 | } 354 | 355 | // parse Protocol::HandshakeResponse320 356 | clientCapabilities = bytesToUint32(in[4:6]) // 2:capability flags, CLIENT_PROTOCOL_41 never set 357 | if clientCapabilities&uint32(clientProtocol41) == 0 { 358 | if clientCapabilities&uint32(clientSSL) > 0 { 359 | // tcpdp mysql dumper not support SSL connection. 360 | err := errors.New("client is trying to connect using SSL. tcpdp mysql dumper not support SSL connection") 361 | return values, err 362 | } 363 | 364 | v := []dumper.DumpValue{} 365 | internal := connMetadata.Internal.(connMetadataInternal) 366 | connMetadata.Internal = internal 367 | connMetadata.Internal.(connMetadataInternal).clientCapabilities[clientProtocol41] = false 368 | buff := bytes.NewBuffer(in[9:]) 369 | readed, _ := buff.ReadBytes(0x00) 370 | username := readString(readed, charSetUtf8) 371 | v = append(v, dumper.DumpValue{ 372 | Key: "username", 373 | Value: username, 374 | }) 375 | if clientCapabilities&uint32(clientConnectWithDB) > 0 { 376 | _, _ = buff.ReadBytes(0x00) 377 | readed, _ := buff.ReadBytes(0x00) 378 | database := readString(readed, charSetUtf8) 379 | v = append(v, dumper.DumpValue{ 380 | Key: "database", 381 | Value: database, 382 | }) 383 | } else { 384 | _, _ = buff.ReadBytes(0x00) 385 | } 386 | if buff.Len() == 0 { 387 | values = append(values, v...) 388 | } 389 | } 390 | 391 | return values, nil 392 | } 393 | 394 | func readMysqlType(buff *bytes.Buffer) dataType { 395 | b, _ := buff.ReadByte() 396 | return dataType(b) 397 | } 398 | 399 | // https://dev.mysql.com/doc/internals/en/integer.html#length-encoded-integer 400 | func readLengthEncodedInteger(buff *bytes.Buffer) uint64 { 401 | l, _ := buff.ReadByte() 402 | n := bytesToUint64([]byte{l}) 403 | if l == 0xfc { 404 | n = bytesToUint64(readBytes(buff, 2)) 405 | } 406 | if l == 0xfd { 407 | n = bytesToUint64(readBytes(buff, 3)) 408 | } 409 | if l == 0xfe { 410 | n = bytesToUint64(readBytes(buff, 8)) 411 | } 412 | return n 413 | } 414 | 415 | // https://dev.mysql.com/doc/internals/en/binary-protocol-value.html 416 | func readBinaryProtocolValue(buff *bytes.Buffer, dataType dataType, cSet charSet) interface{} { 417 | switch dataType { 418 | case typeLonglong: 419 | v := readBytes(buff, 8) 420 | return int64(binary.LittleEndian.Uint64(v)) 421 | case typeLong, typeInt24: 422 | v := readBytes(buff, 4) 423 | return int32(binary.LittleEndian.Uint32(v)) 424 | case typeShort, typeYear: 425 | v := readBytes(buff, 2) 426 | return int16(binary.LittleEndian.Uint16(v)) 427 | case typeTiny: 428 | v := readBytes(buff, 1) 429 | return int8(v[0]) 430 | case typeDouble: 431 | bits := bytesToUint64(readBytes(buff, 8)) 432 | float := math.Float64frombits(bits) 433 | return float 434 | case typeFloat: 435 | bits := bytesToUint64(readBytes(buff, 4)) 436 | float := math.Float32frombits(uint32(bits)) 437 | return float 438 | case typeDate, typeDatetime, typeTimestamp: 439 | return readDatetime(buff, dataType) 440 | case typeTime: 441 | return readTime(buff) 442 | case typeNull: 443 | return nil 444 | default: 445 | l := readLengthEncodedInteger(buff) 446 | v := readBytes(buff, int(l)) 447 | return readString(v, cSet) 448 | } 449 | } 450 | 451 | // ProtocolBinary::MYSQL_TYPE_DATE, ProtocolBinary::MYSQL_TYPE_DATETIME, ProtocolBinary::MYSQL_TYPE_TIMESTAMP 452 | // https://dev.mysql.com/doc/internals/en/binary-protocol-value.html 453 | func readDatetime(buff *bytes.Buffer, dataType dataType) string { 454 | l := bytesToUint64(readBytes(buff, 1)) 455 | year := 0 456 | var month time.Month 457 | day := 0 458 | hour := 0 459 | min := 0 460 | sec := 0 461 | microSecond := 0 462 | switch l { 463 | case 0: 464 | case 4: 465 | year = int(bytesToUint64(readBytes(buff, 2))) 466 | month = time.Month(int(bytesToUint64(readBytes(buff, 1)))) 467 | day = int(bytesToUint64(readBytes(buff, 1))) 468 | case 7: 469 | year = int(bytesToUint64(readBytes(buff, 2))) 470 | month = time.Month(int(bytesToUint64(readBytes(buff, 1)))) 471 | day = int(bytesToUint64(readBytes(buff, 1))) 472 | hour = int(bytesToUint64(readBytes(buff, 1))) 473 | min = int(bytesToUint64(readBytes(buff, 1))) 474 | sec = int(bytesToUint64(readBytes(buff, 1))) 475 | case 11: 476 | year = int(bytesToUint64(readBytes(buff, 2))) 477 | month = time.Month(int(bytesToUint64(readBytes(buff, 1)))) 478 | day = int(bytesToUint64(readBytes(buff, 1))) 479 | hour = int(bytesToUint64(readBytes(buff, 1))) 480 | min = int(bytesToUint64(readBytes(buff, 1))) 481 | sec = int(bytesToUint64(readBytes(buff, 1))) 482 | microSecond = int(bytesToUint64(readBytes(buff, 4))) 483 | } 484 | t := time.Date(year, month, day, hour, min, sec, microSecond*1000, time.UTC) 485 | 486 | if dataType == typeDate { 487 | return t.Format("2006-01-02") 488 | } 489 | ms := fmt.Sprintf("%06d", microSecond) 490 | return fmt.Sprintf("%s.%s %s", t.Format("2006-01-02 15:04:05"), ms[0:3], ms[3:6]) 491 | } 492 | 493 | // ProtocolBinary::MYSQL_TYPE_TIME 494 | // https://dev.mysql.com/doc/internals/en/binary-protocol-value.html 495 | func readTime(buff *bytes.Buffer) string { 496 | l := bytesToUint64(readBytes(buff, 1)) 497 | days := 0 498 | negative := 0 499 | hour := 0 500 | min := 0 501 | sec := 0 502 | microSecond := 0 503 | switch l { 504 | case 0: 505 | case 8: 506 | negative = int(bytesToUint64(readBytes(buff, 1))) 507 | days = int(bytesToUint64(readBytes(buff, 4))) 508 | hour = int(bytesToUint64(readBytes(buff, 1))) 509 | min = int(bytesToUint64(readBytes(buff, 1))) 510 | sec = int(bytesToUint64(readBytes(buff, 1))) 511 | case 12: 512 | negative = int(bytesToUint64(readBytes(buff, 1))) 513 | days = int(bytesToUint64(readBytes(buff, 4))) 514 | hour = int(bytesToUint64(readBytes(buff, 1))) 515 | min = int(bytesToUint64(readBytes(buff, 1))) 516 | sec = int(bytesToUint64(readBytes(buff, 1))) 517 | microSecond = int(bytesToUint64(readBytes(buff, 4))) 518 | } 519 | op := "" 520 | if negative == 1 { 521 | op = "-" 522 | } 523 | t := time.Date(0, time.January, 0, hour, min, sec, microSecond*1000, time.UTC) 524 | ms := fmt.Sprintf("%06d", microSecond) 525 | switch l { 526 | case 0: 527 | return "" 528 | case 12: 529 | return fmt.Sprintf("%s%dd %s.%s %s", op, days, t.Format("15:04:05"), ms[0:3], ms[3:6]) 530 | default: 531 | return fmt.Sprintf("%s%dd %s", op, days, t.Format("15:04:05")) 532 | } 533 | 534 | } 535 | 536 | func bytesToUint32(b []byte) uint32 { 537 | c := make([]byte, len(b)) 538 | copy(c, b) 539 | padding := make([]byte, 4-len(c)) 540 | return binary.LittleEndian.Uint32(append(c, padding...)) 541 | } 542 | 543 | func bytesToUint64(b []byte) uint64 { 544 | c := make([]byte, len(b)) 545 | copy(c, b) 546 | padding := make([]byte, 8-len(c)) 547 | return binary.LittleEndian.Uint64(append(c, padding...)) 548 | } 549 | 550 | func readBytes(buff *bytes.Buffer, len int) []byte { 551 | b := make([]byte, len) 552 | _, _ = buff.Read(b) 553 | return b 554 | } 555 | -------------------------------------------------------------------------------- /dumper/pg/pg.go: -------------------------------------------------------------------------------- 1 | package pg 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "strings" 7 | 8 | "github.com/k1LoW/tcpdp/dumper" 9 | "github.com/k1LoW/tcpdp/logger" 10 | "github.com/pkg/errors" 11 | "go.uber.org/zap" 12 | "go.uber.org/zap/zapcore" 13 | ) 14 | 15 | const ( 16 | messageQuery = 'Q' 17 | messageParse = 'P' 18 | messageBind = 'B' 19 | messageExecute = 'E' 20 | ) 21 | 22 | type dataType int16 23 | 24 | const ( 25 | typeString dataType = iota 26 | typeBinary 27 | ) 28 | 29 | // Dumper struct 30 | type Dumper struct { 31 | name string 32 | logger *zap.Logger 33 | } 34 | 35 | type connMetadataInternal struct { 36 | messageLength uint32 37 | longPacketCache []byte 38 | } 39 | 40 | // NewDumper returns a Dumper 41 | func NewDumper() *Dumper { 42 | dumper := &Dumper{ 43 | name: "pg", 44 | logger: logger.NewQueryLogger(), 45 | } 46 | return dumper 47 | } 48 | 49 | // Name return dumper name 50 | func (p *Dumper) Name() string { 51 | return p.name 52 | } 53 | 54 | // Dump query of PostgreSQL 55 | func (p *Dumper) Dump(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata, additional []dumper.DumpValue) error { 56 | read, _ := p.Read(in, direction, connMetadata) 57 | if len(read) == 0 { 58 | return nil 59 | } 60 | 61 | values := []dumper.DumpValue{} 62 | values = append(values, read...) 63 | values = append(values, connMetadata.DumpValues...) 64 | values = append(values, additional...) 65 | 66 | p.Log(values) 67 | return nil 68 | } 69 | 70 | // Read return byte to analyzed string 71 | func (p *Dumper) Read(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata) ([]dumper.DumpValue, error) { 72 | values, handshakeErr := p.readHandshake(in, direction, connMetadata) 73 | connMetadata.DumpValues = append(connMetadata.DumpValues, values...) 74 | 75 | if handshakeErr != nil { 76 | return values, handshakeErr 77 | } 78 | 79 | if direction == dumper.RemoteToClient || direction == dumper.DstToSrc || direction == dumper.Unknown { 80 | return []dumper.DumpValue{}, nil 81 | } 82 | 83 | if len(connMetadata.Internal.(connMetadataInternal).longPacketCache) > 0 { 84 | internal := connMetadata.Internal.(connMetadataInternal) 85 | in = append(internal.longPacketCache, in...) 86 | internal.longPacketCache = nil 87 | connMetadata.Internal = internal 88 | } 89 | 90 | if len(in) == 0 { 91 | return []dumper.DumpValue{}, nil 92 | } 93 | 94 | messageType := in[0] 95 | 96 | switch messageType { 97 | case messageQuery, messageParse, messageBind, messageExecute: 98 | var messageLength uint32 99 | internal := connMetadata.Internal.(connMetadataInternal) 100 | if internal.messageLength > 0 { 101 | messageLength = internal.messageLength 102 | } else { 103 | ml := make([]byte, 4) 104 | copy(ml, in[1:5]) 105 | messageLength = binary.BigEndian.Uint32(ml) 106 | } 107 | if uint32(len(in[1:])) < messageLength { 108 | internal.messageLength = messageLength 109 | internal.longPacketCache = append(internal.longPacketCache, in...) 110 | connMetadata.Internal = internal 111 | return []dumper.DumpValue{}, nil 112 | } 113 | internal.messageLength = uint32(0) 114 | connMetadata.Internal = internal 115 | } 116 | 117 | var dumps = []dumper.DumpValue{} 118 | // https://www.postgresql.org/docs/10/static/protocol-message-formats.html 119 | switch messageType { 120 | case messageQuery: 121 | query := strings.TrimRight(string(in[5:]), "\x00") 122 | 123 | dumps = []dumper.DumpValue{ 124 | dumper.DumpValue{ 125 | Key: "query", 126 | Value: query, 127 | }, 128 | } 129 | case messageParse: 130 | buff := bytes.NewBuffer(in[5:]) 131 | b, _ := buff.ReadString(0x00) 132 | stmtName := strings.TrimRight(b, "\x00") 133 | b, _ = buff.ReadString(0x00) 134 | query := strings.TrimRight(b, "\x00") 135 | numParams := int(binary.BigEndian.Uint16(readBytes(buff, 2))) 136 | for i := 0; i < numParams; i++ { 137 | // TODO 138 | // Int32: Specifies the object ID of the parameter data type. Placing a zero here is equivalent to leaving the type unspecified. 139 | } 140 | 141 | dumps = []dumper.DumpValue{ 142 | dumper.DumpValue{ 143 | Key: "stmt_name", 144 | Value: stmtName, 145 | }, 146 | dumper.DumpValue{ 147 | Key: "parse_query", 148 | Value: query, 149 | }, 150 | } 151 | case messageBind: 152 | buff := bytes.NewBuffer(in[5:]) 153 | b, _ := buff.ReadString(0x00) 154 | portalName := strings.TrimRight(b, "\x00") 155 | b, _ = buff.ReadString(0x00) 156 | stmtName := strings.TrimRight(b, "\x00") 157 | c := int(binary.BigEndian.Uint16(readBytes(buff, 2))) 158 | dataTypes := []dataType{} 159 | for i := 0; i < c; i++ { 160 | t := dataType(binary.BigEndian.Uint16(readBytes(buff, 2))) 161 | dataTypes = append(dataTypes, t) 162 | } 163 | numParams := int(binary.BigEndian.Uint16(readBytes(buff, 2))) 164 | if c == 0 { 165 | for i := 0; i < numParams; i++ { 166 | dataTypes = append(dataTypes, typeString) 167 | } 168 | } 169 | values := []interface{}{} 170 | for i := 0; i < numParams; i++ { 171 | n := int32(binary.BigEndian.Uint32(readBytes(buff, 4))) 172 | if n == -1 { 173 | continue 174 | } 175 | v := readBytes(buff, int(n)) 176 | if dataTypes[i] == typeString { 177 | values = append(values, string(v)) 178 | } else { 179 | values = append(values, v) 180 | } 181 | } 182 | 183 | dumps = []dumper.DumpValue{ 184 | dumper.DumpValue{ 185 | Key: "portal_name", 186 | Value: portalName, 187 | }, 188 | dumper.DumpValue{ 189 | Key: "stmt_name", 190 | Value: stmtName, 191 | }, 192 | dumper.DumpValue{ 193 | Key: "bind_values", 194 | Value: values, 195 | }, 196 | } 197 | case messageExecute: 198 | buff := bytes.NewBuffer(in[5:]) 199 | b, _ := buff.ReadString(0x00) 200 | portalName := strings.Trim(b, "\x00") 201 | 202 | dumps = []dumper.DumpValue{ 203 | dumper.DumpValue{ 204 | Key: "portal_name", 205 | Value: portalName, 206 | }, 207 | dumper.DumpValue{ 208 | Key: "execute_query", 209 | Value: "", 210 | }, 211 | } 212 | default: 213 | return []dumper.DumpValue{}, nil 214 | } 215 | return append(dumps, dumper.DumpValue{ 216 | Key: "message_type", 217 | Value: string(messageType), 218 | }), nil 219 | } 220 | 221 | // Log values 222 | func (p *Dumper) Log(values []dumper.DumpValue) { 223 | fields := []zapcore.Field{} 224 | for _, kv := range values { 225 | fields = append(fields, zap.Any(kv.Key, kv.Value)) 226 | } 227 | p.logger.Info("-", fields...) 228 | } 229 | 230 | // NewConnMetadata return metadata per TCP connection 231 | func (p *Dumper) NewConnMetadata() *dumper.ConnMetadata { 232 | return &dumper.ConnMetadata{ 233 | DumpValues: []dumper.DumpValue{}, 234 | Internal: connMetadataInternal{ 235 | messageLength: uint32(0), 236 | }, 237 | } 238 | } 239 | 240 | func (p *Dumper) readHandshake(in []byte, direction dumper.Direction, connMetadata *dumper.ConnMetadata) ([]dumper.DumpValue, error) { 241 | values := []dumper.DumpValue{} 242 | if direction == dumper.RemoteToClient || direction == dumper.DstToSrc { 243 | return values, nil 244 | } 245 | if len(in) < 8 { 246 | return values, nil 247 | } 248 | b := make([]byte, 2) 249 | copy(b, in[4:6]) 250 | pNo := binary.BigEndian.Uint16(b) 251 | if pNo == 1234 { 252 | // SSLRequest 253 | b := make([]byte, 2) 254 | copy(b, in[6:8]) 255 | uNo := binary.BigEndian.Uint16(b) 256 | if uNo == 5679 { 257 | // tcpdp pg dumper not support SSL connection. 258 | err := errors.New("client is trying to connect using SSL. tcpdp pg dumper not support SSL connection") 259 | return values, err 260 | } 261 | } 262 | // parse StartupMessage to get username, database 263 | if pNo != 3 { 264 | return values, nil 265 | } 266 | splited := bytes.Split(in[8:], []byte{0x00}) 267 | if len(splited) > 0 { 268 | for i, keyOrValue := range splited { 269 | if i%2 != 0 { 270 | continue 271 | } 272 | if string(keyOrValue) == "user" { 273 | values = append(values, dumper.DumpValue{ 274 | Key: "username", 275 | Value: string(splited[i+1]), 276 | }) 277 | } 278 | if string(keyOrValue) == "database" { 279 | values = append(values, dumper.DumpValue{ 280 | Key: "database", 281 | Value: string(splited[i+1]), 282 | }) 283 | } 284 | } 285 | } 286 | return values, nil 287 | } 288 | 289 | func readBytes(buff *bytes.Buffer, len int) []byte { 290 | b := make([]byte, len) 291 | _, _ = buff.Read(b) 292 | return b 293 | } 294 | -------------------------------------------------------------------------------- /dumper/pg/pg_test.go: -------------------------------------------------------------------------------- 1 | package pg 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | 8 | "github.com/k1LoW/tcpdp/dumper" 9 | "go.uber.org/zap" 10 | "go.uber.org/zap/zapcore" 11 | ) 12 | 13 | var pgValueTests = []struct { 14 | description string 15 | in []byte 16 | direction dumper.Direction 17 | connMetadata dumper.ConnMetadata 18 | expected []dumper.DumpValue 19 | expectedQuery []dumper.DumpValue 20 | }{ 21 | { 22 | "Parse username/database from StartupMessage packet", 23 | []byte{ 24 | 0x00, 0x00, 0x00, 0x64, 0x00, 0x03, 0x00, 0x00, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x66, 0x6c, 25 | 0x6f, 0x61, 0x74, 0x5f, 0x64, 0x69, 0x67, 0x69, 0x74, 0x73, 0x00, 0x32, 0x00, 0x75, 0x73, 0x65, 26 | 0x72, 0x00, 0x70, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x00, 0x64, 0x61, 0x74, 0x61, 0x62, 27 | 0x61, 0x73, 0x65, 0x00, 0x74, 0x65, 0x73, 0x74, 0x64, 0x62, 0x00, 0x63, 0x6c, 0x69, 0x65, 0x6e, 28 | 0x74, 0x5f, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x00, 0x55, 0x54, 0x46, 0x38, 0x00, 29 | 0x64, 0x61, 0x74, 0x65, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x00, 0x49, 0x53, 0x4f, 0x2c, 0x20, 0x4d, 30 | 0x44, 0x59, 0x00, 0x00, 31 | }, 32 | dumper.SrcToDst, 33 | dumper.ConnMetadata{ 34 | DumpValues: []dumper.DumpValue{}, 35 | Internal: connMetadataInternal{ 36 | messageLength: uint32(0), 37 | }, 38 | }, 39 | []dumper.DumpValue{ 40 | dumper.DumpValue{ 41 | Key: "username", 42 | Value: "postgres", 43 | }, 44 | dumper.DumpValue{ 45 | Key: "database", 46 | Value: "testdb", 47 | }, 48 | }, 49 | []dumper.DumpValue{}, 50 | }, 51 | { 52 | "Parse query from MessageQuery packet", 53 | []byte{ 54 | 0x51, 0x00, 0x00, 0x00, 0x19, 0x53, 0x45, 0x4c, 0x45, 0x43, 0x54, 0x20, 0x2a, 0x20, 0x46, 0x52, 55 | 0x4f, 0x4d, 0x20, 0x75, 0x73, 0x65, 0x72, 0x73, 0x3b, 0x00, 56 | }, 57 | dumper.SrcToDst, 58 | dumper.ConnMetadata{ 59 | DumpValues: []dumper.DumpValue{}, 60 | Internal: connMetadataInternal{ 61 | messageLength: uint32(0), 62 | }, 63 | }, 64 | []dumper.DumpValue{}, 65 | []dumper.DumpValue{ 66 | dumper.DumpValue{ 67 | Key: "query", 68 | Value: "SELECT * FROM users;", 69 | }, 70 | dumper.DumpValue{ 71 | Key: "message_type", 72 | Value: "Q", 73 | }, 74 | }, 75 | }, 76 | { 77 | "Parse query from MessageParse packet", 78 | []byte{ 79 | 0x50, 0x00, 0x00, 0x00, 0x34, 0x00, 0x53, 0x45, 0x4c, 0x45, 0x43, 0x54, 0x20, 0x43, 0x4f, 0x4e, 80 | 0x43, 0x41, 0x54, 0x28, 0x24, 0x31, 0x3a, 0x3a, 0x74, 0x65, 0x78, 0x74, 0x2c, 0x20, 0x24, 0x32, 81 | 0x3a, 0x3a, 0x74, 0x65, 0x78, 0x74, 0x2c, 0x20, 0x24, 0x33, 0x3a, 0x3a, 0x74, 0x65, 0x78, 0x74, 82 | 0x29, 0x3b, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x06, 0x53, 0x00, 0x53, 0x00, 0x00, 0x00, 83 | 0x04, 84 | }, 85 | dumper.SrcToDst, 86 | dumper.ConnMetadata{ 87 | DumpValues: []dumper.DumpValue{}, 88 | Internal: connMetadataInternal{ 89 | messageLength: uint32(0), 90 | }, 91 | }, 92 | []dumper.DumpValue{}, 93 | []dumper.DumpValue{ 94 | dumper.DumpValue{ 95 | Key: "stmt_name", 96 | Value: "", 97 | }, 98 | dumper.DumpValue{ 99 | Key: "parse", 100 | Value: "SELECT CONCAT($1::text, $2::text, $3::text);", 101 | }, 102 | dumper.DumpValue{ 103 | Key: "message_type", 104 | Value: "P", 105 | }, 106 | }, 107 | }, 108 | { 109 | "Parse query from MessageBind packet", 110 | []byte{ 111 | 0x42, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x09, 0x30, 112 | 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x39, 0x00, 0x00, 0x00, 0x1e, 0xe3, 0x81, 0x82, 0xe3, 113 | 0x81, 0x84, 0xe3, 0x81, 0x86, 0xe3, 0x81, 0x88, 0xe3, 0x81, 0x8a, 0xe3, 0x81, 0x8b, 0xe3, 0x81, 114 | 0x8d, 0xe3, 0x81, 0x8f, 0xe3, 0x81, 0x91, 0xe3, 0x81, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 115 | 0x45, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, 0x04, 116 | }, 117 | dumper.SrcToDst, 118 | dumper.ConnMetadata{ 119 | DumpValues: []dumper.DumpValue{}, 120 | Internal: connMetadataInternal{ 121 | messageLength: uint32(0), 122 | }, 123 | }, 124 | []dumper.DumpValue{}, 125 | []dumper.DumpValue{ 126 | dumper.DumpValue{ 127 | Key: "portal_name", 128 | Value: "", 129 | }, 130 | dumper.DumpValue{ 131 | Key: "stmt_name", 132 | Value: "", 133 | }, 134 | dumper.DumpValue{ 135 | Key: "bind_values", 136 | Value: []string{"012345679", "あいうえおかきくけこ", ""}, 137 | }, 138 | dumper.DumpValue{ 139 | Key: "message_type", 140 | Value: "B", 141 | }, 142 | }, 143 | }, 144 | { 145 | "When direction = dumper.RemoteToClient do not parse query", 146 | []byte{ 147 | 0x51, 0x00, 0x00, 0x00, 0x19, 0x53, 0x45, 0x4c, 0x45, 0x43, 0x54, 0x20, 0x2a, 0x20, 0x46, 0x52, 148 | 0x4f, 0x4d, 0x20, 0x75, 0x73, 0x65, 0x72, 0x73, 0x3b, 0x00, 149 | }, 150 | dumper.RemoteToClient, 151 | dumper.ConnMetadata{ 152 | DumpValues: []dumper.DumpValue{}, 153 | Internal: connMetadataInternal{ 154 | messageLength: uint32(0), 155 | }, 156 | }, 157 | []dumper.DumpValue{}, 158 | []dumper.DumpValue{}, 159 | }, 160 | } 161 | 162 | func TestPgReadHandshakeStartupMessage(t *testing.T) { 163 | for _, tt := range pgValueTests { 164 | t.Run(tt.description, func(t *testing.T) { 165 | out := new(bytes.Buffer) 166 | dumper := &Dumper{ 167 | logger: newTestLogger(out), 168 | } 169 | in := tt.in 170 | direction := tt.direction 171 | connMetadata := &tt.connMetadata 172 | 173 | actual, err := dumper.readHandshake(in, direction, connMetadata) 174 | if err != nil { 175 | t.Errorf("%v", err) 176 | } 177 | expected := tt.expected 178 | 179 | if len(actual) != len(expected) { 180 | t.Errorf("actual %v\nwant %v", actual, expected) 181 | } 182 | if len(actual) == 2 { 183 | if actual[0] != expected[0] { 184 | t.Errorf("actual %v\nwant %v", actual, expected) 185 | } 186 | if actual[1] != expected[1] { 187 | t.Errorf("actual %v\nwant %v", actual, expected) 188 | } 189 | } 190 | }) 191 | } 192 | } 193 | 194 | func TestPgRead(t *testing.T) { 195 | for _, tt := range pgValueTests { 196 | t.Run(tt.description, func(t *testing.T) { 197 | out := new(bytes.Buffer) 198 | dumper := &Dumper{ 199 | logger: newTestLogger(out), 200 | } 201 | in := tt.in 202 | direction := tt.direction 203 | connMetadata := &tt.connMetadata 204 | 205 | actual, err := dumper.Read(in, direction, connMetadata) 206 | if err != nil { 207 | t.Errorf("%v", err) 208 | } 209 | expected := tt.expectedQuery 210 | 211 | if len(actual) != len(expected) { 212 | t.Errorf("actual %v\nwant %v", actual, expected) 213 | } 214 | if len(actual) == 2 { 215 | if actual[0] != expected[0] { 216 | t.Errorf("actual %#v\nwant %#v", actual[0], expected[0]) 217 | } 218 | if actual[1] != expected[1] { 219 | t.Errorf("actual %#v\nwant %#v", actual[1], expected[1]) 220 | } 221 | } 222 | }) 223 | } 224 | } 225 | 226 | var readBytesTests = []struct { 227 | in []byte 228 | len int 229 | expected []byte 230 | }{ 231 | { 232 | []byte{0x12, 0x34, 0x56, 0x78}, 233 | 2, 234 | []byte{0x12, 0x34}, 235 | }, 236 | { 237 | []byte{0x12, 0x34, 0x56, 0x78}, 238 | 0, 239 | []byte{}, 240 | }, 241 | } 242 | 243 | func TestReadBytes(t *testing.T) { 244 | for _, tt := range readBytesTests { 245 | buff := bytes.NewBuffer(tt.in) 246 | actual := readBytes(buff, tt.len) 247 | if !bytes.Equal(actual, tt.expected) { 248 | t.Errorf("actual %#v\nwant %#v", actual, tt.expected) 249 | } 250 | } 251 | } 252 | 253 | // newTestLogger return zap.Logger for test 254 | func newTestLogger(out io.Writer) *zap.Logger { 255 | encoderConfig := zapcore.EncoderConfig{ 256 | TimeKey: "ts", 257 | LevelKey: "level", 258 | NameKey: "logger", 259 | CallerKey: "caller", 260 | MessageKey: "msg", 261 | StacktraceKey: "stacktrace", 262 | EncodeLevel: zapcore.LowercaseLevelEncoder, 263 | EncodeTime: zapcore.ISO8601TimeEncoder, 264 | EncodeDuration: zapcore.StringDurationEncoder, 265 | EncodeCaller: zapcore.ShortCallerEncoder, 266 | } 267 | 268 | logger := zap.New(zapcore.NewCore( 269 | zapcore.NewJSONEncoder(encoderConfig), 270 | zapcore.AddSync(out), 271 | zapcore.DebugLevel, 272 | )) 273 | 274 | return logger 275 | } 276 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/k1LoW/tcpdp 2 | 3 | go 1.22.1 4 | 5 | require ( 6 | code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6 7 | github.com/bLamarche413/mysql v1.0.3 8 | github.com/google/gopacket v1.1.17 9 | github.com/hnakamur/zap-ltsv v0.0.0-20170731143423-10a3dd1d839c 10 | github.com/lestrrat-go/file-rotatelogs v2.2.1-0.20180926095352-d72d6cf46fc8+incompatible 11 | github.com/lestrrat-go/server-starter v0.0.0-20181210024821-8564cc80d990 12 | github.com/pkg/errors v0.9.1 13 | github.com/rs/xid v1.2.1 14 | github.com/spf13/cobra v1.1.1 15 | github.com/spf13/viper v1.7.0 16 | github.com/xo/dburl v0.0.0-20190203050942-98997a05b24f 17 | go.uber.org/zap v1.13.0 18 | golang.org/x/text v0.3.8 19 | ) 20 | 21 | require ( 22 | github.com/BurntSushi/toml v0.3.1 // indirect 23 | github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect 24 | github.com/fsnotify/fsnotify v1.4.9 // indirect 25 | github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect 26 | github.com/hashicorp/hcl v1.0.0 // indirect 27 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 28 | github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect 29 | github.com/jonboulle/clockwork v0.2.0 // indirect 30 | github.com/kr/text v0.2.0 // indirect 31 | github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc // indirect 32 | github.com/lestrrat-go/strftime v0.0.0-20180821113735-8b31f9c59b0f // indirect 33 | github.com/magiconair/properties v1.8.1 // indirect 34 | github.com/mitchellh/mapstructure v1.4.0 // indirect 35 | github.com/onsi/ginkgo v1.16.4 // indirect 36 | github.com/onsi/gomega v1.13.0 // indirect 37 | github.com/pelletier/go-toml v1.4.0 // indirect 38 | github.com/smartystreets/assertions v1.0.1 // indirect 39 | github.com/spf13/afero v1.5.1 // indirect 40 | github.com/spf13/cast v1.3.0 // indirect 41 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 42 | github.com/spf13/pflag v1.0.5 // indirect 43 | github.com/stretchr/testify v1.7.0 // indirect 44 | github.com/subosito/gotenv v1.2.0 // indirect 45 | github.com/tebeka/strftime v0.0.0-20140926081919-3f9c7761e312 // indirect 46 | go.uber.org/atomic v1.5.0 // indirect 47 | go.uber.org/multierr v1.4.0 // indirect 48 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect 49 | golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect 50 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect 51 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect 52 | golang.org/x/tools v0.1.12 // indirect 53 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 54 | gopkg.in/ini.v1 v1.51.0 // indirect 55 | gopkg.in/yaml.v2 v2.4.0 // indirect 56 | gopkg.in/yaml.v3 v3.0.0 // indirect 57 | honnef.co/go/tools v0.0.1-2020.1.4 // indirect 58 | ) 59 | -------------------------------------------------------------------------------- /integration_test.go: -------------------------------------------------------------------------------- 1 | // +build integration 2 | 3 | package main 4 | 5 | import ( 6 | "bytes" 7 | "context" 8 | "fmt" 9 | "log" 10 | "os" 11 | "os/exec" 12 | "regexp" 13 | "runtime" 14 | "testing" 15 | "time" 16 | ) 17 | 18 | func TestMain(m *testing.M) { 19 | clean() 20 | code := m.Run() 21 | clean() 22 | os.Exit(code) 23 | } 24 | 25 | var proxyTests = []struct { 26 | description string 27 | tcpdpCmd string 28 | benchCmd string 29 | benchMatchString string 30 | }{ 31 | { 32 | "tcpdp proxy -> postgresql", 33 | "./tcpdp proxy -l localhost:54321 -r localhost:$POSTGRES_PORT -d pg --stdout", 34 | "PGPASSWORD=$POSTGRES_PASSWORD pgbench -i postgresql://$POSTGRES_USER@127.0.0.1:54321/$POSTGRES_DB?sslmode=disable && PGPASSWORD=$POSTGRES_PASSWORD pgbench -c 100 -t 10 postgresql://$POSTGRES_USER@127.0.0.1:54321/$POSTGRES_DB?sslmode=disable", 35 | "number of transactions actually processed: 1000/1000", 36 | }, 37 | { 38 | "tcpdp proxy -> mysql", 39 | "./tcpdp proxy -l localhost:33065 -r localhost:$MYSQL_PORT -d mysql --stdout", 40 | "mysqlslap --no-defaults --concurrency=100 --iterations=1 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed --auto-generate-sql-write-number=100 --number-of-queries=1000 --host=127.0.0.1 --port=33065 --user=root --password=$MYSQL_ROOT_PASSWORD $MYSQL_DISABLE_SSL", 41 | "Number of clients running queries: 100", 42 | }, 43 | } 44 | 45 | func TestProxy(t *testing.T) { 46 | for _, tt := range proxyTests { 47 | t.Run(tt.description, func(t *testing.T) { 48 | clean() 49 | ctx, cancel := context.WithCancel(context.Background()) 50 | cmd := exec.CommandContext(ctx, "bash", "-c", tt.tcpdpCmd) 51 | stdout := new(bytes.Buffer) 52 | cmd.Stdout = stdout 53 | err := cmd.Start() 54 | if err != nil { 55 | cancel() 56 | t.Errorf("%v", err) 57 | } 58 | time.Sleep(1 * time.Second) 59 | out, err := exec.CommandContext(ctx, "bash", "-c", tt.benchCmd).CombinedOutput() 60 | if err != nil { 61 | cancel() 62 | t.Errorf("%v:%s", err, out) 63 | } 64 | if !regexp.MustCompile(fmt.Sprintf("%s%s", `(?m)`, tt.benchMatchString)).Match(out) { 65 | t.Errorf("%s", "bench command failed") 66 | } 67 | results := regexp.MustCompile(`(?m)proxy_listen_addr`).FindAllStringSubmatch(stdout.String(), -1) 68 | if len(results) < 100 { 69 | t.Errorf("%s:%s", "parse protocol failed", stdout.String()) 70 | } 71 | cancel() 72 | }) 73 | } 74 | } 75 | 76 | var probeTests = []struct { 77 | description string 78 | tcpdpCmd string 79 | benchCmd string 80 | benchMatchString string 81 | linuxOnly bool 82 | }{ 83 | { 84 | "tcpdp probe - lo -> postgresql", 85 | "sudo ./tcpdp probe -i $LO -t $POSTGRES_PORT -d pg -B 64MB --stdout", 86 | "PGPASSWORD=$POSTGRES_PASSWORD pgbench -i postgresql://$POSTGRES_USER@127.0.0.1:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable && PGPASSWORD=$POSTGRES_PASSWORD pgbench -c 100 -t 10 postgresql://$POSTGRES_USER@127.0.0.1:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable", 87 | "number of transactions actually processed: 1000/1000", 88 | false, 89 | }, 90 | { 91 | "tcpdp probe - lo -> mysql", 92 | "sudo ./tcpdp probe -i $LO -t $MYSQL_PORT -d mysql -B 64MB --stdout", 93 | "mysqlslap --no-defaults --concurrency=100 --iterations=1 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed --auto-generate-sql-write-number=100 --number-of-queries=1000 --host=127.0.0.1 --port=$MYSQL_PORT --user=root --password=$MYSQL_ROOT_PASSWORD $MYSQL_DISABLE_SSL", 94 | "Number of clients running queries: 100", 95 | false, 96 | }, 97 | { 98 | "tcpdp probe - any -> postgresql", 99 | "sudo ./tcpdp probe -i any -t $POSTGRES_PORT -d pg -B 64MB --stdout", 100 | "PGPASSWORD=$POSTGRES_PASSWORD pgbench -i postgresql://$POSTGRES_USER@127.0.0.1:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable && PGPASSWORD=$POSTGRES_PASSWORD pgbench -c 100 -t 10 postgresql://$POSTGRES_USER@127.0.0.1:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable", 101 | "number of transactions actually processed: 1000/1000", 102 | true, 103 | }, 104 | { 105 | "tcpdp probe - any -> mysql", 106 | "sudo ./tcpdp probe -i any -t $MYSQL_PORT -d mysql -B 64MB --stdout", 107 | "mysqlslap --no-defaults --concurrency=100 --iterations=1 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed --auto-generate-sql-write-number=100 --number-of-queries=1000 --host=127.0.0.1 --port=$MYSQL_PORT --user=root --password=$MYSQL_ROOT_PASSWORD $MYSQL_DISABLE_SSL", 108 | "Number of clients running queries: 100", 109 | true, 110 | }, 111 | } 112 | 113 | func TestProbe(t *testing.T) { 114 | for _, tt := range probeTests { 115 | if tt.linuxOnly && runtime.GOOS != "linux" { 116 | t.Skip() 117 | } 118 | 119 | t.Run(tt.description, func(t *testing.T) { 120 | clean() 121 | ctx, cancel := context.WithCancel(context.Background()) 122 | cmd := exec.CommandContext(ctx, "bash", "-c", tt.tcpdpCmd) 123 | stdout := new(bytes.Buffer) 124 | cmd.Stdout = stdout 125 | err := cmd.Start() 126 | if err != nil { 127 | cancel() 128 | t.Errorf("%v", err) 129 | } 130 | time.Sleep(1 * time.Second) 131 | out, err := exec.CommandContext(ctx, "bash", "-c", tt.benchCmd).CombinedOutput() 132 | if err != nil { 133 | cancel() 134 | t.Errorf("%v:%s", err, out) 135 | } 136 | if !regexp.MustCompile(fmt.Sprintf("%s%s", `(?m)`, tt.benchMatchString)).Match(out) { 137 | t.Errorf("%s", "bench command failed") 138 | } 139 | results := regexp.MustCompile(`(?m)probe_target_addr`).FindAllStringSubmatch(stdout.String(), -1) 140 | if len(results) < 100 { 141 | t.Errorf("%s:%s", "parse protocol failed", stdout.String()) 142 | } 143 | cancel() 144 | }) 145 | } 146 | } 147 | 148 | var readTests = []struct { 149 | description string 150 | tcpdpCmd string 151 | }{ 152 | { 153 | "tcpdp read pg_prepare.pcap", 154 | "./tcpdp read -t $POSTGRES_PORT -d pg ./testdata/pcap/pg_prepare.pcap", 155 | }, 156 | { 157 | "tcpdp read mysql_prepare.pcap", 158 | "./tcpdp read -t $MYSQL_PORT -d mysql ./testdata/pcap/mysql_prepare.pcap", 159 | }, 160 | } 161 | 162 | func TestRead(t *testing.T) { 163 | for _, tt := range readTests { 164 | t.Run(tt.description, func(t *testing.T) { 165 | clean() 166 | cmd := exec.Command("bash", "-c", tt.tcpdpCmd) 167 | cmd.Env = os.Environ() 168 | out, err := cmd.CombinedOutput() 169 | if err != nil { 170 | t.Errorf("%v:%s", err, out) 171 | } 172 | results := regexp.MustCompile(`(?m)query`).FindAllStringSubmatch(string(out), -1) 173 | if len(results) < 10 { 174 | t.Errorf("%s:%s", "parse protocol failed", string(out)) 175 | } 176 | }) 177 | } 178 | } 179 | 180 | var longQueryTests = []struct { 181 | description string 182 | tcpdpCmd string 183 | benchCmd string 184 | }{ 185 | { 186 | "tcpdp proxy mysql long query", 187 | "./tcpdp proxy -l localhost:33065 -r localhost:$MYSQL_PORT -d mysql --stdout", 188 | "mysql --host=127.0.0.1 --port=33065 --user=root --password=$MYSQL_ROOT_PASSWORD testdb $MYSQL_DISABLE_SSL < ./testdata/query/long.sql 2>&1 > /dev/null", 189 | }, 190 | { 191 | "tcpdp probe mysql long query", 192 | "sudo ./tcpdp probe -i $LO -t $MYSQL_PORT -d mysql -B 64MB --stdout", 193 | "mysql --host=127.0.0.1 --port=$MYSQL_PORT --user=root --password=$MYSQL_ROOT_PASSWORD testdb $MYSQL_DISABLE_SSL < ./testdata/query/long.sql 2>&1 > /dev/null", 194 | }, 195 | { 196 | "tcpdp proxy postgresql long query", 197 | "./tcpdp proxy -l localhost:54321 -r localhost:$POSTGRES_PORT -d pg --stdout", 198 | "PGPASSWORD=$POSTGRES_PASSWORD psql postgresql://$POSTGRES_USER@127.0.0.1:54321/$POSTGRES_DB?sslmode=disable < ./testdata/query/long.sql 2>&1 > /dev/null", 199 | }, 200 | { 201 | "tcpdp probe postgresql long query", 202 | "sudo ./tcpdp probe -i $LO -t $POSTGRES_PORT -d pg -B 64MB --stdout", 203 | "PGPASSWORD=$POSTGRES_PASSWORD psql postgresql://$POSTGRES_USER@127.0.0.1:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable < ./testdata/query/long.sql 2>&1 > /dev/null", 204 | }, 205 | } 206 | 207 | func TestLongQuery(t *testing.T) { 208 | for _, tt := range longQueryTests { 209 | t.Run(tt.description, func(t *testing.T) { 210 | clean() 211 | ctx, cancel := context.WithCancel(context.Background()) 212 | cmd := exec.CommandContext(ctx, "bash", "-c", tt.tcpdpCmd) 213 | stdout := new(bytes.Buffer) 214 | cmd.Stdout = stdout 215 | err := cmd.Start() 216 | if err != nil { 217 | cancel() 218 | t.Errorf("%v", err) 219 | } 220 | time.Sleep(1 * time.Second) 221 | err = exec.CommandContext(ctx, "bash", "-c", tt.benchCmd).Run() 222 | if err != nil { 223 | cancel() 224 | t.Errorf("%v", err) 225 | } 226 | time.Sleep(1 * time.Second) 227 | if !regexp.MustCompile(`(?m)query_start`).MatchString(stdout.String()) { 228 | t.Errorf("%s:%s", "parse long query failed", stdout.String()) 229 | } 230 | if !regexp.MustCompile(`(?m)query_last`).MatchString(stdout.String()) { 231 | t.Errorf("%s:%s", "parse long query failed", stdout.String()) 232 | } 233 | cancel() 234 | }) 235 | } 236 | } 237 | 238 | var proxyProtocolTests = []struct { 239 | description string 240 | tcpdpCmd string 241 | benchCmd string 242 | benchMatchString string 243 | }{ 244 | { 245 | "haproxy[port:33068 send-proxy upstream:33080] -> tcpdp proxy -> mariadb[port:33081]", 246 | "./tcpdp proxy -l localhost:33080 -r localhost:33081 -d mysql --proxy-protocol --stdout", 247 | "mysqlslap --no-defaults --concurrency=100 --iterations=1 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed --auto-generate-sql-write-number=100 --number-of-queries=1000 --host=127.0.0.1 --port=33068 --user=root --password=$MYSQL_ROOT_PASSWORD $MYSQL_DISABLE_SSL", 248 | "Number of clients running queries: 100", 249 | }, 250 | { 251 | "haproxy[port:33069 send-proxy upstream:33081] -> mariadb[port:33081]", 252 | "sudo ./tcpdp probe -i $LO -t 33081 -d mysql -B 64MB --proxy-protocol --stdout", 253 | "mysqlslap --no-defaults --concurrency=100 --iterations=1 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed --auto-generate-sql-write-number=100 --number-of-queries=1000 --host=127.0.0.1 --port=33069 --user=root --password=$MYSQL_ROOT_PASSWORD $MYSQL_DISABLE_SSL", 254 | "Number of clients running queries: 100", 255 | }, 256 | { 257 | "haproxy[port:33070 send-proxy-v2 upstream:33080] -> tcpdp proxy -> mariadb[port:33081]", 258 | "./tcpdp proxy -l localhost:33080 -r localhost:33081 -d mysql --proxy-protocol --stdout", 259 | "mysqlslap --no-defaults --concurrency=100 --iterations=1 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed --auto-generate-sql-write-number=100 --number-of-queries=1000 --host=127.0.0.1 --port=33070 --user=root --password=$MYSQL_ROOT_PASSWORD $MYSQL_DISABLE_SSL", 260 | "Number of clients running queries: 100", 261 | }, 262 | { 263 | "haproxy[port:33071 send-proxy-v2 upstream:33081] -> mariadb[port:33081]", 264 | "sudo ./tcpdp probe -i $LO -t 33081 -d mysql -B 64MB --proxy-protocol --stdout", 265 | "mysqlslap --no-defaults --concurrency=100 --iterations=1 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed --auto-generate-sql-write-number=100 --number-of-queries=1000 --host=127.0.0.1 --port=33071 --user=root --password=$MYSQL_ROOT_PASSWORD $MYSQL_DISABLE_SSL", 266 | "Number of clients running queries: 100", 267 | }, 268 | } 269 | 270 | func TestProxyProtocol(t *testing.T) { 271 | for _, tt := range proxyProtocolTests { 272 | t.Run(tt.description, func(t *testing.T) { 273 | clean() 274 | ctx, cancel := context.WithCancel(context.Background()) 275 | cmd := exec.CommandContext(ctx, "bash", "-c", tt.tcpdpCmd) 276 | stdout := new(bytes.Buffer) 277 | cmd.Stdout = stdout 278 | if stdout.String() != "" { 279 | t.Fatalf("%s:%s", "stdout not empty", stdout.String()) 280 | } 281 | err := cmd.Start() 282 | if err != nil { 283 | cancel() 284 | t.Errorf("%v", err) 285 | } 286 | time.Sleep(1 * time.Second) 287 | out, err := exec.CommandContext(ctx, "bash", "-c", tt.benchCmd).CombinedOutput() 288 | if err != nil { 289 | cancel() 290 | t.Errorf("%v:%s", err, out) 291 | } 292 | if !regexp.MustCompile(fmt.Sprintf("%s%s", `(?m)`, tt.benchMatchString)).Match(out) { 293 | t.Errorf("%s", "bench command failed") 294 | } 295 | results := regexp.MustCompile(`(?m)proxy_protocol_src_addr`).FindAllStringSubmatch(stdout.String(), -1) 296 | if len(results) < 100 { 297 | t.Errorf("%s:%s", "parse proxy protocol failed", stdout.String()) 298 | } 299 | cancel() 300 | }) 301 | } 302 | } 303 | 304 | var connTests = []struct { 305 | description string 306 | tcpdpCmd string 307 | benchCmd string 308 | benchMatchString string 309 | }{ 310 | { 311 | "tcpdp probe - lo -> postgresql", 312 | "sudo ./tcpdp probe -i $LO -t $POSTGRES_PORT -d conn -B 64MB --stdout", 313 | "PGPASSWORD=$POSTGRES_PASSWORD pgbench -i postgresql://$POSTGRES_USER@127.0.0.1:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable && PGPASSWORD=$POSTGRES_PASSWORD pgbench -c 100 -t 10 postgresql://$POSTGRES_USER@127.0.0.1:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable", 314 | "number of transactions actually processed: 1000/1000", 315 | }, 316 | } 317 | 318 | func TestConn(t *testing.T) { 319 | for _, tt := range connTests { 320 | t.Run(tt.description, func(t *testing.T) { 321 | clean() 322 | ctx, cancel := context.WithCancel(context.Background()) 323 | cmd := exec.CommandContext(ctx, "bash", "-c", tt.tcpdpCmd) 324 | stdout := new(bytes.Buffer) 325 | cmd.Stdout = stdout 326 | err := cmd.Start() 327 | if err != nil { 328 | cancel() 329 | t.Errorf("%v", err) 330 | } 331 | time.Sleep(1 * time.Second) 332 | out, err := exec.CommandContext(ctx, "bash", "-c", tt.benchCmd).CombinedOutput() 333 | if err != nil { 334 | cancel() 335 | t.Errorf("%v:%s", err, out) 336 | } 337 | if !regexp.MustCompile(fmt.Sprintf("%s%s", `(?m)`, tt.benchMatchString)).Match(out) { 338 | t.Errorf("%s", "bench command failed") 339 | } 340 | results := regexp.MustCompile(`(?m)conn_id`).FindAllStringSubmatch(stdout.String(), -1) 341 | if len(results) < 100 { 342 | t.Errorf("%s:%s", "track connection failed", stdout.String()) 343 | } 344 | cancel() 345 | }) 346 | } 347 | } 348 | 349 | func clean() { 350 | cmd := exec.Command("sudo", "rm", "-f", "tcpdp.log*", "dump.log*", "tcpdp.pid") 351 | cmd.Env = os.Environ() 352 | err := cmd.Run() 353 | if err != nil { 354 | log.Fatal(err) 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "time" 11 | 12 | ltsv "github.com/hnakamur/zap-ltsv" 13 | rotatelogs "github.com/lestrrat-go/file-rotatelogs" 14 | "github.com/spf13/viper" 15 | "go.uber.org/zap" 16 | "go.uber.org/zap/zapcore" 17 | ) 18 | 19 | // LogFormatJSON for format JSON 20 | const LogFormatJSON = "json" 21 | 22 | // LogFormatLTSV for format LTSV 23 | const LogFormatLTSV = "ltsv" 24 | 25 | // LogTypeLog for tcpdp.log 26 | const LogTypeLog = "log" 27 | 28 | // LogTypeDumpLog for dump.log 29 | const LogTypeDumpLog = "dumpLog" 30 | 31 | // NewLogger returns logger 32 | func NewLogger() *zap.Logger { 33 | encoderConfig := zapcore.EncoderConfig{ 34 | TimeKey: "ts", 35 | LevelKey: "level", 36 | NameKey: "logger", 37 | CallerKey: "caller", 38 | MessageKey: "msg", 39 | StacktraceKey: "stacktrace", 40 | EncodeLevel: zapcore.LowercaseLevelEncoder, 41 | EncodeTime: zapcore.ISO8601TimeEncoder, 42 | EncodeDuration: zapcore.StringDurationEncoder, 43 | EncodeCaller: zapcore.ShortCallerEncoder, 44 | } 45 | 46 | stdout := viper.GetBool(fmt.Sprintf("%s.stdout", LogTypeLog)) 47 | enable := viper.GetBool(fmt.Sprintf("%s.enable", LogTypeLog)) 48 | format := viper.GetString(fmt.Sprintf("%s.format", LogTypeLog)) 49 | cores := []zapcore.Core{} 50 | 51 | if stdout { 52 | stdoutCore := zapcore.NewCore( 53 | zapcore.NewConsoleEncoder(encoderConfig), 54 | zapcore.AddSync(os.Stdout), 55 | zapcore.DebugLevel, 56 | ) 57 | cores = append(cores, stdoutCore) 58 | } 59 | 60 | if enable { 61 | w := newLogWriter(LogTypeLog) 62 | var encoder zapcore.Encoder 63 | switch format { 64 | case LogFormatJSON: 65 | encoder = zapcore.NewJSONEncoder(encoderConfig) 66 | case LogFormatLTSV: 67 | encoder = ltsv.NewLTSVEncoder(encoderConfig) 68 | } 69 | 70 | logCore := zapcore.NewCore( 71 | encoder, 72 | zapcore.AddSync(w), 73 | zapcore.InfoLevel, 74 | ) 75 | cores = append(cores, logCore) 76 | } 77 | 78 | logger := zap.New(zapcore.NewTee(cores...)) 79 | 80 | return logger 81 | } 82 | 83 | // NewHexLogger returns logger for hex 84 | func NewHexLogger() *zap.Logger { 85 | encoderConfig := zapcore.EncoderConfig{ 86 | EncodeLevel: zapcore.LowercaseLevelEncoder, 87 | EncodeTime: zapcore.ISO8601TimeEncoder, 88 | EncodeDuration: zapcore.StringDurationEncoder, 89 | EncodeCaller: zapcore.ShortCallerEncoder, 90 | } 91 | 92 | return newDumpLogger(encoderConfig) 93 | } 94 | 95 | // NewQueryLogger returns logger for mysql/pg 96 | func NewQueryLogger() *zap.Logger { 97 | encoderConfig := zapcore.EncoderConfig{ 98 | EncodeLevel: zapcore.LowercaseLevelEncoder, 99 | EncodeTime: zapcore.ISO8601TimeEncoder, 100 | EncodeDuration: zapcore.StringDurationEncoder, 101 | EncodeCaller: zapcore.ShortCallerEncoder, 102 | } 103 | 104 | return newDumpLogger(encoderConfig) 105 | } 106 | 107 | func newDumpLogger(encoderConfig zapcore.EncoderConfig) *zap.Logger { 108 | stdout := viper.GetBool(fmt.Sprintf("%s.stdout", LogTypeDumpLog)) 109 | enable := viper.GetBool(fmt.Sprintf("%s.enable", LogTypeDumpLog)) 110 | format := viper.GetString(fmt.Sprintf("%s.format", LogTypeDumpLog)) 111 | stdoutFormat := viper.GetString(fmt.Sprintf("%s.stdoutFormat", LogTypeDumpLog)) 112 | cores := []zapcore.Core{} 113 | 114 | if stdout { 115 | var encoder zapcore.Encoder 116 | switch stdoutFormat { 117 | case LogFormatJSON: 118 | encoder = zapcore.NewJSONEncoder(encoderConfig) 119 | case LogFormatLTSV: 120 | encoder = ltsv.NewLTSVEncoder(encoderConfig) 121 | default: 122 | encoder = zapcore.NewConsoleEncoder(encoderConfig) 123 | } 124 | stdoutCore := zapcore.NewCore( 125 | encoder, 126 | zapcore.AddSync(os.Stdout), 127 | zapcore.DebugLevel, 128 | ) 129 | cores = append(cores, stdoutCore) 130 | } 131 | 132 | if enable { 133 | w := newLogWriter(LogTypeDumpLog) 134 | var encoder zapcore.Encoder 135 | switch format { 136 | case LogFormatJSON: 137 | encoder = zapcore.NewJSONEncoder(encoderConfig) 138 | case LogFormatLTSV: 139 | encoder = ltsv.NewLTSVEncoder(encoderConfig) 140 | } 141 | 142 | logCore := zapcore.NewCore( 143 | encoder, 144 | zapcore.AddSync(w), 145 | zapcore.InfoLevel, 146 | ) 147 | cores = append(cores, logCore) 148 | } 149 | 150 | logger := zap.New(zapcore.NewTee(cores...)) 151 | 152 | return logger 153 | } 154 | 155 | func newLogWriter(logType string) io.Writer { 156 | dir := viper.GetString(fmt.Sprintf("%s.dir", logType)) 157 | rotateEnable := viper.GetBool(fmt.Sprintf("%s.rotateEnable", logType)) 158 | rotationTime := viper.GetString(fmt.Sprintf("%s.rotationTime", logType)) 159 | rotationCount := uint(viper.GetInt(fmt.Sprintf("%s.rotationCount", logType))) 160 | rotationHook := viper.GetString(fmt.Sprintf("%s.rotationHook", logType)) 161 | fileName := viper.GetString(fmt.Sprintf("%s.fileName", logType)) 162 | 163 | path, err := filepath.Abs(filepath.Join(dir, fileName)) 164 | if err != nil { 165 | log.Fatalf("Log setting error %v", err) 166 | } 167 | 168 | logSuffix := "" 169 | options := []rotatelogs.Option{ 170 | rotatelogs.WithClock(rotatelogs.Local), 171 | rotatelogs.WithMaxAge(-1), 172 | } 173 | if rotationCount > 0 { 174 | options = append(options, rotatelogs.WithRotationCount(rotationCount)) 175 | } 176 | 177 | if rotationHook != "" { 178 | options = append(options, rotatelogs.WithHandler(NewRotateHandler(rotationHook))) 179 | } 180 | var w io.Writer 181 | var t time.Duration 182 | if rotateEnable { 183 | switch rotationTime { 184 | case "minutely": 185 | logSuffix = ".%Y%m%d%H%M" 186 | t = 1 * time.Minute 187 | case "hourly": 188 | logSuffix = ".%Y%m%d%H" 189 | t = 1 * time.Hour 190 | case "daily": 191 | logSuffix = ".%Y%m%d" 192 | t = 24 * time.Hour 193 | default: 194 | log.Fatal("Log setting error, please specify one of the periods [daily, hourly, minutely]") 195 | } 196 | options = append(options, rotatelogs.WithLinkName(path)) 197 | options = append(options, rotatelogs.WithRotationTime(t)) 198 | w, err = rotatelogs.New( 199 | path+logSuffix, 200 | options..., 201 | ) 202 | if err != nil { 203 | log.Fatalf("Log setting error %v", err) 204 | } 205 | } else { 206 | // #nosec 207 | w, err = os.Open(path) 208 | if err != nil { 209 | log.Fatalf("Log setting error %v", err) 210 | } 211 | } 212 | 213 | return w 214 | } 215 | 216 | // NewRotateHandler return RotateHandler 217 | func NewRotateHandler(c string) *RotateHandler { 218 | return &RotateHandler{ 219 | command: c, 220 | } 221 | } 222 | 223 | // RotateHandler struct 224 | type RotateHandler struct { 225 | command string 226 | } 227 | 228 | // Handle rotatelogs.Event 229 | func (r *RotateHandler) Handle(e rotatelogs.Event) { 230 | if e.Type() == rotatelogs.FileRotatedEventType { 231 | fre := e.(*rotatelogs.FileRotatedEvent) 232 | // #nosec 233 | out, err := exec.Command(r.command, fre.PreviousFile(), fre.CurrentFile()).CombinedOutput() 234 | if err != nil { 235 | log.Printf("Log rotate event error %v\n", err) 236 | } else { 237 | log.Printf("Log rotate event success %v\n", out) 238 | } 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 Ken'ichiro Oyama 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package main 22 | 23 | import "github.com/k1LoW/tcpdp/cmd" 24 | 25 | func main() { 26 | cmd.Execute() 27 | } 28 | -------------------------------------------------------------------------------- /misc/pcap/mysql/prepare/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | 6 | _ "github.com/bLamarche413/mysql" 7 | ) 8 | 9 | func main() { 10 | for i := 0; i < 5; i++ { 11 | db, err := sql.Open("mysql", "root:mypass@tcp(127.0.0.1:33306)/testdb") 12 | //urlstr := "my://root:mypass@127.0.0.1:33306/testdb" 13 | //db, err := dburl.Open(urlstr) 14 | if err != nil { 15 | panic(err) 16 | } 17 | 18 | tableRows, err := db.Query(`SELECT CONCAT(?, ?, ?);`, "012345679", "あいうえおかきくけこ", "") 19 | if err != nil { 20 | panic(err) 21 | } 22 | for tableRows.Next() { 23 | } 24 | err = tableRows.Close() 25 | if err != nil { 26 | panic(err) 27 | } 28 | 29 | tableRows, err = db.Query(`SELECT ? + ? + ?`, 1, 23.4, 0) 30 | if err != nil { 31 | panic(err) 32 | } 33 | for tableRows.Next() { 34 | } 35 | err = tableRows.Close() 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | tableRows, err = db.Query(`SELECT CONCAT(?, ?, ?, " tcpdp is TCP dump tool with custom dumper written in Go.", " tcpdp is TCP dump tool with custom dumper written in Go.", " tcpdp is TCP dump tool with custom dumper written in Go.", " tcpdp is TCP dump tool with custom dumper written in Go.");`, 41 | "tcpdp", "ティーシーピーディーピーティーシーピーディーピーティーシーピーディーピーティーシーピーディーピーティーシーピーディーピーティーシーピーディーピーティーシーピーディーピー", "") 42 | if err != nil { 43 | panic(err) 44 | } 45 | for tableRows.Next() { 46 | } 47 | err = tableRows.Close() 48 | if err != nil { 49 | panic(err) 50 | } 51 | 52 | err = db.Close() 53 | if err != nil { 54 | panic(err) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /misc/pcap/pg/prepare/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/xo/dburl" 7 | ) 8 | 9 | func main() { 10 | for i := 0; i < 5; i++ { 11 | urlstr := "pg://postgres:pgpass@127.0.0.1:54322/testdb?sslmode=disable" 12 | db, err := dburl.Open(urlstr) 13 | if err != nil { 14 | panic(err) 15 | } 16 | 17 | tableRows, err := db.Query(`SELECT CONCAT($1::text, $2::text, $3::text);`, "012345679", "あいうえおかきくけこ", "") 18 | if err != nil { 19 | panic(err) 20 | } 21 | for tableRows.Next() { 22 | var ( 23 | res string 24 | ) 25 | err := tableRows.Scan(&res) 26 | if err != nil { 27 | panic(err) 28 | } 29 | fmt.Printf("%s\n", res) 30 | } 31 | err = tableRows.Close() 32 | if err != nil { 33 | panic(err) 34 | } 35 | 36 | tableRows, err = db.Query(`SELECT $1::int + $2::float + $3::int`, 1, 23.4, 0) 37 | if err != nil { 38 | panic(err) 39 | } 40 | for tableRows.Next() { 41 | } 42 | err = tableRows.Close() 43 | if err != nil { 44 | panic(err) 45 | } 46 | 47 | err = db.Close() 48 | if err != nil { 49 | panic(err) 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /reader/payload_buffer.go: -------------------------------------------------------------------------------- 1 | package reader 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/k1LoW/tcpdp/dumper" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | type payloadBuffer struct { 13 | srcToDst []byte 14 | dstToSrc []byte 15 | unknown []byte 16 | expires time.Time 17 | // created time.Time 18 | } 19 | 20 | func newPayloadBuffer() *payloadBuffer { 21 | p := payloadBuffer{ 22 | // created: time.Now() 23 | } 24 | p.updateExpires() 25 | return &p 26 | } 27 | 28 | func (p *payloadBuffer) updateExpires() { 29 | if p == nil { 30 | return 31 | } 32 | p.expires = time.Now().Add(time.Duration(packetTTL) * time.Second) 33 | } 34 | 35 | func (p *payloadBuffer) Expired() bool { 36 | if p == nil { 37 | return true 38 | } 39 | return p.expires.Before(time.Now()) 40 | } 41 | 42 | func (p *payloadBuffer) Get(direction dumper.Direction) []byte { 43 | if p == nil { 44 | return nil 45 | } 46 | switch direction { 47 | case dumper.SrcToDst: 48 | if len(p.srcToDst) > 0 { 49 | p.updateExpires() 50 | } 51 | return p.srcToDst 52 | case dumper.DstToSrc: 53 | if len(p.dstToSrc) > 0 { 54 | p.updateExpires() 55 | } 56 | return p.dstToSrc 57 | case dumper.Unknown: 58 | if len(p.unknown) > 0 { 59 | p.updateExpires() 60 | } 61 | return p.unknown 62 | } 63 | return nil 64 | } 65 | 66 | func (p *payloadBuffer) Delete(direction dumper.Direction) { 67 | if p == nil { 68 | return 69 | } 70 | p.updateExpires() 71 | switch direction { 72 | case dumper.SrcToDst: 73 | p.srcToDst = nil 74 | case dumper.DstToSrc: 75 | p.dstToSrc = nil 76 | case dumper.Unknown: 77 | p.unknown = nil 78 | } 79 | } 80 | 81 | func (p *payloadBuffer) Append(direction dumper.Direction, in []byte) { 82 | if p == nil { 83 | return 84 | } 85 | p.updateExpires() 86 | switch direction { 87 | case dumper.SrcToDst: 88 | p.srcToDst = append(p.srcToDst, in...) 89 | case dumper.DstToSrc: 90 | p.dstToSrc = append(p.dstToSrc, in...) 91 | case dumper.Unknown: 92 | p.unknown = append(p.unknown, in...) 93 | } 94 | } 95 | 96 | func (p *payloadBuffer) Size() int { 97 | return len(p.srcToDst) + len(p.dstToSrc) + len(p.unknown) 98 | } 99 | 100 | type payloadBufferManager struct { 101 | buffers map[string]*payloadBuffer 102 | mutex *sync.Mutex 103 | } 104 | 105 | func newPayloadBufferManager() *payloadBufferManager { 106 | return &payloadBufferManager{ 107 | buffers: map[string]*payloadBuffer{}, 108 | mutex: new(sync.Mutex), 109 | } 110 | } 111 | 112 | func (m *payloadBufferManager) lock() { 113 | m.mutex.Lock() 114 | } 115 | 116 | func (m *payloadBufferManager) unlock() { 117 | m.mutex.Unlock() 118 | } 119 | 120 | func (m *payloadBufferManager) newBuffer(key string, force bool) { 121 | m.lock() 122 | if _, ok := m.buffers[key]; !ok || force { 123 | m.buffers[key] = newPayloadBuffer() 124 | } 125 | m.unlock() 126 | } 127 | 128 | func (m *payloadBufferManager) Append(key string, direction dumper.Direction, in []byte) { 129 | m.lock() 130 | m.buffers[key].Append(direction, in) 131 | m.unlock() 132 | } 133 | 134 | func (m *payloadBufferManager) deleteBuffer(key string) { 135 | m.lock() 136 | delete(m.buffers, key) 137 | m.unlock() 138 | } 139 | 140 | func (m *payloadBufferManager) startPurgeTicker(ctx context.Context, logger *zap.Logger) { 141 | t := time.NewTicker(time.Duration(packetTTL/10) * time.Second) 142 | for { 143 | select { 144 | case <-ctx.Done(): 145 | return 146 | case <-t.C: 147 | // purge expired packet buffer cache 148 | purgedSize := 0 149 | m.lock() 150 | for key, b := range m.buffers { 151 | bSize := b.Size() 152 | if b.Expired() || bSize == 0 { 153 | if bSize > 0 { 154 | purgedSize = purgedSize + bSize 155 | } 156 | delete(m.buffers, key) 157 | } 158 | } 159 | m.unlock() 160 | if purgedSize > 0 { 161 | logger.Info("purge expired packet buffer cache", zap.Int("purged_size", purgedSize)) 162 | } 163 | } 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /reader/proxy_protocol.go: -------------------------------------------------------------------------------- 1 | package reader 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "fmt" 8 | "net" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/k1LoW/tcpdp/dumper" 13 | ) 14 | 15 | // https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt 16 | // union { 17 | // struct { 18 | // char line[108]; 19 | // } v1; 20 | // struct { 21 | // uint8_t sig[12]; 22 | // uint8_t ver_cmd; 23 | // uint8_t fam; 24 | // uint16_t len; 25 | // union { 26 | // struct { /* for TCP/UDP over IPv4, len = 12 */ 27 | // uint32_t src_addr; 28 | // uint32_t dst_addr; 29 | // uint16_t src_port; 30 | // uint16_t dst_port; 31 | // } ip4; 32 | // struct { /* for TCP/UDP over IPv6, len = 36 */ 33 | // uint8_t src_addr[16]; 34 | // uint8_t dst_addr[16]; 35 | // uint16_t src_port; 36 | // uint16_t dst_port; 37 | // } ip6; 38 | // struct { /* for AF_UNIX sockets, len = 216 */ 39 | // uint8_t src_addr[108]; 40 | // uint8_t dst_addr[108]; 41 | // } unx; 42 | // } addr; 43 | // } v2; 44 | // } hdr; 45 | 46 | var ( 47 | v1Prefix = []byte("PROXY") 48 | v2Prefix = []byte{0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54, 0x0a} 49 | ) 50 | 51 | // ParseProxyProtocolHeader ... 52 | func ParseProxyProtocolHeader(in []byte) (int, []dumper.DumpValue, error) { 53 | if bytes.Index(in, v1Prefix) == 0 { 54 | return parseProxyProtocolV1Header(in) 55 | } 56 | if bytes.Index(in, v2Prefix) == 0 { 57 | return parseProxyProtocolV2Header(in) 58 | } 59 | return 0, []dumper.DumpValue{}, nil 60 | } 61 | 62 | func parseProxyProtocolV1Header(in []byte) (int, []dumper.DumpValue, error) { 63 | idx := bytes.Index(in, []byte("\r\n")) 64 | values := strings.Split(string(in[0:idx]), " ") 65 | if len(values) == 6 { 66 | srcPort, _ := strconv.ParseUint(values[4], 10, 16) 67 | dstPort, _ := strconv.ParseUint(values[5], 10, 16) 68 | return idx + 2, []dumper.DumpValue{ 69 | dumper.DumpValue{ 70 | Key: "proxy_protocol_src_addr", 71 | Value: fmt.Sprintf("%s:%d", values[2], srcPort), 72 | }, 73 | dumper.DumpValue{ 74 | Key: "proxy_protocol_dst_addr", 75 | Value: fmt.Sprintf("%s:%d", values[3], dstPort), 76 | }, 77 | }, nil 78 | } 79 | return 0, []dumper.DumpValue{}, nil 80 | } 81 | 82 | func parseProxyProtocolV2Header(in []byte) (int, []dumper.DumpValue, error) { 83 | byte13 := in[12] 84 | if !(0x20 == byte13 || 0x21 == byte13) { 85 | return 0, []dumper.DumpValue{}, errors.New("unexpected values") 86 | } 87 | // PROXY or LOCAL 88 | byte14 := in[13] 89 | length := int(binary.BigEndian.Uint16(in[14:16])) 90 | idx := 16 91 | 92 | if byte14 == 0x00 { 93 | // UNSPEC 94 | return idx + length, []dumper.DumpValue{}, errors.New("unexpected values") 95 | } 96 | 97 | if 0x10 == byte14&0xf0 { 98 | // IPv4: 12byte 99 | var ( 100 | srcAddr net.IP // 4 101 | dstAddr net.IP // 4 102 | srcPort uint16 // 2 103 | dstPort uint16 // 2 104 | ) 105 | 106 | srcAddr = in[idx : idx+4] 107 | idx = idx + 4 108 | 109 | dstAddr = in[idx : idx+4] 110 | idx = idx + 4 111 | 112 | srcPort = binary.BigEndian.Uint16(in[idx : idx+2]) 113 | idx = idx + 2 114 | 115 | dstPort = binary.BigEndian.Uint16(in[idx : idx+2]) 116 | idx = idx + 2 117 | return idx, []dumper.DumpValue{ 118 | dumper.DumpValue{ 119 | Key: "proxy_protocol_src_addr", 120 | Value: fmt.Sprintf("%s:%d", srcAddr.String(), srcPort), 121 | }, 122 | dumper.DumpValue{ 123 | Key: "proxy_protocol_dst_addr", 124 | Value: fmt.Sprintf("%s:%d", dstAddr.String(), dstPort), 125 | }, 126 | }, nil 127 | } else if 0x20 == byte14&0xf0 { 128 | // IPv6: 36byte 129 | var ( 130 | srcAddr net.IP // 16 131 | dstAddr net.IP // 16 132 | srcPort uint16 // 2 133 | dstPort uint16 // 2 134 | ) 135 | 136 | srcAddr = in[idx : idx+16] 137 | idx = idx + 16 138 | 139 | dstAddr = in[idx : idx+16] 140 | idx = idx + 16 141 | 142 | srcPort = binary.BigEndian.Uint16(in[idx : idx+2]) 143 | idx = idx + 2 144 | 145 | dstPort = binary.BigEndian.Uint16(in[idx : idx+2]) 146 | idx = idx + 2 147 | return idx, []dumper.DumpValue{ 148 | dumper.DumpValue{ 149 | Key: "proxy_protocol_src_addr", 150 | Value: fmt.Sprintf("%s:%d", srcAddr.String(), srcPort), 151 | }, 152 | dumper.DumpValue{ 153 | Key: "proxy_protocol_dst_addr", 154 | Value: fmt.Sprintf("%s:%d", dstAddr.String(), dstPort), 155 | }, 156 | }, nil 157 | } else if 0x30 == byte14&0xf0 { 158 | // AF_UNIX: 216byte 159 | var ( 160 | srcAddr []byte 161 | dstAddr []byte 162 | ) 163 | 164 | srcAddr = in[idx : idx+108] 165 | idx = idx + 108 166 | 167 | dstAddr = in[idx : idx+108] 168 | idx = idx + 108 169 | return idx, []dumper.DumpValue{ 170 | dumper.DumpValue{ 171 | Key: "proxy_protocol_src_addr", 172 | Value: string(bytes.TrimRight(srcAddr, "\x00")), 173 | }, 174 | dumper.DumpValue{ 175 | Key: "proxy_protocol_dst_addr", 176 | Value: string(bytes.TrimRight(dstAddr, "\x00")), 177 | }, 178 | }, nil 179 | } 180 | 181 | return idx + length, []dumper.DumpValue{}, errors.New("unsupported values") 182 | } 183 | -------------------------------------------------------------------------------- /reader/proxy_protocol_test.go: -------------------------------------------------------------------------------- 1 | package reader 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/k1LoW/tcpdp/dumper" 8 | ) 9 | 10 | var parseProxyProtocolHeaderTests = []struct { 11 | in []byte 12 | wantSeek int 13 | wantDumpValues []dumper.DumpValue 14 | wantError error 15 | }{ 16 | { 17 | // https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-proxy-protocol.html 18 | []byte("PROXY TCP4 198.51.100.22 203.0.113.7 35646 80\r\n"), 19 | 47, 20 | []dumper.DumpValue{ 21 | dumper.DumpValue{ 22 | Key: "proxy_protocol_src_addr", 23 | Value: "198.51.100.22:35646", 24 | }, 25 | dumper.DumpValue{ 26 | Key: "proxy_protocol_dst_addr", 27 | Value: "203.0.113.7:80", 28 | }, 29 | }, 30 | nil, 31 | }, 32 | { 33 | []byte{ 34 | 0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54, 0x0a, 35 | 0x21, 36 | 0x11, 37 | 0x00, 0x0c, 38 | 0x7d, 0x19, 0x0a, 0x01, 39 | 0x0a, 0x04, 0x05, 0x08, 40 | 0x1f, 0x90, 41 | 0x10, 0x68, 42 | }, 43 | 28, 44 | []dumper.DumpValue{ 45 | dumper.DumpValue{ 46 | Key: "proxy_protocol_src_addr", 47 | Value: "125.25.10.1:8080", 48 | }, 49 | dumper.DumpValue{ 50 | Key: "proxy_protocol_dst_addr", 51 | Value: "10.4.5.8:4200", 52 | }, 53 | }, 54 | nil, 55 | }, 56 | } 57 | 58 | // TestParseProxyProtocolHeaderTest ... 59 | func TestParseProxyProtocolHeaderTest(t *testing.T) { 60 | for _, tt := range parseProxyProtocolHeaderTests { 61 | seek, dumpValues, err := ParseProxyProtocolHeader(tt.in) 62 | 63 | if seek != tt.wantSeek { 64 | t.Errorf("got %v\nwant %v", seek, tt.wantSeek) 65 | } 66 | 67 | if !reflect.DeepEqual(dumpValues, tt.wantDumpValues) { 68 | t.Errorf("\ngot %#v\nwant %#v", dumpValues, tt.wantDumpValues) 69 | } 70 | 71 | if err != tt.wantError { 72 | t.Errorf("got %v\nwant %v", err, tt.wantError) 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /reader/reader.go: -------------------------------------------------------------------------------- 1 | package reader 2 | 3 | import ( 4 | "context" 5 | "encoding/binary" 6 | "fmt" 7 | "net" 8 | "runtime" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/google/gopacket" 14 | "github.com/google/gopacket/layers" 15 | "github.com/k1LoW/tcpdp/dumper" 16 | "github.com/rs/xid" 17 | "go.uber.org/zap" 18 | ) 19 | 20 | const anyIP = "0.0.0.0" 21 | 22 | var packetTTL = 600 // second 23 | var maxPacketLen = 0xFFFF // 65535 24 | 25 | // Target struct 26 | type Target struct { 27 | TargetHosts []TargetHost 28 | } 29 | 30 | // TargetHost struct 31 | type TargetHost struct { 32 | Host string 33 | Port uint16 34 | } 35 | 36 | // Match return true if TargetHost match 37 | func (t Target) Match(host string, port uint16) bool { 38 | for _, h := range t.TargetHosts { 39 | if (h.Host == "" || h.Host == host) && h.Port == port { 40 | return true 41 | } 42 | } 43 | return false 44 | } 45 | 46 | // ParseTarget parse target to host:port 47 | func ParseTarget(t string) (Target, error) { 48 | ts := strings.Split(strings.Replace(t, " ", "", -1), "||") 49 | targets := []TargetHost{} 50 | for _, t := range ts { 51 | var port uint16 52 | var host string 53 | if t == "" { 54 | host = "" 55 | port = uint16(0) 56 | } else if strings.Contains(t, ":") { 57 | tAddr, err := net.ResolveTCPAddr("tcp", t) 58 | if err != nil { 59 | return Target{}, err 60 | } 61 | host = tAddr.IP.String() 62 | port = uint16(tAddr.Port) 63 | } else if strings.Contains(t, ".") { 64 | host = t 65 | port = uint16(0) 66 | } else { 67 | host = "" 68 | port64, err := strconv.ParseUint(t, 10, 64) 69 | if err != nil { 70 | return Target{}, err 71 | } 72 | port = uint16(port64) 73 | } 74 | targets = append(targets, TargetHost{ 75 | Host: host, 76 | Port: port, 77 | }) 78 | } 79 | return Target{ 80 | TargetHosts: targets, 81 | }, nil 82 | } 83 | 84 | // NewBPFFilterString return string for BPF 85 | func NewBPFFilterString(target Target) string { 86 | targets := target.TargetHosts 87 | fs := []string{} 88 | for _, target := range targets { 89 | host := target.Host 90 | port := target.Port 91 | f := fmt.Sprintf("(host %s and port %d)", host, port) 92 | if (host == "" || host == anyIP) && port > 0 { 93 | f = fmt.Sprintf("(port %d)", port) 94 | } else if (host != "" && host != anyIP) && port == 0 { 95 | f = fmt.Sprintf("(host %s)", host) 96 | } else if (host == "" || host == anyIP) && port == 0 { 97 | return "tcp" 98 | } 99 | fs = append(fs, f) 100 | } 101 | 102 | return fmt.Sprintf("tcp and (%s)", strings.Join(fs, " or ")) 103 | } 104 | 105 | // PacketReader struct 106 | type PacketReader struct { 107 | ctx context.Context 108 | cancel context.CancelFunc 109 | packetSource *gopacket.PacketSource 110 | dumper dumper.Dumper 111 | pValues []dumper.DumpValue 112 | logger *zap.Logger 113 | packetBuffer chan gopacket.Packet 114 | proxyProtocol bool 115 | enableInternal bool 116 | } 117 | 118 | // NewPacketReader return PacketReader 119 | func NewPacketReader( 120 | ctx context.Context, 121 | cancel context.CancelFunc, 122 | packetSource *gopacket.PacketSource, 123 | dumper dumper.Dumper, 124 | pValues []dumper.DumpValue, 125 | logger *zap.Logger, 126 | internalBufferLength int, 127 | proxyProtocol bool, 128 | enableInternal bool, 129 | ) PacketReader { 130 | internalPacketBuffer := make(chan gopacket.Packet, internalBufferLength) 131 | 132 | reader := PacketReader{ 133 | ctx: ctx, 134 | cancel: cancel, 135 | packetSource: packetSource, 136 | dumper: dumper, 137 | pValues: pValues, 138 | logger: logger, 139 | packetBuffer: internalPacketBuffer, 140 | proxyProtocol: proxyProtocol, 141 | enableInternal: enableInternal, 142 | } 143 | 144 | return reader 145 | } 146 | 147 | // ReadAndDump from gopacket.PacketSource 148 | func (r *PacketReader) ReadAndDump(target Target) error { 149 | packetChan := r.packetSource.Packets() 150 | 151 | if r.dumper.Name() == "conn" { 152 | go r.handleConn(target) 153 | } else { 154 | go r.handlePacket(target) 155 | } 156 | go r.checkBufferdPacket(packetChan) 157 | 158 | for { 159 | select { 160 | case <-r.ctx.Done(): 161 | return nil 162 | case packet := <-packetChan: 163 | r.packetBuffer <- packet 164 | } 165 | } 166 | } 167 | 168 | func (r *PacketReader) handlePacket(target Target) error { 169 | innerCtx, cancel := context.WithCancel(r.ctx) 170 | defer cancel() 171 | mMap := map[string]*dumper.ConnMetadata{} // metadata map per connection 172 | mssMap := map[string]int{} // TCP MSS map per connection 173 | pMap := newPayloadBufferManager() // long payload map per direction 174 | var mem runtime.MemStats 175 | 176 | go pMap.startPurgeTicker(innerCtx, r.logger) 177 | 178 | if r.enableInternal { 179 | go func() { 180 | t := time.NewTicker(1 * time.Minute) 181 | for { 182 | select { 183 | case <-innerCtx.Done(): 184 | return 185 | case <-t.C: 186 | runtime.ReadMemStats(&mem) 187 | bSize := 0 188 | pMap.lock() 189 | for _, b := range pMap.buffers { 190 | bSize = bSize + b.Size() 191 | } 192 | pMap.unlock() 193 | 194 | r.logger.Info("tcpdp internal stats", 195 | zap.Uint64("tcpdp Alloc", mem.Alloc), 196 | zap.Uint64("tcpdp TotalAlloc", mem.TotalAlloc), 197 | zap.Uint64("tcpdp Sys", mem.Sys), 198 | zap.Uint64("tcpdp Lookups", mem.Lookups), 199 | zap.Uint64("tcpdp Frees", mem.Frees), 200 | zap.Uint64("tcpdp HeapAlloc", mem.HeapAlloc), 201 | zap.Uint64("tcpdp HeapSys", mem.HeapSys), 202 | zap.Uint64("tcpdp HeapIdle", mem.HeapIdle), 203 | zap.Uint64("tcpdp HeapInuse", mem.HeapInuse), 204 | zap.Uint64("tcpdp HeapReleased", mem.HeapReleased), 205 | zap.Uint64("tcpdp HeapObjects", mem.HeapObjects), 206 | zap.Uint64("tcpdp StackInuse", mem.StackInuse), 207 | zap.Uint64("tcpdp StackSys", mem.StackSys), 208 | zap.Int("packet handler metadata cache (mMap) length", len(mMap)), 209 | zap.Int("packet handler TCP MSS cache (mssMap) length", len(mssMap)), 210 | zap.Int("packet handler payload buffer cache (pMap) length", len(pMap.buffers)), 211 | zap.Int("packet handler payload buffer cache (pMap) size", bSize)) 212 | } 213 | } 214 | }() 215 | } 216 | 217 | for { 218 | select { 219 | case <-r.ctx.Done(): 220 | return nil 221 | case packet := <-r.packetBuffer: 222 | if packet == nil { 223 | r.cancel() 224 | return nil 225 | } 226 | ipLayer := packet.Layer(layers.LayerTypeIPv4) 227 | if ipLayer == nil { 228 | continue 229 | } 230 | tcpLayer := packet.Layer(layers.LayerTypeTCP) 231 | if tcpLayer == nil { 232 | continue 233 | } 234 | ip, _ := ipLayer.(*layers.IPv4) 235 | tcp, _ := tcpLayer.(*layers.TCP) 236 | 237 | var key string 238 | var direction dumper.Direction 239 | srcToDstKey := fmt.Sprintf("%s:%d->%s:%d", ip.SrcIP.String(), tcp.SrcPort, ip.DstIP.String(), tcp.DstPort) 240 | dstToSrcKey := fmt.Sprintf("%s:%d->%s:%d", ip.DstIP.String(), tcp.DstPort, ip.SrcIP.String(), tcp.SrcPort) 241 | if target.Match(ip.DstIP.String(), uint16(tcp.DstPort)) { 242 | key = srcToDstKey 243 | direction = dumper.SrcToDst 244 | } else if target.Match(ip.SrcIP.String(), uint16(tcp.SrcPort)) { 245 | key = dstToSrcKey 246 | direction = dumper.DstToSrc 247 | } else { 248 | key = "-" 249 | direction = dumper.Unknown 250 | } 251 | 252 | if tcp.SYN && !tcp.ACK { 253 | if direction == dumper.Unknown { 254 | key = srcToDstKey 255 | } 256 | 257 | // TCP connection start 258 | delete(mMap, key) 259 | delete(mssMap, key) 260 | pMap.deleteBuffer(key) 261 | 262 | // TCP connection start ( hex, mysql, pg ) 263 | connID := xid.New().String() 264 | mss := int(binary.BigEndian.Uint16(tcp.LayerContents()[22:24])) 265 | connMetadata := r.dumper.NewConnMetadata() 266 | connMetadata.DumpValues = []dumper.DumpValue{ 267 | dumper.DumpValue{ 268 | Key: "conn_id", 269 | Value: connID, 270 | }, 271 | } 272 | mMap[key] = connMetadata 273 | mssMap[key] = mss 274 | pMap.newBuffer(key, true) 275 | } else if tcp.SYN && tcp.ACK { 276 | if direction == dumper.Unknown { 277 | key = dstToSrcKey 278 | } 279 | 280 | if _, ok := mMap[key]; !ok { 281 | // TCP connection start ( hex, mysql, pg ) 282 | connID := xid.New().String() 283 | connMetadata := r.dumper.NewConnMetadata() 284 | connMetadata.DumpValues = []dumper.DumpValue{ 285 | dumper.DumpValue{ 286 | Key: "conn_id", 287 | Value: connID, 288 | }, 289 | } 290 | mMap[key] = connMetadata 291 | } 292 | 293 | mss := int(binary.BigEndian.Uint16(tcp.LayerContents()[22:24])) 294 | current, ok := mssMap[key] 295 | if !ok || mss < current { 296 | mssMap[key] = mss 297 | } 298 | mMap[key].DumpValues = append(mMap[key].DumpValues, dumper.DumpValue{ 299 | Key: "mss", 300 | Value: mss, 301 | }) 302 | } else if tcp.FIN { 303 | // TCP connection end (FIN=1) 304 | delete(mssMap, key) 305 | if _, ok := mMap[key]; ok { 306 | mMap[key].Fin = true 307 | } 308 | pMap.deleteBuffer(key) 309 | } else if _, ok := mMap[key]; ok && tcp.ACK && mMap[key].Fin { 310 | // TCP connection end (ACK=1) 311 | delete(mMap, key) 312 | if direction == dumper.Unknown { 313 | for _, key := range []string{srcToDstKey, dstToSrcKey} { 314 | delete(mMap, key) 315 | } 316 | } 317 | continue 318 | } else if tcp.RST { 319 | delete(mssMap, key) 320 | delete(mMap, key) 321 | if direction == dumper.Unknown { 322 | for _, key := range []string{srcToDstKey, dstToSrcKey} { 323 | delete(mMap, key) 324 | } 325 | } 326 | pMap.deleteBuffer(key) 327 | continue 328 | } 329 | 330 | in := tcpLayer.LayerPayload() 331 | if len(in) == 0 { 332 | continue 333 | } 334 | 335 | pMap.newBuffer(key, false) 336 | 337 | mss, ok := mssMap[key] 338 | if ok { 339 | maxPacketLen = mss - (len(tcp.LayerContents()) - 20) 340 | } 341 | if len(in) == maxPacketLen { 342 | pMap.lock() 343 | pMap.buffers[key].Append(direction, in) 344 | pMap.unlock() 345 | continue 346 | } 347 | pMap.lock() 348 | bb := pMap.buffers[key].Get(direction) 349 | if len(bb) > 0 { 350 | in = append(bb, in...) 351 | } 352 | pMap.buffers[key].Delete(direction) 353 | pMap.unlock() 354 | 355 | if direction == dumper.Unknown { 356 | for _, k := range []string{srcToDstKey, dstToSrcKey} { 357 | _, ok := mMap[k] 358 | if ok { 359 | key = k 360 | } 361 | } 362 | } 363 | 364 | connMetadata, ok := mMap[key] 365 | if !ok { 366 | connMetadata = r.dumper.NewConnMetadata() 367 | } 368 | 369 | ts := packet.Metadata().CaptureInfo.Timestamp 370 | 371 | values := []dumper.DumpValue{ 372 | dumper.DumpValue{ 373 | Key: "ts", 374 | Value: ts, 375 | }, 376 | dumper.DumpValue{ 377 | Key: "src_addr", 378 | Value: fmt.Sprintf("%s:%d", ip.SrcIP.String(), tcp.SrcPort), 379 | }, 380 | dumper.DumpValue{ 381 | Key: "dst_addr", 382 | Value: fmt.Sprintf("%s:%d", ip.DstIP.String(), tcp.DstPort), 383 | }, 384 | } 385 | 386 | var read []dumper.DumpValue 387 | var err error 388 | if r.proxyProtocol { 389 | seek, ppValues, err := ParseProxyProtocolHeader(in) 390 | if err != nil { 391 | r.cancel() 392 | r.logger.WithOptions(zap.AddCaller()).Fatal("error", zap.Error(err)) 393 | return err 394 | } 395 | connMetadata.DumpValues = append(connMetadata.DumpValues, ppValues...) 396 | read, err = r.dumper.Read(in[seek:], direction, connMetadata) 397 | if err != nil { 398 | 399 | values = append(values, dumper.DumpValue{ 400 | Key: "error", 401 | Value: err, 402 | }) 403 | values = append(values, read...) 404 | values = append(values, r.pValues...) 405 | values = append(values, connMetadata.DumpValues...) 406 | r.dumper.Log(values) 407 | 408 | pMap.deleteBuffer(key) 409 | // error but continue 410 | continue 411 | } 412 | } else { 413 | read, err = r.dumper.Read(in, direction, connMetadata) 414 | if err != nil { 415 | 416 | values = append(values, dumper.DumpValue{ 417 | Key: "error", 418 | Value: err, 419 | }) 420 | values = append(values, read...) 421 | values = append(values, r.pValues...) 422 | values = append(values, connMetadata.DumpValues...) 423 | r.dumper.Log(values) 424 | 425 | pMap.deleteBuffer(key) 426 | // error but continue 427 | continue 428 | } 429 | } 430 | mMap[key] = connMetadata 431 | if len(read) == 0 { 432 | continue 433 | } 434 | 435 | values = append(values, read...) 436 | values = append(values, r.pValues...) 437 | values = append(values, connMetadata.DumpValues...) 438 | 439 | r.dumper.Log(values) 440 | } 441 | } 442 | } 443 | 444 | func (r *PacketReader) handleConn(target Target) error { 445 | innerCtx, cancel := context.WithCancel(r.ctx) 446 | defer cancel() 447 | var mem runtime.MemStats 448 | 449 | if r.enableInternal { 450 | go func() { 451 | t := time.NewTicker(1 * time.Minute) 452 | for { 453 | select { 454 | case <-innerCtx.Done(): 455 | return 456 | case <-t.C: 457 | runtime.ReadMemStats(&mem) 458 | r.logger.Info("tcpdp internal stats", 459 | zap.Uint64("tcpdp Alloc", mem.Alloc), 460 | zap.Uint64("tcpdp TotalAlloc", mem.TotalAlloc), 461 | zap.Uint64("tcpdp Sys", mem.Sys), 462 | zap.Uint64("tcpdp Lookups", mem.Lookups), 463 | zap.Uint64("tcpdp Frees", mem.Frees), 464 | zap.Uint64("tcpdp HeapAlloc", mem.HeapAlloc), 465 | zap.Uint64("tcpdp HeapSys", mem.HeapSys), 466 | zap.Uint64("tcpdp HeapIdle", mem.HeapIdle), 467 | zap.Uint64("tcpdp HeapInuse", mem.HeapInuse), 468 | zap.Uint64("tcpdp HeapReleased", mem.HeapReleased), 469 | zap.Uint64("tcpdp HeapObjects", mem.HeapObjects), 470 | zap.Uint64("tcpdp StackInuse", mem.StackInuse), 471 | zap.Uint64("tcpdp StackSys", mem.StackSys), 472 | ) 473 | } 474 | } 475 | }() 476 | } 477 | 478 | for { 479 | select { 480 | case <-r.ctx.Done(): 481 | return nil 482 | case packet := <-r.packetBuffer: 483 | if packet == nil { 484 | r.cancel() 485 | return nil 486 | } 487 | ipLayer := packet.Layer(layers.LayerTypeIPv4) 488 | if ipLayer == nil { 489 | continue 490 | } 491 | tcpLayer := packet.Layer(layers.LayerTypeTCP) 492 | if tcpLayer == nil { 493 | continue 494 | } 495 | ip, _ := ipLayer.(*layers.IPv4) 496 | tcp, _ := tcpLayer.(*layers.TCP) 497 | 498 | if !(tcp.SYN && !tcp.ACK) { 499 | continue 500 | } 501 | 502 | // TCP connection start ( hex, mysql, pg ) 503 | connID := xid.New().String() 504 | connMetadata := r.dumper.NewConnMetadata() 505 | connMetadata.DumpValues = []dumper.DumpValue{ 506 | dumper.DumpValue{ 507 | Key: "conn_id", 508 | Value: connID, 509 | }, 510 | } 511 | in := tcpLayer.LayerPayload() 512 | ts := packet.Metadata().CaptureInfo.Timestamp 513 | values := []dumper.DumpValue{ 514 | dumper.DumpValue{ 515 | Key: "ts", 516 | Value: ts, 517 | }, 518 | dumper.DumpValue{ 519 | Key: "src_addr", 520 | Value: fmt.Sprintf("%s:%d", ip.SrcIP.String(), tcp.SrcPort), 521 | }, 522 | dumper.DumpValue{ 523 | Key: "dst_addr", 524 | Value: fmt.Sprintf("%s:%d", ip.DstIP.String(), tcp.DstPort), 525 | }, 526 | } 527 | 528 | if r.proxyProtocol { 529 | _, ppValues, err := ParseProxyProtocolHeader(in) 530 | if err != nil { 531 | r.cancel() 532 | r.logger.WithOptions(zap.AddCaller()).Fatal("error", zap.Error(err)) 533 | return err 534 | } 535 | connMetadata.DumpValues = append(connMetadata.DumpValues, ppValues...) 536 | } 537 | values = append(values, r.pValues...) 538 | values = append(values, connMetadata.DumpValues...) 539 | 540 | r.dumper.Log(values) 541 | } 542 | } 543 | } 544 | 545 | func (r *PacketReader) checkBufferdPacket(packetChan chan gopacket.Packet) { 546 | t := time.NewTicker(1 * time.Second) 547 | L: 548 | for { 549 | select { 550 | case <-r.ctx.Done(): 551 | break L 552 | case <-t.C: 553 | packetBuffered := len(packetChan) 554 | internalPacketBuffered := len(r.packetBuffer) 555 | if internalPacketBuffered > (cap(r.packetBuffer)/10) || packetBuffered > (cap(packetChan)/10) { 556 | r.logger.Info("buffered packet stats", zap.Int("internal_buffered", internalPacketBuffered), zap.Int("packet_buffered", packetBuffered)) 557 | } 558 | } 559 | } 560 | t.Stop() 561 | } 562 | -------------------------------------------------------------------------------- /reader/reader_test.go: -------------------------------------------------------------------------------- 1 | package reader 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | var parseTargetTests = []struct { 9 | target string 10 | wantTarget Target 11 | wantBPFFilter string 12 | }{ 13 | { 14 | "localhost:80", 15 | Target{ 16 | TargetHosts: []TargetHost{ 17 | TargetHost{ 18 | Host: "127.0.0.1", 19 | Port: uint16(80), 20 | }, 21 | }, 22 | }, 23 | "tcp and ((host 127.0.0.1 and port 80))", 24 | }, 25 | { 26 | "0.0.0.0:80", 27 | Target{ 28 | TargetHosts: []TargetHost{ 29 | TargetHost{ 30 | Host: "0.0.0.0", 31 | Port: uint16(80), 32 | }, 33 | }, 34 | }, 35 | "tcp and ((port 80))", 36 | }, 37 | { 38 | "80", 39 | Target{ 40 | TargetHosts: []TargetHost{ 41 | TargetHost{ 42 | Host: "", 43 | Port: uint16(80), 44 | }, 45 | }, 46 | }, 47 | "tcp and ((port 80))", 48 | }, 49 | { 50 | "127.0.0.1", 51 | Target{ 52 | TargetHosts: []TargetHost{ 53 | TargetHost{ 54 | Host: "127.0.0.1", 55 | Port: uint16(0), 56 | }, 57 | }, 58 | }, 59 | "tcp and ((host 127.0.0.1))", 60 | }, 61 | { 62 | "", 63 | Target{ 64 | TargetHosts: []TargetHost{ 65 | TargetHost{ 66 | Host: "", 67 | Port: uint16(0), 68 | }, 69 | }, 70 | }, 71 | "tcp", 72 | }, 73 | { 74 | "0.0.0.0:0", 75 | Target{ 76 | TargetHosts: []TargetHost{ 77 | TargetHost{ 78 | Host: "0.0.0.0", 79 | Port: uint16(0), 80 | }, 81 | }, 82 | }, 83 | "tcp", 84 | }, 85 | { 86 | "0.0.0.0", 87 | Target{ 88 | TargetHosts: []TargetHost{ 89 | TargetHost{ 90 | Host: "0.0.0.0", 91 | Port: uint16(0), 92 | }, 93 | }, 94 | }, 95 | "tcp", 96 | }, 97 | { 98 | "127.0.0.1||203.0.113.1", 99 | Target{ 100 | TargetHosts: []TargetHost{ 101 | TargetHost{ 102 | Host: "127.0.0.1", 103 | Port: uint16(0), 104 | }, 105 | TargetHost{ 106 | Host: "203.0.113.1", 107 | Port: uint16(0), 108 | }, 109 | }, 110 | }, 111 | "tcp and ((host 127.0.0.1) or (host 203.0.113.1))", 112 | }, 113 | { 114 | "127.0.0.1 || 203.0.113.1", 115 | Target{ 116 | TargetHosts: []TargetHost{ 117 | TargetHost{ 118 | Host: "127.0.0.1", 119 | Port: uint16(0), 120 | }, 121 | TargetHost{ 122 | Host: "203.0.113.1", 123 | Port: uint16(0), 124 | }, 125 | }, 126 | }, 127 | "tcp and ((host 127.0.0.1) or (host 203.0.113.1))", 128 | }, 129 | { 130 | "127.0.0.1:80 || 203.0.113.1:80", 131 | Target{ 132 | TargetHosts: []TargetHost{ 133 | TargetHost{ 134 | Host: "127.0.0.1", 135 | Port: uint16(80), 136 | }, 137 | TargetHost{ 138 | Host: "203.0.113.1", 139 | Port: uint16(80), 140 | }, 141 | }, 142 | }, 143 | "tcp and ((host 127.0.0.1 and port 80) or (host 203.0.113.1 and port 80))", 144 | }, 145 | { 146 | "127.0.0.1:80 || 127.0.0.1:443", 147 | Target{ 148 | TargetHosts: []TargetHost{ 149 | TargetHost{ 150 | Host: "127.0.0.1", 151 | Port: uint16(80), 152 | }, 153 | TargetHost{ 154 | Host: "127.0.0.1", 155 | Port: uint16(443), 156 | }, 157 | }, 158 | }, 159 | "tcp and ((host 127.0.0.1 and port 80) or (host 127.0.0.1 and port 443))", 160 | }, 161 | { 162 | "80 || 127.0.0.1:443", 163 | Target{ 164 | TargetHosts: []TargetHost{ 165 | TargetHost{ 166 | Host: "", 167 | Port: uint16(80), 168 | }, 169 | TargetHost{ 170 | Host: "127.0.0.1", 171 | Port: uint16(443), 172 | }, 173 | }, 174 | }, 175 | "tcp and ((port 80) or (host 127.0.0.1 and port 443))", 176 | }, 177 | { 178 | "0.0.0.0 || 127.0.0.1:443", 179 | Target{ 180 | TargetHosts: []TargetHost{ 181 | TargetHost{ 182 | Host: "0.0.0.0", 183 | Port: uint16(0), 184 | }, 185 | TargetHost{ 186 | Host: "127.0.0.1", 187 | Port: uint16(443), 188 | }, 189 | }, 190 | }, 191 | "tcp", 192 | }, 193 | } 194 | 195 | func TestParseTarget(t *testing.T) { 196 | for _, tt := range parseTargetTests { 197 | target := tt.target 198 | gotTarget, err := ParseTarget(target) 199 | 200 | if err != nil { 201 | t.Errorf("%v", err) 202 | } 203 | 204 | if !reflect.DeepEqual(gotTarget, tt.wantTarget) { 205 | t.Errorf("got %v\nwant %v", gotTarget, tt.wantTarget) 206 | } 207 | } 208 | } 209 | 210 | func TestNewBPFFilterString(t *testing.T) { 211 | for _, tt := range parseTargetTests { 212 | target := tt.target 213 | gotTarget, err := ParseTarget(target) 214 | 215 | if err != nil { 216 | t.Errorf("%v", err) 217 | } 218 | 219 | want := tt.wantBPFFilter 220 | got := NewBPFFilterString(gotTarget) 221 | 222 | if got != want { 223 | t.Errorf("got %v\nwant %v", got, want) 224 | } 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /server/probe_server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "regexp" 10 | "strconv" 11 | "sync" 12 | "syscall" 13 | "time" 14 | 15 | "code.cloudfoundry.org/bytefmt" 16 | "github.com/google/gopacket" 17 | "github.com/google/gopacket/pcap" 18 | "github.com/k1LoW/tcpdp/dumper" 19 | "github.com/k1LoW/tcpdp/dumper/conn" 20 | "github.com/k1LoW/tcpdp/dumper/hex" 21 | "github.com/k1LoW/tcpdp/dumper/mysql" 22 | "github.com/k1LoW/tcpdp/dumper/pg" 23 | "github.com/k1LoW/tcpdp/reader" 24 | "github.com/spf13/viper" 25 | "go.uber.org/zap" 26 | "go.uber.org/zap/zapcore" 27 | ) 28 | 29 | var numberRegexp = regexp.MustCompile(`^\d+$`) 30 | 31 | const promiscuous = false 32 | const timeout = pcap.BlockForever 33 | 34 | // PcapConfig struct 35 | type PcapConfig struct { 36 | Device string 37 | BufferSize string 38 | ImmediateMode bool 39 | SnapshotLength string 40 | Promiscuous bool 41 | Timeout time.Duration 42 | Filter string 43 | } 44 | 45 | // ProbeServer struct 46 | type ProbeServer struct { 47 | pidfile string 48 | ctx context.Context 49 | shutdown context.CancelFunc 50 | Wg *sync.WaitGroup 51 | ClosedChan chan struct{} 52 | logger *zap.Logger 53 | dumper dumper.Dumper 54 | target reader.Target 55 | pcapConfig PcapConfig 56 | proxyProtocol bool 57 | } 58 | 59 | // NewProbeServer returns a new Server 60 | func NewProbeServer(ctx context.Context, logger *zap.Logger) (*ProbeServer, error) { 61 | innerCtx, shutdown := context.WithCancel(ctx) 62 | closedChan := make(chan struct{}) 63 | 64 | var d dumper.Dumper 65 | dumpType := viper.GetString("tcpdp.dumper") 66 | 67 | switch dumpType { 68 | case "hex": 69 | d = hex.NewDumper() 70 | case "pg": 71 | d = pg.NewDumper() 72 | case "mysql": 73 | d = mysql.NewDumper() 74 | case "conn": 75 | d = conn.NewDumper() 76 | default: 77 | d = hex.NewDumper() 78 | } 79 | 80 | pidfile, err := filepath.Abs(viper.GetString("tcpdp.pidfile")) 81 | if err != nil { 82 | logger.WithOptions(zap.AddCaller()).Fatal("pidfile path error", zap.Error(err)) 83 | shutdown() 84 | return nil, err 85 | } 86 | 87 | target := viper.GetString("probe.target") 88 | t, err := reader.ParseTarget(target) 89 | if err != nil { 90 | logger.WithOptions(zap.AddCaller()).Fatal("parse target error", zap.Error(err)) 91 | shutdown() 92 | return nil, err 93 | } 94 | 95 | filter := viper.GetString("probe.filter") 96 | if filter == "" { 97 | filter = reader.NewBPFFilterString(t) 98 | } else { 99 | filter = fmt.Sprintf("tcp and (%s)", filter) 100 | } 101 | 102 | pcapConfig := PcapConfig{ 103 | Device: viper.GetString("probe.interface"), 104 | BufferSize: viper.GetString("probe.bufferSize"), 105 | ImmediateMode: viper.GetBool("probe.immediateMode"), 106 | SnapshotLength: viper.GetString("probe.snapshotLength"), 107 | Promiscuous: promiscuous, 108 | Timeout: timeout, 109 | Filter: filter, 110 | } 111 | 112 | return &ProbeServer{ 113 | pidfile: pidfile, 114 | ctx: innerCtx, 115 | shutdown: shutdown, 116 | ClosedChan: closedChan, 117 | logger: logger, 118 | dumper: d, 119 | target: t, 120 | pcapConfig: pcapConfig, 121 | proxyProtocol: viper.GetBool("tcpdp.proxyProtocol"), 122 | }, nil 123 | } 124 | 125 | // Start probe server. 126 | func (s *ProbeServer) Start() error { 127 | if err := s.writePID(); err != nil { 128 | s.logger.WithOptions(zap.AddCaller()).Fatal(fmt.Sprintf("can not write %s", s.pidfile), zap.Error(err)) 129 | return err 130 | } 131 | defer s.deletePID() 132 | 133 | defer func() { 134 | close(s.ClosedChan) 135 | }() 136 | 137 | pcapBufferSize, err := byteFormat(s.pcapConfig.BufferSize) 138 | if err != nil { 139 | s.logger.WithOptions(zap.AddCaller()).Fatal("parse buffer-size error", zap.Error(err)) 140 | return err 141 | } 142 | immediateMode := s.pcapConfig.ImmediateMode 143 | snapshotLength, err := byteFormat(s.pcapConfig.SnapshotLength) 144 | if err != nil { 145 | s.logger.WithOptions(zap.AddCaller()).Fatal("parse snapshot-length error", zap.Error(err)) 146 | return err 147 | } 148 | internalBufferLength := viper.GetInt("probe.internalBufferLength") 149 | 150 | target := viper.GetString("probe.target") 151 | 152 | pValues := []dumper.DumpValue{ 153 | dumper.DumpValue{ 154 | Key: "interface", 155 | Value: s.pcapConfig.Device, 156 | }, 157 | dumper.DumpValue{ 158 | Key: "probe_target_addr", 159 | Value: target, 160 | }, 161 | } 162 | 163 | inactiveHandle, err := pcap.NewInactiveHandle(s.pcapConfig.Device) 164 | if err != nil { 165 | fields := s.fieldsWithErrorAndValues(err, pValues) 166 | s.logger.WithOptions(zap.AddCaller()).Fatal("pcap create error", fields...) 167 | return err 168 | } 169 | if err := inactiveHandle.SetSnapLen(int(snapshotLength)); err != nil { 170 | fields := s.fieldsWithErrorAndValues(err, pValues) 171 | s.logger.WithOptions(zap.AddCaller()).Fatal("pcap create error (snaplen)", fields...) 172 | return err 173 | } 174 | if err := inactiveHandle.SetPromisc(s.pcapConfig.Promiscuous); err != nil { 175 | fields := s.fieldsWithErrorAndValues(err, pValues) 176 | s.logger.WithOptions(zap.AddCaller()).Fatal("pcap create error (promiscuous)", fields...) 177 | return err 178 | } 179 | if err := inactiveHandle.SetTimeout(s.pcapConfig.Timeout); err != nil { 180 | fields := s.fieldsWithErrorAndValues(err, pValues) 181 | s.logger.WithOptions(zap.AddCaller()).Fatal("pcap create error (timeout)", fields...) 182 | return err 183 | } 184 | if err := inactiveHandle.SetBufferSize(int(pcapBufferSize)); err != nil { 185 | fields := s.fieldsWithErrorAndValues(err, pValues) 186 | s.logger.WithOptions(zap.AddCaller()).Fatal("pcap create error (pcap_buffer_size)", fields...) 187 | return err 188 | } 189 | if err := inactiveHandle.SetImmediateMode(immediateMode); err != nil { 190 | fields := s.fieldsWithErrorAndValues(err, pValues) 191 | s.logger.WithOptions(zap.AddCaller()).Fatal("pcap create error (pcap_set_immediate_mode)", fields...) 192 | return err 193 | } 194 | 195 | handle, err := inactiveHandle.Activate() 196 | if err != nil { 197 | fields := s.fieldsWithErrorAndValues(err, pValues) 198 | s.logger.WithOptions(zap.AddCaller()).Fatal("pcap handle activate error", fields...) 199 | return err 200 | } 201 | 202 | s.checkStats(handle) 203 | defer func() { 204 | stats, _ := handle.Stats() 205 | s.logger.Info("pcap Stats", zap.Int("packet_received", stats.PacketsReceived), zap.Int("packet_dropped", stats.PacketsDropped), zap.Int("packet_if_dropped", stats.PacketsIfDropped)) 206 | handle.Close() 207 | }() 208 | 209 | if err := handle.SetBPFFilter(s.pcapConfig.Filter); err != nil { 210 | fields := s.fieldsWithErrorAndValues(err, pValues) 211 | s.logger.WithOptions(zap.AddCaller()).Fatal("Set BPF error", fields...) 212 | return err 213 | } 214 | 215 | proxyProtocol := viper.GetBool("tcpdp.proxyProtocol") 216 | enableInternal := viper.GetBool("log.enableInternal") 217 | 218 | packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) 219 | r := reader.NewPacketReader( 220 | s.ctx, 221 | s.shutdown, 222 | packetSource, 223 | s.dumper, 224 | pValues, 225 | s.logger, 226 | internalBufferLength, 227 | proxyProtocol, 228 | enableInternal, 229 | ) 230 | 231 | if err := r.ReadAndDump(s.target); err != nil { 232 | fields := s.fieldsWithErrorAndValues(err, pValues) 233 | s.logger.WithOptions(zap.AddCaller()).Fatal("ReadAndDump error", fields...) 234 | return err 235 | } 236 | 237 | return err 238 | } 239 | 240 | // Shutdown server. 241 | func (s *ProbeServer) Shutdown() { 242 | s.shutdown() 243 | } 244 | 245 | func (s *ProbeServer) checkStats(handle *pcap.Handle) { 246 | go func() { 247 | t := time.NewTicker(1 * time.Second) 248 | packetsDropped := 0 249 | packetsIfDropped := 0 250 | L: 251 | for { 252 | select { 253 | case <-s.ctx.Done(): 254 | break L 255 | case <-t.C: 256 | stats, _ := handle.Stats() 257 | if stats.PacketsDropped > packetsDropped || stats.PacketsIfDropped > packetsIfDropped { 258 | s.logger.Error("pcap packets dropped", zap.Int("packet_received", stats.PacketsReceived), zap.Int("packet_dropped", stats.PacketsDropped), zap.Int("packet_if_dropped", stats.PacketsIfDropped)) 259 | } 260 | packetsDropped = stats.PacketsDropped 261 | packetsIfDropped = stats.PacketsIfDropped 262 | } 263 | } 264 | t.Stop() 265 | }() 266 | } 267 | 268 | // https://gist.github.com/davidnewhall/3627895a9fc8fa0affbd747183abca39 269 | func (s *ProbeServer) writePID() error { 270 | if data, err := ioutil.ReadFile(s.pidfile); err == nil { 271 | if pid, err := strconv.Atoi(string(data)); err == nil { 272 | if process, err := os.FindProcess(pid); err == nil { 273 | if err := process.Signal(syscall.Signal(0)); err == nil { 274 | return fmt.Errorf("pid already running: %d", pid) 275 | } 276 | } 277 | } 278 | } 279 | return ioutil.WriteFile(s.pidfile, []byte(fmt.Sprintf("%d\n", os.Getpid())), 0664) // #nosec 280 | } 281 | 282 | func (s *ProbeServer) deletePID() { 283 | if err := os.Remove(s.pidfile); err != nil { 284 | s.logger.WithOptions(zap.AddCaller()).Fatal(fmt.Sprintf("can not delete %s", s.pidfile), zap.Error(err)) 285 | } 286 | } 287 | 288 | func (s *ProbeServer) fieldsWithErrorAndValues(err error, pValues []dumper.DumpValue) []zapcore.Field { 289 | fields := []zapcore.Field{ 290 | zap.Error(err), 291 | } 292 | 293 | for _, kv := range pValues { 294 | fields = append(fields, zap.Any(kv.Key, kv.Value)) 295 | } 296 | 297 | return fields 298 | } 299 | 300 | // PcapConfig return ProbeServer.pcapConfig 301 | func (s *ProbeServer) PcapConfig() PcapConfig { 302 | return s.pcapConfig 303 | } 304 | 305 | func byteFormat(s string) (int, error) { 306 | if numberRegexp.MatchString(s) { 307 | s = s + "B" 308 | } 309 | i, err := bytefmt.ToBytes(s) 310 | if err != nil { 311 | return -1, err 312 | } 313 | return int(i), nil 314 | } 315 | -------------------------------------------------------------------------------- /server/proxy.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "strings" 7 | "time" 8 | 9 | "github.com/k1LoW/tcpdp/dumper" 10 | "github.com/k1LoW/tcpdp/reader" 11 | "github.com/rs/xid" 12 | "github.com/spf13/viper" 13 | "go.uber.org/zap" 14 | "go.uber.org/zap/zapcore" 15 | ) 16 | 17 | const maxPacketLen = 0xFFFF 18 | 19 | // Proxy struct 20 | type Proxy struct { 21 | server *Server 22 | ctx context.Context 23 | Close context.CancelFunc 24 | connID string 25 | conn *net.TCPConn 26 | remoteConn *net.TCPConn 27 | connMetadata *dumper.ConnMetadata 28 | seqNum uint64 29 | proxyProtocol bool 30 | } 31 | 32 | // NewProxy returns a new Proxy 33 | func NewProxy(s *Server, conn, remoteConn *net.TCPConn) *Proxy { 34 | innerCtx, close := context.WithCancel(s.ctx) 35 | 36 | connID := xid.New().String() 37 | 38 | connMetadata := s.dumper.NewConnMetadata() 39 | connMetadata.DumpValues = []dumper.DumpValue{ 40 | dumper.DumpValue{ 41 | Key: "conn_id", 42 | Value: connID, 43 | }, 44 | dumper.DumpValue{ 45 | Key: "client_addr", 46 | Value: conn.RemoteAddr().String(), 47 | }, 48 | dumper.DumpValue{ 49 | Key: "proxy_listen_addr", 50 | Value: conn.LocalAddr().String(), 51 | }, 52 | dumper.DumpValue{ 53 | Key: "proxy_client_addr", 54 | Value: remoteConn.LocalAddr().String(), 55 | }, 56 | dumper.DumpValue{ 57 | Key: "remote_addr", 58 | Value: remoteConn.RemoteAddr().String(), 59 | }, 60 | } 61 | 62 | return &Proxy{ 63 | server: s, 64 | ctx: innerCtx, 65 | Close: close, 66 | connID: connID, 67 | conn: conn, 68 | remoteConn: remoteConn, 69 | connMetadata: connMetadata, 70 | seqNum: 0, 71 | proxyProtocol: viper.GetBool("tcpdp.proxyProtocol"), 72 | } 73 | } 74 | 75 | // Start proxy 76 | func (p *Proxy) Start() { 77 | defer func() { 78 | if err := p.conn.Close(); err != nil { 79 | p.server.logger.WithOptions(zap.AddCaller()).Error("proxy conn Close error") 80 | } 81 | if err := p.remoteConn.Close(); err != nil { 82 | p.server.logger.WithOptions(zap.AddCaller()).Error("proxy remoteConn Close error") 83 | } 84 | }() 85 | 86 | go p.pipe(p.conn, p.remoteConn) 87 | go p.pipe(p.remoteConn, p.conn) 88 | 89 | select { 90 | case <-p.ctx.Done(): 91 | return 92 | } 93 | } 94 | 95 | func (p *Proxy) dump(b []byte, direction dumper.Direction) error { 96 | kvs := []dumper.DumpValue{ 97 | dumper.DumpValue{ 98 | Key: "conn_seq_num", 99 | Value: p.seqNum, 100 | }, 101 | dumper.DumpValue{ 102 | Key: "direction", 103 | Value: direction.String(), 104 | }, 105 | dumper.DumpValue{ 106 | Key: "ts", 107 | Value: time.Now(), 108 | }, 109 | } 110 | 111 | if p.proxyProtocol { 112 | seek, ppValues, err := reader.ParseProxyProtocolHeader(b) 113 | if err != nil { 114 | p.Close() 115 | return err 116 | } 117 | p.connMetadata.DumpValues = append(p.connMetadata.DumpValues, ppValues...) 118 | return p.server.dumper.Dump(b[seek:], direction, p.connMetadata, kvs) 119 | } 120 | return p.server.dumper.Dump(b, direction, p.connMetadata, kvs) 121 | } 122 | 123 | func (p *Proxy) pipe(srcConn, destConn *net.TCPConn) { 124 | defer p.Close() 125 | 126 | var direction dumper.Direction 127 | if p.server.remoteAddr.String() == destConn.RemoteAddr().String() { 128 | direction = dumper.ClientToRemote 129 | } else { 130 | direction = dumper.RemoteToClient 131 | } 132 | 133 | buff := make([]byte, maxPacketLen) 134 | longB := []byte{} 135 | for { 136 | n, err := srcConn.Read(buff) 137 | if err != nil { 138 | if err.Error() != "EOF" && !strings.Contains(err.Error(), "use of closed network connection") { 139 | fields := p.fieldsWithErrorAndDirection(err, direction) 140 | p.server.logger.WithOptions(zap.AddCaller()).Error("strCon Read error", fields...) 141 | } 142 | break 143 | } 144 | 145 | b := buff[:n] 146 | if n == maxPacketLen && buff[n-1] != 0x00 { 147 | longB = append(longB, b...) 148 | } else { 149 | if len(longB) > 0 { 150 | longB = append(longB, b...) 151 | err = p.dump(longB, direction) 152 | longB = nil 153 | } else { 154 | err = p.dump(b, direction) 155 | } 156 | if err != nil { 157 | fields := p.fieldsWithErrorAndDirection(err, direction) 158 | p.server.logger.WithOptions(zap.AddCaller()).Error("dumper Dump error", fields...) 159 | break 160 | } 161 | } 162 | 163 | if _, err := destConn.Write(b); err != nil { 164 | fields := p.fieldsWithErrorAndDirection(err, direction) 165 | p.server.logger.WithOptions(zap.AddCaller()).Error("destCon Write error", fields...) 166 | break 167 | } 168 | 169 | select { 170 | case <-p.ctx.Done(): 171 | break 172 | default: 173 | p.seqNum++ 174 | } 175 | } 176 | } 177 | 178 | func (p *Proxy) fieldsWithErrorAndDirection(err error, direction dumper.Direction) []zapcore.Field { 179 | fields := []zapcore.Field{ 180 | zap.Error(err), 181 | zap.Uint64("conn_seq_num", p.seqNum), 182 | zap.String("direction", direction.String()), 183 | } 184 | 185 | for _, kv := range p.connMetadata.DumpValues { 186 | fields = append(fields, zap.Any(kv.Key, kv.Value)) 187 | } 188 | 189 | return fields 190 | } 191 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "net" 8 | "os" 9 | "path/filepath" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | "syscall" 14 | 15 | "github.com/k1LoW/tcpdp/dumper" 16 | "github.com/k1LoW/tcpdp/dumper/conn" 17 | "github.com/k1LoW/tcpdp/dumper/hex" 18 | "github.com/k1LoW/tcpdp/dumper/mysql" 19 | "github.com/k1LoW/tcpdp/dumper/pg" 20 | "github.com/lestrrat-go/server-starter/listener" 21 | "github.com/spf13/viper" 22 | "go.uber.org/zap" 23 | "go.uber.org/zap/zapcore" 24 | ) 25 | 26 | // Server struct 27 | type Server struct { 28 | pidfile string 29 | listenAddr *net.TCPAddr 30 | remoteAddr *net.TCPAddr 31 | ctx context.Context 32 | shutdown context.CancelFunc 33 | Wg *sync.WaitGroup 34 | ClosedChan chan struct{} 35 | listener *net.TCPListener 36 | logger *zap.Logger 37 | dumper dumper.Dumper 38 | } 39 | 40 | // NewServer returns a new Server 41 | func NewServer(ctx context.Context, lAddr, rAddr *net.TCPAddr, logger *zap.Logger) *Server { 42 | innerCtx, shutdown := context.WithCancel(ctx) 43 | wg := &sync.WaitGroup{} 44 | closedChan := make(chan struct{}) 45 | 46 | var d dumper.Dumper 47 | dumpType := viper.GetString("tcpdp.dumper") 48 | 49 | switch dumpType { 50 | case "hex": 51 | d = hex.NewDumper() 52 | case "pg": 53 | d = pg.NewDumper() 54 | case "mysql": 55 | d = mysql.NewDumper() 56 | case "conn": 57 | d = conn.NewDumper() 58 | default: 59 | d = hex.NewDumper() 60 | } 61 | 62 | pidfile, err := filepath.Abs(viper.GetString("tcpdp.pidfile")) 63 | if err != nil { 64 | logger.WithOptions(zap.AddCaller()).Fatal("pidfile path error", zap.Error(err)) 65 | } 66 | 67 | return &Server{ 68 | pidfile: pidfile, 69 | listenAddr: lAddr, 70 | remoteAddr: rAddr, 71 | ctx: innerCtx, 72 | shutdown: shutdown, 73 | Wg: wg, 74 | ClosedChan: closedChan, 75 | logger: logger, 76 | dumper: d, 77 | } 78 | } 79 | 80 | // Start server. 81 | func (s *Server) Start() error { 82 | err := s.writePID() 83 | if err != nil { 84 | s.logger.WithOptions(zap.AddCaller()).Fatal(fmt.Sprintf("can not write %s", s.pidfile), zap.Error(err)) 85 | return err 86 | } 87 | defer s.deletePID() 88 | useServerStarter := viper.GetBool("proxy.useServerStarter") 89 | 90 | if useServerStarter { 91 | listeners, err := listener.ListenAll() 92 | if listeners == nil || err != nil { 93 | s.logger.WithOptions(zap.AddCaller()).Fatal("server-starter listen error", zap.Error(err)) 94 | return err 95 | } 96 | lt := listeners[0].(*net.TCPListener) 97 | s.listener = lt 98 | } else { 99 | lt, err := net.ListenTCP("tcp", s.listenAddr) 100 | if err != nil { 101 | s.logger.WithOptions(zap.AddCaller()).Fatal("listenAddr ListenTCP error", zap.Error(err)) 102 | return err 103 | } 104 | s.listener = lt 105 | } 106 | 107 | defer func() { 108 | if err := s.listener.Close(); err != nil && !strings.Contains(err.Error(), "use of closed network connection") { 109 | s.logger.WithOptions(zap.AddCaller()).Error("server listener Close error", zap.Error(err)) 110 | } 111 | close(s.ClosedChan) 112 | }() 113 | 114 | for { 115 | conn, err := s.listener.AcceptTCP() 116 | if err != nil { 117 | if ne, ok := err.(net.Error); ok { 118 | if ne.Temporary() { 119 | continue 120 | } 121 | if !strings.Contains(err.Error(), "use of closed network connection") { 122 | select { 123 | case <-s.ctx.Done(): 124 | break 125 | default: 126 | s.logger.WithOptions(zap.AddCaller()).Fatal("listener AcceptTCP error", zap.Error(err)) 127 | } 128 | } 129 | } 130 | return err 131 | } 132 | s.Wg.Add(1) 133 | go s.handleConn(conn) 134 | } 135 | } 136 | 137 | // Shutdown server. 138 | func (s *Server) Shutdown() { 139 | select { 140 | case <-s.ctx.Done(): 141 | default: 142 | s.shutdown() 143 | if err := s.listener.Close(); err != nil && !strings.Contains(err.Error(), "use of closed network connection") { 144 | s.logger.WithOptions(zap.AddCaller()).Error("server listener Close error", zap.Error(err)) 145 | } 146 | } 147 | } 148 | 149 | // GracefulShutdown server. 150 | func (s *Server) GracefulShutdown() { 151 | select { 152 | case <-s.ctx.Done(): 153 | default: 154 | if err := s.listener.Close(); err != nil && !strings.Contains(err.Error(), "use of closed network connection") { 155 | s.logger.WithOptions(zap.AddCaller()).Error("server listener Close error", zap.Error(err)) 156 | } 157 | } 158 | } 159 | 160 | func (s *Server) handleConn(conn *net.TCPConn) { 161 | defer s.Wg.Done() 162 | 163 | remoteConn, err := net.DialTCP("tcp", nil, s.remoteAddr) 164 | if err != nil { 165 | fields := s.fieldsWithErrorAndConn(err, conn) 166 | s.logger.WithOptions(zap.AddCaller()).Error("remoteAddr DialTCP error", fields...) 167 | if err := conn.Close(); err != nil { 168 | s.logger.WithOptions(zap.AddCaller()).Error("server conn Close error", fields...) 169 | } 170 | return 171 | } 172 | 173 | p := NewProxy(s, conn, remoteConn) 174 | p.Start() 175 | } 176 | 177 | func (s *Server) fieldsWithErrorAndConn(err error, conn *net.TCPConn) []zapcore.Field { 178 | fields := []zapcore.Field{ 179 | zap.Error(err), 180 | zap.String("client_addr", conn.RemoteAddr().String()), 181 | zap.String("proxy_listen_addr", conn.LocalAddr().String()), 182 | zap.String("remote_addr", s.remoteAddr.String()), 183 | } 184 | return fields 185 | } 186 | 187 | // https://gist.github.com/davidnewhall/3627895a9fc8fa0affbd747183abca39 188 | func (s *Server) writePID() error { 189 | if data, err := ioutil.ReadFile(s.pidfile); err == nil { 190 | if pid, err := strconv.Atoi(string(data)); err == nil { 191 | if process, err := os.FindProcess(pid); err == nil { 192 | if err := process.Signal(syscall.Signal(0)); err == nil { 193 | return fmt.Errorf("pid already running: %d", pid) 194 | } 195 | } 196 | } 197 | } 198 | return ioutil.WriteFile(s.pidfile, []byte(fmt.Sprintf("%d\n", os.Getpid())), 0664) // #nosec 199 | } 200 | 201 | func (s *Server) deletePID() { 202 | if err := os.Remove(s.pidfile); err != nil { 203 | s.logger.WithOptions(zap.AddCaller()).Fatal(fmt.Sprintf("can not delete %s", s.pidfile), zap.Error(err)) 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /template/control.template: -------------------------------------------------------------------------------- 1 | Package: tcpdp 2 | Maintainer: Ken'ichiro OYAMA 3 | Architecture: amd64 4 | Priority: optional 5 | Section: net 6 | Version: {{ .Env.VERSION }}-1 7 | Build-Depends: libpcap-dev 8 | Depends: libpcap0.8 9 | Description: tcpdp is TCP dump tool with custom dumper written in Go. 10 | Homepage: https://github.com/k1LoW/tcpdp 11 | -------------------------------------------------------------------------------- /template/tcpdp.spec.template: -------------------------------------------------------------------------------- 1 | %define name tcpdp 2 | %define release 1.{{ .Env.DIST }} 3 | %define version {{ .Env.VERSION }} 4 | %define buildroot %{_tmppath}/%{name}-%{version}-buildroot 5 | %define debug_package %{nil} 6 | 7 | BuildRoot: %{buildroot} 8 | Summary: tcpdp is TCP dump tool with custom dumper written in Go. 9 | License: MIT 10 | Packager: Ken'ichiro OYAMA 11 | Source: %{name}-%{version}.tar.gz 12 | Name: %{name} 13 | Version: %{version} 14 | Release: %{release} 15 | Prefix: %{_prefix} 16 | Group: Applications/Internet 17 | Requires: libpcap 18 | BuildRequires: make libpcap-devel 19 | 20 | %description 21 | tcpdp is TCP dump tool with custom dumper written in Go. 22 | 23 | %prep 24 | %setup -q -n %{name}-%{version} 25 | 26 | %build 27 | make 28 | 29 | %install 30 | %{__rm} -rf %{buildroot} 31 | mkdir -p %{buildroot}%{_bindir} 32 | make BINDIR=%{buildroot}%{_bindir} install 33 | 34 | %clean 35 | %{__rm} -rf %{buildroot} 36 | 37 | %files 38 | %{_bindir}/tcpdp 39 | -------------------------------------------------------------------------------- /testdata/haproxy/haproxy.cfg: -------------------------------------------------------------------------------- 1 | global 2 | log 127.0.0.1 local0 debug 3 | 4 | defaults 5 | log global 6 | option tcplog 7 | mode tcp 8 | maxconn 5000 9 | 10 | timeout connect 5s 11 | timeout client 5s 12 | timeout server 5s 13 | 14 | frontend frontend-proxy 15 | bind *:33068 16 | default_backend backend-proxy 17 | 18 | frontend frontend-mariadb 19 | bind *:33069 20 | default_backend backend-mariadb 21 | 22 | frontend frontend-proxy-v2 23 | bind *:33070 24 | default_backend backend-proxy-v2 25 | 26 | frontend frontend-mariadb-v2 27 | bind *:33071 28 | default_backend backend-mariadb-v2 29 | 30 | backend backend-proxy 31 | server proxy host.docker.internal:33080 send-proxy 32 | 33 | backend backend-mariadb 34 | server mariadb host.docker.internal:33081 send-proxy 35 | 36 | backend backend-proxy-v2 37 | server proxy host.docker.internal:33080 send-proxy-v2 38 | 39 | backend backend-mariadb-v2 40 | server mariadb host.docker.internal:33081 send-proxy-v2 41 | -------------------------------------------------------------------------------- /testdata/mariadb.conf.d/custom.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | proxy-protocol-networks=::1, 0.0.0.0/0 ,localhost 3 | -------------------------------------------------------------------------------- /testdata/mysql.conf.d/custom.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | max_connections = 500 3 | max_connect_errors = 1000 -------------------------------------------------------------------------------- /testdata/pcap/mysql_prepare.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/k1LoW/tcpdp/f921059db8951deadde75a4abf535776456e34da/testdata/pcap/mysql_prepare.pcap -------------------------------------------------------------------------------- /testdata/pcap/pg_prepare.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/k1LoW/tcpdp/f921059db8951deadde75a4abf535776456e34da/testdata/pcap/pg_prepare.pcap -------------------------------------------------------------------------------- /testdata/terraform/benchmark.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "ap-northeast-1" 3 | } 4 | 5 | resource "aws_vpc" "default" { 6 | cidr_block = "10.0.0.0/16" 7 | enable_dns_hostnames = true 8 | tags { 9 | Name = "vpc_tcpdp" 10 | } 11 | } 12 | 13 | resource "aws_internet_gateway" "igw" { 14 | vpc_id = "${aws_vpc.default.id}" 15 | tags { 16 | Name = "igw_tcpdp" 17 | } 18 | } 19 | 20 | resource "aws_subnet" "public_b" { 21 | vpc_id = "${aws_vpc.default.id}" 22 | cidr_block = "10.0.1.0/24" 23 | availability_zone = "ap-northeast-1b" 24 | 25 | tags { 26 | Name = "subnet_tcpdp_public_a" 27 | } 28 | } 29 | 30 | resource "aws_subnet" "public_c" { 31 | vpc_id = "${aws_vpc.default.id}" 32 | cidr_block = "10.0.2.0/24" 33 | availability_zone = "ap-northeast-1c" 34 | 35 | tags { 36 | Name = "subnet_tcpdp_public_c" 37 | } 38 | } 39 | 40 | resource "aws_subnet" "private_b" { 41 | vpc_id = "${aws_vpc.default.id}" 42 | cidr_block = "10.0.3.0/24" 43 | availability_zone = "ap-northeast-1b" 44 | 45 | tags { 46 | Name = "subnet_tcpdp_private_a" 47 | } 48 | } 49 | 50 | resource "aws_subnet" "private_c" { 51 | vpc_id = "${aws_vpc.default.id}" 52 | cidr_block = "10.0.4.0/24" 53 | availability_zone = "ap-northeast-1c" 54 | 55 | tags { 56 | Name = "subnet_tcpdp_private_c" 57 | } 58 | } 59 | 60 | resource "aws_route_table" "public-rt" { 61 | vpc_id = "${aws_vpc.default.id}" 62 | route { 63 | cidr_block = "0.0.0.0/0" 64 | gateway_id = "${aws_internet_gateway.igw.id}" 65 | } 66 | tags { 67 | Name = "public-rt" 68 | } 69 | } 70 | 71 | resource "aws_route_table_association" "rta-1b" { 72 | subnet_id = "${aws_subnet.public_b.id}" 73 | route_table_id = "${aws_route_table.public-rt.id}" 74 | } 75 | 76 | resource "aws_route_table_association" "rta-1c" { 77 | subnet_id = "${aws_subnet.public_c.id}" 78 | route_table_id = "${aws_route_table.public-rt.id}" 79 | } 80 | 81 | resource "aws_security_group" "ec2" { 82 | name = "sg_ec2_tcpdp" 83 | vpc_id = "${aws_vpc.default.id}" 84 | 85 | ingress { 86 | from_port = 22 87 | to_port = 22 88 | protocol = "tcp" 89 | description = "SSH" 90 | cidr_blocks = ["0.0.0.0/0"] 91 | } 92 | 93 | egress { 94 | from_port = 0 95 | to_port = 0 96 | protocol = "-1" 97 | cidr_blocks = ["0.0.0.0/0"] 98 | } 99 | 100 | tags { 101 | Name = "sg_ec2_tcpdp" 102 | } 103 | 104 | lifecycle { 105 | create_before_destroy = true 106 | } 107 | } 108 | 109 | resource "aws_security_group" "rds" { 110 | name = "sg_rds_tcpdp" 111 | vpc_id = "${aws_vpc.default.id}" 112 | 113 | ingress { 114 | from_port = 3306 115 | to_port = 3306 116 | protocol = "tcp" 117 | description = "tcpdp" 118 | security_groups = ["${aws_security_group.ec2.id}"] 119 | cidr_blocks = ["0.0.0.0/0"] 120 | } 121 | 122 | egress { 123 | from_port = 0 124 | to_port = 0 125 | protocol = "-1" 126 | cidr_blocks = ["0.0.0.0/0"] 127 | } 128 | 129 | tags { 130 | Name = "sg_rds_tcpdp" 131 | } 132 | 133 | lifecycle { 134 | create_before_destroy = true 135 | } 136 | } 137 | 138 | resource "aws_key_pair" "default" { 139 | key_name = "key_tcpdp" 140 | public_key = "${file("~/.ssh/id_rsa.pub")}" 141 | } 142 | 143 | resource "aws_instance" "default" { 144 | ami = "ami-e99f4896" // Amazon Linux 2 AMI (HVM), SSD Volume Type 145 | instance_type = "m4.large" 146 | associate_public_ip_address = "true" 147 | key_name = "${aws_key_pair.default.key_name}" 148 | subnet_id = "${aws_subnet.public_b.id}" 149 | vpc_security_group_ids = ["${aws_security_group.ec2.id}"] 150 | 151 | root_block_device { 152 | volume_type = "standard" 153 | volume_size = "50" 154 | delete_on_termination = "false" 155 | } 156 | 157 | tags { 158 | "Name" = "tcpdp" 159 | } 160 | } 161 | 162 | resource "aws_db_subnet_group" "default" { 163 | name = "db_subnet_group_tcpdp" 164 | subnet_ids = ["${aws_subnet.private_b.id}", "${aws_subnet.private_c.id}"] 165 | 166 | tags { 167 | Name = "db_subnet_group_tcpdp" 168 | } 169 | } 170 | 171 | resource "aws_db_parameter_group" "default" { 172 | name = "pgtcpdp" 173 | family = "mysql5.7" 174 | 175 | parameter { 176 | name = "max_connect_errors" 177 | value = "1000" 178 | } 179 | } 180 | 181 | resource "aws_db_instance" "tcpdp" { 182 | allocated_storage = 10 183 | storage_type = "gp2" 184 | engine = "mysql" 185 | engine_version = "5.7.22" 186 | instance_class = "db.m4.large" 187 | name = "tcpdp" 188 | username = "tcpdp" 189 | password = "tcpdppass" 190 | parameter_group_name = "${aws_db_parameter_group.default.id}" 191 | apply_immediately = true 192 | skip_final_snapshot = true 193 | vpc_security_group_ids = ["${aws_security_group.rds.id}"] 194 | db_subnet_group_name = "${aws_db_subnet_group.default.id}" 195 | publicly_accessible = true 196 | } 197 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | // Name for this 4 | const Name string = "tcpdp" 5 | 6 | // Version for this 7 | const Version string = "0.23.9" 8 | --------------------------------------------------------------------------------