├── integration_test ├── test.pcap ├── test.expected └── test.sh ├── .travis.yml ├── configs ├── testimony.conf ├── upstart └── systemd.conf ├── install.sh ├── go ├── testimony │ ├── Makefile │ └── testimony.go ├── testclient │ ├── Makefile │ └── testclient.go ├── testimonyd │ ├── Makefile │ ├── internal │ │ ├── vlog │ │ │ └── vlog.go │ │ └── socket │ │ │ ├── socket.c │ │ │ ├── daemon.go │ │ │ └── socket.go │ └── daemon.go ├── protocol │ ├── to_c │ │ └── to_c.go │ └── protocol.go └── Makefile ├── Makefile ├── c ├── Makefile ├── testimony_client.c ├── testimony.h └── testimony.c ├── CONTRIBUTING.md ├── README.md └── LICENSE /integration_test/test.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/testimony/HEAD/integration_test/test.pcap -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: required 3 | dist: trusty 4 | before_install: 5 | - sudo apt-get update -qq 6 | - sudo apt-get install tcpreplay 7 | script: 8 | - go test ./... 9 | - DUMMY=dummy1 bash integration_test/test.sh 10 | -------------------------------------------------------------------------------- /configs/testimony.conf: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "SocketName": "/tmp/testimony.sock", 4 | "Interface": "eth0", 5 | "BlockSize": 1048576, 6 | "NumBlocks": 16, 7 | "FanoutSize": 1, 8 | "BlockTimeoutMillis": 1000, 9 | "User": "root" 10 | } 11 | ] -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ ! -f /usr/sbin/testimonyd ]; then 4 | sudo cp -v go/testimonyd/testimonyd /usr/sbin/testimonyd 5 | fi 6 | 7 | if [ ! -f /etc/testimony.conf ]; then 8 | sudo cp -v configs/testimony.conf /etc/testimony.conf 9 | fi 10 | 11 | if [ ! -f /etc/systemd/system/testimony.service ]; then 12 | sudo cp -v configs/systemd.conf /etc/systemd/system/testimony.service 13 | sudo chmod 0644 /etc/systemd/system/testimony.service 14 | fi 15 | 16 | sudo service testimony start -------------------------------------------------------------------------------- /go/testimony/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | all: 16 | go build 17 | 18 | test: 19 | go test 20 | 21 | clean: 22 | 23 | .PHONY: all clean 24 | -------------------------------------------------------------------------------- /go/testclient/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | all: *.go 16 | go build -o testclient 17 | 18 | test: 19 | go test 20 | 21 | clean: 22 | rm -f testclient 23 | 24 | .PHONY: all clean 25 | -------------------------------------------------------------------------------- /configs/upstart: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # testimony - cross-process sharing of AF_PACKET data 16 | 17 | description "sharing packets in memory" 18 | 19 | start on runlevel [2345] 20 | stop on runlevel [!2345] 21 | 22 | respawn 23 | respawn limit 1 300 # At most once per 5 minutes 24 | 25 | exec /usr/sbin/testimonyd 26 | -------------------------------------------------------------------------------- /go/testimonyd/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | GO=go 16 | 17 | all: testimonyd 18 | 19 | .PHONY: all clean install 20 | 21 | clean: 22 | rm -rf testimonyd 23 | 24 | testimonyd: *.go internal/*/* 25 | $(GO) build -o testimonyd 26 | 27 | install: testimonyd 28 | cp -fv testimonyd /usr/sbin/testimonyd 29 | 30 | test: 31 | go test ./... 32 | -------------------------------------------------------------------------------- /go/protocol/to_c/to_c.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "fmt" 19 | "sort" 20 | 21 | "github.com/google/testimony/go/protocol" 22 | ) 23 | 24 | func main() { 25 | keys := []int{} 26 | for t := range protocol.TypeNames { 27 | keys = append(keys, int(t)) 28 | } 29 | sort.Ints(keys) 30 | for _, k := range keys { 31 | fmt.Printf("#define TESTIMONY_PROTOCOL_TYPE_%s %d\n", protocol.TypeNames[protocol.Type(k)], k) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | all: go c 16 | 17 | clean: go_clean c_clean 18 | 19 | install: go_install c_install 20 | 21 | test: go_test c_test 22 | 23 | .PHONY: c go 24 | 25 | go: 26 | $(MAKE) -C go 27 | 28 | go_test: 29 | $(MAKE) -C go test 30 | 31 | go_clean: 32 | $(MAKE) -C go clean 33 | 34 | go_install: 35 | $(MAKE) -C go install 36 | 37 | c: 38 | $(MAKE) -C c 39 | 40 | c_test: 41 | $(MAKE) -C c test 42 | 43 | c_clean: 44 | $(MAKE) -C c clean 45 | 46 | c_install: 47 | $(MAKE) -C c install 48 | -------------------------------------------------------------------------------- /configs/systemd.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # stenographer - full packet to disk capture 16 | # 17 | # stenographer is a simple, fast method of writing live packets to disk, 18 | # then requesting those packets after-the-fact for post-hoc analysis. 19 | 20 | [Unit] 21 | Description=packet capture to disk 22 | After=network.target 23 | 24 | [Service] 25 | User=root 26 | Group=root 27 | SyslogIdentifier=testimony 28 | LimitFSIZE=4294967296 29 | LimitNOFILE=1000000 30 | ExecStart=/usr/sbin/testimonyd -config /etc/testimony.conf 31 | ExecStopPost=/bin/pkill -9 testimony 32 | 33 | [Install] 34 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /go/testimonyd/internal/vlog/vlog.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package vlog 16 | 17 | import ( 18 | "flag" 19 | "log" 20 | "path/filepath" 21 | "runtime" 22 | ) 23 | 24 | var verbose = flag.Int("v", 0, "Verbose logging, increase for more logs") 25 | 26 | // V logs a message based on the --v command line flag. 27 | func V(level int, format string, args ...interface{}) { 28 | VUp(level, 1, format, args...) 29 | } 30 | 31 | // Vup logs a message based on the --v command line flag, using the n'th 32 | // caller's file/line number instead of this one. 33 | func VUp(level int, caller int, format string, args ...interface{}) { 34 | if level <= *verbose { 35 | _, file, line, _ := runtime.Caller(caller + 1) 36 | args = append([]interface{}{filepath.Base(file), line}, args...) 37 | log.Printf("%s:%d -\t"+format, args...) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /go/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | all: testimony testimonyd buildtestclient 16 | 17 | test: testimony testimonyd testimony_test testimonyd_test testclient_test 18 | 19 | clean: testimony_clean testimonyd_clean testclient_clean 20 | 21 | install: testimonyd_install 22 | 23 | .PHONY: testimony testimonyd 24 | 25 | testimony: 26 | $(MAKE) -C testimony 27 | 28 | testimony_test: 29 | $(MAKE) -C testimony test 30 | 31 | testimony_clean: 32 | $(MAKE) -C testimony clean 33 | 34 | testimonyd: 35 | $(MAKE) -C testimonyd 36 | 37 | testimonyd_test: 38 | $(MAKE) -C testimonyd test 39 | 40 | testimonyd_clean: 41 | $(MAKE) -C testimonyd clean 42 | 43 | testimonyd_install: 44 | $(MAKE) -C testimonyd install 45 | 46 | buildtestclient: 47 | $(MAKE) -C testclient 48 | 49 | testclient_test: 50 | $(MAKE) -C testclient test 51 | 52 | testclient_clean: 53 | $(MAKE) -C testclient clean 54 | -------------------------------------------------------------------------------- /c/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | AR=ar 16 | RANLIB=ranlib 17 | CC=gcc 18 | CFLAGS=-fPIC -Wall -g -O2 -I. -fno-strict-aliasing 19 | LDFLAGS= 20 | SHELL=/bin/bash 21 | 22 | all: libtestimony.a libtestimony.so testimony_client 23 | 24 | .PHONY: all clean install 25 | 26 | clean: 27 | rm -rf *.o *.a *.so *.so.* testimony_client 28 | 29 | %.o: %.c 30 | $(CC) $(CFLAGS) -c -o $@ $< 31 | 32 | testimony_client: testimony_client.o libtestimony.a libtestimony.so 33 | $(CC) -o testimony_client testimony_client.o $(LDFLAGS) -L. -ltestimony 34 | 35 | libtestimony.a: testimony.o 36 | $(AR) cr $@ $^ && \ 37 | $(RANLIB) $@ 38 | 39 | libtestimony.so: testimony.o 40 | $(CC) -shared -o $@ $^ 41 | 42 | install: libtestimony.a libtestimony.so 43 | cp -f -v libtestimony.{a,so} /usr/lib && \ 44 | chmod 644 /usr/lib/libtestimony.{a,so} && \ 45 | cp -f -v testimony.h /usr/include && \ 46 | chmod 644 /usr/include/testimony.h 47 | 48 | test: 49 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing To Testimony 2 | ========================= 3 | 4 | Want to contribute? Great! First, read this page (including the small print at the end). 5 | 6 | ### Before you contribute ### 7 | Before we can use your code, you must sign the 8 | [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual?csw=1) 9 | (CLA), which you can do online. The CLA is necessary mainly because you own the 10 | copyright to your changes, even after your contribution becomes part of our 11 | codebase, so we need your permission to use and distribute your code. We also 12 | need to be sure of various other things—for instance that you'll tell us if you 13 | know that your code infringes on other people's patents. You don't have to sign 14 | the CLA until after you've submitted your code for review and a member has 15 | approved it, but you must do it before we can put your code into our codebase. 16 | Before you start working on a larger contribution, you should get in touch with 17 | us first through the issue tracker with your idea so that we can help out and 18 | possibly guide you. Coordinating up front makes it much easier to avoid 19 | frustration later on. 20 | 21 | ### Code reviews ### 22 | All submissions, including submissions by project members, require review. We 23 | use Github pull requests for this purpose. 24 | 25 | ### The small print ### 26 | Contributions made by corporations are covered by a different agreement than 27 | the one above, the Software Grant and Corporate Contributor License Agreement. 28 | -------------------------------------------------------------------------------- /go/testimonyd/daemon.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "bytes" 19 | "encoding/json" 20 | "flag" 21 | "io/ioutil" 22 | "log" 23 | "log/syslog" 24 | "syscall" 25 | 26 | "github.com/google/testimony/go/testimonyd/internal/socket" 27 | ) 28 | 29 | var ( 30 | confFilename = flag.String("config", "/etc/testimony.conf", "Testimony config") 31 | logToSyslog = flag.Bool("syslog", true, "log messages to syslog") 32 | ) 33 | 34 | func main() { 35 | flag.Parse() 36 | if *logToSyslog { 37 | s, err := syslog.New(syslog.LOG_USER|syslog.LOG_INFO, "testimonyd") 38 | if err != nil { 39 | log.Fatalf("could not set up syslog logging: %v", err) 40 | } 41 | log.SetOutput(s) 42 | } 43 | log.Printf("Starting testimonyd...") 44 | confdata, err := ioutil.ReadFile(*confFilename) 45 | if err != nil { 46 | log.Fatalf("could not read configuration %q: %v", *confFilename, err) 47 | } 48 | // Set umask which will affect all of the sockets we create: 49 | syscall.Umask(0177) 50 | var t socket.Testimony 51 | if err := json.NewDecoder(bytes.NewBuffer(confdata)).Decode(&t); err != nil { 52 | log.Fatalf("could not parse configuration %q: %v", *confFilename, err) 53 | } 54 | for i, _ := range t { 55 | if t[i].FanoutSize == 0 { 56 | t[i].FanoutSize = 1 57 | } 58 | } 59 | socket.RunTestimony(t) 60 | } 61 | -------------------------------------------------------------------------------- /integration_test/test.expected: -------------------------------------------------------------------------------- 1 | 0060dd46998890e2ba4832380800450000541b8b40004001c91ea9fe0101a9fe01020800c4d8023700019e559956000000002c700e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 2 | 90e2ba4832380060dd4699880800450000540dd70000400116d3a9fe0102a9fe01010000ccd8023700019e559956000000002c700e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 3 | 0060dd46998890e2ba4832380800450000541c1c40004001c88da9fe0101a9fe0102080085d8023700029f559956000000006a6f0e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 4 | 90e2ba4832380060dd4699880800450000540e7d00004001162da9fe0102a9fe010100008dd8023700029f559956000000006a6f0e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 5 | 0060dd46998890e2ba4832380800450000541cea40004001c7bfa9fe0101a9fe0102080084d702370003a0559956000000006a6f0e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 6 | 90e2ba4832380060dd4699880800450000540f6000004001154aa9fe0102a9fe010100008cd702370003a0559956000000006a6f0e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 7 | 0060dd46998890e2ba4832380800450000541d2240004001c787a9fe0101a9fe010208006bd602370004a155995600000000826f0e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 8 | 90e2ba4832380060dd469988080045000054101d00004001148da9fe0102a9fe0101000073d602370004a155995600000000826f0e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 9 | 0060dd46998890e2ba4832380800450000541d4040004001c769a9fe0101a9fe010208004bd502370005a255995600000000a16f0e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 10 | 90e2ba4832380060dd469988080045000054106d00004001143da9fe0102a9fe0101000053d502370005a255995600000000a16f0e0000000000101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637 11 | -------------------------------------------------------------------------------- /go/testclient/testclient.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // testimony_testclient provides a simple client for testing block processing. 16 | package main 17 | 18 | import ( 19 | "flag" 20 | "fmt" 21 | "log" 22 | "time" 23 | 24 | "github.com/google/testimony/go/testimony" 25 | ) 26 | 27 | var ( 28 | socketName = flag.String("socket", "", "Name of testimony socket") 29 | fanoutInt = flag.Int("fanout", 0, "Fanout number, if applicable") 30 | dump = flag.Bool("dump", false, "If true, output packet dump as hex") 31 | count = flag.Int("count", -1, "If == 0, number of packets to read in") 32 | ) 33 | 34 | func main() { 35 | flag.Parse() 36 | 37 | log.Printf("connecting to %q", *socketName) 38 | conn, err := testimony.Connect(*socketName) 39 | if err != nil { 40 | log.Fatalf("failed to connect: %v", err) 41 | } 42 | log.Printf("setting fanout to %d", *fanoutInt) 43 | if err := conn.Init(*fanoutInt); err != nil { 44 | log.Fatalf("failed to set fanout: %v", err) 45 | } 46 | 47 | log.Printf("reading blocks") 48 | totalCount := 0 49 | blockNum := 0 50 | start := time.Now() 51 | for { 52 | log.Printf("getting block") 53 | block, err := conn.Block() 54 | if err != nil { 55 | log.Fatalf("block reading failed: %v", err) 56 | } 57 | log.Printf("processing block") 58 | blockNum++ 59 | blockCount := 0 60 | for block.Next() { 61 | if *count == 0 { 62 | break 63 | } 64 | *count-- 65 | if *dump { 66 | fmt.Printf("%x\n", block.PacketData()) 67 | } 68 | blockCount++ 69 | } 70 | log.Printf("returning block") 71 | if err := block.Return(); err != nil { 72 | log.Fatalf("block return failed: %v", err) 73 | } 74 | totalCount += blockCount 75 | log.Printf("block %d had %d packets, %d total in %v", blockNum, blockCount, totalCount, time.Since(start)) 76 | if *count == 0 { 77 | break 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /integration_test/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2015 Google Inc. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | DUMMY="${DUMMY-dummy0}" 18 | DIR="$(mktemp -d)" 19 | SOCKET="$DIR/socket" 20 | CONFIG="$DIR/config" 21 | BASEDIR="${BASEDIR-/tmp}" 22 | 23 | set -e 24 | cd $(dirname $0) 25 | 26 | function Log { 27 | FORMAT=$1 28 | shift 29 | echo -e "${FORMAT}$(date +%H:%M:%S.%N) --- $@\e[0m" 30 | } 31 | function Info { 32 | Log "\e[7m" "$@" 33 | } 34 | function Error { 35 | Log "\e[41m" "$@" 36 | } 37 | function Die { 38 | Error "$@" 39 | for file in `find $DIR -type f | sort`; do 40 | Info "$file" 41 | cat $file 42 | done 43 | exit 1 44 | } 45 | function Kill { 46 | sudo kill "$@" && sleep 1 && (sudo kill -9 "$@" || true) 47 | } 48 | 49 | Info "Testing sudo access" 50 | sudo cat /dev/null 51 | Info "Installing tcpreplay" 52 | sudo apt-get install -y tcpreplay 53 | 54 | Info "Building" 55 | pushd ../ 56 | make 57 | popd 58 | 59 | cat > $CONFIG << EOF 60 | [ 61 | { 62 | "SocketName": "$SOCKET" 63 | , "Interface": "$DUMMY" 64 | , "BlockSize": 1048576 65 | , "NumBlocks": 16 66 | , "BlockTimeoutMillis": 1000 67 | , "FanoutSize": 1 68 | , "User": "$(whoami)" 69 | , "Filter": "host 169.254.1.1 and host 169.254.1.2" 70 | } 71 | ] 72 | EOF 73 | 74 | Info "Setting up $DUMMY interface" 75 | sudo /sbin/modprobe dummy 76 | sudo ip link add $DUMMY type dummy || Error "$DUMMY may already exist" 77 | sudo ifconfig $DUMMY promisc up 78 | 79 | Info "Starting testimony" 80 | sudo ../go/testimonyd/testimonyd --config=$CONFIG --syslog=false 2>&1 & 81 | DAEMON_PID="$!" 82 | sleep 1 83 | sudo kill -0 $DAEMON_PID || Die "Daemon not running" 84 | 85 | Info "Starting clients" 86 | CLIENT_PIDS="" 87 | for i in {1..5}; do 88 | ../go/testclient/testclient --socket=$SOCKET --dump --count=10 >$DIR/out$i 2>$DIR/err$i & 89 | CLIENT_PIDS="$CLIENT_PIDS $!" 90 | done 91 | for i in {6..10}; do 92 | LD_LIBRARY_PATH=../c strace -e recvfrom,sendto -o $DIR/strace$i ../c/testimony_client --socket=$SOCKET --dump --count=10 >$DIR/out$i 2>$DIR/err$i & 93 | CLIENT_PIDS="$CLIENT_PIDS $!" 94 | done 95 | sleep 1 96 | sudo kill -0 $CLIENT_PIDS || Die "Client not running" 97 | 98 | Info "Sending packets to $DUMMY" 99 | sudo tcpreplay -i $DUMMY --topspeed test.pcap 100 | sleep 2 101 | 102 | Info "Turning off client" 103 | Kill $CLIENT_PIDS || Info "Failed to stop client, expected" 104 | Info "Turning off daemon" 105 | Kill $DAEMON_PID || Error "Failed to stop daemon" 106 | 107 | Info "Testing client output" 108 | for i in {1..10}; do 109 | diff -Naur $DIR/out$i test.expected || Die "Output from client $i failed, see $DIR/out$i and $DIR/err$i" 110 | done 111 | rm -rf $DIR 112 | sudo ip link delete $DUMMY || Error "Failed to clean up dummy $DUMMY" 113 | Info "SUCCESS" 114 | exit 0 115 | -------------------------------------------------------------------------------- /go/protocol/protocol.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package protocol 16 | 17 | import ( 18 | "encoding/binary" 19 | "fmt" 20 | "io" 21 | ) 22 | 23 | // Types used when sending requests/data between client and server. 24 | type Type uint16 25 | 26 | // Types used for sending data between client and server. 27 | const ( 28 | TypeBlockIndex Type = 0 29 | 30 | TypeServerToClient Type = 0x8000 + iota 31 | 32 | TypeWaitingForFanoutIndex 33 | TypeFanoutSize 34 | TypeBlockSize 35 | TypeNumBlocks 36 | 37 | TypeClientToServer Type = 0xC000 + iota 38 | 39 | TypeFanoutIndex 40 | 41 | TypeError Type = 0xFFFF 42 | ) 43 | 44 | // TypeNames allows for printing of protocols. 45 | var TypeNames = map[Type]string{ 46 | TypeBlockIndex: "BlockIndex", 47 | TypeServerToClient: "ServerToClient", 48 | TypeWaitingForFanoutIndex: "WaitingForFanoutIndex", 49 | TypeFanoutSize: "FanoutSize", 50 | TypeBlockSize: "BlockSize", 51 | TypeNumBlocks: "NumBlocks", 52 | TypeClientToServer: "ClientToServer", 53 | TypeFanoutIndex: "FanoutIndex", 54 | TypeError: "Error", 55 | } 56 | 57 | // TypeOf returns the high-level type (TypeServerToClient, TypeClientToServer, 58 | // TypeBlockIndex, or TypeError) of a type. 59 | func TypeOf(t Type) Type { 60 | switch { 61 | case t&0x8000 == 0: 62 | return TypeBlockIndex 63 | case t > TypeServerToClient && t < TypeClientToServer: 64 | return TypeServerToClient 65 | case t > TypeClientToServer && t < TypeError: 66 | return TypeClientToServer 67 | } 68 | return TypeError 69 | } 70 | 71 | // SendType sends a given type with a nil value to the given writer. 72 | func SendType(to io.Writer, typ Type) error { 73 | return SendTLV(to, typ, nil) 74 | } 75 | 76 | // SendUint32 sends a given type with a uint32 value to the given writer. 77 | func SendUint32(to io.Writer, typ Type, val uint32) error { 78 | var buf [4]byte 79 | binary.BigEndian.PutUint32(buf[:], val) 80 | return SendTLV(to, typ, buf[:]) 81 | } 82 | 83 | // SendTLV sends an arbitrary-length value with the given type type to a writer. 84 | func SendTLV(to io.Writer, typ Type, val []byte) error { 85 | if TypeOf(typ) != TypeServerToClient && TypeOf(typ) != TypeClientToServer { 86 | return fmt.Errorf("invalid send type %d", typ) 87 | } else if len(val) > 0xFFFF { 88 | return fmt.Errorf("too-long value (%d > %d)", len(val), 0xFFFF) 89 | } 90 | var buf [64]byte 91 | binary.BigEndian.PutUint32(buf[:], ToTL(typ, len(val))) 92 | if _, err := to.Write(append(buf[:4], val...)); err != nil { 93 | return fmt.Errorf("writing %d (len %d): %v", typ, len(val), err) 94 | } 95 | return nil 96 | } 97 | 98 | // TLFrom splits a uint32 into a type and length. 99 | func TLFrom(from uint32) (typ Type, length int) { 100 | if from&0x80000000 == 0 { 101 | return TypeBlockIndex, 0 102 | } 103 | return Type(from >> 16), int(from & 0xFFFF) 104 | } 105 | 106 | // ToTL converts a type/length to a TL uint32. 107 | func ToTL(typ Type, length int) uint32 { 108 | return (uint32(typ) << 16) + uint32(length) 109 | } 110 | -------------------------------------------------------------------------------- /c/testimony_client.c: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include // fprintf() 17 | #include // strerror() 18 | #include // atoi() 19 | #include // argp_parse() 20 | 21 | #define SOCKET_BUF_SIZE 256 22 | char* flag_socket = "/path/to/socket"; 23 | int flag_fanout_index = 0; 24 | int flag_count = 0; 25 | int flag_dump = 0; 26 | 27 | int ParseOption(int key, char* arg, struct argp_state* state) { 28 | switch (key) { 29 | case 300: 30 | flag_socket = arg; break; 31 | case 301: 32 | flag_fanout_index = atoi(arg); break; 33 | case 302: 34 | flag_count = atoi(arg); break; 35 | case 303: 36 | flag_dump = 1; break; 37 | } 38 | return 0; 39 | } 40 | 41 | int main(int argc, char** argv) { 42 | int r; 43 | const struct tpacket_block_desc* block; 44 | const struct tpacket3_hdr* packet; 45 | const uint8_t *packet_data; 46 | const uint8_t *packet_data_limit; 47 | testimony t; 48 | 49 | const char* s = "STRING"; 50 | const char* n = "NUM"; 51 | struct argp_option options[] = { 52 | {"socket", 300, s, 0, "Socket to connect to"}, 53 | {"index", 301, n, 0, "Fanout index to request"}, 54 | {"count", 302, n, 0, "Number of packets to process before exiting"}, 55 | {"dump", 303, 0, 0, "Dump packet hex to STDOUT"}, 56 | {0}, 57 | }; 58 | struct argp argp = {options, &ParseOption}; 59 | argp_parse(&argp, argc, argv, 0, 0, 0); 60 | 61 | fprintf(stderr, "Connecting to '%s'\n", flag_socket); 62 | r = testimony_connect(&t, flag_socket); 63 | if (r < 0) { 64 | fprintf(stderr, "Error with connect: %s\n", strerror(-r)); 65 | return 1; 66 | } 67 | testimony_conn(t)->fanout_index = flag_fanout_index; 68 | r = testimony_init(t); 69 | if (r < 0) { 70 | fprintf(stderr, "Error with init: %s: %s\n", testimony_error(t), 71 | strerror(-r)); 72 | return 1; 73 | } 74 | fprintf(stderr, "Init complete\n"); 75 | testimony_iter iter; 76 | testimony_iter_init(&iter); 77 | while (1) { 78 | r = testimony_get_block(t, -1, &block); 79 | if (r < 0) { 80 | fprintf(stderr, "Error with get: %s: %s\n", testimony_error(t), 81 | strerror(-r)); 82 | return 1; 83 | } 84 | fprintf(stderr, "got block %p with %d packets\n", block, 85 | block->hdr.bh1.num_pkts); 86 | testimony_iter_reset(iter, block); 87 | while ((packet = testimony_iter_next(iter)) != NULL) { 88 | if (flag_dump) { 89 | packet_data = testimony_packet_data(packet); 90 | packet_data_limit = packet_data + packet->tp_snaplen; 91 | for (; packet_data < packet_data_limit; packet_data++) { 92 | printf("%02x", *packet_data); 93 | } 94 | printf("\n"); 95 | } 96 | if (--flag_count == 0) { 97 | goto done; 98 | } 99 | } 100 | r = testimony_return_block(t, block); 101 | if (r < 0) { 102 | fprintf(stderr, "Error with return: %s: %s\n", testimony_error(t), 103 | strerror(-r)); 104 | return 1; 105 | } 106 | } 107 | done: 108 | testimony_close(t); 109 | return 0; 110 | } 111 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/google/testimony.svg?branch=master)](https://travis-ci.org/google/testimony) 2 | 3 | Testimony 4 | ========= 5 | 6 | Testimony is a single-machine, multi-process architecture for sharing AF_PACKET 7 | data across processes. This allows packets to be copied from NICs into memory 8 | a single time. Then, multiple processes can process this packet data in 9 | parallel without the need for additional copies. 10 | 11 | Testimony allows users to configure multiple different AF_PACKET sockets with 12 | different filters, blocks, etc. Administrators can specify BPF filters for 13 | these sockets, as well as which user should have access to which socket. This 14 | allows admins to easily set up access to a restricted set of packets for 15 | specific users. 16 | 17 | For discussion, questions, etc, use testimony-discuss@googlegroups.com. 18 | 19 | Detailed Design 20 | --------------- 21 | 22 | Testimony is implemented in a very simple client/server model. A single server, 23 | `testimonyd`, creates AF_PACKET sockets. Client processes then talk to it over 24 | AF_UNIX sockets. The client processes are passed the socket file descriptors; 25 | each then mmap's the socket into its own memory space. This done, the server 26 | then watches for new packet blocks and serves indexes to those blocks out to 27 | each client, reference-counting the blocks and returning them to the kernel only 28 | when all clients have released them. 29 | 30 | ### Configuration ### 31 | 32 | On creation, sockets are configured based on the /etc/testimony.conf 33 | configuration file, which lists all sockets to be created. Each socket contains 34 | these configuration options: 35 | 36 | * **SocketName:** Name of the socket file to create. `/tmp/foo.sock`, that kind 37 | of thing. This socket name is given to a connecting client so it can find 38 | where/how to communicate with `testimonyd`. 39 | * **Interface:** Name of the interface to sniff packets on, e.g. `eth0`, `em1`, 40 | etc. 41 | * **BlockSize:** AF_PACKET provides packets to user-space by filling up 42 | memory blocks of a specific size, until it either can't fit the next packet 43 | into the current block or a timeout is reached. The larger the block, the 44 | more packets can be passed to the user at once. BlockSize is in bytes. 45 | * **NumBlocks:** Number of blocks to allocate in memory. `NumBlocks * 46 | BlockSize` is the total size in memory of the AF_PACKET packet memory 47 | region for a single fanout. 48 | * **BlockTimeoutMillis:** If fewer than BlockSize bytes are sniffed by AF_PACKET 49 | before this number of milliseconds passes, AF_PACKET provides the current 50 | block to users in a less-than-full state. 51 | * **FanoutType:** AF_PACKET allows fanout, where multiple memory regions of the 52 | same size are created and packets are load-balanced between them. For 53 | fanout possibilities, see `/usr/include/linux/if_packet.h` 54 | * **FanoutSize:** The number of memory regions to fan out to. Total memory 55 | usage of AF_PACKET is `FanoutSize * MemoryRegionSize`, where 56 | `MemoryRegionSize` is `BlockSize * NumBlocks`. FanoutSize can be 57 | considered the number of parallel processes that want to access packet 58 | data. 59 | * **FanoutID:** Integer fanout ID to use when setting socket options. These 60 | are globally unique so it can be tuned to avoid conflicts with other 61 | processes that use AF_PACKET. If unspecified or 0 an ID will be auto 62 | assigned starting with 1. 63 | * **User:** This socket will be owned by the given user, mode `0600`. This 64 | allows root to provide different sockets with different capabilities to 65 | specific users. 66 | * **Filter:** BPF filter for this socket. If this is set, testimony will 67 | guarantee that the socket passed to child processes has this filter locked 68 | in such a way that clients cannot remove it. 69 | 70 | ### Wire Protocol ### 71 | 72 | Testimony uses an extremely simple wire protocol for establishing client 73 | connections and passing memory regions back and forth. 74 | 75 | Most values are passed as TLV, in the form: 76 | 77 | ``` 78 | | 0 | 1 | 2 | 3 | 4 | 5 | 6 | ... | 79 | | type | len | value .... | 80 | ``` 81 | 82 | Where type and length are big-endian uint32, and value is a set of bytes `len` 83 | bytes long. The type will always have its highest-order bit set, to 84 | differentiate between a TLV and a block index. 85 | 86 | SERVER CLIENT 87 | ------ ------ 88 | <-- initial connection --- 89 | --- version byte (1 byte == 2) ---> 90 | --- fanout size, block size, num blocks --> 91 | --- waiting for fanout index --> 92 | <-- fanout index --- 93 | --- socket FD, + 1 dummy byte (ignored) --> 94 | 95 | At this point, the client is connected. 96 | 97 | --- block index for client (4BE) --> 98 | <-- block index to return (4BE) --- 99 | 100 | Post-connection, most communication is 4-byte block indexes passed back 101 | and forth. At any time post-connection, either the server or client may 102 | send arbitrary TLV values across the wire... the other side should handle 103 | them if it knows how and ignore them if it doesn't. 104 | 105 | The server sends a block index to the client when that block is 106 | available to process (and it references the block internally). The client 107 | returns the block index to the server when it's done processing it, and the 108 | server unrefs that block. When a block has no more references, it is returned 109 | to the kernel to be refilled with packets. 110 | 111 | 112 | ### Installation ### 113 | 114 | Run install.sh after testimony building testimony. 115 | This script will create default config file `/etc/testimony.conf` and will add new testimony service. -------------------------------------------------------------------------------- /go/testimonyd/internal/socket/socket.c: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include // htons() 16 | #include // socket(), bind(), listen(), accept(), SOCK_STREAM 17 | #include // See socket man page 18 | #include // strerror() 19 | #include // ETH_P_ALL 20 | #include // TPACKET_V3, PACKET_FANOUT_LB 21 | #include // errno 22 | #include // if_nametoindex() 23 | #include // mmap(), PROT_*, MAP_* 24 | #include // close() 25 | #include // sock_fprog, sock_filter 26 | 27 | #ifndef UNIX_PATH_MAX 28 | #define UNIX_PATH_MAX 108 29 | #endif 30 | 31 | #define SOCKET_READY_OR_TIMEOUT 0 32 | 33 | // Function for waiting for new blocks using select() function 34 | // Blocks until a block is ready from AF_PACKET socket. 35 | // Returns 0 if socket is ready or timeout, errno on failure (errno > 0). 36 | int WaitForBlocks(int sock_fd) { 37 | struct timeval tv; 38 | fd_set fds; 39 | 40 | FD_ZERO(&fds); 41 | FD_SET(sock_fd, &fds); 42 | 43 | tv.tv_sec = 1; 44 | tv.tv_usec = 0; 45 | 46 | // return error code if select() returns not 0 or 1 47 | if (select(sock_fd + 1, &fds, NULL, NULL, &tv) < 0) { 48 | return errno; 49 | } 50 | 51 | // defined in go side 52 | return SOCKET_READY_OR_TIMEOUT; 53 | } 54 | 55 | 56 | // AFPacket does all of the necessary construction of an AF_PACKET socket 57 | // in C, to avoid a bunch of C.blah cgo stuff in daemon.go. It takes in a bunch 58 | // of arguments and outputs an AF_PACKET socket file descriptor, a void* 59 | // pointing to the mmap'd region of that socket, and any error message. 60 | // Returns zero on success, on error returns -1 and sets errno. 61 | int AFPacket(const char* iface, int block_size, int block_nr, int block_ms, 62 | int fanout_id, int fanout_size, int fanout_type, 63 | int filter_size, struct sock_filter* filters, 64 | // outputs: 65 | int* fd, void** ring, const char** err) { 66 | // Set up the initial socket. 67 | *fd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); 68 | if (*fd < 0) { 69 | *err = "socket creation failure"; 70 | return -1; 71 | } 72 | 73 | // Request TPACKET_V3. 74 | int v = TPACKET_V3; 75 | int r = setsockopt(*fd, SOL_PACKET, PACKET_VERSION, &v, sizeof(v)); 76 | if (r < 0) { 77 | *err = "setsockopt PACKET_VERSION failure"; 78 | goto fail1; 79 | } 80 | 81 | // If requested, set up and lock a BPF filter on the socket. 82 | if (filter_size) { 83 | #if defined(SO_ATTACH_FILTER) && defined(SO_LOCK_FILTER) 84 | struct sock_fprog filter; 85 | filter.filter = filters; 86 | filter.len = filter_size; 87 | 88 | r = setsockopt(*fd, SOL_SOCKET, SO_ATTACH_FILTER, &filter, sizeof(filter)); 89 | if (r < 0) { 90 | *err = "setsockopt SO_ATTACH_FILTER error"; 91 | goto fail1; 92 | } 93 | v = 1; 94 | r = setsockopt(*fd, SOL_SOCKET, SO_LOCK_FILTER, &v, sizeof(v)); 95 | if (r < 0) { 96 | *err = "setsockopt SO_LOCK_FILTER error"; 97 | goto fail1; 98 | } 99 | #else 100 | // If folks want a filter, that means they want to give access to specific 101 | // packets to a specific user. If we can't attach a filter, we give them 102 | // too many packets. If we can't lock the filter, they can change the 103 | // filter on the socket they receive and elevate their permissions. In 104 | // either case, fail hard. If this isn't supported, folks can still use 105 | // testimonyd without filters. 106 | *err = "filter requested, but BPF filtering or filter locking unsupported"; 107 | errno = ENOSYS; 108 | goto fail1; 109 | #endif 110 | } 111 | 112 | // Request a RX_RING so we can mmap the socket. 113 | struct tpacket_req3 tp3; 114 | memset(&tp3, 0, sizeof(tp3)); 115 | tp3.tp_block_size = block_size; 116 | tp3.tp_frame_size = block_size; 117 | tp3.tp_block_nr = block_nr; 118 | tp3.tp_frame_nr = block_nr; 119 | tp3.tp_retire_blk_tov = block_ms; // timeout, ms 120 | r = setsockopt(*fd, SOL_PACKET, PACKET_RX_RING, &tp3, sizeof(tp3)); 121 | if (r < 0) { 122 | *err = "setsockopt PACKET_RX_RING failure"; 123 | goto fail1; 124 | } 125 | 126 | // MMap the RX_RING to create a packet memory region. 127 | *ring = 128 | mmap(NULL, (size_t) tp3.tp_block_size * tp3.tp_block_nr, 129 | PROT_READ | PROT_WRITE, MAP_SHARED | MAP_LOCKED | MAP_NORESERVE, 130 | *fd, 0); 131 | if (*ring == MAP_FAILED) { 132 | *err = "ring mmap failed"; 133 | errno = EINVAL; 134 | goto fail1; 135 | } 136 | 137 | // Bind the socket to a single interface. 138 | struct sockaddr_ll ll; 139 | memset(&ll, 0, sizeof(ll)); 140 | ll.sll_family = AF_PACKET; 141 | ll.sll_protocol = htons(ETH_P_ALL); 142 | ll.sll_ifindex = if_nametoindex(iface); 143 | if (ll.sll_ifindex == 0) { 144 | *err = "if_nametoindex failed"; 145 | errno = EINVAL; 146 | goto fail2; 147 | } 148 | r = bind(*fd, (struct sockaddr*)&ll, sizeof(ll)); 149 | if (r < 0) { 150 | *err = "bind failed"; 151 | goto fail2; 152 | } 153 | 154 | // Set up fanout. 155 | // If fanout size is 1, there's no point in trying to set fanout. 156 | if (fanout_size != 1) { 157 | int fanout = (fanout_id & 0xFFFF) | (fanout_type << 16); 158 | r = setsockopt(*fd, SOL_PACKET, PACKET_FANOUT, &fanout, sizeof(fanout)); 159 | if (r < 0) { 160 | *err = "setsockopt PACKET_FANOUT failed"; 161 | goto fail2; 162 | } 163 | } 164 | return 0; 165 | 166 | fail2 : { 167 | int err = errno; 168 | munmap(*ring, block_size * block_nr); 169 | errno = err; 170 | } 171 | fail1 : { 172 | int err = errno; 173 | close(*fd); 174 | errno = err; 175 | } 176 | return -1; 177 | } 178 | -------------------------------------------------------------------------------- /c/testimony.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef __TESTIMONY_H__ 16 | #define __TESTIMONY_H__ 17 | 18 | #define TESTIMONY_VERSION 2 // Current highest supported protocol version. 19 | 20 | #include // tpacket_block_desc, tpacket3_hdr 21 | #include // int64_t, uint8_t 22 | #include // size_t 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | // All functions that return an int return 0 on success, -errno on failure. 29 | 30 | struct testimony_internal; 31 | // testimony provides a link to a local testimony server, which serves up 32 | // AF_PACKET packet blocks. Usage: 33 | // 34 | // testimony t; 35 | // CHECK(testimony_init(&t, "/tmp/socketname") == 0); 36 | // testimony_connection* conn = testimony_conn(t); 37 | // printf("Fanout size: %d\n", conn->fanout_size); 38 | // printf("Block size: %d\n", conn->block_size); 39 | // // Set fanout index you'd like to use, must be [0, fanout_size). 40 | // conn->fanout_index = 2; 41 | // CHECK(testimony_init(t) == 0); 42 | // 43 | // // Now, you're connected to Testimony and ready to start reading packets. 44 | // 45 | // struct tpacket_block_desc* block; 46 | // while (x) { 47 | // CHECK(testimony_get_block(t, 1000 /* timeout, millis */, &block) == 0); 48 | // if (!block) { continue; } 49 | // // use block... 50 | // CHECK(testimony_return_block(t, block) == 0); 51 | // } 52 | // CHECK(testimony_close(t) == 0); 53 | typedef struct testimony_internal* testimony; 54 | 55 | typedef struct { 56 | // Filled in by server, shouldn't be modified by client: 57 | int fanout_size; // set by testimony_connect 58 | size_t block_size; // set by testimony_init 59 | size_t block_nr; // set by testimony_init 60 | // Settable by client to modify behavior of testimony_init: 61 | int fanout_index; 62 | } testimony_connection; 63 | 64 | // Initializes a connection to the testimony server. 65 | // After a successful call to testimony_connect, testimony_close should be 66 | // called on t should any future error occur. 67 | int testimony_connect(testimony* t, const char* socket_name); 68 | // Requests information about the connection. This can be called after connect 69 | // and before init. All other functions should be called after init. 70 | // Modifications to the returned data will modify the behavior of init. 71 | testimony_connection* testimony_conn(testimony t); 72 | // Returns a human-readable error message related to the last issue. 73 | char* testimony_error(testimony t); 74 | // Initiates block reads. The behavior of this function will change 75 | // based on modifications the client has made to testimony_conn(t). 76 | // On error, testimony_error may be called and testimony_close must be. 77 | int testimony_init(testimony t); 78 | // Closes a connection to the testimony server. t should not be reused after 79 | // this call. 80 | int testimony_close(testimony t); 81 | // Gets a new block of packets from testimony. 82 | // If timeout_millis < 0, block forever. If == 0, don't block. If > 0, block 83 | // for at most the given number of milliseconds. 84 | int testimony_get_block(testimony t, int timeout_millis, const struct tpacket_block_desc** block); 85 | // Returns a processed block of packets back to testimony. 86 | int testimony_return_block(testimony t, const struct tpacket_block_desc* block); 87 | 88 | // testimony_return_packets counts the number of packets processed in a 89 | // testimony block and auto-returns the block after the Nth call, where N is the 90 | // number of packets in the given block. 91 | // 92 | // Usage: 93 | // while (...) { 94 | // struct tpacket_block_desc* block; 95 | // CHECK(testimony_get_block(t, 1000, &block) == 0); 96 | // for (... iterate over packets in block ...) { 97 | // ... handle packet in block ... 98 | // CHECK(testimony_return_packets(t, block, 1) == 0); 99 | // } 100 | // // If you call return_packet, do NOT call testimony_return_block. 101 | // // Block will automatically be returned after Nth call to 102 | // // testimony_return_packets(t, block, 1), where N is the number of 103 | // // packets in the block. 104 | // } 105 | int testimony_return_packets(testimony t, const struct tpacket_block_desc* block, uint32_t packets); 106 | 107 | struct testimony_iter_internal; 108 | // testimony_iter provides an easy method for iterating over packets 109 | // in a tpacket3 block. 110 | // 111 | // Usage: 112 | // testimony_iter iter; 113 | // CHECK(testimony_iter_init(&iter) == 0); 114 | // while (...) { 115 | // struct tpacket_block_desc* block; 116 | // CHECK(testimony_get_block(t, 1000, &block) == 0); 117 | // if (!block) { continue; } 118 | // CHECK(testimony_iter_reset(iter, block) == 0); 119 | // struct tpacket3_hdr* packet; 120 | // while ((packet = testimony_iter_next(iter)) != NULL) { 121 | // handle_packet(packet); 122 | // } 123 | // CHECK(testimony_return_block(t, block) == 0); 124 | // } 125 | // CHECK(testimony_iter_close(iter)); 126 | // 127 | typedef struct testimony_iter_internal* testimony_iter; 128 | 129 | // Initiate iterator. Returns 0 on success, -errno on failure. 130 | int testimony_iter_init(testimony_iter* iter); 131 | // Reset iterator to iterate over a new block. 132 | int testimony_iter_reset( 133 | testimony_iter iter, const struct tpacket_block_desc* block); 134 | // Return the next packet in the block, or NULL if we have no more packets. 135 | const struct tpacket3_hdr* testimony_iter_next(testimony_iter iter); 136 | // Clean up the iterator. Use of iter after this call will break. 137 | int testimony_iter_close(testimony_iter iter); 138 | 139 | // testimony_packet_data is a helper function to extract the packet data 140 | // from a tpacket3 packet header. The returned buffer will be pkt->tp_snaplen 141 | // bytes long. pkt->tp_len is the length of the original packet, and may 142 | // be >= tp_snaplen. 143 | const uint8_t* testimony_packet_data(const struct tpacket3_hdr* pkt); 144 | // testimony_packet_nanos is the nanosecond timestamp for the given packet. 145 | int64_t testimony_packet_nanos(const struct tpacket3_hdr* pkt); 146 | 147 | #ifdef __cplusplus 148 | } 149 | #endif 150 | 151 | #endif 152 | -------------------------------------------------------------------------------- /go/testimonyd/internal/socket/daemon.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package socket 16 | 17 | import ( 18 | "encoding/binary" 19 | "fmt" 20 | "io" 21 | "log" 22 | "net" 23 | "os" 24 | "os/user" 25 | "strconv" 26 | "syscall" 27 | "unsafe" 28 | 29 | "github.com/google/testimony/go/protocol" 30 | "github.com/google/testimony/go/testimonyd/internal/vlog" 31 | ) 32 | 33 | // #include 34 | // #include 35 | import "C" 36 | 37 | // Testimony is the configuration parsed from the config file. 38 | type Testimony []SocketConfig 39 | 40 | const protocolVersion = 2 41 | 42 | // SocketConfig defines how an individual socket should be set up. 43 | type SocketConfig struct { 44 | SocketName string // filename for the socket 45 | Interface string // interface to sniff packets on 46 | BlockSize int // block size (in bytes) of a single packet block 47 | NumBlocks int // number of packet blocks in the memory region 48 | BlockTimeoutMillis int // timeout for filling up a single block 49 | FanoutType int // which type of fanout to use (see linux/if_packet.h) 50 | FanoutSize int // number of threads to fan out to 51 | FanoutID int // fanout id to avoid conflicts 52 | User, Group string // user/group to provide the socket to (will chown it) 53 | Filter string // BPF filter to apply to this socket 54 | } 55 | 56 | func (s SocketConfig) uid() (int, error) { 57 | var u *user.User 58 | var err error 59 | if s.User == "" { 60 | u, err = user.Current() 61 | } else { 62 | u, err = user.Lookup(s.User) 63 | } 64 | if err != nil { 65 | return 0, fmt.Errorf("could not get user: %v", err) 66 | } 67 | return strconv.Atoi(u.Uid) 68 | } 69 | 70 | func (s SocketConfig) gid() (int, error) { 71 | // Sadly, at present Go doesn't have group functions to match its os/user 72 | // functions... so we jump down into C. 73 | if s.Group == "" { 74 | return 0, nil 75 | } 76 | groupName := C.CString(s.Group) 77 | defer C.free(unsafe.Pointer(groupName)) 78 | var buf [2048]byte 79 | var grp C.struct_group 80 | var grpPtr *C.struct_group 81 | if _, err := C.getgrnam_r(groupName, &grp, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)), &grpPtr); err != nil { 82 | return -1, err 83 | } else if grpPtr == nil { 84 | return -1, fmt.Errorf("group %q not found", s.Group) 85 | } 86 | return int(grpPtr.gr_gid), nil 87 | } 88 | 89 | // RunTestimony runs the testimonyd server given the passed in configuration. 90 | func RunTestimony(t Testimony) { 91 | fanoutID := 0 92 | autoID := 1 93 | names := map[string]bool{} 94 | // Populate a map of manually defined IDs to avoid auto assignment conflicts. 95 | ids := map[int]bool{} 96 | for _, sc := range t { 97 | if sc.FanoutID < 0 { 98 | log.Fatalf("invalid config: %d is not a valid FanoutID", sc.FanoutID) 99 | } 100 | if ids[sc.FanoutID] { 101 | log.Fatalf("invalid config: duplicate FanoutID %d", sc.FanoutID) 102 | } 103 | if sc.FanoutID > 0 { 104 | ids[sc.FanoutID] = true 105 | } 106 | } 107 | 108 | for _, sc := range t { 109 | // Check for duplicate socket names 110 | if names[sc.SocketName] { 111 | log.Fatalf("invalid config: duplicate socket name %q", sc.SocketName) 112 | } 113 | names[sc.SocketName] = true 114 | 115 | // Set fanoutID from config or find next available id. 116 | if sc.FanoutID > 0 { 117 | fanoutID = sc.FanoutID 118 | } else { 119 | for ids[autoID] { 120 | autoID++ 121 | } 122 | fanoutID = autoID 123 | autoID++ 124 | } 125 | // Set up FanoutSize sockets and start goroutines to manage each. 126 | var socks []*socket 127 | for i := 0; i < sc.FanoutSize; i++ { 128 | sock, err := newSocket(sc, fanoutID, i) 129 | if err != nil { 130 | log.Fatalf("invalid config %+v: %v", sc, err) 131 | } 132 | socks = append(socks, sock) 133 | go sock.run() 134 | } 135 | 136 | // Set up UNIX socket to serve these AF_PACKET sockets on, and start 137 | // goroutine to manage its connections. 138 | _ignore_error_ := os.Remove(sc.SocketName) 139 | _ = _ignore_error_ 140 | list, err := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: sc.SocketName}) 141 | if err != nil { 142 | log.Fatalf("failed to listen on socket: %v", err) 143 | } else if err := setPermissions(sc); err != nil { 144 | log.Fatalf("failed to set socket permissions: %v", err) 145 | } 146 | go t.run(list, sc, socks) 147 | } 148 | // We'd love to drop privs here, but thanks to 149 | // https://github.com/golang/go/issues/1435 we can't :( 150 | select {} // Block (serving) forever. 151 | } 152 | 153 | func setPermissions(sc SocketConfig) error { 154 | uid, err := sc.uid() 155 | if err != nil { 156 | return fmt.Errorf("could not get uid to change to: %v", err) 157 | } 158 | gid, err := sc.gid() 159 | if err != nil { 160 | return fmt.Errorf("could not get gid to change to: %v", err) 161 | } 162 | vlog.V(1, "chowning %q to %d/%d", sc.SocketName, uid, gid) 163 | if err := syscall.Chown(sc.SocketName, uid, gid); err != nil { 164 | return fmt.Errorf("unable to chown to (%d, 0): %v", uid, err) 165 | } 166 | return nil 167 | } 168 | 169 | func (t Testimony) run(list *net.UnixListener, sc SocketConfig, socks []*socket) { 170 | for { 171 | c, err := list.AcceptUnix() 172 | if err != nil { 173 | log.Fatalf("failed to accept connection: %v", err) 174 | } 175 | go t.handle(socks, c) 176 | } 177 | } 178 | 179 | func (t Testimony) handle(socks []*socket, c *net.UnixConn) { 180 | defer func() { 181 | if c != nil { 182 | c.Close() 183 | } 184 | }() 185 | connStr := c.RemoteAddr().String() 186 | log.Printf("Received new connection %q", connStr) 187 | var version [1]byte 188 | version[0] = protocolVersion 189 | if _, err := c.Write(version[:]); err != nil { 190 | log.Printf("new conn %q failed to write version: %v", connStr, err) 191 | return 192 | } 193 | conf := socks[0].conf 194 | if err := protocol.SendUint32(c, protocol.TypeFanoutSize, uint32(len(socks))); err != nil { 195 | log.Printf("new conn %q failed to send fanout size: %v", connStr, err) 196 | return 197 | } 198 | if err := protocol.SendUint32(c, protocol.TypeBlockSize, uint32(conf.BlockSize)); err != nil { 199 | log.Printf("new conn %q failed to send block size: %v", connStr, err) 200 | return 201 | } 202 | if err := protocol.SendUint32(c, protocol.TypeNumBlocks, uint32(conf.NumBlocks)); err != nil { 203 | log.Printf("new conn %q failed to send number of blocks: %v", connStr, err) 204 | return 205 | } 206 | if err := protocol.SendType(c, protocol.TypeWaitingForFanoutIndex); err != nil { 207 | log.Printf("new conn %q failed to send wait: %v", connStr, err) 208 | return 209 | } 210 | var fanoutMsg [8]byte 211 | if _, err := io.ReadFull(c, fanoutMsg[:]); err == io.EOF { 212 | log.Printf("new conn %q closed early, probably just gathering connection data", connStr) 213 | return 214 | } else if err != nil { 215 | log.Printf("new conn %q failed to read fanout index: %v", connStr, err) 216 | return 217 | } 218 | valA, valB := binary.BigEndian.Uint32(fanoutMsg[:4]), binary.BigEndian.Uint32(fanoutMsg[4:]) 219 | if valA != protocol.ToTL(protocol.TypeFanoutIndex, 4) { 220 | log.Printf("new conn %q got unexpected type/value waiting for fanout message: %d/%d", connStr, valA>>16, valA&0xFFFF) 221 | return 222 | } 223 | idx := int(valB) 224 | if idx < 0 || idx >= len(socks) { 225 | log.Printf("new conn %q invalid index %v", connStr, idx) 226 | return 227 | } 228 | sock := socks[idx] 229 | fdMsg := syscall.UnixRights(sock.fd) 230 | var msg [1]byte // dummy byte 231 | n, n2, err := c.WriteMsgUnix( 232 | msg[:], fdMsg, nil) 233 | if err != nil || n != len(msg) || n2 != len(fdMsg) { 234 | log.Printf("new conn %q failed to send file descriptor: %v", connStr, err) 235 | return 236 | } 237 | vlog.V(2, "new conn %q spun up, passing off to socket", connStr) 238 | sock.newConns <- c 239 | c = nil // so it doesn't get closed by deferred func. 240 | } 241 | -------------------------------------------------------------------------------- /go/testimony/testimony.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package testimony provides a method for sharing AF_PACKET memory regions 16 | // across multiple processes. 17 | package testimony 18 | 19 | // #include 20 | import "C" 21 | 22 | import ( 23 | "crypto/rand" 24 | "encoding/binary" 25 | "encoding/hex" 26 | "fmt" 27 | "io" 28 | "net" 29 | "os" 30 | "path/filepath" 31 | "syscall" 32 | "unsafe" 33 | 34 | "github.com/google/testimony/go/protocol" 35 | ) 36 | 37 | const protocolVersion = 2 38 | 39 | func localSocketName() string { 40 | var randbytes [8]byte 41 | if n, err := rand.Read(randbytes[:]); err != nil || n != len(randbytes) { 42 | panic("random bytes failure") 43 | } 44 | return filepath.Join(os.TempDir(), fmt.Sprintf("testimony_go_client_%s", hex.EncodeToString(randbytes[:]))) 45 | } 46 | 47 | // Conn is a connection to the testimonyd server. It allows the current process 48 | // to share testimonyd AF_PACKET sockets. 49 | type Conn struct { 50 | c *net.UnixConn 51 | fd int 52 | ring []byte 53 | 54 | numBlocks int 55 | blockSize int 56 | fanoutSize int 57 | } 58 | 59 | func (c *Conn) NumBlocks() int { return c.numBlocks } 60 | func (c *Conn) BlockSize() int { return c.blockSize } 61 | func (c *Conn) FanoutSize() int { return c.fanoutSize } 62 | 63 | // Close closes the connection to the testimonyd server. 64 | func (t *Conn) Close() (ret error) { 65 | if t.ring != nil { 66 | if err := syscall.Munmap(t.ring); err != nil { 67 | ret = err 68 | } else { 69 | t.ring = nil 70 | } 71 | } 72 | if t.fd != 0 { 73 | if err := syscall.Close(t.fd); err != nil { 74 | ret = err 75 | } else { 76 | t.fd = 0 77 | } 78 | } 79 | if t.c != nil { 80 | if err := t.c.Close(); err != nil { 81 | ret = err 82 | } else { 83 | t.c = nil 84 | } 85 | } 86 | return 87 | } 88 | 89 | // Block is an AF_PACKET TPACKETv3 block, and provides access to the packets in 90 | // that block. 91 | type Block struct { 92 | t *Conn 93 | i int 94 | B []byte 95 | offset int 96 | left int 97 | pkt *C.struct_tpacket3_hdr 98 | } 99 | 100 | // Connect connects to the testimonyd server. 101 | func Connect(socketname string) (*Conn, error) { 102 | t := &Conn{} 103 | done := false 104 | defer func() { 105 | if !done { 106 | t.Close() 107 | } 108 | }() 109 | var err error 110 | t.c, err = net.DialUnix("unix", 111 | &net.UnixAddr{Net: "unix", Name: localSocketName()}, 112 | &net.UnixAddr{Net: "unix", Name: socketname}) 113 | if err != nil { 114 | return nil, fmt.Errorf("error connecting: %v", err) 115 | } 116 | var version [1]byte 117 | if _, err := io.ReadFull(t.c, version[:]); err != nil { 118 | return nil, fmt.Errorf("error reading initial byte: %v", err) 119 | } else if version[0] != protocolVersion { 120 | return nil, fmt.Errorf("protocol mismatch, want %v got %v", protocolVersion, version[0]) 121 | } 122 | var buf [4]byte 123 | tlvLoop: 124 | for { 125 | if _, err := io.ReadFull(t.c, buf[:]); err != nil { 126 | return nil, fmt.Errorf("reading initial TLV: %v", err) 127 | } 128 | typ, length := protocol.TLFrom(binary.BigEndian.Uint32(buf[:])) 129 | if protocol.TypeOf(typ) != protocol.TypeServerToClient { 130 | return nil, fmt.Errorf("bad initial type %d", typ) 131 | } 132 | val := make([]byte, int(length)) 133 | if _, err := io.ReadFull(t.c, val); err != nil { 134 | return nil, fmt.Errorf("reading initial val: %v", err) 135 | } 136 | switch typ { 137 | case protocol.TypeWaitingForFanoutIndex: 138 | break tlvLoop 139 | case protocol.TypeFanoutSize: 140 | if length != 4 { 141 | return nil, fmt.Errorf("invalid fanout size length %d", length) 142 | } 143 | t.fanoutSize = int(binary.BigEndian.Uint32(val)) 144 | case protocol.TypeBlockSize: 145 | if length != 4 { 146 | return nil, fmt.Errorf("invalid block size length %d", length) 147 | } 148 | t.blockSize = int(binary.BigEndian.Uint32(val)) 149 | case protocol.TypeNumBlocks: 150 | if length != 4 { 151 | return nil, fmt.Errorf("invalid num blocks length %d", length) 152 | } 153 | t.numBlocks = int(binary.BigEndian.Uint32(val)) 154 | default: 155 | // ignore 156 | } 157 | } 158 | if t.fanoutSize <= 0 || t.blockSize <= 0 || t.numBlocks <= 0 { 159 | return nil, fmt.Errorf("missing fanout/block size or num blocks") 160 | } 161 | done = true 162 | return t, nil 163 | } 164 | 165 | func (t *Conn) Init(fanoutIndex int) (err error) { 166 | // TODO: Parse fanout size, allow client to chose fanout number based on it. 167 | defer func() { 168 | if err != nil { 169 | t.Close() 170 | } 171 | }() 172 | if err := protocol.SendUint32(t.c, protocol.TypeFanoutIndex, uint32(fanoutIndex)); err != nil { 173 | return fmt.Errorf("error writing fanout index: %v", err) 174 | } 175 | var msg [1]byte 176 | var oob [1024]byte 177 | n, n2, _, _, err := t.c.ReadMsgUnix(msg[:], oob[:]) 178 | if err != nil { 179 | return fmt.Errorf("error reading fd: %v", err) 180 | } else if n != len(msg) { 181 | return fmt.Errorf("got wrong number of initial bytes: %d", n) 182 | } else if n2 >= len(oob) { 183 | return fmt.Errorf("got too many oob bytes: %d", n2) 184 | } 185 | if msgs, err := syscall.ParseSocketControlMessage(oob[:n2]); err != nil { 186 | return fmt.Errorf("could not parse socket control msg: %v", err) 187 | } else if len(msgs) != 1 { 188 | return fmt.Errorf("wrong number of control messages: %d", len(msgs)) 189 | } else if fds, err := syscall.ParseUnixRights(&msgs[0]); err != nil { 190 | return fmt.Errorf("could not parse unix rights: %v", err) 191 | } else if len(fds) != 1 { 192 | return fmt.Errorf("wrong number of fds: %d", len(fds)) 193 | } else { 194 | t.fd = fds[0] 195 | } 196 | if t.ring, err = syscall.Mmap(t.fd, 0, t.blockSize*t.numBlocks, syscall.PROT_READ, syscall.MAP_SHARED|syscall.MAP_NORESERVE); err != nil { 197 | return fmt.Errorf("mmap failed: %v", err) 198 | } 199 | return nil 200 | } 201 | 202 | // Block gets the next block of packets from testimonyd. 203 | func (t *Conn) Block() (*Block, error) { 204 | var idx int 205 | readLoop: 206 | for { 207 | var m [4]byte 208 | if _, err := io.ReadFull(t.c, m[:]); err != nil { 209 | return nil, fmt.Errorf("error reading block index: %v", err) 210 | } 211 | num := binary.BigEndian.Uint32(m[:]) 212 | typ, length := protocol.TLFrom(num) 213 | switch protocol.TypeOf(typ) { 214 | case protocol.TypeBlockIndex: 215 | idx = int(num) 216 | break readLoop 217 | case protocol.TypeServerToClient: 218 | if _, err := io.ReadFull(t.c, make([]byte, int(length))); err != nil { 219 | return nil, fmt.Errorf("error reading type %d value of length %d: %v", typ, length, err) 220 | } 221 | default: 222 | return nil, fmt.Errorf("received non-server-to-client message: %d", typ) 223 | } 224 | } 225 | if idx < 0 || idx >= t.numBlocks { 226 | return nil, fmt.Errorf("read invalid index %d", idx) 227 | } 228 | start := idx * t.blockSize 229 | return &Block{ 230 | t: t, 231 | i: idx, 232 | B: t.ring[start : start+t.blockSize], 233 | }, nil 234 | } 235 | 236 | // Return returns this block to the testimonyd server. 237 | func (b *Block) Return() error { 238 | var m [4]byte 239 | binary.BigEndian.PutUint32(m[:], uint32(b.i)) 240 | if _, err := b.t.c.Write(m[:]); err != nil { 241 | return fmt.Errorf("error writing index: %v", err) 242 | } 243 | b.t, b.i, b.B = nil, 0, nil 244 | return nil 245 | } 246 | 247 | func (b *Block) header() *C.struct_tpacket_hdr_v1 { 248 | desc := (*C.struct_tpacket_block_desc)(unsafe.Pointer(&b.B[0])) 249 | return (*C.struct_tpacket_hdr_v1)(unsafe.Pointer(&desc.hdr[0])) 250 | } 251 | 252 | // Next allows the user to iterate through the set of packets in this Block, 253 | // changing the value returned by Packet. 254 | func (b *Block) Next() bool { 255 | if b.offset == 0 { 256 | b.left = int(b.header().num_pkts) 257 | b.offset = int(b.header().offset_to_first_pkt) 258 | } else { 259 | b.offset += int(b.pkt.tp_next_offset) 260 | } 261 | if b.left <= 0 { 262 | return false 263 | } 264 | b.left-- 265 | b.pkt = (*C.struct_tpacket3_hdr)(unsafe.Pointer(&b.B[b.offset])) 266 | return true 267 | } 268 | 269 | // Packet provides access to the current packet. Next calls change this to 270 | // point to the next packet in the block. 271 | func (b *Block) Packet() *C.struct_tpacket3_hdr { 272 | return b.pkt 273 | } 274 | 275 | // PacketData provides access to the current packet's data. 276 | func (b *Block) PacketData() []byte { 277 | if b.pkt == nil { 278 | return nil 279 | } 280 | start := b.offset + int(b.pkt.tp_mac) 281 | return b.B[start : start+int(b.pkt.tp_snaplen)] 282 | } 283 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /go/testimonyd/internal/socket/socket.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package socket 16 | 17 | /* 18 | #include 19 | #include 20 | #include // for C.free 21 | #include // for SOL_PACKET, getsockopt 22 | 23 | struct sock_fprog; 24 | 25 | // See comments in socket.cc 26 | int AFPacket(const char* iface, int block_size, int block_nr, int block_ms, 27 | int fanout_id, int fanout_size, int fanout_type, 28 | int filter_size, struct sock_filter* filters, 29 | // Outputs: 30 | int* fd, void** ring, const char** err); 31 | 32 | int WaitForBlocks(int sock_fd); 33 | */ 34 | import "C" 35 | 36 | import ( 37 | "bufio" 38 | "bytes" 39 | "encoding/binary" 40 | "fmt" 41 | "io" 42 | "log" 43 | "net" 44 | "os/exec" 45 | "strconv" 46 | "sync/atomic" 47 | "syscall" 48 | "time" 49 | "unsafe" 50 | 51 | "github.com/google/testimony/go/protocol" 52 | "github.com/google/testimony/go/testimonyd/internal/vlog" 53 | ) 54 | 55 | // socket handles a single AF_PACKET socket. There will be N Socket objects for 56 | // each SocketConfig, where N == FanoutSize. This Socket stores the file 57 | // descriptor and memory region of a single underlying AF_PACKET socket. 58 | type socket struct { 59 | num int // fanout index for this socket 60 | conf SocketConfig // configuration 61 | fd int // file descriptor for AF_PACKET socket 62 | newConns chan *net.UnixConn // new client connections come in here 63 | oldConns chan *conn // old client connections come in here for cleanup 64 | newBlocks chan *block // when a new block is available, it comes in here 65 | blocks []*block // all blocks in the memory region 66 | currentConns map[*conn]bool // list of current connections a new block will be sent to 67 | ring uintptr // pointer to memory region 68 | } 69 | 70 | // newSocket creates a new Socket object based on a config. 71 | func newSocket(sc SocketConfig, fanoutID int, num int) (*socket, error) { 72 | s := &socket{ 73 | num: num, 74 | conf: sc, 75 | newConns: make(chan *net.UnixConn), 76 | oldConns: make(chan *conn), 77 | newBlocks: make(chan *block, sc.NumBlocks), 78 | currentConns: map[*conn]bool{}, 79 | blocks: make([]*block, sc.NumBlocks), 80 | } 81 | 82 | // Compile the BPF filter, if it was requested. 83 | var filt *C.struct_sock_filter 84 | var filtsize C.int 85 | if sc.Filter != "" { 86 | f, err := compileFilter(sc.Interface, sc.Filter) 87 | if err != nil { 88 | return nil, fmt.Errorf("unable to compile filter %q on interface %q: %v", sc.Filter, sc.Interface, err) 89 | } 90 | filt = &f[0] 91 | filtsize = C.int(len(f)) 92 | } 93 | 94 | // Set up block objects, used to reference count blocks for clients. 95 | for i := 0; i < sc.NumBlocks; i++ { 96 | s.blocks[i] = &block{s: s, index: i} 97 | } 98 | 99 | // Call into our C code to actually create the socket. 100 | iface := C.CString(sc.Interface) 101 | defer C.free(unsafe.Pointer(iface)) 102 | var fd C.int 103 | var ring unsafe.Pointer 104 | var errStr *C.char 105 | if _, err := C.AFPacket(iface, C.int(sc.BlockSize), C.int(sc.NumBlocks), 106 | C.int(sc.BlockTimeoutMillis), C.int(fanoutID), C.int(sc.FanoutSize), C.int(sc.FanoutType), 107 | filtsize, filt, 108 | &fd, &ring, &errStr); err != nil { 109 | return nil, fmt.Errorf("C AFPacket call failed: %v: %v", C.GoString(errStr), err) 110 | } 111 | s.fd = int(fd) 112 | s.ring = uintptr(ring) 113 | log.Printf("%v set up with %+v", s, sc) 114 | return s, nil 115 | } 116 | 117 | // String returns a unique string for this socket. 118 | func (s *socket) String() string { 119 | return fmt.Sprintf("[S:%v:%v]", s.conf.SocketName, s.num) 120 | } 121 | 122 | // getNewBlocks is a goroutine that watches for new available packet blocks, 123 | // which the run() method passes to clients. 124 | func (s *socket) getNewBlocks() { 125 | blockIndex := 0 126 | for { 127 | b := s.blocks[blockIndex] 128 | for !b.ready() { 129 | if err := C.WaitForBlocks(C.int(s.fd)); err != 0 { 130 | log.Panicf("C WaitForBlocks failed: %s", syscall.Errno(err).Error()) 131 | } 132 | } 133 | b.ref() 134 | vlog.V(3, "%v got new block %v", s, b) 135 | s.newBlocks <- b 136 | blockIndex = (blockIndex + 1) % s.conf.NumBlocks 137 | } 138 | } 139 | 140 | func (s *socket) reportStats() { 141 | var totalPackets, totalDrops uint64 142 | // getting statistics returns the stats since the last invocation. We clear 143 | // counters by doing an initial read we ignore. 144 | s.stats() 145 | const seconds = 60 146 | for range time.Tick(time.Second * seconds) { 147 | // Output stats to log on each full round of a ring. 148 | stats, err := s.stats() 149 | if err != nil { 150 | log.Printf("error getting statistics: %v", err) 151 | } else { 152 | totalPackets += uint64(stats.tp_packets) 153 | totalDrops += uint64(stats.tp_drops) 154 | vlog.V(1, "%v stats: %d packets (%.02fpps), %d drops (%.02fpps) (%.02f%% dropped) since last log, %d packets, %d drops total (%.02f%% dropped)", s, 155 | stats.tp_packets, float64(stats.tp_packets)/seconds, stats.tp_drops, float64(stats.tp_drops)/seconds, float64(stats.tp_drops)/float64(stats.tp_drops+stats.tp_packets)*100, 156 | totalPackets, totalDrops, float64(totalDrops)/float64(totalPackets+totalDrops)*100) 157 | } 158 | } 159 | } 160 | 161 | // run handles new connections, old connections, new blocks... basically 162 | // everything. 163 | func (s *socket) run() { 164 | go func() { 165 | s.getNewBlocks() 166 | }() 167 | go s.reportStats() 168 | for { 169 | select { 170 | case c := <-s.newConns: 171 | // register a new client connection 172 | s.addNewConn(c) 173 | case c := <-s.oldConns: 174 | // unregister an old client connection and close its blocks 175 | close(c.newBlocks) 176 | delete(s.currentConns, c) 177 | case b := <-s.newBlocks: 178 | // a new block is avaiable, send it out to all clients 179 | for c, _ := range s.currentConns { 180 | b.ref() 181 | select { 182 | case c.newBlocks <- b: 183 | default: 184 | vlog.V(1, "failed to send %v to %v", b, c) 185 | b.unref() 186 | } 187 | } 188 | b.unref() 189 | } 190 | } 191 | } 192 | 193 | // conn represents a set-up client connection (already initiated and with the 194 | // file descriptor passed through). 195 | type conn struct { 196 | s *socket 197 | c *net.UnixConn 198 | newBlocks chan *block 199 | oldBlocks chan int 200 | } 201 | 202 | // String returns a unique string for this connection. 203 | func (c *conn) String() string { 204 | return fmt.Sprintf("[C:%v:%v]", c.s, c.c.RemoteAddr()) 205 | } 206 | 207 | // handleReads handles client->server communication. 208 | func (c *conn) handleReads() { 209 | defer close(c.oldBlocks) 210 | for { 211 | // Wait for a block index to be passed back from the client. 212 | var buf [4]byte 213 | n, err := c.c.Read(buf[:]) 214 | if err == io.EOF { 215 | return 216 | } else if err != nil || n != len(buf) { 217 | vlog.V(1, "%v read error (%d bytes): %v", c, n, err) 218 | return 219 | } 220 | num := binary.BigEndian.Uint32(buf[:]) 221 | if num&0x80000000 != 0 { 222 | typ, length := protocol.TLFrom(num) 223 | if protocol.TypeOf(typ) != protocol.TypeClientToServer { 224 | vlog.V(1, "%v client sent bad type %d", c, typ) 225 | } 226 | var val []byte 227 | if length != 0 { 228 | val = make([]byte, length) 229 | if _, err := io.ReadFull(c.c, val); err != nil { 230 | vlog.V(2, "%v read TLV %d length %d: %v", c, typ, length, err) 231 | return 232 | } 233 | } 234 | if err := c.handleTLV(typ, val); err != nil { 235 | vlog.V(2, "%v handling type %d: %v", c, typ, val) 236 | return 237 | } 238 | } else { 239 | i := int(num) 240 | if i < 0 || i >= c.s.conf.NumBlocks { 241 | log.Printf("%v got invalid block %d", c, i) 242 | return 243 | } 244 | // We add one to the returned int so we can detect a closed channel (which 245 | // returns 0, the zero-value for ints). 246 | c.oldBlocks <- i + 1 247 | } 248 | } 249 | } 250 | 251 | func (c *conn) handleTLV(typ protocol.Type, val []byte) error { 252 | log.Printf("IGNORING TLV: %d = %x", typ, val) 253 | return nil 254 | } 255 | 256 | // run handles communicating with a single external client via a single 257 | // connection. It maintains the invariant that every block it gets via the 258 | // newBlocks channel will be unref'd exactly once. It's up to the block sender 259 | // to ref the blocks for the conn. 260 | func (c *conn) run() { 261 | go c.handleReads() 262 | outstanding := make([]time.Time, len(c.s.blocks)) 263 | 264 | // Wait for either the reader or writer to stop. 265 | var out []byte 266 | loop: 267 | for { 268 | select { 269 | case b := <-c.newBlocks: 270 | out = out[:0] 271 | vlog.V(2, "%v writing %v", c, b) 272 | blockLoop: 273 | for { 274 | if !outstanding[b.index].IsZero() { 275 | log.Fatalf("%v received already outstanding block %v", c, b) 276 | } 277 | outstanding[b.index] = time.Now() 278 | idx := len(out) 279 | out = append(out, 0, 0, 0, 0) 280 | binary.BigEndian.PutUint32(out[idx:], uint32(b.index)) 281 | select { 282 | case b = <-c.newBlocks: 283 | vlog.V(2, "%v batching %v", c, b) 284 | default: 285 | break blockLoop 286 | } 287 | } 288 | if _, err := c.c.Write(out); err != nil { 289 | vlog.V(1, "%v write error: %v", c, err) 290 | break loop 291 | } 292 | case i := <-c.oldBlocks: 293 | if i == 0 { 294 | // read loop is closed 295 | break loop 296 | } 297 | i-- // We added 1 to index in handleReads, remove 1 to get back to correct index. 298 | if outstanding[i].IsZero() { 299 | log.Printf("%v received non-outstanding block %v from client", c, i) 300 | break loop 301 | } 302 | b := c.s.blocks[i] 303 | vlog.V(3, "%v took %v to process block %v", c, time.Since(outstanding[i]), b) 304 | outstanding[i] = time.Time{} 305 | b.unref() // MOST IMPORTANT LINE EVER 306 | } 307 | } 308 | 309 | // Close things down. 310 | log.Printf("Connection %v closing", c) 311 | c.c.Close() 312 | vlog.V(3, "%v marking self old", c) 313 | c.s.oldConns <- c 314 | vlog.V(3, "%v waiting for reads", c) 315 | for b := range c.newBlocks { 316 | vlog.V(3, "%v returning unsent %v", c, b) 317 | b.unref() 318 | } 319 | // empty out oldBlocks to allow handleReads to finish, but don't do anything 320 | // with them. the next loop (over outstanding) will unref and return all 321 | // remaining blocks. 322 | for _ = range c.oldBlocks { 323 | } 324 | for i, t := range outstanding { 325 | if !t.IsZero() { 326 | b := c.s.blocks[i] 327 | vlog.V(3, "%v returning outstanding %v after %v", c, b, time.Since(t)) 328 | b.unref() 329 | } 330 | } 331 | } 332 | 333 | // addNewConn is called by the testimonyd server when a new connection has been 334 | // initiated. The passed-in conn should already have done the initial 335 | // configuration handshake, and be ready to start receiving blocks. 336 | func (s *socket) addNewConn(c *net.UnixConn) { 337 | newConn := &conn{ 338 | s: s, 339 | c: c, 340 | newBlocks: make(chan *block, len(s.blocks)), 341 | oldBlocks: make(chan int, len(s.blocks)), 342 | } 343 | log.Printf("%v new connection %v", s, newConn) 344 | s.currentConns[newConn] = true 345 | go newConn.run() 346 | } 347 | 348 | // block stores ilocal information on a single block within the memory region. 349 | type block struct { 350 | s *socket 351 | index int // my index within the memory block 352 | 353 | r int32 // reference count for this block, uses atomic 354 | } 355 | 356 | // ref reference the block. 357 | func (b *block) ref() { 358 | refs := atomic.AddInt32(&b.r, 1) 359 | vlog.VUp(5, 1, "%v refs = %d", b, refs) 360 | } 361 | 362 | // unref dereferences the block. When the refcount reaches zero, the block is 363 | // returned to the kernel via clear(). 364 | func (b *block) unref() { 365 | refs := atomic.AddInt32(&b.r, -1) 366 | vlog.VUp(5, 1, "%v unref = %d", b, refs) 367 | if refs == 0 { 368 | b.clear() 369 | } else if refs < 0 { 370 | panic(fmt.Sprintf("invalid unref of %v to %d", b, refs)) 371 | } 372 | } 373 | 374 | // String provides a unique human-readable string. 375 | func (b *block) String() string { 376 | return fmt.Sprintf("[B:%v:%v]", b.s, b.index) 377 | } 378 | 379 | // cblock provides this block as a C tpacket pointer. 380 | func (b *block) cblock() *C.struct_tpacket_hdr_v1 { 381 | blockDesc := (*C.struct_tpacket_block_desc)(unsafe.Pointer(b.s.ring + uintptr(b.s.conf.BlockSize)*uintptr(b.index))) 382 | hdr := (*C.struct_tpacket_hdr_v1)(unsafe.Pointer(&blockDesc.hdr[0])) 383 | return hdr 384 | } 385 | 386 | // clear clears the block's block status, returning the block to the kernel so 387 | // it can add additional packets. 388 | func (b *block) clear() { 389 | vlog.VUp(3, 2, "%v clear", b) 390 | b.cblock().block_status = 0 391 | } 392 | 393 | // ready returns true when the block status has been set by the kernel, saying 394 | // that packets are ready for processing. 395 | func (b *block) ready() bool { 396 | return atomic.LoadInt32(&b.r) == 0 && b.cblock().block_status != 0 397 | } 398 | 399 | // compileFilter compiles a BPF filter, currently by calling tcpdump externally. 400 | func compileFilter(iface, filt string) ([]C.struct_sock_filter, error) { 401 | cmd := exec.Command("/usr/sbin/tcpdump", "-i", iface, "-ddd", filt) 402 | var out bytes.Buffer 403 | cmd.Stdout = &out 404 | if err := cmd.Run(); err != nil { 405 | return nil, fmt.Errorf("could not run tcpdump to compile BPF: %v", err) 406 | } 407 | ints := []int{} 408 | scanner := bufio.NewScanner(&out) 409 | scanner.Split(bufio.ScanWords) 410 | for scanner.Scan() { 411 | i, err := strconv.Atoi(scanner.Text()) 412 | if err != nil { 413 | return nil, fmt.Errorf("error scanning token %q: %v", scanner.Text(), err) 414 | } 415 | ints = append(ints, i) 416 | } 417 | if len(ints) == 0 || len(ints) != ints[0]*4+1 { 418 | return nil, fmt.Errorf("invalid length of tcpdump ints") 419 | } 420 | ints = ints[1:] 421 | var bpfs []C.struct_sock_filter 422 | for i := 0; i < len(ints); i += 4 { 423 | bpfs = append(bpfs, C.struct_sock_filter{ 424 | code: C.__u16(ints[i]), 425 | jt: C.__u8(ints[i+1]), 426 | jf: C.__u8(ints[i+2]), 427 | k: C.__u32(ints[i+3]), 428 | }) 429 | } 430 | return bpfs, nil 431 | } 432 | 433 | func (s *socket) stats() (*C.struct_tpacket_stats_v3, error) { 434 | var out C.struct_tpacket_stats_v3 435 | size := C.socklen_t(unsafe.Sizeof(out)) 436 | if _, err := C.getsockopt(C.int(s.fd), C.SOL_PACKET, C.PACKET_STATISTICS, unsafe.Pointer(&out), &size); err != nil { 437 | return nil, err 438 | } 439 | return &out, nil 440 | } 441 | -------------------------------------------------------------------------------- /c/testimony.c: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include // socket(), connect() 16 | #include // See socket man page 17 | #include // sockaddr_un 18 | #include // errno 19 | #include // close() 20 | #include // printf() 21 | #include // mmap(), MAP_*, PROT_* 22 | #include // uint32_t 23 | #include // poll(), POLLIN, pollfd 24 | #include // malloc(), free() 25 | 26 | #include 27 | 28 | #ifdef __cplusplus 29 | extern "C" { 30 | #endif 31 | 32 | #define TESTIMONY_ERRBUF_SIZE 256 33 | #define TESTIMONY_BUF_SIZE 256 34 | 35 | /* This list generated by go/protocol/to_c binary */ 36 | #define TESTIMONY_PROTOCOL_TYPE_BlockIndex 0 37 | #define TESTIMONY_PROTOCOL_TYPE_ServerToClient 32769 38 | #define TESTIMONY_PROTOCOL_TYPE_WaitingForFanoutIndex 32770 39 | #define TESTIMONY_PROTOCOL_TYPE_FanoutSize 32771 40 | #define TESTIMONY_PROTOCOL_TYPE_BlockSize 32772 41 | #define TESTIMONY_PROTOCOL_TYPE_NumBlocks 32773 42 | #define TESTIMONY_PROTOCOL_TYPE_ClientToServer 49158 43 | #define TESTIMONY_PROTOCOL_TYPE_FanoutIndex 49159 44 | #define TESTIMONY_PROTOCOL_TYPE_Error 65535 45 | 46 | struct testimony_internal { 47 | testimony_connection conn; 48 | int sock_fd; 49 | int afpacket_fd; 50 | uint8_t* ring; 51 | char errbuf[TESTIMONY_ERRBUF_SIZE]; 52 | uint32_t* block_counts; 53 | uint8_t buf[TESTIMONY_BUF_SIZE]; 54 | uint8_t* buf_start; 55 | uint8_t* buf_limit; 56 | }; 57 | 58 | static uint32_t protocol_type(uint32_t x) { 59 | if ((x & 0x80000000) == 0) { return TESTIMONY_PROTOCOL_TYPE_BlockIndex; } 60 | return x >> 16; 61 | } 62 | 63 | static uint32_t protocol_length(uint32_t x) { 64 | if ((x & 0x80000000) == 0) { return 0; } 65 | return x & 0xFFFF; 66 | } 67 | 68 | static void set_be_32(uint8_t* a, uint32_t x) { 69 | a[0] = x >> 24; 70 | a[1] = x >> 16; 71 | a[2] = x >> 8; 72 | a[3] = x; 73 | } 74 | #define TERR(msg, ...) \ 75 | do { \ 76 | int err = errno; \ 77 | snprintf(t->errbuf, TESTIMONY_ERRBUF_SIZE, \ 78 | msg ": %s", ##__VA_ARGS__, strerror(err)); \ 79 | errno = err; \ 80 | } while (0) 81 | #define TERR_SET(errno_val, msg, ...) \ 82 | do { \ 83 | errno = errno_val; \ 84 | TERR(msg, ##__VA_ARGS__); \ 85 | } while (0) 86 | static int recv_t(testimony t, uint8_t* out, size_t s) { 87 | int r; 88 | uint8_t* out_limit = out + s; 89 | size_t to_copy_out, to_copy_buf, to_copy; 90 | while (out < out_limit) { 91 | while (t->buf_start >= t->buf_limit) { 92 | t->buf_start = t->buf; 93 | t->buf_limit = t->buf; 94 | r = recv(t->sock_fd, t->buf_start, TESTIMONY_BUF_SIZE, 0); 95 | if (r == 0) { 96 | errno = ECANCELED; 97 | return -1; 98 | } else if (r < 0) { 99 | if (errno == EINTR) { continue; } 100 | return -1; 101 | } 102 | t->buf_limit = t->buf_start + r; 103 | } 104 | to_copy_out = out_limit - out; 105 | to_copy_buf = t->buf_limit - t->buf_start; 106 | to_copy = to_copy_out < to_copy_buf ? to_copy_out : to_copy_buf; 107 | memcpy(out, t->buf_start, to_copy); 108 | out += to_copy; 109 | t->buf_start += to_copy; 110 | } 111 | return 0; 112 | } 113 | static int recv_be_32(testimony t, uint32_t* out) { 114 | uint8_t msg[4]; 115 | int r, i; 116 | r = recv_t(t, msg, 4); 117 | if (r < 0) { return r; } 118 | *out = 0; 119 | for (i = 0; i < 4; i++) { 120 | *out = (*out << 8) + msg[i]; 121 | } 122 | return 0; 123 | } 124 | 125 | static int send_be_32(int fd, size_t in) { 126 | uint8_t msg[4]; 127 | uint8_t* readfrom = msg; 128 | uint8_t* limit = msg + 4; 129 | int r; 130 | set_be_32(msg, in); 131 | while (readfrom < limit) { 132 | r = send(fd, readfrom, limit - readfrom, 0); 133 | if (r < 0) { 134 | return -1; 135 | } 136 | readfrom += r; 137 | } 138 | return 0; 139 | } 140 | 141 | // With much thanks to 142 | // http://blog.varunajayasiri.com/passing-file-descriptors-between-processes-using-sendmsg-and-recvmsg 143 | static int recv_file_descriptor(int socket) { 144 | struct msghdr message; 145 | struct iovec iov[1]; 146 | struct cmsghdr* control_message = NULL; 147 | uint8_t ctrl_buf[CMSG_SPACE(sizeof(int))]; 148 | uint8_t data[1]; 149 | int r; 150 | 151 | memset(&message, 0, sizeof(struct msghdr)); 152 | memset(ctrl_buf, 0, CMSG_SPACE(sizeof(int))); 153 | 154 | /* For the block data */ 155 | iov[0].iov_base = data; 156 | iov[0].iov_len = sizeof(data); 157 | 158 | message.msg_name = NULL; 159 | message.msg_namelen = 0; 160 | message.msg_control = ctrl_buf; 161 | message.msg_controllen = CMSG_SPACE(sizeof(int)); 162 | message.msg_iov = iov; 163 | message.msg_iovlen = 1; 164 | 165 | r = recvmsg(socket, &message, 0); 166 | if (r != sizeof(data)) { 167 | return -1; 168 | } 169 | 170 | /* Iterate through header to find if there is a file descriptor */ 171 | for (control_message = CMSG_FIRSTHDR(&message); control_message != NULL; 172 | control_message = CMSG_NXTHDR(&message, control_message)) { 173 | if ((control_message->cmsg_level == SOL_SOCKET) && 174 | (control_message->cmsg_type == SCM_RIGHTS)) { 175 | return *((int*)CMSG_DATA(control_message)); 176 | } 177 | } 178 | 179 | errno = EIO; 180 | return -1; 181 | } 182 | 183 | static int discard_bytes(testimony t, size_t n) { 184 | uint8_t val[TESTIMONY_BUF_SIZE]; 185 | size_t len; 186 | int r; 187 | while (n) { 188 | len = n > TESTIMONY_BUF_SIZE ? TESTIMONY_BUF_SIZE : n; 189 | r = recv_t(t, val, len); 190 | if (r < 0) { 191 | TERR("recv of val length %d", (int)n); 192 | return r; 193 | } 194 | n -= len; 195 | } 196 | return 0; 197 | } 198 | 199 | int testimony_connect(testimony* tp, const char* socket_name) { 200 | struct sockaddr_un saddr; 201 | sa_family_t laddr = AF_UNIX; // Use unnamed socket on client side 202 | int r, err; 203 | uint8_t version; 204 | uint32_t msg, proto_typ, proto_len; 205 | testimony t = (testimony)malloc(sizeof(struct testimony_internal)); 206 | if (t == NULL) { 207 | return -ENOMEM; 208 | } 209 | memset(t, 0, sizeof(*t)); 210 | 211 | saddr.sun_family = AF_UNIX; 212 | strncpy(saddr.sun_path, socket_name, sizeof(saddr.sun_path) - 1); 213 | saddr.sun_path[sizeof(saddr.sun_path) - 1] = 0; 214 | 215 | t->sock_fd = socket(AF_UNIX, SOCK_STREAM, 0); 216 | if (t->sock_fd < 0) { 217 | TERR("socket creation failed"); 218 | goto fail; 219 | } 220 | r = bind(t->sock_fd, (struct sockaddr*)&laddr, sizeof(laddr)); 221 | if (r < 0) { 222 | TERR("bind failed"); 223 | goto fail; 224 | } 225 | 226 | r = connect(t->sock_fd, (struct sockaddr*)&saddr, sizeof(saddr)); 227 | if (r < 0) { 228 | TERR("connect to '%s' failed", saddr.sun_path); 229 | goto fail; 230 | } 231 | r = recv_t(t, &version, 1); 232 | if (r < 0) { 233 | TERR("recv of protocol version failed"); 234 | goto fail; 235 | } else if (version != TESTIMONY_VERSION) { 236 | TERR_SET(EPROTONOSUPPORT, "received unsupported protocol version %d", version); 237 | goto fail; 238 | } 239 | proto_typ = TESTIMONY_PROTOCOL_TYPE_Error; 240 | while (1) { 241 | r = recv_be_32(t, &msg); 242 | if (r < 0) { 243 | TERR("did not receive fanout size"); 244 | goto fail; 245 | } 246 | proto_typ = protocol_type(msg); 247 | proto_len = protocol_length(msg); 248 | if (proto_typ == TESTIMONY_PROTOCOL_TYPE_WaitingForFanoutIndex && proto_len == 0) { 249 | break; 250 | } 251 | if (proto_len == 4) { 252 | r = recv_be_32(t, &msg); 253 | if (r < 0) { 254 | TERR("did not receive type %d uint32 value", proto_typ); 255 | goto fail; 256 | } 257 | switch (proto_typ) { 258 | case TESTIMONY_PROTOCOL_TYPE_FanoutSize: 259 | t->conn.fanout_size = msg; 260 | break; 261 | case TESTIMONY_PROTOCOL_TYPE_BlockSize: 262 | t->conn.block_size = msg; 263 | break; 264 | case TESTIMONY_PROTOCOL_TYPE_NumBlocks: 265 | t->conn.block_nr = msg; 266 | break; 267 | default: 268 | // ignore 269 | break; 270 | } 271 | } else { 272 | r = discard_bytes(t, proto_len); 273 | if (r < 0) { 274 | TERR("did not discard type %d value", proto_typ); 275 | goto fail; 276 | } 277 | } 278 | } 279 | if (t->conn.fanout_size == 0 || t->conn.block_size == 0 || t->conn.block_nr == 0) { 280 | TERR_SET(EINVAL, "didn't get fanout size and block size/nr"); 281 | goto fail; 282 | } 283 | *tp = t; 284 | return 0; 285 | fail: 286 | err = errno; 287 | testimony_close(t); 288 | return -err; 289 | } 290 | 291 | testimony_connection* testimony_conn(testimony t) { return &t->conn; } 292 | 293 | int testimony_init(testimony t) { 294 | uint8_t msg[4]; 295 | int r; 296 | if (t->ring) { 297 | TERR_SET(EINVAL, "testimony has already been initiated"); 298 | return -errno; 299 | } 300 | set_be_32(msg, (TESTIMONY_PROTOCOL_TYPE_FanoutIndex << 16) + 4); 301 | r = send(t->sock_fd, &msg, sizeof(msg), 0); 302 | if (r < 0) { 303 | TERR("send of fanout index type"); 304 | return -errno; 305 | } 306 | set_be_32(msg, t->conn.fanout_index); 307 | r = send(t->sock_fd, &msg, sizeof(msg), 0); 308 | if (r < 0) { 309 | TERR("send of fanout index failed"); 310 | return -errno; 311 | } 312 | 313 | t->afpacket_fd = recv_file_descriptor(t->sock_fd); 314 | if (t->afpacket_fd < 0) { 315 | TERR("recv of file descriptor failed"); 316 | return -errno; 317 | } 318 | 319 | // calloc inits memory to zero 320 | t->block_counts = (uint32_t*)calloc(t->conn.block_nr, sizeof(uint32_t)); 321 | 322 | t->ring = mmap(NULL, t->conn.block_size * t->conn.block_nr, PROT_READ, 323 | MAP_SHARED | MAP_NORESERVE, t->afpacket_fd, 0); 324 | if (t->ring == MAP_FAILED) { 325 | t->ring = 0; 326 | TERR("local mmap of file descriptor failed"); 327 | return -errno; 328 | } 329 | return 0; 330 | } 331 | 332 | int testimony_close(testimony t) { 333 | if (t->ring != 0) { 334 | if (munmap(t->ring, t->conn.block_nr * t->conn.block_size) < 0) 335 | return -errno; 336 | } 337 | if (close(t->sock_fd) < 0) return -errno; 338 | free(t->block_counts); 339 | free(t); 340 | return 0; 341 | } 342 | 343 | int testimony_get_block(testimony t, int timeout_millis, 344 | const struct tpacket_block_desc** block) { 345 | // TODO: Currently, testimony_get_block may block for longer than 346 | // timeout_millis, if either: 347 | // 1) the poll() is interrupted with a signal and restarts (EINTR handling) 348 | // 2) the recv (in recv_t) is unblocked by poll, but blocks before it can 349 | // read in all the bytes it needs 350 | // 3) we block on the poll, read in a TLV, then block again 351 | // In short, we're really bad at using timeout_millis... should probably do 352 | // something about that. 353 | struct pollfd pfd; 354 | uint32_t blockidx, old_count; 355 | int r; 356 | uint32_t typ; 357 | *block = NULL; 358 | if (t->sock_fd == 0 || t->ring == 0) { 359 | TERR_SET(EINVAL, "testimony is not yet initiated, run testimony_init"); 360 | return -errno; 361 | } 362 | while (1) { 363 | if (timeout_millis >= 0 && t->buf_start == t->buf_limit) { 364 | memset(&pfd, 0, sizeof(pfd)); 365 | pfd.fd = t->sock_fd; 366 | pfd.events = POLLIN; 367 | r = poll(&pfd, 1, timeout_millis); 368 | if (r < 0) { 369 | if (errno == EINTR) { continue; } 370 | TERR("testimony poll of socket failed"); 371 | return -errno; 372 | } 373 | if (r == 0) { 374 | return 0; // Timed out, no block ready yet. 375 | } 376 | // A read is ready, fall through. 377 | } 378 | r = recv_be_32(t, &blockidx); 379 | if (r < 0) { 380 | TERR("recv of block index failed"); 381 | return -errno; 382 | } 383 | typ = protocol_type(blockidx); 384 | if (typ == TESTIMONY_PROTOCOL_TYPE_BlockIndex) { 385 | break; 386 | } 387 | r = discard_bytes(t, protocol_length(blockidx)); 388 | if (r < 0) { 389 | TERR("recv of block index failed"); 390 | return -errno; 391 | } 392 | } 393 | if (blockidx >= t->conn.block_nr) { 394 | TERR_SET(EIO, "received invalid block index %d, should be [0, %d)", 395 | (int)blockidx, (int)t->conn.block_nr); 396 | return -errno; 397 | } 398 | *block = 399 | (const struct tpacket_block_desc*)(t->ring + t->conn.block_size * blockidx); 400 | #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 401 | if ((old_count = __sync_val_compare_and_swap( 402 | t->block_counts + blockidx, 0, (*block)->hdr.bh1.num_pkts)) != 0) { 403 | TERR_SET(EIO, "block count CAS failed for block %d, current count %d != 0", 404 | (int)blockidx, (int)old_count); 405 | return -errno; 406 | } 407 | #endif 408 | return 0; 409 | } 410 | 411 | static const uint32_t kInvalidBlockIndex = 0xFFFFFFFF; 412 | 413 | static uint32_t testimony_block_index(testimony t, const struct tpacket_block_desc* block) { 414 | size_t blockptr = (size_t)block; 415 | blockptr -= (size_t)t->ring; 416 | blockptr /= t->conn.block_size; 417 | if (blockptr >= t->conn.block_nr) { 418 | return kInvalidBlockIndex; 419 | } 420 | return blockptr; 421 | } 422 | 423 | int testimony_return_block(testimony t, const struct tpacket_block_desc* block) { 424 | int r; 425 | uint32_t old_count; 426 | uint32_t blockidx = testimony_block_index(t, block); 427 | if (blockidx == kInvalidBlockIndex) { 428 | TERR_SET(EINVAL, "block does not appear to have come from this testimony instance"); 429 | return -errno; 430 | } 431 | #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 432 | // Set block count for this block to zero (& with 0), and make sure the packet 433 | // count was sane. 434 | old_count = __sync_fetch_and_and(t->block_counts + blockidx, 0); 435 | if (old_count != 0 && old_count != block->hdr.bh1.num_pkts) { 436 | TERR_SET(EINVAL, "block count invalid... maybe testimony_return_block and " 437 | "testimony_return_packet were both called?"); 438 | return -errno; 439 | } 440 | #endif 441 | r = send_be_32(t->sock_fd, blockidx); 442 | if (r < 0) { 443 | TERR("send of block index failed"); 444 | return -errno; 445 | } 446 | return 0; 447 | } 448 | 449 | int testimony_return_packets(testimony t, const struct tpacket_block_desc* block, uint32_t packets) { 450 | #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 451 | uint32_t blockidx = testimony_block_index(t, block); 452 | uint32_t count; 453 | if (blockidx == kInvalidBlockIndex) { 454 | TERR_SET(EINVAL, "block does not appear to have come from this testimony instance"); 455 | return -errno; 456 | } 457 | count = __sync_fetch_and_sub(t->block_counts + blockidx, packets); 458 | if (count == packets) { 459 | return testimony_return_block(t, block); 460 | } else if (count < packets) { 461 | TERR_SET(EINVAL, "return_packets amount overflows number of packets"); 462 | return -errno; 463 | } 464 | return 0; 465 | #else 466 | TERR_SET(EINVAL, "compiler does not support atomics, " 467 | "testimony_return_packet not available"); 468 | return -errno; 469 | #endif 470 | } 471 | 472 | char* testimony_error(testimony t) { return t->errbuf; } 473 | 474 | struct testimony_iter_internal { 475 | const struct tpacket_block_desc* block; 476 | uint8_t* pkt; 477 | int left; 478 | }; 479 | 480 | int testimony_iter_init(testimony_iter* iter) { 481 | *iter = (testimony_iter)malloc(sizeof(struct testimony_iter_internal)); 482 | if (*iter == NULL) { 483 | return -ENOMEM; 484 | } 485 | memset(*iter, 0, sizeof(struct testimony_iter_internal)); 486 | return 0; 487 | } 488 | 489 | int testimony_iter_reset(testimony_iter iter, 490 | const struct tpacket_block_desc* block) { 491 | if (block->version != TPACKET_V3) { 492 | return -EPROTONOSUPPORT; 493 | } 494 | iter->block = block; 495 | iter->left = block->hdr.bh1.num_pkts; 496 | iter->pkt = NULL; 497 | return 0; 498 | } 499 | 500 | const struct tpacket3_hdr* testimony_iter_next(testimony_iter iter) { 501 | if (!iter->left) { 502 | return NULL; 503 | } 504 | iter->left--; 505 | if (iter->pkt) { 506 | iter->pkt += ((const struct tpacket3_hdr*)iter->pkt)->tp_next_offset; 507 | } else { 508 | iter->pkt = 509 | (uint8_t*)iter->block + iter->block->hdr.bh1.offset_to_first_pkt; 510 | } 511 | return (const struct tpacket3_hdr*)iter->pkt; 512 | } 513 | 514 | int testimony_iter_close(testimony_iter iter) { 515 | free(iter); 516 | return 0; 517 | } 518 | 519 | const uint8_t* testimony_packet_data(const struct tpacket3_hdr* pkt) { 520 | return (const uint8_t*)pkt + pkt->tp_mac; 521 | } 522 | int64_t testimony_packet_nanos(const struct tpacket3_hdr* pkt) { 523 | return (int64_t)pkt->tp_sec * 1000000000LL + pkt->tp_nsec; 524 | } 525 | 526 | #ifdef __cplusplus 527 | } 528 | #endif 529 | --------------------------------------------------------------------------------