├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── bbr └── bbr.go ├── ccp ├── ccp.go ├── handleFlow.go └── handler.go ├── ccpFlow ├── flow.go ├── flow_test.go └── pattern │ ├── pattern.go │ └── pattern_test.go ├── compound └── compound.go ├── cubic └── cubic.go ├── ipc ├── bench_test.go ├── ipc.go ├── msg.go ├── serialize.go └── serialize_test.go ├── ipcBackend ├── ipc.go └── lib_test.go ├── netlinkipc ├── netlink.go ├── nonlinux.go └── setup.go ├── nl_userapp ├── nl_userapp.go └── nonlinux.go ├── reno ├── cc_test.go └── reno.go ├── tests ├── start-test.sh ├── testClient │ └── client.go └── testServer │ └── server.go ├── udpDataplane ├── bench_test.go ├── notify.go ├── packet.go ├── packet_test.go ├── rx.go ├── socket.go ├── socket_test.go ├── tx.go ├── window.go └── window_test.go ├── unixsocket ├── bench_test.go ├── socket.go └── socket_test.go └── vegas └── vegas.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.capnp.* 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 mit-nms 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PACKAGES = ./ccpFlow \ 2 | ./ccpFlow/pattern \ 3 | ./ipcBackend \ 4 | ./ipc \ 5 | ./udpDataplane \ 6 | ./unixsocket \ 7 | ./netlinkipc \ 8 | ./reno \ 9 | ./vegas \ 10 | ./cubic \ 11 | ./compound \ 12 | ./bbr \ 13 | ./nl_userapp \ 14 | ./ccp 15 | 16 | all: compile test 17 | 18 | compile: ccpl testClient testServer nltest 19 | 20 | ccpl: build 21 | go build -o ./ccpl ccp/ccp 22 | 23 | build: 24 | go build $(PACKAGES) 25 | 26 | test: ccpl testClient testServer 27 | go test $(PACKAGES) 28 | go vet $(PACKAGES) 29 | #golint $(PACKAGES) 30 | 31 | testClient: build 32 | go build -o ./testClient ./tests/testClient/client.go 33 | 34 | testServer: build 35 | go build -o ./testServer ./tests/testServer/server.go 36 | 37 | nltest: build 38 | go build -o ./nltest ccp/nl_userapp 39 | 40 | clean: 41 | rm -f ./testClient 42 | rm -f ./testServer 43 | rm -f ./nltest 44 | rm -f ./ccpl 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This repository is an older experimental implementation of CCP, and is longer maintained. 2 | Please see https://github.com/ccp-project/portus. 3 | 4 | Congestion Control Plane 5 | ======================== 6 | 7 | The congestion control plane allows out-of-the-loop control over congestion control events in various datapaths. 8 | There is one included datapath, in `udpDataplane`, which implements reliable delivery. 9 | 10 | There are 4 main parts of this repository: 11 | - An IPC layer (`ipc`) allowing for different IPC backends (`ipcBackend` package interface). Currently unix sockets (`unixsocket`) and netlink sockets (`netlinkipc`) are implented 12 | - A sample UDP datapath with reliable delivery (`udpDataplane`) 13 | - Note: the UDP datapath does not have full functionality. 14 | - An executable congestion control plane (`ccp`), and interface for defining congestion control schemes (`ccpFlow`) 15 | - Various congestion control schemes (`reno`, `cubic`, `vegas`, etc). 16 | 17 | How to run 18 | ---------- 19 | 20 | - `go get ./...` 21 | - `make` 22 | - `./ccpl --datapath= --congAlg=<...>` 23 | 24 | 25 | How to write a new congestion control algorithm 26 | ------------------------------------------------ 27 | 28 | - Make a new sub-package: `mkdir ` 29 | - Define an exported type in your subpackage which implements `ccpFlow.Flow` 30 | - Export a function with signature `func Init();` which calls `ccpFlow.Register()`. It takes: 31 | - A name for your algorithm (used by the `--congAlg` flag) 32 | - A closure which returns an instance of your type. 33 | - In `ccp/ccp.go`: 34 | - Import your package: `import "ccp/"` 35 | - Call `Init()` at the top of `main()`: `.Init()` 36 | 37 | 38 | Dependencies 39 | ------------ 40 | 41 | - Logrus https://github.com/sirupsen/logrus 42 | -------------------------------------------------------------------------------- /bbr/bbr.go: -------------------------------------------------------------------------------- 1 | package bbr 2 | 3 | import ( 4 | "math" 5 | "time" 6 | 7 | "ccp/ccpFlow" 8 | "ccp/ccpFlow/pattern" 9 | "ccp/ipc" 10 | 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // implement ccpFlow.Flow interface 15 | type BBR struct { 16 | pktSize uint32 17 | 18 | lastAck uint32 19 | rtt time.Duration 20 | lastDrop time.Time 21 | lastUpdate time.Time 22 | rcv_rate float32 23 | wait_time time.Duration 24 | 25 | sockid uint32 26 | ipc ipc.SendOnly 27 | } 28 | 29 | func (b *BBR) Name() string { 30 | return "bbr" 31 | } 32 | 33 | func (b *BBR) Create( 34 | socketid uint32, 35 | send ipc.SendOnly, 36 | pktsz uint32, 37 | startSeq uint32, 38 | startCwnd uint32, 39 | ) { 40 | b.sockid = socketid 41 | b.ipc = send 42 | b.pktSize = pktsz 43 | b.rcv_rate = float32(b.pktSize * 100) 44 | b.lastDrop = time.Now() 45 | b.lastUpdate = time.Now() 46 | b.rtt = time.Since(b.lastDrop) 47 | if startSeq == 0 { 48 | b.lastAck = startSeq 49 | } else { 50 | b.lastAck = startSeq - 1 51 | } 52 | b.wait_time = 160 * time.Millisecond 53 | b.sendPattern(0.95*b.rcv_rate, b.wait_time/8) 54 | 55 | } 56 | 57 | func (b *BBR) GotMeasurement(m ccpFlow.Measurement) { 58 | // Ignore out of order netlink messages 59 | // Happens sometimes when the reporting interval is small 60 | // If within 10 packets, assume no integer overflow 61 | if m.Ack < b.lastAck && m.Ack > b.lastAck-b.pktSize*10 { 62 | return 63 | } 64 | 65 | if b.rcv_rate < float32(m.Rout) { 66 | b.rcv_rate = float32(m.Rout) 67 | } 68 | 69 | // Handle integer overflow / sequence wraparound 70 | var newBytesAcked uint64 71 | if m.Ack < b.lastAck { 72 | newBytesAcked = uint64(math.MaxUint32) + uint64(m.Ack) - uint64(b.lastAck) 73 | } else { 74 | newBytesAcked = uint64(m.Ack) - uint64(b.lastAck) 75 | } 76 | 77 | acked := newBytesAcked 78 | b.rtt = m.Rtt 79 | 80 | if time.Since(b.lastUpdate) >= b.wait_time { 81 | b.wait_time = m.Rtt 82 | b.sendPattern(0.95*b.rcv_rate, b.wait_time/8) 83 | b.lastUpdate = time.Now() 84 | 85 | log.WithFields(log.Fields{ 86 | "gotAck": m.Ack, 87 | "currRate (Mbps)": b.rcv_rate / 125000, 88 | "currLastAck": b.lastAck, 89 | "newlyAcked": acked, 90 | "rin (Mbps)": m.Rin / 125000, 91 | "rout (Mbps)": m.Rout / 125000, 92 | "rtt-ns": m.Rtt.Nanoseconds(), 93 | }).Info("[bbr] got ack") 94 | } 95 | 96 | b.lastAck = m.Ack 97 | return 98 | } 99 | 100 | func (b *BBR) Drop(ev ccpFlow.DropEvent) { 101 | if time.Since(b.lastDrop) <= b.rtt { 102 | return 103 | } 104 | 105 | //oldRate := b.rcv_rate 106 | log.WithFields(log.Fields{ 107 | "time since last drop": time.Since(b.lastDrop), 108 | "rtt": b.rtt, 109 | }).Info("[bbr] got drop") 110 | 111 | b.lastDrop = time.Now() 112 | 113 | //switch ev { 114 | //case ccpFlow.DupAck: 115 | // b.rcv_rate /=2 116 | //case ccpFlow.Timeout: 117 | // b.rcv_rate = float32(b.pktSize * 100) 118 | //default: 119 | // log.WithFields(log.Fields{ 120 | // "event": ev, 121 | // }).Warn("[bbr] unknown drop event type") 122 | // return 123 | //} 124 | 125 | //b.lastUpdate = time.Now() 126 | //b.wait_time = 160*time.Millisecond 127 | //b.sendPattern(0.95*b.rcv_rate, b.wait_time/8) 128 | // 129 | //log.WithFields(log.Fields{ 130 | // "oldRate": oldRate, 131 | // "newRate": b.rcv_rate, 132 | // "event": ev, 133 | //}).Info("[bbr] drop") 134 | } 135 | 136 | func (b *BBR) sendPattern(rcv_rate float32, wait_time time.Duration) { 137 | pattern, err := pattern. 138 | NewPattern(). 139 | Rate(1.25 * rcv_rate). 140 | Wait(wait_time / 2). 141 | Report(). 142 | Wait(wait_time / 2). 143 | Report(). 144 | Rate(0.75 * rcv_rate). 145 | Wait(wait_time / 2). 146 | Report(). 147 | Wait(wait_time / 2). 148 | Report(). 149 | Rate(rcv_rate). 150 | Wait(wait_time). 151 | Report(). 152 | Wait(wait_time). 153 | Report(). 154 | Wait(wait_time). 155 | Report(). 156 | Wait(wait_time). 157 | Report(). 158 | Wait(wait_time). 159 | Report(). 160 | Wait(wait_time). 161 | Report(). 162 | Compile() 163 | if err != nil { 164 | log.WithFields(log.Fields{ 165 | "err": err, 166 | "rate": b.rcv_rate, 167 | }).Info("make cwnd msg failed") 168 | return 169 | } 170 | err = b.ipc.SendPatternMsg(b.sockid, pattern) 171 | if err != nil { 172 | log.WithFields(log.Fields{"rate": b.rcv_rate, "name": b.sockid}).Warn(err) 173 | } 174 | } 175 | 176 | func Init() { 177 | ccpFlow.Register("bbr", func() ccpFlow.Flow { 178 | return &BBR{} 179 | }) 180 | } 181 | -------------------------------------------------------------------------------- /ccp/ccp.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "time" 6 | 7 | "ccp/bbr" 8 | "ccp/compound" 9 | "ccp/cubic" 10 | "ccp/ipc" 11 | "ccp/reno" 12 | "ccp/vegas" 13 | 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | func init() { 18 | log.SetLevel(log.InfoLevel) 19 | log.SetFormatter(&log.JSONFormatter{ 20 | TimestampFormat: time.RFC3339Nano, 21 | }) 22 | } 23 | 24 | var datapath = flag.String("datapath", "udp", "which IPC backend to use (udp|kernel)") 25 | var overrideAlg = flag.String("congAlg", "nil", "override the datapath's requested congestion control algorithm for all flows (cubic|reno|vegas|nil)") 26 | var initCwnd = flag.Uint("initCwnd", 10, "override the default starting congestion window") 27 | 28 | var flows map[uint32]flowHandler 29 | var dp ipc.Datapath 30 | 31 | func main() { 32 | flag.Parse() 33 | 34 | log.WithFields(log.Fields{ 35 | "datapath": *datapath, 36 | "overrideAlg": *overrideAlg, 37 | "startCwnd": *initCwnd, 38 | }).Info("parsed flags") 39 | 40 | flows = make(map[uint32]flowHandler) 41 | bbr.Init() 42 | compound.Init() 43 | cubic.Init() 44 | vegas.Init() 45 | reno.Init() 46 | 47 | switch *datapath { 48 | case "udp": 49 | dp = ipc.UNIX 50 | case "kernel": 51 | dp = ipc.NETLINK 52 | default: 53 | log.WithFields(log.Fields{ 54 | "datapath": *datapath, 55 | }).Warn("unknown datapath") 56 | return 57 | } 58 | 59 | com, err := ipc.SetupCcpListen(dp) 60 | if err != nil { 61 | log.Error(err) 62 | return 63 | } 64 | 65 | ackCh, err := com.ListenMeasureMsg() 66 | if err != nil { 67 | log.Error(err) 68 | return 69 | } 70 | 71 | createCh, err := com.ListenCreateMsg() 72 | if err != nil { 73 | log.Error(err) 74 | return 75 | } 76 | 77 | dropCh, err := com.ListenDropMsg() 78 | if err != nil { 79 | log.Error(err) 80 | return 81 | } 82 | 83 | handleMsgs(createCh, ackCh, dropCh) 84 | } 85 | -------------------------------------------------------------------------------- /ccp/handleFlow.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "ccp/ccpFlow" 7 | 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | /* The event loop for a single flow 12 | * Receive filtered messages from the CCP corresp 13 | * to this flow, and call the appropriate handling function 14 | */ 15 | func handleFlow( 16 | sockId uint32, 17 | flow ccpFlow.Flow, 18 | msgs flowHandler, 19 | endFlow chan uint32, 20 | ) { 21 | for { 22 | select { 23 | case m := <-msgs.flowMeasureCh: 24 | log.WithFields(log.Fields{ 25 | "flowid": m.SocketId(), 26 | "ackno": m.AckNo(), 27 | "rtt": m.Rtt(), 28 | "loss": m.Loss(), 29 | "rin": m.Rin(), 30 | "rout": m.Rout(), 31 | }).Debug("handleMeasure") 32 | 33 | flow.GotMeasurement(ccpFlow.Measurement{ 34 | Ack: m.AckNo(), 35 | Rtt: m.Rtt(), 36 | Loss: m.Loss(), 37 | Rin: m.Rin(), 38 | Rout: m.Rout(), 39 | }) 40 | case dr := <-msgs.flowDropCh: 41 | log.WithFields(log.Fields{ 42 | "flowid": dr.SocketId, 43 | "drEvent": dr.Event, 44 | }).Debug("handleDrop") 45 | flow.Drop(ccpFlow.DropEvent(dr.Event())) 46 | case <-time.After(time.Minute): 47 | // garbage collect this goroutine after a minute of inactivity 48 | endFlow <- sockId 49 | return 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /ccp/handler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "ccp/ccpFlow" 5 | "ccp/ipc" 6 | 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | type flowHandler struct { 11 | flowMeasureCh chan ipc.MeasureMsg 12 | flowDropCh chan ipc.DropMsg 13 | } 14 | 15 | /* The event loop for the CCP 16 | * Demultiplex messages across flows, and dispatch new per-flow 17 | * event loops on CREATE messages. 18 | */ 19 | func handleMsgs( 20 | createCh chan ipc.CreateMsg, 21 | measureCh chan ipc.MeasureMsg, 22 | dropCh chan ipc.DropMsg, 23 | ) { 24 | endFlow := make(chan uint32) 25 | for { 26 | select { 27 | case cr := <-createCh: 28 | handleCreate(cr, endFlow) 29 | case m := <-measureCh: 30 | handleMeasure(m) 31 | case dr := <-dropCh: 32 | handleDrop(dr) 33 | case sid := <-endFlow: 34 | handleFlowEnd(sid) 35 | } 36 | } 37 | } 38 | 39 | func handleCreate(cr ipc.CreateMsg, endFlow chan uint32) { 40 | log.WithFields(log.Fields{ 41 | "flowid": cr.SocketId(), 42 | "startseq": cr.StartSeq(), 43 | "alg": cr.CongAlg(), 44 | }).Info("handleCreate") 45 | 46 | if _, ok := flows[cr.SocketId()]; ok { 47 | log.WithFields(log.Fields{ 48 | "flowid": cr.SocketId(), 49 | }).Error("Creating already created flow") 50 | return 51 | } 52 | 53 | var f ccpFlow.Flow 54 | var err error 55 | if *overrideAlg != "nil" { 56 | f, err = ccpFlow.GetFlow(*overrideAlg) 57 | if err != nil { 58 | log.WithFields(log.Fields{ 59 | "datapath request": cr.CongAlg(), 60 | "override request": *overrideAlg, 61 | "error": err, 62 | }).Warn("Unknown flow type, trying datapath request") 63 | } else { 64 | goto gotFlow 65 | } 66 | } 67 | 68 | f, err = ccpFlow.GetFlow(cr.CongAlg()) 69 | if err != nil { 70 | log.WithFields(log.Fields{ 71 | "alg": cr.CongAlg(), 72 | "error": err, 73 | }).Warn("Unknown flow type, using reno") 74 | f, err = ccpFlow.GetFlow("reno") 75 | if err != nil { 76 | log.WithFields(log.Fields{ 77 | "registered": ccpFlow.ListRegistered(), 78 | "asked": "reno", 79 | }).Panic(err) 80 | } 81 | } 82 | 83 | gotFlow: 84 | ipCh, err := ipc.SetupCcpSend(dp, cr.SocketId()) 85 | if err != nil { 86 | log.WithFields(log.Fields{ 87 | "flowid": cr.SocketId(), 88 | }).Error("Error creating ccp->socket ipc channel for flow") 89 | } 90 | 91 | switch dp { 92 | case ipc.UNIX: 93 | f.Create(cr.SocketId(), ipCh, 1462, cr.StartSeq(), uint32(*initCwnd)) 94 | case ipc.NETLINK: 95 | f.Create(cr.SocketId(), ipCh, 1460, cr.StartSeq(), uint32(*initCwnd)) 96 | } 97 | 98 | handler := flowHandler{ 99 | flowMeasureCh: make(chan ipc.MeasureMsg), 100 | flowDropCh: make(chan ipc.DropMsg), 101 | } 102 | 103 | go handleFlow(cr.SocketId(), f, handler, endFlow) 104 | flows[cr.SocketId()] = handler 105 | } 106 | 107 | func handleMeasure(m ipc.MeasureMsg) { 108 | if handler, ok := flows[m.SocketId()]; !ok { 109 | log.WithFields(log.Fields{ 110 | "flowid": m.SocketId(), 111 | "msg": "measure", 112 | }).Warn("Unknown flow") 113 | return 114 | } else { 115 | handler.flowMeasureCh <- m 116 | } 117 | } 118 | 119 | func handleDrop(dr ipc.DropMsg) { 120 | if handler, ok := flows[dr.SocketId()]; !ok { 121 | log.WithFields(log.Fields{ 122 | "flowid": dr.SocketId(), 123 | "msg": "drop", 124 | }).Warn("Unknown flow") 125 | return 126 | } else { 127 | handler.flowDropCh <- dr 128 | } 129 | } 130 | 131 | func handleFlowEnd(sid uint32) { 132 | delete(flows, sid) 133 | } 134 | -------------------------------------------------------------------------------- /ccpFlow/flow.go: -------------------------------------------------------------------------------- 1 | package ccpFlow 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "ccp/ipc" 8 | ) 9 | 10 | type DropEvent string 11 | 12 | var DupAck DropEvent = DropEvent("dupack") 13 | var Timeout DropEvent = DropEvent("timeout") 14 | var Ecn DropEvent = DropEvent("ecn") 15 | 16 | type Measurement struct { 17 | Ack uint32 18 | Rtt time.Duration 19 | Rin uint64 20 | Rout uint64 21 | Loss uint32 22 | } 23 | 24 | type Flow interface { 25 | // Name returns a string identifying the CC algorithm 26 | Name() string 27 | // Create takes configuration parameters from the CCP and does initialization 28 | Create( 29 | sockid uint32, 30 | send ipc.SendOnly, 31 | pktsz uint32, 32 | startSeq uint32, 33 | initCwnd uint32, 34 | ) 35 | // Measurement: callback for when a specified measurement is received 36 | GotMeasurement(m Measurement) 37 | // Drop: callback for drop event 38 | Drop(event DropEvent) 39 | } 40 | 41 | // name of flow to function which returns blank instance 42 | var protocolRegistry map[string]func() Flow 43 | 44 | // Register a new type of flow 45 | // name: unique name of the flow type 46 | // f: function which returns a blank instance of an implementing type 47 | func Register(name string, f func() Flow) error { 48 | if protocolRegistry == nil { 49 | protocolRegistry = make(map[string]func() Flow) 50 | } 51 | 52 | if _, ok := protocolRegistry[name]; ok { 53 | return fmt.Errorf("flow algorithm %v already registered", name) 54 | } 55 | protocolRegistry[name] = f 56 | return nil 57 | } 58 | 59 | func ListRegistered() (regs []string) { 60 | for name, _ := range protocolRegistry { 61 | regs = append(regs, name) 62 | } 63 | return 64 | } 65 | 66 | func GetFlow(name string) (Flow, error) { 67 | if f, ok := protocolRegistry[name]; !ok { 68 | return nil, fmt.Errorf("unknown flow algorithm %v", name) 69 | } else { 70 | return f(), nil 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /ccpFlow/flow_test.go: -------------------------------------------------------------------------------- 1 | package ccpFlow 2 | 3 | import ( 4 | "testing" 5 | 6 | "ccp/ipc" 7 | ) 8 | 9 | type TestFlow struct{} 10 | 11 | func (t *TestFlow) Name() string { 12 | return "mock" 13 | } 14 | 15 | func (t *TestFlow) Create( 16 | sockid uint32, 17 | send ipc.SendOnly, 18 | stSq uint32, 19 | pktsz uint32, 20 | startCwnd uint32, 21 | ) { 22 | return 23 | } 24 | 25 | func (t *TestFlow) GotMeasurement(m Measurement) { 26 | return 27 | } 28 | 29 | func (t *TestFlow) Drop(ev DropEvent) { 30 | return 31 | } 32 | 33 | func TestRegister(t *testing.T) { 34 | err := Register("mock", func() Flow { return &TestFlow{} }) 35 | if err != nil { 36 | t.Error(err) 37 | return 38 | } 39 | 40 | rgs := ListRegistered() 41 | 42 | if len(rgs) != 1 { 43 | t.Error("wrong number of registered algorithms, wanted 1", len(rgs)) 44 | return 45 | } else if rgs[0] != "mock" { 46 | t.Error("wrong registered algorithm, wanted \"mock\"", rgs[0]) 47 | return 48 | } 49 | 50 | f, err := GetFlow(rgs[0]) 51 | if err != nil { 52 | t.Error(err) 53 | return 54 | } 55 | 56 | if f.Name() != "mock" { 57 | t.Error("wrong registered algorithm, wanted \"mock\"", rgs[0]) 58 | return 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /ccpFlow/pattern/pattern.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type Pattern struct { 9 | Sequence []PatternEvent 10 | err error 11 | } 12 | 13 | type PatternEventType uint8 14 | 15 | const ( 16 | SETRATEABS PatternEventType = iota 17 | SETCWNDABS 18 | SETRATEREL 19 | WAITABS 20 | WAITREL 21 | REPORT 22 | ) 23 | 24 | type PatternEvent struct { 25 | Type PatternEventType 26 | Duration time.Duration 27 | Cwnd uint32 28 | Rate float32 29 | Factor float32 30 | } 31 | 32 | func NewPattern() *Pattern { 33 | return &Pattern{ 34 | Sequence: make([]PatternEvent, 0), 35 | } 36 | } 37 | 38 | func (p *Pattern) Cwnd(cwnd uint32) *Pattern { 39 | if p.err != nil { 40 | return p 41 | } 42 | 43 | p.Sequence = append(p.Sequence, PatternEvent{ 44 | Type: SETCWNDABS, 45 | Cwnd: cwnd, 46 | }) 47 | 48 | return p 49 | } 50 | 51 | func (p *Pattern) Rate(rate float32) *Pattern { 52 | if p.err != nil { 53 | return p 54 | } 55 | 56 | p.Sequence = append(p.Sequence, PatternEvent{ 57 | Type: SETRATEABS, 58 | Rate: rate, 59 | }) 60 | 61 | return p 62 | } 63 | 64 | func (p *Pattern) RelativeRate(factor float32) *Pattern { 65 | if p.err != nil { 66 | return p 67 | } 68 | 69 | p.Sequence = append(p.Sequence, PatternEvent{ 70 | Type: SETRATEREL, 71 | Factor: factor, 72 | }) 73 | 74 | return p 75 | } 76 | 77 | func (p *Pattern) Wait(wait time.Duration) *Pattern { 78 | if p.err != nil { 79 | return p 80 | } 81 | 82 | if len(p.Sequence) == 0 { 83 | p.err = fmt.Errorf("wait cannot start pattern") 84 | return p 85 | } 86 | 87 | p.Sequence = append(p.Sequence, PatternEvent{ 88 | Type: WAITABS, 89 | Duration: wait, 90 | }) 91 | 92 | return p 93 | } 94 | 95 | func (p *Pattern) WaitRtts(factor float32) *Pattern { 96 | if p.err != nil { 97 | return p 98 | } 99 | 100 | p.Sequence = append(p.Sequence, PatternEvent{ 101 | Type: WAITREL, 102 | Factor: factor, 103 | }) 104 | 105 | return p 106 | } 107 | 108 | func (p *Pattern) Report() *Pattern { 109 | if p.err != nil { 110 | return p 111 | } 112 | 113 | p.Sequence = append(p.Sequence, PatternEvent{ 114 | Type: REPORT, 115 | }) 116 | 117 | return p 118 | } 119 | 120 | func (p *Pattern) Compile() (*Pattern, error) { 121 | if len(p.Sequence) == 1 { 122 | // add a long wait after 123 | // this will function as a one-off set 124 | p = p.Wait(time.Second) 125 | } 126 | 127 | if p.err != nil { 128 | return nil, p.err 129 | } 130 | 131 | return p, nil 132 | } 133 | -------------------------------------------------------------------------------- /ccpFlow/pattern/pattern_test.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestFactor(t *testing.T) { 8 | p, err := NewPattern().Cwnd(42).WaitRtts(1.0).Compile() 9 | if err != nil { 10 | t.Error(err) 11 | } 12 | 13 | if p.err != nil { 14 | t.Errorf("pattern error %s should be cleared", p.err) 15 | } 16 | 17 | if sqlen := len(p.Sequence); sqlen != 2 { 18 | t.Errorf("incorrect pattern length %d", sqlen) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /compound/compound.go: -------------------------------------------------------------------------------- 1 | package compound 2 | 3 | import ( 4 | "math" 5 | "time" 6 | 7 | "ccp/ccpFlow" 8 | "ccp/ccpFlow/pattern" 9 | "ccp/ipc" 10 | 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // Specification from https://tools.ietf.org/html/draft-sridharan-tcpm-ctcp-02#section-3 and http://www.dcs.gla.ac.uk/~lewis/CTCP.pdf 15 | // Add a "delay window" (dwnd) component to standard TCP. 16 | // wnd = min(cwnd + dwnd) 17 | // on ack: cwnd = cwnd + 1/(cwnd+dwnd) 18 | // set dwnd according to a binomial function: 19 | // dwnd(t+1) = 20 | // dwnd(t) + alpha*dwnd(t)^k -1 if diff < gamma 21 | // dwnd(t) - eta*diff if diff >= gamma 22 | // dwnd(t) * (1-beta) if diff < gamma 23 | // alpha = 1/8, beta = 1/2, eta = 1, k = 0.75 24 | // dwnd in packets. 25 | 26 | // implement ccpFlow.Flow interface 27 | type Compound struct { 28 | pktSize uint32 29 | initCwnd float32 30 | 31 | ssthresh float32 32 | wnd float32 33 | cwnd float32 34 | dwnd float32 35 | lastAck uint32 36 | 37 | sockid uint32 38 | ipc ipc.SendOnly 39 | baseRTT float32 40 | alpha float32 41 | beta float32 42 | k float64 43 | eta float32 44 | gamma float32 45 | gamma_low float32 46 | gamma_high float32 47 | diff_reno float32 48 | lastDrop time.Time 49 | } 50 | 51 | func (c *Compound) Name() string { 52 | return "compound" 53 | } 54 | 55 | func (c *Compound) Create( 56 | socketid uint32, 57 | send ipc.SendOnly, 58 | pktsz uint32, 59 | startSeq uint32, 60 | startCwnd uint32, 61 | ) { 62 | c.sockid = socketid 63 | c.ipc = send 64 | c.pktSize = pktsz 65 | c.initCwnd = float32(pktsz * 10) 66 | c.ssthresh = 0x7fffffff 67 | c.wnd = float32(pktsz * startCwnd) 68 | c.cwnd = float32(pktsz * startCwnd) 69 | c.dwnd = 0 70 | if startSeq == 0 { 71 | c.lastAck = startSeq 72 | } else { 73 | c.lastAck = startSeq - 1 74 | } 75 | 76 | c.baseRTT = 0 77 | c.alpha = 0.125 78 | c.beta = 0.5 79 | c.k = 0.8 80 | c.eta = 1 81 | c.gamma = 30 82 | c.gamma_low = 5 83 | c.gamma_high = 30 84 | c.diff_reno = -1 85 | c.lastDrop = time.Now() 86 | c.newPattern() 87 | } 88 | 89 | func (c *Compound) GotMeasurement(m ccpFlow.Measurement) { 90 | // reordering of messsages 91 | // if within 10 packets, assume no integer overflow 92 | if m.Ack < c.lastAck && m.Ack > c.lastAck-c.pktSize*10 { 93 | return 94 | } 95 | 96 | // handle integer overflow / sequence wraparound 97 | var newBytesAcked uint64 98 | if m.Ack < c.lastAck { 99 | newBytesAcked = uint64(math.MaxUint32) + uint64(m.Ack) - uint64(c.lastAck) 100 | } else { 101 | newBytesAcked = uint64(m.Ack) - uint64(c.lastAck) 102 | } 103 | 104 | if c.cwnd < c.ssthresh { 105 | // increase cwnd by 1 per packet 106 | c.cwnd += float32(newBytesAcked) 107 | } else { 108 | // increase cwnd by 1 / cwnd per packet 109 | c.cwnd += float32(c.pktSize) * (float32(newBytesAcked) / c.cwnd) 110 | } 111 | 112 | RTT := float32(m.Rtt.Seconds()) 113 | if c.baseRTT <= 0 || RTT < c.baseRTT { 114 | c.baseRTT = RTT 115 | } 116 | 117 | // increase cwnd by 1 / cwnd per packet 118 | c.cwnd += float32(c.pktSize) * (float32(newBytesAcked) / c.wnd) 119 | 120 | // dwnd update 121 | expected := c.wnd / c.baseRTT 122 | actual := c.wnd / RTT 123 | diff := (expected - actual) * c.baseRTT 124 | c.diff_reno = diff 125 | increment := float32(0.0) 126 | if diff < c.gamma*float32(c.pktSize) { 127 | increment = float32(c.pktSize) * ((c.alpha * float32(math.Pow(float64(c.wnd)/float64(c.pktSize), c.k))) - 1) 128 | if increment < 0 { 129 | increment = 0 130 | } 131 | } else { 132 | increment = -c.eta * diff 133 | } 134 | 135 | c.dwnd += increment * (float32(newBytesAcked) / c.wnd) 136 | if c.dwnd < 0 { 137 | c.dwnd = 0 138 | } 139 | 140 | //wnd update 141 | c.wnd = c.cwnd + c.dwnd 142 | 143 | // notify increased cwnd 144 | c.newPattern() 145 | 146 | log.WithFields(log.Fields{ 147 | "gotAck": m.Ack, 148 | "currCwnd": c.wnd, 149 | "currLastAck": c.lastAck, 150 | "newlyAcked": newBytesAcked, 151 | }).Info("[compound] got ack") 152 | 153 | c.lastAck = m.Ack 154 | return 155 | } 156 | 157 | func (c *Compound) Drop(ev ccpFlow.DropEvent) { 158 | //if float32(time.Since(c.lastDrop).Seconds()) <= c.baseRTT { 159 | // return 160 | //} 161 | //c.lastDrop = time.Now() 162 | 163 | oldCwnd := c.wnd 164 | switch ev { 165 | case ccpFlow.DupAck: 166 | c.cwnd /= 2 167 | c.ssthresh = c.cwnd 168 | if c.cwnd < c.initCwnd { 169 | c.cwnd = c.initCwnd 170 | } 171 | 172 | c.dwnd = c.wnd*(1-c.beta) - c.cwnd 173 | if c.dwnd < 0 { 174 | c.dwnd = 0 175 | } 176 | 177 | c.wnd = c.cwnd + c.dwnd 178 | 179 | // gamma auto tuning 180 | if c.diff_reno >= 0 { 181 | g_sample := 0.75 * (c.diff_reno / float32(c.pktSize)) 182 | lambda := float32(0.8) //couldn't find how to set this 183 | c.gamma = lambda*c.gamma + (1-lambda)*g_sample 184 | if c.gamma < c.gamma_low { 185 | c.gamma = c.gamma_low 186 | } else if c.gamma > c.gamma_high { 187 | c.gamma = c.gamma_high 188 | } 189 | 190 | c.diff_reno = -1 191 | } 192 | case ccpFlow.Timeout: 193 | c.ssthresh = c.wnd / 2 194 | c.wnd = c.initCwnd 195 | c.cwnd = c.initCwnd 196 | c.dwnd = 0 197 | c.gamma = 30 198 | default: 199 | log.WithFields(log.Fields{ 200 | "event": ev, 201 | }).Warn("[reno] unknown drop event type") 202 | return 203 | } 204 | 205 | log.WithFields(log.Fields{ 206 | "oldCwnd": oldCwnd, 207 | "currCwnd": c.wnd, 208 | "event": ev, 209 | }).Info("[compound] drop") 210 | 211 | c.newPattern() 212 | } 213 | 214 | func (c *Compound) newPattern() { 215 | staticPattern, err := pattern. 216 | NewPattern(). 217 | Cwnd(uint32(c.wnd)). 218 | Wait(time.Duration(10) * time.Millisecond). 219 | Report(). 220 | Compile() 221 | if err != nil { 222 | log.WithFields(log.Fields{ 223 | "err": err, 224 | "cwnd": c.wnd, 225 | }).Info("make cwnd msg failed") 226 | return 227 | } 228 | 229 | err = c.ipc.SendPatternMsg(c.sockid, staticPattern) 230 | if err != nil { 231 | log.WithFields(log.Fields{"cwnd": c.wnd, "name": c.sockid}).Warn(err) 232 | } 233 | } 234 | 235 | func Init() { 236 | ccpFlow.Register("compound", func() ccpFlow.Flow { 237 | return &Compound{} 238 | }) 239 | } 240 | -------------------------------------------------------------------------------- /cubic/cubic.go: -------------------------------------------------------------------------------- 1 | package cubic 2 | 3 | //Pseudo Code from https://pdfs.semanticscholar.org/4e8f/00ffc07c77340ba4121b36585754f8b8379a.pdf 4 | 5 | import ( 6 | "math" 7 | "time" 8 | 9 | "ccp/ccpFlow" 10 | "ccp/ccpFlow/pattern" 11 | "ccp/ipc" 12 | 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | // implement ccpFlow.Flow interface 17 | type Cubic struct { 18 | pktSize uint32 19 | initCwnd float64 20 | 21 | cwnd float64 22 | lastAck uint32 23 | lastDrop time.Time 24 | rtt time.Duration 25 | sockid uint32 26 | ipc ipc.SendOnly 27 | 28 | //state for cubic 29 | ssthresh float64 30 | cwnd_cnt float64 31 | tcp_friendliness bool 32 | BETA float64 33 | fast_convergence bool 34 | C float64 35 | Wlast_max float64 36 | epoch_start float64 37 | origin_point float64 38 | dMin float64 39 | Wtcp float64 40 | K float64 41 | ack_cnt float64 42 | cnt float64 43 | } 44 | 45 | func (c *Cubic) Name() string { 46 | return "cubic" 47 | } 48 | 49 | func (c *Cubic) Create( 50 | socketid uint32, 51 | send ipc.SendOnly, 52 | pktsz uint32, 53 | startSeq uint32, 54 | startCwnd uint32, 55 | ) { 56 | c.sockid = socketid 57 | c.pktSize = pktsz 58 | c.lastAck = 0 59 | c.ipc = send 60 | // Pseudo code doesn't specify how to intialize these 61 | c.lastAck = startSeq 62 | c.initCwnd = float64(10) 63 | c.cwnd = float64(startCwnd) 64 | c.ssthresh = (0x7fffffff / float64(pktsz)) 65 | c.lastDrop = time.Now() 66 | c.rtt = time.Duration(0) 67 | // not sure about what this value should be 68 | c.cwnd_cnt = 0 69 | 70 | c.tcp_friendliness = true 71 | c.BETA = 0.2 72 | c.fast_convergence = true 73 | c.C = 0.4 74 | c.cubic_reset() 75 | 76 | pattern, err := pattern. 77 | NewPattern(). 78 | Cwnd(uint32(c.cwnd * float64(c.pktSize))). 79 | WaitRtts(0.1). 80 | Report(). 81 | Compile() 82 | if err != nil { 83 | log.WithFields(log.Fields{ 84 | "err": err, 85 | "cwndPkts": c.cwnd, 86 | }).Info("make cwnd msg failed") 87 | return 88 | } 89 | 90 | c.sendPattern(pattern) 91 | } 92 | 93 | func (c *Cubic) cubic_reset() { 94 | c.Wlast_max = 0 95 | c.epoch_start = 0 96 | c.origin_point = 0 97 | c.dMin = 0 98 | c.Wtcp = 0 99 | c.K = 0 100 | c.ack_cnt = 0 101 | } 102 | 103 | func (c *Cubic) GotMeasurement(m ccpFlow.Measurement) { 104 | // Ignore out of order netlink messages 105 | // Happens sometimes when the reporting interval is small 106 | // If within 10 packets, assume no integer overflow 107 | if m.Ack < c.lastAck && m.Ack > c.lastAck-c.pktSize*10 { 108 | return 109 | } 110 | 111 | // Handle integer overflow / sequence wraparound 112 | var newBytesAcked uint64 113 | if m.Ack < c.lastAck { 114 | newBytesAcked = uint64(math.MaxUint32) + uint64(m.Ack) - uint64(c.lastAck) 115 | } else { 116 | newBytesAcked = uint64(m.Ack) - uint64(c.lastAck) 117 | } 118 | 119 | c.rtt = m.Rtt 120 | RTT := float64(c.rtt.Seconds()) 121 | no_of_acks := float64(newBytesAcked) / float64(c.pktSize) 122 | 123 | if c.cwnd <= c.ssthresh { 124 | if c.cwnd+no_of_acks < c.ssthresh { 125 | c.cwnd += no_of_acks 126 | no_of_acks = 0 127 | } else { 128 | no_of_acks -= (c.ssthresh - c.cwnd) 129 | c.cwnd = c.ssthresh 130 | } 131 | } 132 | 133 | for i := uint32(0); i < uint32(no_of_acks); i++ { 134 | if c.dMin <= 0 || RTT < c.dMin { 135 | c.dMin = RTT 136 | } 137 | 138 | c.cubic_update() 139 | if c.cwnd_cnt > c.cnt { 140 | c.cwnd = c.cwnd + 1 141 | c.cwnd_cnt = 0 142 | } else { 143 | c.cwnd_cnt = c.cwnd_cnt + 1 144 | } 145 | } 146 | 147 | // notify increased cwnd 148 | pattern, err := pattern. 149 | NewPattern(). 150 | Cwnd(uint32(c.cwnd * float64(c.pktSize))). 151 | WaitRtts(0.5). 152 | Report(). 153 | Compile() 154 | if err != nil { 155 | log.WithFields(log.Fields{ 156 | "err": err, 157 | "cwndPkts": c.cwnd, 158 | }).Info("make cwnd msg failed") 159 | return 160 | } 161 | 162 | c.sendPattern(pattern) 163 | 164 | log.WithFields(log.Fields{ 165 | "gotAck": m.Ack, 166 | "rtt-ns": c.rtt.Nanoseconds(), 167 | "currLastAck": c.lastAck, 168 | "newlyAckedPackets": float64(newBytesAcked) / float64(c.pktSize), 169 | "currCwndPkts": c.cwnd, 170 | }).Info("[cubic] got ack") 171 | 172 | c.lastAck = m.Ack 173 | return 174 | } 175 | 176 | func (c *Cubic) Drop(ev ccpFlow.DropEvent) { 177 | if time.Since(c.lastDrop) <= c.rtt { 178 | return 179 | } 180 | 181 | c.lastDrop = time.Now() 182 | 183 | switch ev { 184 | case ccpFlow.DupAck: 185 | c.epoch_start = 0 186 | if c.cwnd < c.Wlast_max && c.fast_convergence { 187 | c.Wlast_max = c.cwnd * ((2 - c.BETA) / 2) 188 | } else { 189 | c.Wlast_max = c.cwnd 190 | } 191 | 192 | c.cwnd = c.cwnd * (1 - c.BETA) 193 | c.ssthresh = c.cwnd 194 | case ccpFlow.Timeout: 195 | c.ssthresh = c.cwnd / 2 196 | c.cwnd = c.initCwnd 197 | c.cubic_reset() 198 | default: 199 | log.WithFields(log.Fields{ 200 | "event": ev, 201 | }).Warn("[cubic] unknown drop event type") 202 | return 203 | } 204 | 205 | pattern, err := pattern. 206 | NewPattern(). 207 | Cwnd(uint32(c.cwnd * float64(c.pktSize))). 208 | WaitRtts(0.1). 209 | Report(). 210 | Compile() 211 | if err != nil { 212 | log.WithFields(log.Fields{ 213 | "err": err, 214 | "cwndPkts": c.cwnd, 215 | }).Info("make cwnd msg failed") 216 | return 217 | } 218 | 219 | c.sendPattern(pattern) 220 | log.WithFields(log.Fields{ 221 | "currCwndPkts": c.cwnd, 222 | "event": ev, 223 | }).Info("[cubic] drop") 224 | } 225 | 226 | func (c *Cubic) cubic_update() { 227 | c.ack_cnt = c.ack_cnt + 1 228 | if c.epoch_start <= 0 { 229 | c.epoch_start = float64(time.Now().UnixNano() / 1e9) 230 | if c.cwnd < c.Wlast_max { 231 | c.K = math.Pow(math.Max(0.0, ((c.Wlast_max-c.cwnd)/c.C)), 1.0/3.0) 232 | c.origin_point = c.Wlast_max 233 | } else { 234 | c.K = 0 235 | c.origin_point = c.cwnd 236 | } 237 | 238 | c.ack_cnt = 1 239 | c.Wtcp = c.cwnd 240 | } 241 | 242 | t := float64(time.Now().UnixNano()/1e9) + c.dMin - c.epoch_start 243 | target := c.origin_point + c.C*((t-c.K)*(t-c.K)*(t-c.K)) 244 | if target > c.cwnd { 245 | c.cnt = c.cwnd / (target - c.cwnd) 246 | } else { 247 | c.cnt = 100 * c.cwnd 248 | } 249 | 250 | if c.tcp_friendliness { 251 | c.cubic_tcp_friendliness() 252 | } 253 | } 254 | 255 | func (c *Cubic) cubic_tcp_friendliness() { 256 | c.Wtcp = c.Wtcp + (((3 * c.BETA) / (2 - c.BETA)) * (c.ack_cnt / c.cwnd)) 257 | c.ack_cnt = 0 258 | if c.Wtcp > c.cwnd { 259 | max_cnt := c.cwnd / (c.Wtcp - c.cwnd) 260 | if c.cnt > max_cnt { 261 | c.cnt = max_cnt 262 | } 263 | } 264 | } 265 | 266 | func (c *Cubic) sendPattern(pattern *pattern.Pattern) { 267 | err := c.ipc.SendPatternMsg(c.sockid, pattern) 268 | if err != nil { 269 | log.WithFields(log.Fields{ 270 | "cwndPkts": c.cwnd, 271 | "name": c.sockid, 272 | }).Warn(err) 273 | } 274 | } 275 | 276 | func Init() { 277 | ccpFlow.Register("cubic", func() ccpFlow.Flow { 278 | return &Cubic{} 279 | }) 280 | } 281 | -------------------------------------------------------------------------------- /ipc/bench_test.go: -------------------------------------------------------------------------------- 1 | package ipc 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "ccp/ccpFlow/pattern" 8 | ) 9 | 10 | func BenchmarkEncodeCreateMsg(b *testing.B) { 11 | i, err := testSetup(false) 12 | if err != nil { 13 | b.Error(err) 14 | return 15 | } 16 | 17 | outMsgCh, err := i.ListenCreateMsg() 18 | if err != nil { 19 | b.Error(err) 20 | return 21 | } 22 | 23 | for k := 0; k < b.N; k++ { 24 | err = i.SendCreateMsg( 25 | testNum, 26 | testNum, 27 | testString, 28 | ) 29 | if err != nil { 30 | b.Error(err) 31 | return 32 | } 33 | 34 | select { 35 | case _ = <-outMsgCh: 36 | continue 37 | case <-time.After(time.Second): 38 | b.Error("timed out") 39 | } 40 | } 41 | } 42 | 43 | func BenchmarkMeasureMsg(b *testing.B) { 44 | i, err := testSetup(false) 45 | if err != nil { 46 | b.Error(err) 47 | return 48 | } 49 | 50 | outMsgCh, _ := i.ListenMeasureMsg() 51 | for k := 0; k < b.N; k++ { 52 | i.SendMeasureMsg( 53 | testNum, 54 | testNum, 55 | testDuration, 56 | testBigNum, 57 | testBigNum, 58 | ) 59 | 60 | select { 61 | case _ = <-outMsgCh: 62 | continue 63 | case <-time.After(time.Second): 64 | b.Error("timed out") 65 | } 66 | } 67 | } 68 | 69 | func BenchmarkEncodeDropMsg(b *testing.B) { 70 | i, err := testSetup(false) 71 | if err != nil { 72 | b.Error(err) 73 | return 74 | } 75 | 76 | outMsgCh, _ := i.ListenDropMsg() 77 | for k := 0; k < b.N; k++ { 78 | i.SendDropMsg( 79 | testNum, 80 | testString, 81 | ) 82 | 83 | select { 84 | case _ = <-outMsgCh: 85 | continue 86 | case <-time.After(time.Second): 87 | b.Error("timed out") 88 | } 89 | } 90 | } 91 | 92 | func BenchmarkEncodePatternMsg(t *testing.B) { 93 | i, err := testSetup(false) 94 | if err != nil { 95 | t.Error(err) 96 | return 97 | } 98 | 99 | outMsgCh, _ := i.ListenPatternMsg() 100 | p, err := pattern. 101 | NewPattern(). 102 | Cwnd(testNum). 103 | WaitRtts(1.0). 104 | Report(). 105 | Compile() 106 | if err != nil { 107 | t.Error(err) 108 | return 109 | } 110 | 111 | for k := 0; k < t.N; k++ { 112 | err = i.SendPatternMsg(testNum, p) 113 | if err != nil { 114 | t.Error(err) 115 | return 116 | } 117 | 118 | select { 119 | case _ = <-outMsgCh: 120 | continue 121 | case <-time.After(time.Second): 122 | t.Error("timed out") 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /ipc/ipc.go: -------------------------------------------------------------------------------- 1 | package ipc 2 | 3 | import ( 4 | "fmt" 5 | 6 | flowPattern "ccp/ccpFlow/pattern" 7 | "ccp/ipcBackend" 8 | "ccp/netlinkipc" 9 | "ccp/unixsocket" 10 | ) 11 | 12 | // setup and teardown logic 13 | 14 | type Datapath int 15 | 16 | const ( 17 | UNIX Datapath = iota 18 | NETLINK 19 | ) 20 | 21 | type Ipc struct { 22 | CreateNotify chan CreateMsg 23 | MeasureNotify chan MeasureMsg 24 | DropNotify chan DropMsg 25 | PatternNotify chan PatternMsg 26 | 27 | backend ipcbackend.Backend 28 | } 29 | 30 | // handle of IPC to pass to CC implementations 31 | type SendOnly interface { 32 | SendPatternMsg(socketId uint32, pattern *flowPattern.Pattern) error 33 | } 34 | 35 | func SetupCcpListen(datapath Datapath) (*Ipc, error) { 36 | var back ipcbackend.Backend 37 | var err error 38 | 39 | switch datapath { 40 | case UNIX: 41 | back, err = unixsocket.New().SetupListen("ccp-in", 0).SetupFinish() 42 | case NETLINK: 43 | back, err = netlinkipc.New().SetupListen("", 0).SetupFinish() 44 | default: 45 | return nil, fmt.Errorf("unknown datapath") 46 | } 47 | 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | return SetupWithBackend(back) 53 | } 54 | 55 | func SetupCcpSend(datapath Datapath, sockid uint32) (SendOnly, error) { 56 | var back ipcbackend.Backend 57 | var err error 58 | 59 | switch datapath { 60 | case UNIX: 61 | back, err = unixsocket.New().SetupSend("ccp-out", sockid).SetupFinish() 62 | case NETLINK: 63 | back, err = netlinkipc.New().SetupSend("", 0).SetupFinish() 64 | default: 65 | return nil, fmt.Errorf("unknown datapath") 66 | } 67 | 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | return SetupWithBackend(back) 73 | } 74 | 75 | // Setup both sending and receiving 76 | // Only useful for UDP datapath 77 | func SetupCli(sockid uint32) (*Ipc, error) { 78 | back, err := unixsocket.New().SetupSend("ccp-in", 0).SetupListen("ccp-out", sockid).SetupFinish() 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | return SetupWithBackend(back) 84 | } 85 | 86 | func SetupWithBackend(back ipcbackend.Backend) (*Ipc, error) { 87 | i := &Ipc{ 88 | CreateNotify: make(chan CreateMsg), 89 | MeasureNotify: make(chan MeasureMsg), 90 | DropNotify: make(chan DropMsg), 91 | PatternNotify: make(chan PatternMsg), 92 | backend: back, 93 | } 94 | 95 | ch := i.backend.Listen() 96 | go i.demux(ch) 97 | return i, nil 98 | } 99 | 100 | func (i *Ipc) Close() error { 101 | return i.backend.Close() 102 | } 103 | -------------------------------------------------------------------------------- /ipc/msg.go: -------------------------------------------------------------------------------- 1 | package ipc 2 | 3 | import ( 4 | "time" 5 | 6 | flowPattern "ccp/ccpFlow/pattern" 7 | ) 8 | 9 | // the external serialization interface 10 | 11 | type CreateMsg struct { 12 | socketId uint32 13 | startSeq uint32 14 | congAlg string 15 | } 16 | 17 | func (c *CreateMsg) New(sid uint32, startSeq uint32, alg string) { 18 | c.socketId = sid 19 | c.startSeq = startSeq 20 | c.congAlg = alg 21 | } 22 | 23 | func (c *CreateMsg) SocketId() uint32 { 24 | return c.socketId 25 | } 26 | 27 | func (c *CreateMsg) StartSeq() uint32 { 28 | return c.startSeq 29 | } 30 | 31 | func (c *CreateMsg) CongAlg() string { 32 | return c.congAlg 33 | } 34 | 35 | func (c *CreateMsg) Serialize() ([]byte, error) { 36 | return msgWriter(ipcMsg{ 37 | typ: CREATE, 38 | socketId: c.socketId, 39 | u32s: []uint32{c.startSeq}, 40 | str: c.congAlg, 41 | }) 42 | } 43 | 44 | type MeasureMsg struct { 45 | socketId uint32 46 | ackNo uint32 47 | rtt time.Duration 48 | loss uint32 49 | rin uint64 50 | rout uint64 51 | } 52 | 53 | func (m *MeasureMsg) New( 54 | sid uint32, 55 | ack uint32, 56 | t time.Duration, 57 | loss uint32, 58 | rin uint64, 59 | rout uint64, 60 | ) { 61 | m.socketId = sid 62 | m.ackNo = ack 63 | m.rtt = t 64 | m.loss = loss 65 | m.rin = rin 66 | m.rout = rout 67 | } 68 | 69 | func (m *MeasureMsg) SocketId() uint32 { 70 | return m.socketId 71 | } 72 | 73 | func (m *MeasureMsg) AckNo() uint32 { 74 | return m.ackNo 75 | } 76 | 77 | func (m *MeasureMsg) Rtt() time.Duration { 78 | return m.rtt 79 | } 80 | 81 | func (m *MeasureMsg) Loss() uint32 { 82 | return m.loss 83 | } 84 | 85 | func (m *MeasureMsg) Rin() uint64 { 86 | return m.rin 87 | } 88 | 89 | func (m *MeasureMsg) Rout() uint64 { 90 | return m.rout 91 | } 92 | 93 | func (m *MeasureMsg) Serialize() ([]byte, error) { 94 | return msgWriter(ipcMsg{ 95 | typ: MEASURE, 96 | socketId: m.socketId, 97 | u32s: []uint32{m.ackNo, uint32(m.rtt.Nanoseconds() / 1000), m.loss}, // microseconds 98 | u64s: []uint64{m.rin, m.rout}, 99 | }) 100 | } 101 | 102 | type DropMsg struct { 103 | socketId uint32 104 | event string 105 | } 106 | 107 | func (d *DropMsg) New(sid uint32, ev string) { 108 | d.socketId = sid 109 | d.event = ev 110 | } 111 | 112 | func (d *DropMsg) SocketId() uint32 { 113 | return d.socketId 114 | } 115 | 116 | func (d *DropMsg) Event() string { 117 | return d.event 118 | } 119 | 120 | func (d *DropMsg) Serialize() ([]byte, error) { 121 | return msgWriter(ipcMsg{ 122 | typ: DROP, 123 | socketId: d.socketId, 124 | str: d.event, 125 | }) 126 | } 127 | 128 | type PatternMsg struct { 129 | socketId uint32 130 | pattern *flowPattern.Pattern 131 | } 132 | 133 | func (p *PatternMsg) New(sid uint32, t *flowPattern.Pattern) { 134 | p.socketId = sid 135 | p.pattern = t 136 | } 137 | 138 | func (p *PatternMsg) SocketId() uint32 { 139 | return p.socketId 140 | } 141 | 142 | func (p *PatternMsg) Pattern() *flowPattern.Pattern { 143 | return p.pattern 144 | } 145 | 146 | func (p *PatternMsg) Serialize() ([]byte, error) { 147 | s, err := serializeSequence(p.pattern.Sequence) 148 | if err != nil { 149 | return nil, err 150 | } 151 | 152 | return msgWriter(ipcMsg{ 153 | typ: PATTERN, 154 | socketId: p.socketId, 155 | u32s: []uint32{uint32(len(p.pattern.Sequence))}, 156 | str: string(s), 157 | }) 158 | } 159 | 160 | func (i *Ipc) SendCreateMsg( 161 | socketId uint32, 162 | startSeq uint32, 163 | alg string, 164 | ) error { 165 | return i.backend.SendMsg(&CreateMsg{ 166 | socketId: socketId, 167 | startSeq: startSeq, 168 | congAlg: alg, 169 | }) 170 | } 171 | 172 | func (i *Ipc) SendMeasureMsg( 173 | socketId uint32, 174 | ack uint32, 175 | rtt time.Duration, 176 | loss uint32, 177 | rin uint64, 178 | rout uint64, 179 | ) error { 180 | return i.backend.SendMsg(&MeasureMsg{ 181 | socketId: socketId, 182 | ackNo: ack, 183 | rtt: rtt, 184 | loss: loss, 185 | rin: rin, 186 | rout: rout, 187 | }) 188 | } 189 | 190 | func (i *Ipc) SendDropMsg(socketId uint32, ev string) error { 191 | return i.backend.SendMsg(&DropMsg{ 192 | socketId: socketId, 193 | event: ev, 194 | }) 195 | } 196 | 197 | func (i *Ipc) SendPatternMsg(socketId uint32, pattern *flowPattern.Pattern) error { 198 | return i.backend.SendMsg(&PatternMsg{ 199 | socketId: socketId, 200 | pattern: pattern, 201 | }) 202 | } 203 | 204 | func (i *Ipc) ListenCreateMsg() (chan CreateMsg, error) { 205 | return i.CreateNotify, nil 206 | } 207 | 208 | func (i *Ipc) ListenDropMsg() (chan DropMsg, error) { 209 | return i.DropNotify, nil 210 | } 211 | 212 | func (i *Ipc) ListenMeasureMsg() (chan MeasureMsg, error) { 213 | return i.MeasureNotify, nil 214 | } 215 | 216 | func (i *Ipc) ListenPatternMsg() (chan PatternMsg, error) { 217 | return i.PatternNotify, nil 218 | } 219 | -------------------------------------------------------------------------------- /ipc/serialize.go: -------------------------------------------------------------------------------- 1 | package ipc 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "time" 8 | 9 | flowPattern "ccp/ccpFlow/pattern" 10 | 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // the internal serialization logic 15 | 16 | type msgType uint8 17 | 18 | const ( 19 | CREATE msgType = iota 20 | MEASURE 21 | DROP 22 | PATTERN 23 | ) 24 | 25 | /* Messages: header followed by 0+ uint32s, then 0+ uint64s, then 0-1 strings 26 | */ 27 | type ipcMsg struct { 28 | typ msgType 29 | len uint8 30 | socketId uint32 31 | u32s []uint32 32 | u64s []uint64 33 | str string 34 | } 35 | 36 | /* (type, len, socket_id) header 37 | * ----------------------------------- 38 | * | Msg Type | Len (B) | Uint32 | 39 | * | (1 B) | (1 B) | (32 bits) | 40 | * ----------------------------------- 41 | * total: 6 Bytes 42 | */ 43 | func readHeader(b []byte) ( 44 | typ msgType, 45 | l uint8, 46 | socketId uint32, 47 | err error, 48 | ) { 49 | err = nil 50 | if len(b) < 6 { 51 | err = fmt.Errorf("unable to read header") 52 | return 53 | } 54 | 55 | hdr := bytes.NewBuffer(b[:6]) 56 | err = binary.Read(hdr, binary.LittleEndian, &typ) 57 | err = binary.Read(hdr, binary.LittleEndian, &l) 58 | err = binary.Read(hdr, binary.LittleEndian, &socketId) 59 | return 60 | } 61 | 62 | func writeHeader( 63 | typ msgType, 64 | len uint8, 65 | socketId uint32, 66 | ) (b []byte) { 67 | buf := new(bytes.Buffer) 68 | 69 | binary.Write(buf, binary.LittleEndian, uint8(typ)) 70 | binary.Write(buf, binary.LittleEndian, len) 71 | binary.Write(buf, binary.LittleEndian, socketId) 72 | 73 | return buf.Bytes() 74 | } 75 | 76 | func msgReader(buf []byte) (msg ipcMsg, err error) { 77 | typ, l, socketId, err := readHeader(buf) 78 | if err != nil { 79 | return ipcMsg{}, err 80 | } 81 | 82 | msg = ipcMsg{ 83 | typ: typ, 84 | len: l, 85 | socketId: socketId, 86 | u32s: make([]uint32, 0), 87 | u64s: make([]uint64, 0), 88 | str: "", 89 | } 90 | 91 | var numU32, numU64 int 92 | var hasStr bool 93 | switch typ { 94 | case CREATE: 95 | numU32 = 1 96 | numU64 = 0 97 | hasStr = true 98 | case DROP: 99 | numU32 = 0 100 | numU64 = 0 101 | hasStr = true 102 | case MEASURE: 103 | numU32 = 3 104 | numU64 = 2 105 | hasStr = false 106 | case PATTERN: 107 | numU32 = 1 108 | numU64 = 0 109 | hasStr = true 110 | default: 111 | return ipcMsg{}, fmt.Errorf("malformed message") 112 | } 113 | 114 | payload := bytes.NewBuffer(buf[6:]) 115 | for i := 0; i < numU32; i++ { 116 | var u uint32 117 | binary.Read(payload, binary.LittleEndian, &u) 118 | msg.u32s = append(msg.u32s, u) 119 | } 120 | 121 | for i := 0; i < numU64; i++ { 122 | var u uint64 123 | binary.Read(payload, binary.LittleEndian, &u) 124 | msg.u64s = append(msg.u64s, u) 125 | } 126 | 127 | if hasStr { 128 | s := make([]byte, int(msg.len)-6-numU32*4-numU64*8) 129 | binary.Read(payload, binary.LittleEndian, &s) 130 | 131 | // remove null terminator 132 | if s[len(s)-1] == byte(0) { 133 | s = s[:len(s)-1] 134 | } 135 | msg.str = string(s) 136 | } 137 | 138 | return 139 | } 140 | 141 | func (i *Ipc) demux(ch chan []byte) { 142 | for buf := range ch { 143 | ipcm, err := msgReader(buf) 144 | if err != nil { 145 | log.WithFields(log.Fields{ 146 | "err": err, 147 | "buf": buf, 148 | }).Warn("failed to parse message") 149 | continue 150 | } 151 | switch ipcm.typ { 152 | case MEASURE: 153 | i.MeasureNotify <- MeasureMsg{ 154 | socketId: ipcm.socketId, 155 | ackNo: ipcm.u32s[0], 156 | rtt: time.Duration(ipcm.u32s[1]) * time.Microsecond, 157 | loss: ipcm.u32s[2], 158 | rin: ipcm.u64s[0], 159 | rout: ipcm.u64s[1], 160 | } 161 | case DROP: 162 | i.DropNotify <- DropMsg{ 163 | socketId: ipcm.socketId, 164 | event: ipcm.str, 165 | } 166 | case CREATE: 167 | i.CreateNotify <- CreateMsg{ 168 | socketId: ipcm.socketId, 169 | startSeq: ipcm.u32s[0], 170 | congAlg: ipcm.str, 171 | } 172 | case PATTERN: 173 | p, err := deserializePattern(ipcm.str, ipcm.u32s[0]) 174 | if err != nil { 175 | continue 176 | } 177 | 178 | i.PatternNotify <- PatternMsg{ 179 | socketId: ipcm.socketId, 180 | pattern: p, 181 | } 182 | } 183 | } 184 | } 185 | 186 | func msgWriter(msg ipcMsg) ([]byte, error) { 187 | // header: 6 Bytes 188 | switch { 189 | case msg.typ == CREATE && len(msg.u32s) == 1 && len(msg.u64s) == 0 && msg.str != "": 190 | // + 1 uint32, + string 191 | msg.len = 10 + uint8(len(msg.str)) 192 | case msg.typ == DROP && len(msg.u32s) == 0 && len(msg.u64s) == 0 && msg.str != "": 193 | // + string 194 | msg.len = uint8(6 + len(msg.str)) 195 | case msg.typ == MEASURE && len(msg.u32s) == 3 && len(msg.u64s) == 2 && msg.str == "": 196 | // + 3 uint32, + 2 uint64, no string 197 | // 6 + 12 + 16 = 34 198 | msg.len = 34 199 | case msg.typ == PATTERN && len(msg.u32s) == 1 && len(msg.u64s) == 0 && msg.str != "": 200 | // + 1 uint32, + string 201 | msg.len = 10 + uint8(len(msg.str)) 202 | default: 203 | return nil, fmt.Errorf("Invalid message") 204 | } 205 | 206 | buf := new(bytes.Buffer) 207 | binary.Write(buf, binary.LittleEndian, uint8(msg.typ)) 208 | binary.Write(buf, binary.LittleEndian, msg.len) 209 | binary.Write(buf, binary.LittleEndian, msg.socketId) 210 | 211 | for _, val := range msg.u32s { 212 | binary.Write(buf, binary.LittleEndian, val) 213 | } 214 | 215 | for _, val := range msg.u64s { 216 | binary.Write(buf, binary.LittleEndian, val) 217 | } 218 | 219 | if msg.str != "" { 220 | binary.Write(buf, binary.LittleEndian, []byte(msg.str)) 221 | } 222 | 223 | return buf.Bytes(), nil 224 | } 225 | 226 | // Pattern serialization 227 | 228 | /* (type, len, value?) event description 229 | * ---------------------------------------- 230 | * | Event Type | Len (B) | Uint32? | 231 | * | (1 B) | (1 B) | (0||32 bits) | 232 | * ---------------------------------------- 233 | * total: 2 || 6 Bytes 234 | */ 235 | func serializePatternEvent(ev flowPattern.PatternEvent) (buf []byte, err error) { 236 | b := new(bytes.Buffer) 237 | binary.Write(b, binary.LittleEndian, uint8(ev.Type)) 238 | var value uint32 239 | switch ev.Type { 240 | case flowPattern.SETRATEABS: 241 | value = uint32(ev.Rate) 242 | 243 | case flowPattern.SETCWNDABS: 244 | value = ev.Cwnd 245 | 246 | case flowPattern.SETRATEREL: 247 | fallthrough 248 | case flowPattern.WAITREL: 249 | value = uint32(ev.Factor * 100) 250 | 251 | case flowPattern.WAITABS: 252 | value = uint32(ev.Duration.Nanoseconds() / 1e3) 253 | 254 | case flowPattern.REPORT: 255 | err = binary.Write(b, binary.LittleEndian, uint8(2)) 256 | goto ret 257 | default: 258 | err = fmt.Errorf("unknown pattern-event type: %v", ev.Type) 259 | } 260 | 261 | err = binary.Write(b, binary.LittleEndian, uint8(6)) 262 | err = binary.Write(b, binary.LittleEndian, value) 263 | ret: 264 | buf = b.Bytes() 265 | return 266 | } 267 | 268 | func serializeSequence(evs []flowPattern.PatternEvent) ([]byte, error) { 269 | buf := make([]byte, 0) 270 | for _, ev := range evs { 271 | b, err := serializePatternEvent(ev) 272 | if err != nil { 273 | return nil, err 274 | } 275 | 276 | buf = append(buf, b...) 277 | } 278 | 279 | return buf, nil 280 | } 281 | 282 | func deserializePattern(msg string, numEvents uint32) (pat *flowPattern.Pattern, err error) { 283 | var evType uint8 284 | var evLength uint8 285 | buf := bytes.NewBuffer([]byte(msg)) 286 | pat = flowPattern.NewPattern() 287 | for i := uint32(0); i < numEvents; i++ { 288 | err = binary.Read(buf, binary.LittleEndian, &evType) 289 | err = binary.Read(buf, binary.LittleEndian, &evLength) 290 | if err != nil { 291 | return nil, err 292 | } 293 | 294 | if evLength == 2 && flowPattern.PatternEventType(evType) == flowPattern.REPORT { 295 | pat = pat.Report() 296 | continue 297 | } else if evLength == 2 { 298 | return nil, fmt.Errorf("could not parse event type %d", evType) 299 | } else if evLength != 6 { 300 | return nil, fmt.Errorf("could not parse event %d of length %d", evType, evLength) 301 | } 302 | 303 | var evVal uint32 304 | err = binary.Read(buf, binary.LittleEndian, &evVal) 305 | if err != nil { 306 | return nil, err 307 | } 308 | 309 | switch flowPattern.PatternEventType(evType) { 310 | case flowPattern.SETRATEABS: 311 | pat = pat.Rate(float32(evVal) / 100.0) 312 | case flowPattern.SETRATEREL: 313 | pat = pat.RelativeRate(float32(evVal) / 100.0) 314 | case flowPattern.SETCWNDABS: 315 | pat = pat.Cwnd(evVal) 316 | case flowPattern.WAITREL: 317 | pat = pat.WaitRtts(float32(evVal) / 100.0) 318 | case flowPattern.WAITABS: 319 | pat = pat.Wait(time.Duration(evVal) * time.Microsecond) 320 | } 321 | } 322 | 323 | return pat.Compile() 324 | } 325 | -------------------------------------------------------------------------------- /ipc/serialize_test.go: -------------------------------------------------------------------------------- 1 | package ipc 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "ccp/ccpFlow/pattern" 8 | "ccp/ipcBackend" 9 | 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // dummy backend 14 | type MockBackend struct { 15 | ch chan []byte 16 | doLog bool 17 | } 18 | 19 | func NewMockBackend(doLog bool) ipcbackend.Backend { 20 | return &MockBackend{ 21 | ch: make(chan []byte), 22 | doLog: doLog, 23 | } 24 | } 25 | 26 | func (d *MockBackend) SetupListen(l string, id uint32) ipcbackend.Backend { 27 | return d 28 | } 29 | 30 | func (d *MockBackend) SetupSend(l string, id uint32) ipcbackend.Backend { 31 | return d 32 | } 33 | 34 | func (d *MockBackend) SetupFinish() (ipcbackend.Backend, error) { 35 | return d, nil 36 | } 37 | 38 | func (d *MockBackend) SendMsg(msg ipcbackend.Msg) error { 39 | s, err := msg.Serialize() 40 | if err != nil { 41 | log.WithFields(log.Fields{ 42 | "err": err, 43 | "buf": s, 44 | "msg": msg, 45 | }).Warn("failed to serialize message") 46 | return err 47 | } 48 | 49 | if d.doLog { 50 | log.WithFields(log.Fields{ 51 | "msg": s, 52 | }).Info("sending message") 53 | } 54 | 55 | go func() { 56 | d.ch <- s 57 | }() 58 | 59 | return nil 60 | } 61 | 62 | func (d *MockBackend) Listen() chan []byte { 63 | ch := make(chan []byte) 64 | go func() { 65 | for b := range d.ch { 66 | ch <- b 67 | } 68 | }() 69 | 70 | return ch 71 | } 72 | 73 | func (d *MockBackend) Close() error { 74 | close(d.ch) 75 | return nil 76 | } 77 | 78 | func testSetup(doLog bool) (i *Ipc, err error) { 79 | back, err := NewMockBackend(doLog).SetupFinish() 80 | if err != nil { 81 | return 82 | } 83 | 84 | i, err = SetupWithBackend(back) 85 | if err != nil { 86 | return 87 | } 88 | 89 | return 90 | } 91 | 92 | // test values 93 | const testNum uint32 = 42 94 | const testDuration time.Duration = time.Duration(42) * time.Millisecond 95 | const testBigNum uint64 = 42 96 | const testString string = "forty-two" 97 | 98 | func TestEncodeMeasureMsg(t *testing.T) { 99 | i, err := testSetup(true) 100 | if err != nil { 101 | t.Error(err) 102 | return 103 | } 104 | 105 | outMsgCh, _ := i.ListenMeasureMsg() 106 | i.SendMeasureMsg( 107 | testNum, 108 | testNum, 109 | testDuration, 110 | testNum, 111 | testBigNum, 112 | testBigNum, 113 | ) 114 | 115 | select { 116 | case out := <-outMsgCh: 117 | if out.SocketId() != testNum || 118 | out.AckNo() != testNum || 119 | out.Rtt() != testDuration || 120 | out.Loss() != testNum || 121 | out.Rin() != testBigNum || 122 | out.Rout() != testBigNum { 123 | t.Errorf( 124 | "wrong message\ngot (%v, %v, %v, %v, %v, %v)\nexpected (%v, %v, %v, %v, %v, %v)", 125 | out.SocketId(), 126 | out.AckNo(), 127 | out.Rtt(), 128 | out.Loss(), 129 | out.Rin(), 130 | out.Rout(), 131 | testNum, 132 | testNum, 133 | testDuration, 134 | testNum, 135 | testBigNum, 136 | testBigNum, 137 | ) 138 | } 139 | case <-time.After(time.Second): 140 | t.Error("timed out") 141 | } 142 | } 143 | 144 | func TestEncodeCreateMsg(t *testing.T) { 145 | i, err := testSetup(true) 146 | if err != nil { 147 | t.Error(err) 148 | return 149 | } 150 | 151 | outMsgCh, err := i.ListenCreateMsg() 152 | if err != nil { 153 | t.Error(err) 154 | return 155 | } 156 | 157 | err = i.SendCreateMsg( 158 | testNum, 159 | testNum, 160 | testString, 161 | ) 162 | if err != nil { 163 | t.Error(err) 164 | return 165 | } 166 | 167 | select { 168 | case out := <-outMsgCh: 169 | if out.SocketId() != testNum || 170 | out.StartSeq() != testNum || 171 | out.CongAlg() != testString { 172 | t.Errorf( 173 | "wrong message\ngot (%v, %v, %v)\nexpected (%v, %v, %v)", 174 | out.SocketId(), 175 | out.StartSeq(), 176 | out.CongAlg(), 177 | testNum, 178 | testNum, 179 | testString, 180 | ) 181 | } 182 | case <-time.After(time.Second): 183 | t.Error("timed out") 184 | } 185 | } 186 | 187 | func TestEncodeDropMsg(t *testing.T) { 188 | i, err := testSetup(true) 189 | if err != nil { 190 | t.Error(err) 191 | return 192 | } 193 | 194 | outMsgCh, _ := i.ListenDropMsg() 195 | i.SendDropMsg( 196 | testNum, 197 | testString, 198 | ) 199 | 200 | select { 201 | case out := <-outMsgCh: 202 | if out.SocketId() != testNum || 203 | out.Event() != testString { 204 | t.Errorf( 205 | "wrong message\ngot (%v, %v)\nexpected (%v, %v)", 206 | out.SocketId(), 207 | out.Event(), 208 | testNum, 209 | testString, 210 | ) 211 | } 212 | case <-time.After(time.Second): 213 | t.Error("timed out") 214 | } 215 | } 216 | 217 | func TestEncodePatternMsg(t *testing.T) { 218 | i, err := testSetup(true) 219 | if err != nil { 220 | t.Error(err) 221 | return 222 | } 223 | 224 | outMsgCh, _ := i.ListenPatternMsg() 225 | p, err := pattern. 226 | NewPattern(). 227 | Cwnd(testNum). 228 | WaitRtts(1.0). 229 | Report(). 230 | Compile() 231 | if err != nil { 232 | t.Error(err) 233 | return 234 | } 235 | 236 | err = i.SendPatternMsg(testNum, p) 237 | if err != nil { 238 | t.Error(err) 239 | return 240 | } 241 | 242 | select { 243 | case out := <-outMsgCh: 244 | if out.SocketId() != testNum { 245 | t.Errorf( 246 | "wrong message\ngot sid (%v)\nexpected (%v)", 247 | out.SocketId(), 248 | testNum, 249 | ) 250 | return 251 | } 252 | 253 | if len(out.Pattern().Sequence) != 3 { 254 | t.Errorf( 255 | "wrong pattern length\ngot %v\nexpected %v", 256 | len(out.Pattern().Sequence), 257 | 3, 258 | ) 259 | return 260 | } 261 | 262 | for i, ev := range out.Pattern().Sequence { 263 | switch i { 264 | case 0: // cwnd 265 | if ev.Type != pattern.SETCWNDABS || ev.Cwnd != testNum { 266 | t.Errorf( 267 | "wrong message\ngot event %d (type %v, cwnd %d)\nexpected (%v, %v)", 268 | i, 269 | ev.Type, 270 | ev.Cwnd, 271 | pattern.SETCWNDABS, 272 | testNum, 273 | ) 274 | return 275 | } 276 | case 1: // WaitRtts 277 | if ev.Type != pattern.WAITREL || ev.Factor != 1.0 { 278 | t.Errorf( 279 | "wrong message\ngot event %d (type %v, factor %v)\nexpected (%v, %v)", 280 | i, 281 | ev.Type, 282 | ev.Factor, 283 | pattern.WAITREL, 284 | 1.0, 285 | ) 286 | return 287 | } 288 | case 2: // Report 289 | if ev.Type != pattern.REPORT { 290 | t.Errorf( 291 | "wrong message\ngot event %d type %d\nexpected %v", 292 | i, 293 | ev.Type, 294 | pattern.REPORT, 295 | ) 296 | return 297 | } 298 | } 299 | } 300 | case <-time.After(time.Second): 301 | t.Error("timed out") 302 | } 303 | } 304 | -------------------------------------------------------------------------------- /ipcBackend/ipc.go: -------------------------------------------------------------------------------- 1 | package ipcbackend 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | type Msg interface { 9 | Serialize() ([]byte, error) 10 | } 11 | 12 | type Backend interface { 13 | SetupListen(l string, id uint32) Backend 14 | SetupSend(l string, id uint32) Backend 15 | SetupFinish() (Backend, error) 16 | 17 | SendMsg(msg Msg) error 18 | Listen() chan []byte 19 | 20 | Close() error 21 | } 22 | 23 | func AddressForListen(loc string, id uint32) (fd string, openFiles string, err error) { 24 | err = nil 25 | if id != 0 { 26 | dirName := fmt.Sprintf("/tmp/ccp-%d", id) 27 | os.RemoveAll(dirName) 28 | err = os.MkdirAll(dirName, 0755) 29 | if err != nil { 30 | return "", "", err 31 | } 32 | 33 | fd = fmt.Sprintf("%s/%s", dirName, loc) 34 | openFiles = dirName 35 | } else { 36 | fd = fmt.Sprintf("/tmp/ccp-%s", loc) 37 | os.RemoveAll(fd) 38 | openFiles = fd 39 | } 40 | 41 | return 42 | } 43 | 44 | func AddressForSend(loc string, id uint32) (fd string) { 45 | if id != 0 { 46 | fd = fmt.Sprintf("/tmp/ccp-%d/%s", id, loc) 47 | } else { 48 | fd = fmt.Sprintf("/tmp/ccp-%s", loc) 49 | } 50 | return 51 | } 52 | -------------------------------------------------------------------------------- /ipcBackend/lib_test.go: -------------------------------------------------------------------------------- 1 | package ipcbackend 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestListenAddr(t *testing.T) { 11 | f, o, err := AddressForListen("addr-test", 2) 12 | if err != nil { 13 | t.Error(err) 14 | return 15 | } 16 | 17 | defer func() { 18 | os.RemoveAll(o) 19 | }() 20 | 21 | if !bytes.Equal([]byte(f), []byte(fmt.Sprintf("/tmp/ccp-2/addr-test"))) { 22 | t.Error("wrong value", f) 23 | return 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /netlinkipc/netlink.go: -------------------------------------------------------------------------------- 1 | package netlinkipc 2 | 3 | import ( 4 | "ccp/ipcBackend" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/mdlayher/netlink" 8 | ) 9 | 10 | const ( 11 | NETLINK_MCAST_GROUP = 22 12 | ) 13 | 14 | type NetlinkIpc struct { 15 | conn *netlink.Conn 16 | 17 | listenCh chan []byte 18 | 19 | err error 20 | killed chan interface{} 21 | } 22 | 23 | func New() ipcbackend.Backend { 24 | return &NetlinkIpc{} 25 | } 26 | 27 | func (n *NetlinkIpc) SetupSend(loc string, id uint32) ipcbackend.Backend { 28 | if n.err != nil { 29 | return n 30 | } 31 | 32 | if n.conn == nil { 33 | nl, err := nlInit() 34 | if err != nil { 35 | n.err = err 36 | return n 37 | } 38 | 39 | n.conn = nl 40 | } 41 | 42 | return n 43 | } 44 | 45 | func (n *NetlinkIpc) SetupListen(loc string, id uint32) ipcbackend.Backend { 46 | if n.err != nil { 47 | return n 48 | } 49 | 50 | if n.conn == nil { 51 | nl, err := nlInit() 52 | if err != nil { 53 | n.err = err 54 | return n 55 | } 56 | 57 | n.conn = nl 58 | } 59 | 60 | n.listenCh = make(chan []byte) 61 | go n.listen() 62 | 63 | return n 64 | } 65 | 66 | func (s *NetlinkIpc) SetupFinish() (ipcbackend.Backend, error) { 67 | if s.err != nil { 68 | log.WithFields(log.Fields{ 69 | "err": s.err, 70 | }).Error("error setting up IPC") 71 | return s, s.err 72 | } else { 73 | s.killed = make(chan interface{}) 74 | return s, nil 75 | } 76 | } 77 | 78 | func (n *NetlinkIpc) SendMsg(msg ipcbackend.Msg) error { 79 | buf, err := msg.Serialize() 80 | if err != nil { 81 | return err 82 | } 83 | 84 | resp := netlink.Message{ 85 | Header: netlink.Header{ 86 | Flags: netlink.HeaderFlagsRequest | netlink.HeaderFlagsAcknowledge, 87 | }, 88 | Data: buf, 89 | } 90 | 91 | _, err = n.conn.Send(resp) 92 | return err 93 | } 94 | 95 | func (n *NetlinkIpc) Listen() chan []byte { 96 | msgCh := make(chan []byte) 97 | 98 | go func() { 99 | for { 100 | select { 101 | case <-n.killed: 102 | close(msgCh) 103 | return 104 | case buf := <-n.listenCh: 105 | msgCh <- buf 106 | } 107 | } 108 | }() 109 | 110 | return msgCh 111 | } 112 | 113 | func (n *NetlinkIpc) Close() error { 114 | close(n.killed) 115 | return nil 116 | } 117 | 118 | func (n *NetlinkIpc) listen() { 119 | for { 120 | select { 121 | case <-n.killed: 122 | n.conn.Close() 123 | close(n.listenCh) 124 | return 125 | default: 126 | } 127 | 128 | msgs, err := n.conn.Receive() 129 | if err != nil { 130 | log.WithFields(log.Fields{ 131 | "where": "netlinkipc.listen", 132 | }).Warn(err) 133 | continue 134 | } 135 | 136 | for _, msg := range msgs { 137 | select { 138 | case n.listenCh <- msg.Data: 139 | default: // ok to drop messages 140 | } 141 | } 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /netlinkipc/nonlinux.go: -------------------------------------------------------------------------------- 1 | // +build !linux 2 | 3 | package netlinkipc 4 | 5 | import ( 6 | "fmt" 7 | 8 | "github.com/mdlayher/netlink" 9 | ) 10 | 11 | func nlInit() (*netlink.Conn, error) { 12 | return nil, fmt.Errorf("netlink only supported on linux") 13 | } 14 | -------------------------------------------------------------------------------- /netlinkipc/setup.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package netlinkipc 4 | 5 | import ( 6 | "golang.org/x/sys/unix" 7 | 8 | "github.com/mdlayher/netlink" 9 | ) 10 | 11 | func nlInit() (*netlink.Conn, error) { 12 | nl, err := netlink.Dial( 13 | unix.NETLINK_USERSOCK, 14 | &netlink.Config{ 15 | Groups: 0, 16 | }, 17 | ) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | nl.JoinGroup(NETLINK_MCAST_GROUP) 23 | 24 | return nl, nil 25 | } 26 | -------------------------------------------------------------------------------- /nl_userapp/nl_userapp.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | 8 | "ccp/ccpFlow/pattern" 9 | "ccp/ipc" 10 | "ccp/netlinkipc" 11 | 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | func main() { 16 | nl, err := netlinkipc.New().SetupSend("", 33).SetupListen("", 33).SetupFinish() 17 | if err != nil { 18 | log.WithFields(log.Fields{ 19 | "where": "recv setup", 20 | }).Warn(err) 21 | return 22 | } 23 | 24 | i, _ := ipc.SetupWithBackend(nl) 25 | testControl(i) 26 | } 27 | 28 | func testControl(i *ipc.Ipc) { 29 | cwnd := uint32(10) * 1460 30 | crs, _ := i.ListenCreateMsg() 31 | mrs, _ := i.ListenMeasureMsg() 32 | for { 33 | select { 34 | case cr := <-crs: 35 | log.WithFields(log.Fields{ 36 | "got": fmt.Sprintf("(%v, %v)", cr.SocketId(), cr.CongAlg()), 37 | }).Info("rcvd create") 38 | case ack := <-mrs: 39 | log.WithFields(log.Fields{ 40 | "got": fmt.Sprintf("(%v, %v, %v)", ack.SocketId(), ack.AckNo(), ack.Rtt()), 41 | "cwnd": cwnd, 42 | }).Info("rcvd ack") 43 | } 44 | 45 | cwnd += 2 * 1460 46 | staticPattern, err := pattern.NewPattern(). 47 | Cwnd(cwnd). 48 | Compile() 49 | if err != nil { 50 | log.WithFields(log.Fields{ 51 | "err": err, 52 | "cwnd": cwnd, 53 | }).Info("send cwnd msg failed") 54 | continue 55 | } 56 | 57 | i.SendPatternMsg(42, staticPattern) 58 | log.WithFields(log.Fields{ 59 | "cwnd": cwnd, 60 | }).Info("sent cwnd msg") 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /nl_userapp/nonlinux.go: -------------------------------------------------------------------------------- 1 | // +build !linux 2 | 3 | package main 4 | 5 | import "fmt" 6 | 7 | func main() { 8 | fmt.Println("netlink only supported on linux") 9 | } 10 | -------------------------------------------------------------------------------- /reno/cc_test.go: -------------------------------------------------------------------------------- 1 | package reno 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | "time" 7 | 8 | "ccp/ccpFlow" 9 | flowPattern "ccp/ccpFlow/pattern" 10 | ) 11 | 12 | // mock ipc.SendOnly 13 | type MockSendOnly struct { 14 | ch chan *flowPattern.Pattern 15 | } 16 | 17 | func (m *MockSendOnly) SendPatternMsg(socketId uint32, pattern *flowPattern.Pattern) error { 18 | go func() { 19 | m.ch <- pattern 20 | }() 21 | return nil 22 | } 23 | 24 | func TestReno(t *testing.T) { 25 | t.Log("init") 26 | Init() 27 | f, err := ccpFlow.GetFlow("reno") 28 | if err != nil { 29 | t.Error(err) 30 | return 31 | } 32 | 33 | if n := f.Name(); !bytes.Equal([]byte(n), []byte("reno")) { 34 | t.Errorf("got \"%s\", expected \"reno\"", n) 35 | return 36 | } 37 | 38 | ipcMockCh := make(chan *flowPattern.Pattern) 39 | mockIpc := &MockSendOnly{ch: ipcMockCh} 40 | f.Create(42, mockIpc, 1462, 0, 10) 41 | <-ipcMockCh // ignore the first initial cwnd set 42 | 43 | if f.(*Reno).lastAck != 0 || f.(*Reno).sockid != 42 { 44 | t.Errorf("got \"%v\", expected lastAck=0 and sockid=42", f) 45 | return 46 | } 47 | 48 | f.GotMeasurement(ccpFlow.Measurement{ 49 | Ack: uint32(292400), 50 | Rtt: time.Microsecond, 51 | }) 52 | if f.(*Reno).lastAck != 292400 || f.(*Reno).sockid != 42 { 53 | t.Errorf("got \"%v\", expected lastAck=292400 and sockid=42", f) 54 | return 55 | } 56 | 57 | p := <-ipcMockCh 58 | if len(p.Sequence) != 3 { 59 | t.Errorf("expected sequence length 3, got %d", len(p.Sequence)) 60 | return 61 | } else if p.Sequence[0].Type != flowPattern.SETCWNDABS { 62 | t.Errorf("expected event type %d, got %d", flowPattern.SETCWNDABS, p.Sequence[0].Type) 63 | return 64 | } 65 | 66 | t.Log("isolated drop") 67 | f.Drop(ccpFlow.DupAck) 68 | p = <-ipcMockCh 69 | if len(p.Sequence) != 3 { 70 | t.Errorf("expected sequence length 2, got %d", len(p.Sequence)) 71 | return 72 | } else if p.Sequence[0].Type != flowPattern.SETCWNDABS { 73 | t.Errorf("expected event type %d, got %d", flowPattern.SETCWNDABS, p.Sequence[0].Type) 74 | return 75 | } 76 | 77 | t.Log("complete drop") 78 | f.Drop(ccpFlow.Timeout) 79 | p = <-ipcMockCh 80 | if len(p.Sequence) != 3 { 81 | t.Errorf("expected sequence length 2, got %d", len(p.Sequence)) 82 | return 83 | } else if p.Sequence[0].Type != flowPattern.SETCWNDABS { 84 | t.Errorf("expected event type %d, got %d", flowPattern.SETCWNDABS, p.Sequence[0].Type) 85 | return 86 | } else if p.Sequence[0].Cwnd != 14620 { 87 | t.Errorf("expected cwnd 14620, got %d", p.Sequence[0].Cwnd) 88 | return 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /reno/reno.go: -------------------------------------------------------------------------------- 1 | package reno 2 | 3 | import ( 4 | "math" 5 | "time" 6 | 7 | "ccp/ccpFlow" 8 | "ccp/ccpFlow/pattern" 9 | "ccp/ipc" 10 | 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // implement ccpFlow.Flow interface 15 | type Reno struct { 16 | pktSize uint32 17 | initCwnd float32 18 | 19 | ssthresh uint32 20 | cwndClamp float32 21 | cwnd float32 22 | lastAck uint32 23 | rtt time.Duration 24 | lastDrop time.Time 25 | 26 | sockid uint32 27 | ipc ipc.SendOnly 28 | } 29 | 30 | func (r *Reno) Name() string { 31 | return "reno" 32 | } 33 | 34 | func (r *Reno) Create( 35 | socketid uint32, 36 | send ipc.SendOnly, 37 | pktsz uint32, 38 | startSeq uint32, 39 | startCwnd uint32, 40 | ) { 41 | r.sockid = socketid 42 | r.ipc = send 43 | r.pktSize = pktsz 44 | r.ssthresh = 0x7fffffff 45 | r.cwndClamp = 2e5 * float32(pktsz) 46 | r.initCwnd = float32(pktsz * 10) 47 | r.cwnd = float32(pktsz * startCwnd) 48 | r.lastDrop = time.Now() 49 | r.rtt = time.Since(r.lastDrop) 50 | if startSeq == 0 { 51 | r.lastAck = startSeq 52 | } else { 53 | r.lastAck = startSeq - 1 54 | } 55 | 56 | pattern, err := pattern. 57 | NewPattern(). 58 | Cwnd(uint32(r.cwnd)). 59 | WaitRtts(0.1). 60 | Report(). 61 | Compile() 62 | if err != nil { 63 | log.WithFields(log.Fields{ 64 | "err": err, 65 | "cwnd": r.cwnd, 66 | }).Info("make cwnd msg failed") 67 | return 68 | } 69 | 70 | r.sendPattern(pattern) 71 | } 72 | 73 | func (r *Reno) GotMeasurement(m ccpFlow.Measurement) { 74 | // Ignore out of order netlink messages 75 | // Happens sometimes when the reporting interval is small 76 | // If within 10 packets, assume no integer overflow 77 | if m.Ack < r.lastAck && m.Ack > r.lastAck-r.pktSize*10 { 78 | return 79 | } 80 | 81 | // Handle integer overflow / sequence wraparound 82 | var newBytesAcked uint64 83 | if m.Ack < r.lastAck { 84 | newBytesAcked = uint64(math.MaxUint32) + uint64(m.Ack) - uint64(r.lastAck) 85 | } else { 86 | newBytesAcked = uint64(m.Ack) - uint64(r.lastAck) 87 | } 88 | 89 | acked := newBytesAcked 90 | 91 | if uint32(r.cwnd) < r.ssthresh { 92 | // increase cwnd by 1 per packet, until ssthresh 93 | if uint64(r.cwnd)+newBytesAcked > uint64(r.ssthresh) { 94 | newBytesAcked -= uint64(r.ssthresh - uint32(r.cwnd)) 95 | r.cwnd = float32(r.ssthresh) 96 | } else { 97 | r.cwnd += float32(newBytesAcked) 98 | newBytesAcked = 0 99 | } 100 | } 101 | 102 | // increase cwnd by 1 / cwnd per packet 103 | r.cwnd += float32(r.pktSize) * (float32(newBytesAcked) / r.cwnd) 104 | 105 | if r.cwnd > r.cwndClamp { 106 | r.cwnd = r.cwndClamp 107 | } 108 | 109 | // notify increased cwnd 110 | pattern, err := pattern. 111 | NewPattern(). 112 | Cwnd(uint32(r.cwnd)). 113 | WaitRtts(0.5). 114 | Report(). 115 | Compile() 116 | if err != nil { 117 | log.WithFields(log.Fields{ 118 | "err": err, 119 | "cwnd": r.cwnd, 120 | }).Info("make cwnd msg failed") 121 | return 122 | } 123 | 124 | r.sendPattern(pattern) 125 | 126 | r.rtt = m.Rtt 127 | 128 | log.WithFields(log.Fields{ 129 | "gotAck": m.Ack, 130 | "currCwndPkts": r.cwnd / float32(r.pktSize), 131 | "currLastAck": r.lastAck, 132 | "newlyAcked": acked, 133 | "ssThresh": r.ssthresh, 134 | "rtt-ns": r.rtt.Nanoseconds(), 135 | }).Info("[reno] got ack") 136 | 137 | r.lastAck = m.Ack 138 | return 139 | } 140 | 141 | func (r *Reno) Drop(ev ccpFlow.DropEvent) { 142 | if time.Since(r.lastDrop) <= r.rtt { 143 | return 144 | } 145 | 146 | log.WithFields(log.Fields{ 147 | "time since last drop": time.Since(r.lastDrop), 148 | "rtt": r.rtt, 149 | }).Info("[reno] got drop") 150 | 151 | r.lastDrop = time.Now() 152 | 153 | oldCwnd := r.cwnd 154 | switch ev { 155 | case ccpFlow.DupAck: 156 | r.cwnd /= 2 157 | r.ssthresh = uint32(r.cwnd) 158 | if r.cwnd < r.initCwnd { 159 | r.cwnd = r.initCwnd 160 | } 161 | case ccpFlow.Timeout: 162 | r.ssthresh = uint32(r.cwnd / 2) 163 | r.cwnd = r.initCwnd 164 | default: 165 | log.WithFields(log.Fields{ 166 | "event": ev, 167 | }).Warn("[reno] unknown drop event type") 168 | return 169 | } 170 | 171 | pattern, err := pattern. 172 | NewPattern(). 173 | Cwnd(uint32(r.cwnd)). 174 | WaitRtts(0.1). 175 | Report(). 176 | Compile() 177 | if err != nil { 178 | log.WithFields(log.Fields{ 179 | "err": err, 180 | "cwnd": r.cwnd, 181 | }).Info("make cwnd msg failed") 182 | return 183 | } 184 | 185 | r.sendPattern(pattern) 186 | 187 | log.WithFields(log.Fields{ 188 | "oldCwndPkts": oldCwnd / float32(r.pktSize), 189 | "currCwndPkts": r.cwnd / float32(r.pktSize), 190 | "event": ev, 191 | "ssThresh": r.ssthresh, 192 | }).Info("[reno] drop") 193 | } 194 | 195 | func (r *Reno) sendPattern(pattern *pattern.Pattern) { 196 | err := r.ipc.SendPatternMsg(r.sockid, pattern) 197 | if err != nil { 198 | log.WithFields(log.Fields{"cwnd": r.cwnd, "name": r.sockid}).Warn(err) 199 | } 200 | } 201 | 202 | func Init() { 203 | ccpFlow.Register("reno", func() ccpFlow.Flow { 204 | return &Reno{} 205 | }) 206 | } 207 | -------------------------------------------------------------------------------- /tests/start-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /tests/testClient/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | 8 | "ccp/udpDataplane" 9 | 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | var ip = flag.String("ip", "127.0.0.1", "ip address to connect to") 14 | var size = flag.Int("size", 11200, "amount of data to request") 15 | 16 | func main() { 17 | flag.Parse() 18 | sock, err := udpDataplane.Socket(*ip, "40000", "CLIENT") 19 | if err != nil { 20 | log.Error(err) 21 | return 22 | } 23 | 24 | req := []byte(fmt.Sprintf("testRequest: size %d", *size)) 25 | _, err = sock.Write(req) 26 | if err != nil { 27 | log.Error(err) 28 | return 29 | } 30 | 31 | response := new(bytes.Buffer) 32 | rcvd := sock.Read(100) 33 | for rb := range rcvd { 34 | response.Write(rb) 35 | if response.Len() >= *size-13 { 36 | break 37 | } 38 | } 39 | 40 | log.WithFields(log.Fields{ 41 | "got": response.Len(), 42 | }).Info("done") 43 | 44 | sock.Fin() 45 | } 46 | -------------------------------------------------------------------------------- /tests/testServer/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | "strings" 7 | 8 | "ccp/udpDataplane" 9 | 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | func main() { 14 | sock, err := udpDataplane.Socket("", "40000", "SERVER") 15 | if err != nil { 16 | log.Error(err) 17 | return 18 | } 19 | 20 | rcvd := sock.Read(1) 21 | rb := <-rcvd 22 | req := strings.Split(string(rb), " ") 23 | if len(req) != 3 || req[0] != "testRequest:" || req[1] != "size" { 24 | log.WithFields(log.Fields{ 25 | "got": rb, 26 | "expected": "'test request: size {x}'", 27 | }).Warn("malformed request") 28 | } 29 | 30 | sz, err := strconv.Atoi(req[2]) 31 | if err != nil { 32 | log.Error(err) 33 | return 34 | } 35 | 36 | log.WithFields(log.Fields{ 37 | "req": string(rb), 38 | }).Info("got req") 39 | 40 | resp := bytes.Repeat([]byte("test response\n"), sz/14) 41 | sent, err := sock.Write(resp) 42 | if err != nil { 43 | log.Error(err) 44 | return 45 | } 46 | 47 | log.WithFields(log.Fields{ 48 | "len": len(resp), 49 | "asked len": sz, 50 | }).Info("started write") 51 | 52 | for a := range sent { 53 | log.WithFields(log.Fields{ 54 | "ack": a, 55 | "total": len(resp), 56 | }).Info("server acked") 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /udpDataplane/bench_test.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "ccp/ccpFlow" 12 | "ccp/ipc" 13 | "ccp/reno" 14 | 15 | log "github.com/sirupsen/logrus" 16 | ) 17 | 18 | func init() { 19 | log.SetLevel(log.WarnLevel) 20 | } 21 | 22 | func BenchmarkDoRx(b *testing.B) { 23 | // setup dummy socket 24 | s := &Sock{ 25 | name: "test", 26 | 27 | conn: nil, 28 | writeBuf: make([]byte, 2e6), 29 | readBuf: make([]byte, 2e6), 30 | 31 | //sender 32 | cwnd: 5 * PACKET_SIZE, // init_cwnd = ~ 1 pkt 33 | lastAckedSeqNo: 0, 34 | dupAckCnt: 0, 35 | nextSeqNo: 0, 36 | inFlight: makeWindow(), 37 | 38 | //receiver 39 | lastAck: 0, 40 | rcvWindow: makeWindow(), 41 | 42 | // ccp communication 43 | ackNotifyThresh: PACKET_SIZE * 10, // ~ 10 pkts 44 | 45 | // synchronization 46 | shouldTx: make(chan interface{}, 1), 47 | shouldPass: make(chan uint32, 1), 48 | notifyAcks: make(chan notifyAck, 1), 49 | notifyDrops: make(chan notifyDrop, 1), 50 | ackedData: make(chan uint32), 51 | closed: make(chan interface{}), 52 | } 53 | 54 | b.ResetTimer() 55 | 56 | for i := 0; i < b.N; i++ { 57 | doBenchmarkRx(s) 58 | s.inFlight = makeWindow() 59 | } 60 | } 61 | 62 | func doBenchmarkRx(s *Sock) { 63 | // check sender side (receiving acks) 64 | s.inFlight.addPkt(time.Now(), &Packet{ 65 | SeqNo: uint32(1460), 66 | AckNo: 0, 67 | Flag: ACK, 68 | Length: 1462, 69 | Payload: bytes.Repeat([]byte("t"), 1462), 70 | }) 71 | 72 | s.doRx(&Packet{ 73 | SeqNo: 0, 74 | AckNo: uint32(1460), 75 | Flag: ACK, 76 | Length: 0, 77 | }) 78 | } 79 | 80 | func BenchmarkSocket(b *testing.B) { 81 | b.SetBytes(1e6) 82 | for i := 0; i < b.N; i++ { 83 | kill := make(chan interface{}) 84 | ready := make(chan interface{}) 85 | go renoCcp(kill, ready) 86 | <-ready 87 | log.Debug("ccp ready") 88 | go server(kill, ready) 89 | <-ready 90 | done := make(chan interface{}) 91 | go client(done) 92 | <-done 93 | close(kill) 94 | } 95 | } 96 | 97 | // from ccp/ccp.go 98 | func renoCcp(killed chan interface{}, ready chan interface{}) { 99 | com, err := ipc.SetupCcpListen(ipc.UNIX) 100 | if err != nil { 101 | log.Error(err) 102 | return 103 | } 104 | 105 | ackCh, err := com.ListenMeasureMsg() 106 | if err != nil { 107 | log.Error(err) 108 | return 109 | } 110 | 111 | dropCh, err := com.ListenDropMsg() 112 | if err != nil { 113 | log.Error(err) 114 | return 115 | } 116 | 117 | createCh, err := com.ListenCreateMsg() 118 | if err != nil { 119 | log.Error(err) 120 | return 121 | } 122 | 123 | ready <- struct{}{} 124 | 125 | r := &reno.Reno{} 126 | for { 127 | select { 128 | case <-killed: 129 | return 130 | case cr := <-createCh: 131 | log.Info("got create") 132 | ipCh, err := ipc.SetupCcpSend(ipc.UNIX, cr.SocketId()) 133 | if err != nil { 134 | log.WithFields(log.Fields{"flowid": cr.SocketId()}).Error("Error creating ccp->socket ipc channel for flow") 135 | } 136 | r.Create(40000, ipCh, 1462, 0, 10) 137 | case ack := <-ackCh: 138 | log.Info("got ack") 139 | r.GotMeasurement(ccpFlow.Measurement{ 140 | Ack: ack.AckNo(), 141 | Rtt: ack.Rtt(), 142 | }) 143 | case dr := <-dropCh: 144 | log.Info("got drop") 145 | r.Drop(ccpFlow.DropEvent(dr.Event())) 146 | } 147 | } 148 | } 149 | 150 | // from test/testServer/server.go 151 | func server(killed chan interface{}, ready chan interface{}) { 152 | sockCh := socketNonBlocking("", "40000", "SERVER") 153 | ready <- struct{}{} 154 | sock := <-sockCh 155 | 156 | rcvd := sock.Read(1) 157 | rb := <-rcvd 158 | req := strings.Split(string(rb), " ") 159 | if len(req) != 3 || req[0] != "testRequest:" || req[1] != "size" { 160 | log.WithFields(log.Fields{ 161 | "got": rb, 162 | "expected": "'test request: size {x}'", 163 | }).Warn("malformed request") 164 | } 165 | 166 | sz, err := strconv.Atoi(req[2]) 167 | if err != nil { 168 | log.Error(err) 169 | return 170 | } 171 | 172 | log.WithFields(log.Fields{ 173 | "req": string(rb), 174 | }).Info("got req") 175 | 176 | resp := bytes.Repeat([]byte("test response\n"), sz/14) 177 | sent, err := sock.Write(resp) 178 | if err != nil { 179 | log.Error(err) 180 | return 181 | } 182 | 183 | log.WithFields(log.Fields{ 184 | "len": len(resp), 185 | "asked len": sz, 186 | }).Info("started write") 187 | 188 | for a := range sent { 189 | select { 190 | case <-killed: 191 | return 192 | default: 193 | } 194 | 195 | log.WithFields(log.Fields{ 196 | "ack": a, 197 | "total": len(resp), 198 | }).Info("server acked") 199 | } 200 | } 201 | 202 | // from test/testClient/client.go 203 | func client(done chan interface{}) { 204 | size := int(1e6) 205 | sock, err := Socket("127.0.0.1", "40000", "CLIENT") 206 | if err != nil { 207 | log.Error(err) 208 | return 209 | } 210 | 211 | req := []byte(fmt.Sprintf("testRequest: size %d", size)) 212 | _, err = sock.Write(req) 213 | if err != nil { 214 | log.Error(err) 215 | return 216 | } 217 | 218 | response := new(bytes.Buffer) 219 | rcvd := sock.Read(100) 220 | for rb := range rcvd { 221 | response.Write(rb) 222 | if response.Len() >= size-13 { 223 | break 224 | } 225 | } 226 | 227 | log.WithFields(log.Fields{ 228 | "got": response.Len(), 229 | }).Info("done") 230 | 231 | sock.Fin() 232 | 233 | done <- struct{}{} 234 | } 235 | -------------------------------------------------------------------------------- /udpDataplane/notify.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "time" 5 | 6 | "ccp/ccpFlow" 7 | "ccp/ccpFlow/pattern" 8 | "ccp/ipc" 9 | 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | type notifyAck struct { 14 | ack uint32 15 | rtt time.Duration 16 | } 17 | 18 | type notifyDrop struct { 19 | lastAck uint32 20 | ev string 21 | } 22 | 23 | func (sock *Sock) ipcListen(patternCh chan ipc.PatternMsg, measureMsgs chan notifyAck) { 24 | patternChanged := make(chan *pattern.Pattern) 25 | go func() { 26 | stopPattern := make(chan interface{}) 27 | for newPattern := range patternChanged { 28 | select { 29 | case stopPattern <- struct{}{}: 30 | default: 31 | } 32 | 33 | go sock.doPattern(newPattern, measureMsgs, stopPattern) 34 | } 35 | }() 36 | 37 | for { 38 | select { 39 | case pmsg := <-patternCh: 40 | patternChanged <- pmsg.Pattern() 41 | case <-sock.closed: 42 | close(patternChanged) 43 | return 44 | } 45 | } 46 | } 47 | 48 | func (sock *Sock) setupIpc() error { 49 | ipcL, err := ipc.SetupCli(sock.port) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | sock.ipc = ipcL 55 | 56 | // start listening for pattern specifications 57 | patternSet, err := sock.ipc.ListenPatternMsg() 58 | if err != nil { 59 | return err 60 | } 61 | 62 | measureMsgWaiter := make(chan notifyAck) 63 | go sock.doNotify(measureMsgWaiter) 64 | go sock.ipcListen(patternSet, measureMsgWaiter) 65 | 66 | sock.ipc.SendCreateMsg(sock.port, 0, "reno") 67 | return nil 68 | } 69 | 70 | func (sock *Sock) doNotify(measureMsg chan notifyAck) { 71 | totAck := uint32(0) 72 | droppedPktNo := uint32(0) 73 | timeout := time.NewTimer(time.Second) 74 | for { 75 | select { 76 | case notifAck := <-sock.notifyAcks: 77 | if notifAck.ack > totAck { 78 | totAck = notifAck.ack 79 | } else { 80 | continue 81 | } 82 | 83 | log.WithFields(log.Fields{ 84 | "name": sock.name, 85 | "acked": totAck, 86 | "rtt": notifAck.rtt, 87 | }).Info("notifyAcks") 88 | 89 | select { 90 | case measureMsg <- notifAck: 91 | default: 92 | } 93 | 94 | select { 95 | case sock.ackedData <- totAck: 96 | default: 97 | } 98 | case dropEv := <-sock.notifyDrops: 99 | if dropEv.lastAck > droppedPktNo { 100 | log.WithFields(log.Fields{ 101 | "name": sock.name, 102 | "event": dropEv, 103 | }).Info("notifyDrops") 104 | writeDropMsg(sock.name, sock.port, sock.ipc, dropEv.ev) 105 | droppedPktNo = dropEv.lastAck 106 | } 107 | case <-timeout.C: 108 | timeout.Reset(time.Second) 109 | case <-sock.closed: 110 | log.WithFields(log.Fields{"where": "doNotify", "name": sock.name}).Debug("closed, exiting") 111 | close(sock.ackedData) 112 | return 113 | } 114 | if !timeout.Stop() { 115 | <-timeout.C 116 | } 117 | timeout.Reset(time.Second) 118 | } 119 | } 120 | 121 | func (sock *Sock) doNonWaitPatternEvent(ev pattern.PatternEvent, currRtt time.Duration) { 122 | var cwnd uint32 123 | switch ev.Type { 124 | case pattern.REPORT: 125 | fallthrough 126 | case pattern.WAITREL: 127 | fallthrough 128 | case pattern.WAITABS: 129 | return 130 | 131 | case pattern.SETRATEABS: 132 | // set cwnd = rate * rtt 133 | cwnd = uint32(float64(ev.Rate) * currRtt.Seconds()) 134 | 135 | case pattern.SETCWNDABS: 136 | cwnd = ev.Cwnd 137 | 138 | case pattern.SETRATEREL: 139 | cwnd = uint32(float64(sock.cwnd) * float64(ev.Factor)) 140 | } 141 | 142 | log.WithFields(log.Fields{ 143 | "cwnd": cwnd, 144 | }).Info("set cwnd") 145 | sock.mux.Lock() 146 | sock.cwnd = cwnd 147 | sock.mux.Unlock() 148 | } 149 | 150 | func (sock *Sock) doPattern(p *pattern.Pattern, measureMsgs chan notifyAck, stopPattern chan interface{}) { 151 | // infinitely loop through events in the sequence 152 | var currRtt time.Duration 153 | for { 154 | for _, ev := range p.Sequence { 155 | wait := time.Duration(0) 156 | switch ev.Type { 157 | case pattern.WAITABS: 158 | wait = ev.Duration 159 | case pattern.WAITREL: 160 | wait = time.Duration(currRtt.Seconds()*float64(ev.Factor)*1e6) * time.Microsecond 161 | 162 | case pattern.REPORT: 163 | select { 164 | case meas := <-measureMsgs: 165 | currRtt = meas.rtt 166 | writeMeasureMsg(sock.name, sock.port, sock.ipc, meas.ack, meas.rtt) 167 | continue 168 | case <-stopPattern: 169 | return 170 | } 171 | 172 | default: 173 | sock.doNonWaitPatternEvent(ev, currRtt) 174 | continue 175 | } 176 | 177 | select { 178 | case <-time.After(wait): 179 | continue 180 | case <-stopPattern: 181 | return 182 | } 183 | } 184 | } 185 | } 186 | 187 | func writeMeasureMsg( 188 | name string, 189 | id uint32, 190 | out *ipc.Ipc, 191 | ack uint32, 192 | rtt time.Duration, 193 | ) { 194 | err := out.SendMeasureMsg(id, ack, rtt, 0, 0, 0) 195 | if err != nil { 196 | log.WithFields(log.Fields{"ack": ack, "name": name, "id": id, "where": "notify.writeMeasureMsg"}).Warn(err) 197 | return 198 | } 199 | } 200 | 201 | func writeDropMsg(name string, id uint32, out *ipc.Ipc, event string) { 202 | switch event { 203 | case "timeout": 204 | event = string(ccpFlow.Timeout) 205 | case "3xdupack": 206 | event = string(ccpFlow.DupAck) 207 | default: 208 | log.WithFields(log.Fields{ 209 | "event": event, 210 | "id": id, 211 | "name": name, 212 | }).Panic("unknown event") 213 | } 214 | err := out.SendDropMsg(id, event) 215 | if err != nil { 216 | log.WithFields(log.Fields{"event": event, "name": name, "id": id, "where": "notify.writeDropMsg"}).Warn(err) 217 | return 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /udpDataplane/packet.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | 7 | "github.com/akshayknarayan/udp/packetops" 8 | ) 9 | 10 | // 1463 = 1500 - 28 (ip + udp) - 12 (my header) 11 | const PACKET_SIZE = 1460 12 | 13 | type PacketFlag uint8 14 | 15 | const ( 16 | SYN PacketFlag = iota 17 | SYNACK 18 | ACK 19 | FIN 20 | ) 21 | 22 | // Implement packetops.Packet 23 | type Packet struct { 24 | SeqNo uint32 // 32 bits = 4 bytes 25 | AckNo uint32 // 32 bits = 4 bytes 26 | Flag PacketFlag // Upper 4 bits of Length int below = 0.5 byte 27 | Length uint16 // Only use bottom 12 bits! Max size = 2^12 = 4096. 12 bits = 1.5 bytes 28 | Sack []bool // bit vector, 16 bits = 2 bytes 29 | Payload []byte 30 | } 31 | 32 | func (pkt *Packet) Encode( 33 | size int, 34 | ) (*packetops.RawPacket, error) { 35 | b, err := encode(*pkt) 36 | return &packetops.RawPacket{Buf: b}, err 37 | } 38 | 39 | func encode(p Packet) ([]byte, error) { 40 | buf := new(bytes.Buffer) 41 | err := binary.Write(buf, binary.LittleEndian, p.SeqNo) 42 | if err != nil { 43 | return buf.Bytes(), err 44 | } 45 | 46 | err = binary.Write(buf, binary.LittleEndian, p.AckNo) 47 | if err != nil { 48 | return buf.Bytes(), err 49 | } 50 | 51 | // ensure only bottom 12 bits used 52 | p.Length = p.Length & 0x0fff 53 | // ensure only bottom 4 bits used 54 | flag := (uint16(p.Flag) & 0x3) << 12 55 | 56 | field := flag | p.Length // uint16, top 4 bits flag, bottom 12 bits len 57 | 58 | err = binary.Write(buf, binary.LittleEndian, field) 59 | if err != nil { 60 | return buf.Bytes(), err 61 | } 62 | 63 | // make bit vector in uint16 64 | sack := uint16(0) 65 | for i, v := range p.Sack { 66 | if v { 67 | sack |= 1 << uint(i) 68 | } 69 | } 70 | 71 | err = binary.Write(buf, binary.LittleEndian, sack) 72 | if err != nil { 73 | return buf.Bytes(), err 74 | } 75 | 76 | buf.Write(p.Payload) 77 | 78 | return buf.Bytes(), err 79 | } 80 | 81 | func (pkt *Packet) Decode( 82 | r *packetops.RawPacket, 83 | ) error { 84 | p, err := decode(r.Buf) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | pkt.SeqNo = p.SeqNo 90 | pkt.AckNo = p.AckNo 91 | pkt.Flag = p.Flag 92 | pkt.Length = p.Length 93 | pkt.Sack = p.Sack 94 | pkt.Payload = p.Payload 95 | 96 | return nil 97 | } 98 | 99 | func decode(b []byte) (Packet, error) { 100 | var p Packet 101 | buf := bytes.NewReader(b) 102 | 103 | err := binary.Read(buf, binary.LittleEndian, &p.SeqNo) 104 | if err != nil { 105 | return p, err 106 | } 107 | 108 | err = binary.Read(buf, binary.LittleEndian, &p.AckNo) 109 | if err != nil { 110 | return p, err 111 | } 112 | 113 | var field uint16 114 | err = binary.Read(buf, binary.LittleEndian, &field) 115 | if err != nil { 116 | return p, err 117 | } 118 | 119 | p.Length = field & 0xfff 120 | p.Flag = PacketFlag((field & 0xf000) >> 12) 121 | 122 | var sack uint16 123 | err = binary.Read(buf, binary.LittleEndian, &sack) 124 | if err != nil { 125 | return p, err 126 | } 127 | 128 | p.Sack = make([]bool, 0, 16) 129 | for i := 0; i < 16; i++ { 130 | p.Sack = append(p.Sack, ((sack>>uint(i))&1) == 1) 131 | } 132 | 133 | // SeqNo + AckNo + (Flag,Length) + SACK vector = 12 bytes 134 | p.Payload = make([]byte, len(b)-12) 135 | copy(p.Payload, b[12:]) 136 | 137 | return p, err 138 | } 139 | -------------------------------------------------------------------------------- /udpDataplane/packet_test.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/akshayknarayan/udp/packetops" 8 | ) 9 | 10 | func equal(a []bool, b []bool) bool { 11 | if len(a) != len(b) { 12 | return false 13 | } 14 | 15 | for i, v := range a { 16 | if v != b[i] { 17 | return false 18 | } 19 | } 20 | return true 21 | } 22 | 23 | func TestEncodePacket(t *testing.T) { 24 | p := &Packet{ 25 | SeqNo: 0, 26 | AckNo: 0, 27 | Flag: ACK, 28 | Length: 10, 29 | Sack: []bool{}, 30 | Payload: bytes.Repeat([]byte{'t'}, 10), 31 | } 32 | 33 | enc, err := p.Encode(0) 34 | if err != nil { 35 | t.Errorf("encoding error: %v", err) 36 | } 37 | 38 | expected := bytes.Repeat([]byte{0}, 8) // seq and ack 39 | expected = append(expected, []byte{0xa, 0x20}...) 40 | expected = append(expected, []byte{0x00, 0x00}...) 41 | expected = append(expected, bytes.Repeat([]byte{'t'}, 10)...) 42 | 43 | if len(enc.Buf) != len(expected) { 44 | t.Errorf("wrong length:\n%v\n%v", enc, expected) 45 | return 46 | } 47 | 48 | for i := 0; i < len(expected); i++ { 49 | if enc.Buf[i] != expected[i] { 50 | t.Errorf("expected %v\ngot %v", enc, expected) 51 | return 52 | } 53 | } 54 | } 55 | 56 | func TestDecodePacket(t *testing.T) { 57 | r := bytes.Repeat([]byte{0}, 8) // seq and ack 58 | r = append(r, []byte{0xa, 0x20}...) 59 | r = append(r, []byte{0x01, 0x00}...) 60 | r = append(r, bytes.Repeat([]byte{'t'}, 10)...) 61 | 62 | raw := &packetops.RawPacket{ 63 | Buf: r, 64 | From: nil, 65 | } 66 | 67 | expected := &Packet{ 68 | SeqNo: 0, 69 | AckNo: 0, 70 | Flag: ACK, 71 | Length: 10, 72 | Sack: []bool{true, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false}, 73 | Payload: bytes.Repeat([]byte{'t'}, 10), 74 | } 75 | 76 | got := &Packet{} 77 | err := got.Decode(raw) 78 | if err != nil { 79 | t.Errorf("decoding error: %v", err) 80 | } 81 | 82 | if got.SeqNo != expected.SeqNo || 83 | got.AckNo != expected.AckNo || 84 | got.Flag != expected.Flag || 85 | got.Length != expected.Length || 86 | !equal(got.Sack, expected.Sack) { 87 | t.Errorf("expected: %v\ngot: %v", expected, got) 88 | return 89 | } 90 | 91 | for i := 0; i < int(got.Length); i++ { 92 | if got.Payload[i] != expected.Payload[i] { 93 | t.Errorf("expected %v\ngot %v", expected, got) 94 | return 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /udpDataplane/rx.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/akshayknarayan/udp/packetops" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | func (sock *Sock) rx() { 12 | rcvdPkts := make(chan *Packet) 13 | go func() { 14 | for { 15 | select { 16 | case <-sock.closed: 17 | return 18 | default: 19 | } 20 | 21 | rcvd := &Packet{} 22 | _, err := packetops.RecvPacket(sock.conn, rcvd) 23 | if err != nil { 24 | log.WithFields(log.Fields{"where": "rx"}).Warn(err) 25 | continue 26 | } 27 | 28 | rcvdPkts <- rcvd 29 | } 30 | }() 31 | 32 | for { 33 | select { 34 | case <-sock.closed: 35 | return 36 | case rcvd := <-rcvdPkts: 37 | err := sock.doRx(rcvd) 38 | if err != nil { 39 | log.Warn(err) 40 | } 41 | } 42 | } 43 | } 44 | 45 | func (sock *Sock) doRx(rcvd *Packet) error { 46 | if rcvd.Flag == FIN { 47 | sock.Close() 48 | return nil 49 | } else if rcvd.Flag != ACK { 50 | var flag string 51 | switch rcvd.Flag { 52 | case SYN: 53 | flag = "SYN" 54 | case SYNACK: 55 | flag = "SYNACK" 56 | default: 57 | flag = "unknown" 58 | } 59 | 60 | err := fmt.Errorf("connection in unknown state") 61 | log.WithFields(log.Fields{ 62 | "flag": flag, 63 | "pkt": rcvd, 64 | }).Panic(err) 65 | return err 66 | } 67 | 68 | rcvd.Payload = rcvd.Payload[:rcvd.Length] 69 | 70 | sock.mux.Lock() 71 | sock.handleAck(rcvd) 72 | sock.handleData(rcvd) 73 | sock.mux.Unlock() 74 | 75 | return nil 76 | } 77 | 78 | // process ack 79 | // sender 80 | func (sock *Sock) handleAck(rcvd *Packet) { 81 | firstUnacked, err := sock.inFlight.start() 82 | if err != nil { 83 | return 84 | } 85 | 86 | if rcvd.AckNo < firstUnacked { 87 | return 88 | } 89 | 90 | lastAcked, rtt, err := sock.inFlight.rcvdPkt(time.Now(), rcvd) 91 | if err != nil { 92 | // there were no packets in flight 93 | // so we got an ack to a packet we didn't send 94 | log.WithFields(log.Fields{ 95 | "name": sock.name, 96 | "firstUnacked": firstUnacked, 97 | "rcvd.ackno": rcvd.AckNo, 98 | "inFlight": sock.inFlight.order, 99 | }).Panic("unknown packet") 100 | } 101 | 102 | if lastAcked == sock.lastAckedSeqNo && sock.nextSeqNo > lastAcked { 103 | sock.dupAckCnt++ 104 | log.WithFields(log.Fields{ 105 | "name": sock.name, 106 | "sock.lastAcked": lastAcked, 107 | "firstUnacked": firstUnacked, 108 | "rcvd.ackno": rcvd.AckNo, 109 | "inFlight": sock.inFlight.order, 110 | "dupAcks": sock.dupAckCnt, 111 | "sack": rcvd.Sack, 112 | }).Debug("dup ack") 113 | 114 | if sock.dupAckCnt >= 3 { 115 | // dupAckCnt >= 3 -> packet drop 116 | log.WithFields(log.Fields{ 117 | "name": sock.name, 118 | "sock.dupAckCnt": sock.dupAckCnt, 119 | "sock.lastAcked": lastAcked, 120 | }).Debug("drop detected") 121 | sock.inFlight.drop(lastAcked, rcvd) 122 | select { 123 | case sock.notifyDrops <- notifyDrop{ev: "3xdupack", lastAck: lastAcked}: 124 | default: 125 | } 126 | } 127 | 128 | select { 129 | case sock.shouldTx <- struct{}{}: 130 | default: 131 | } 132 | 133 | return 134 | } else { 135 | sock.dupAckCnt = 0 136 | } 137 | 138 | sock.lastAckedSeqNo = lastAcked 139 | select { 140 | case sock.shouldTx <- struct{}{}: 141 | default: 142 | } 143 | 144 | log.WithFields(log.Fields{ 145 | "name": sock.name, 146 | "lastAcked": sock.lastAckedSeqNo, 147 | "rcvd.ackno": rcvd.AckNo, 148 | "inFlight": sock.inFlight.order, 149 | "rtt": rtt, 150 | }).Info("new ack") 151 | 152 | select { 153 | case sock.notifyAcks <- notifyAck{ack: lastAcked, rtt: rtt}: 154 | default: 155 | } 156 | } 157 | 158 | // process received payload 159 | func (sock *Sock) handleData(rcvd *Packet) { 160 | if rcvd.Length > 0 && rcvd.SeqNo >= sock.lastAck { // relevant data packet 161 | if _, ok := sock.rcvWindow.pkts[rcvd.SeqNo]; ok { 162 | // spurious retransmission 163 | return 164 | } 165 | 166 | // new data! 167 | sock.rcvWindow.addPkt(time.Now(), rcvd) 168 | ackNo, err := sock.rcvWindow.cumAck(sock.lastAck) 169 | if err != nil { 170 | ackNo = sock.lastAck 171 | log.WithFields(log.Fields{"name": sock.name, "ackNo": ackNo}).Warn(err) 172 | } 173 | 174 | sock.lastAck = ackNo 175 | log.WithFields(log.Fields{ 176 | "name": sock.name, 177 | "rcvd.seqno": rcvd.SeqNo, 178 | "rcvd.length": rcvd.Length, 179 | "sock.lastAck": sock.lastAck, 180 | }).Info("new data") 181 | 182 | copy(sock.readBuf[rcvd.SeqNo:], rcvd.Payload[:rcvd.Length]) 183 | select { 184 | case sock.shouldPass <- sock.lastAck: 185 | log.Debug("sent on shouldPass") 186 | case <-sock.closed: 187 | close(sock.shouldPass) 188 | return 189 | default: 190 | log.Debug("skipping shouldPass") 191 | } 192 | 193 | select { 194 | case sock.shouldTx <- struct{}{}: 195 | default: 196 | } 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /udpDataplane/socket.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "strconv" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "ccp/ipc" 12 | 13 | "github.com/akshayknarayan/udp/packetops" 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | func init() { 18 | log.SetLevel(log.InfoLevel) 19 | } 20 | 21 | type Sock struct { 22 | name string 23 | port uint32 24 | 25 | conn *net.UDPConn // the underlying connection 26 | writeBuf []byte // TODO make it a ring buffer 27 | writeBufPos int 28 | readBuf []byte // TODO make it a ring buffer 29 | 30 | // sender 31 | cwnd uint32 32 | lastAckedSeqNo uint32 33 | dupAckCnt uint8 34 | nextSeqNo uint32 35 | inFlight *window 36 | 37 | // receiver 38 | lastAck uint32 39 | rcvWindow *window 40 | 41 | // communication with CCP 42 | ackNotifyThresh uint32 43 | ipc *ipc.Ipc 44 | 45 | // synchronization 46 | shouldTx chan interface{} 47 | shouldPass chan uint32 48 | notifyAcks chan notifyAck 49 | notifyDrops chan notifyDrop 50 | ackedData chan uint32 51 | closed chan interface{} 52 | 53 | mux sync.Mutex 54 | } 55 | 56 | // create and connect a socket. 57 | // if ip == "", will listen on port until a SYN arrives 58 | // else, will send a SYN to ip:port to establish a connection 59 | func Socket(ip string, port string, name string) (s *Sock, err error) { 60 | s = <-socketNonBlocking(ip, port, name) 61 | if s == nil { 62 | err = fmt.Errorf("could not create socket") 63 | } 64 | 65 | return 66 | } 67 | 68 | func socketNonBlocking(ip string, port string, name string) (ch chan *Sock) { 69 | log.WithFields(log.Fields{ 70 | "ip": ip, 71 | "port": port, 72 | }).Info("creating socket...") 73 | 74 | ch = make(chan *Sock) 75 | if ip != "" { 76 | conn, err := makeConnClient(ip, port) 77 | if err != nil { 78 | log.WithFields(log.Fields{ 79 | "ip": ip, 80 | "port": port, 81 | "name": name, 82 | "err": err, 83 | }).Warn("makeConnClient failed") 84 | goto fail 85 | } 86 | 87 | sk, err := mkSocket(conn, name) 88 | if err != nil { 89 | log.WithFields(log.Fields{ 90 | "ip": ip, 91 | "port": port, 92 | "name": name, 93 | "err": err, 94 | }).Warn("mkSocket failed") 95 | goto fail 96 | } 97 | 98 | log.WithFields(log.Fields{ 99 | "ip": ip, 100 | "port": port, 101 | "name": name, 102 | }).Info("created socket!") 103 | 104 | go func() { ch <- sk }() 105 | return 106 | } else { 107 | connCh, err := makeConnServer(ip, port) 108 | if err != nil { 109 | log.WithFields(log.Fields{ 110 | "ip": ip, 111 | "port": port, 112 | "name": name, 113 | "err": err, 114 | }).Warn("makeConnServer failed") 115 | goto fail 116 | } 117 | 118 | go func() { 119 | conn := <-connCh 120 | if conn == nil { 121 | ch <- nil 122 | return 123 | } 124 | 125 | log.WithFields(log.Fields{ 126 | "ip": ip, 127 | "port": port, 128 | "name": name, 129 | }).Info("created socket!") 130 | 131 | sk, err := mkSocket(conn, name) 132 | if err != nil { 133 | log.WithFields(log.Fields{ 134 | "ip": ip, 135 | "port": port, 136 | "name": name, 137 | "err": err, 138 | }).Warn("mkSocket failed") 139 | ch <- nil 140 | return 141 | } 142 | 143 | ch <- sk 144 | }() 145 | return 146 | } 147 | 148 | fail: 149 | go func() { 150 | ch <- nil 151 | }() 152 | return ch 153 | } 154 | 155 | func makeConnServer(ip string, port string) (connCh chan *net.UDPConn, err error) { 156 | conn, addr, err := packetops.SetupListeningSock(port) 157 | if err != nil { 158 | return nil, err 159 | } 160 | 161 | connCh = make(chan *net.UDPConn) 162 | 163 | go func() { 164 | log.Info("Listening for SYN") 165 | syn := &Packet{} 166 | conn, err := packetops.ListenForSyn(conn, addr, syn) 167 | if err != nil { 168 | log.WithFields(log.Fields{ 169 | "ip": ip, 170 | "port": port, 171 | "err": err, 172 | }).Warn("makeConnServer failed") 173 | connCh <- nil 174 | return 175 | } 176 | 177 | syn.Flag = SYNACK 178 | syn.AckNo = 1 179 | 180 | log.Info("Sending SYNACK") 181 | err = packetops.SendSyn(conn, syn) 182 | if err != nil { 183 | log.WithFields(log.Fields{ 184 | "ip": ip, 185 | "port": port, 186 | "err": err, 187 | }).Warn("makeConnServer failed") 188 | connCh <- nil 189 | return 190 | } 191 | 192 | connCh <- conn 193 | }() 194 | 195 | return 196 | } 197 | 198 | func makeConnClient(ip string, port string) (conn *net.UDPConn, err error) { 199 | var addr *net.UDPAddr 200 | conn, addr, err = packetops.SetupClientSock(ip, port) 201 | if err != nil { 202 | return nil, err 203 | } 204 | 205 | syn := &Packet{ 206 | SeqNo: 0, 207 | AckNo: 0, 208 | Flag: SYN, 209 | } 210 | 211 | log.Info("Sending SYN and expecting ACK") 212 | packetops.SynAckExchange(conn, addr, syn) 213 | return 214 | } 215 | 216 | func mkSocket(conn *net.UDPConn, name string) (*Sock, error) { 217 | s := &Sock{ 218 | name: name, 219 | 220 | conn: conn, 221 | writeBuf: make([]byte, 2e6), 222 | readBuf: make([]byte, 2e6), 223 | 224 | //sender 225 | cwnd: 5 * PACKET_SIZE, // init_cwnd = ~ 1 pkt 226 | lastAckedSeqNo: 0, 227 | dupAckCnt: 0, 228 | nextSeqNo: 0, 229 | inFlight: makeWindow(), 230 | 231 | //receiver 232 | lastAck: 0, 233 | rcvWindow: makeWindow(), 234 | 235 | // ccp communication 236 | ackNotifyThresh: PACKET_SIZE * 10, // ~ 10 pkts 237 | // ipc initialized later 238 | 239 | // synchronization 240 | shouldTx: make(chan interface{}, 1), 241 | shouldPass: make(chan uint32, 1), 242 | notifyAcks: make(chan notifyAck, 1), 243 | notifyDrops: make(chan notifyDrop, 1), 244 | ackedData: make(chan uint32), 245 | closed: make(chan interface{}), 246 | } 247 | 248 | addr := s.conn.LocalAddr().String() 249 | spl := strings.Split(addr, ":") 250 | lport, err := strconv.Atoi(spl[1]) 251 | if err != nil { 252 | log.WithFields(log.Fields{ 253 | "name": s.name, 254 | "addr": spl, 255 | "lport": lport, 256 | }).Warn(err) 257 | lport = 42424 258 | } 259 | 260 | s.port = uint32(lport) 261 | err = s.setupIpc() 262 | if err != nil { 263 | log.WithFields(log.Fields{ 264 | "name": s.name, 265 | }).Error(err) 266 | return nil, err 267 | } 268 | 269 | go s.rx() 270 | go s.tx() 271 | 272 | return s, nil 273 | } 274 | 275 | // currently only supports writing once 276 | func (sock *Sock) Write(b []byte) (chan uint32, error) { 277 | if len(b) > len(sock.writeBuf) { 278 | return sock.ackedData, fmt.Errorf("Write exceeded buffer: %d > %d", len(b), len(sock.writeBuf)) 279 | } 280 | 281 | sock.mux.Lock() 282 | copy(sock.writeBuf, b) 283 | sock.writeBufPos = len(b) 284 | sock.mux.Unlock() 285 | select { 286 | case sock.shouldTx <- struct{}{}: 287 | case <-sock.closed: 288 | return nil, fmt.Errorf("socket closed") 289 | } 290 | return sock.ackedData, nil 291 | } 292 | 293 | // receiver 294 | func (sock *Sock) Read(returnGranularity uint32) chan []byte { 295 | passUp := make(chan []byte, 1) 296 | go func() { 297 | totAck := uint32(0) 298 | notifiedDataNo := uint32(0) 299 | timeout := time.NewTimer(time.Second) 300 | loop: 301 | for { 302 | select { 303 | case lastAck, ok := <-sock.shouldPass: 304 | if !ok { 305 | break loop 306 | } 307 | 308 | log.WithFields(log.Fields{ 309 | "name": sock.name, 310 | "acked": totAck, 311 | "notifiedDataNo": notifiedDataNo, 312 | }).Debug("got send on shouldPass") 313 | 314 | if lastAck > totAck { 315 | totAck = lastAck 316 | } 317 | 318 | if !timeout.Stop() { 319 | <-timeout.C 320 | } 321 | timeout.Reset(time.Second) 322 | case <-timeout.C: 323 | timeout.Reset(time.Second) 324 | } 325 | 326 | if totAck-notifiedDataNo > returnGranularity { 327 | select { 328 | case passUp <- sock.readBuf[notifiedDataNo:totAck]: 329 | notifiedDataNo = totAck 330 | log.Debug("notified application of rcvd data") 331 | default: 332 | } 333 | } 334 | } 335 | 336 | close(passUp) 337 | }() 338 | 339 | return passUp 340 | } 341 | 342 | func (sock *Sock) Fin() error { 343 | sock.mux.Lock() 344 | pkt := &Packet{ 345 | SeqNo: sock.nextSeqNo, 346 | AckNo: sock.lastAck, 347 | Flag: FIN, 348 | Length: 0, 349 | Payload: []byte{}, 350 | } 351 | sock.mux.Unlock() 352 | packetops.SendPacket(sock.conn, pkt, 0) 353 | return sock.Close() 354 | } 355 | 356 | func (sock *Sock) Close() error { 357 | log.WithFields(log.Fields{ 358 | "name": sock.name, 359 | }).Info("closing") 360 | 361 | close(sock.closed) 362 | err := sock.conn.Close() 363 | if err != nil { 364 | log.WithFields(log.Fields{ 365 | "name": sock.name, 366 | "where": "closing", 367 | }).Warn(err) 368 | } 369 | 370 | sock.ipc.Close() 371 | return nil 372 | } 373 | -------------------------------------------------------------------------------- /udpDataplane/socket_test.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "ccp/ipc" 10 | 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | func TestBasicTransfer(t *testing.T) { 15 | fmt.Println("starting...") 16 | // create dummy ccp 17 | ccp, err := ipc.SetupCcpListen(ipc.UNIX) 18 | if err != nil { 19 | t.Error(err) 20 | return 21 | } 22 | 23 | go dummyCcp(ccp) 24 | testTransfer(t, "40000", bytes.Repeat([]byte{'a'}, 100), true) 25 | ccp.Close() 26 | } 27 | 28 | func TestLongerTransfer(t *testing.T) { 29 | // create dummy ccp 30 | ccp, err := ipc.SetupCcpListen(ipc.UNIX) 31 | if err != nil { 32 | t.Error(err) 33 | return 34 | } 35 | 36 | go dummyCcp(ccp) 37 | testTransfer(t, "40001", bytes.Repeat([]byte{'b'}, 10000), true) 38 | ccp.Close() 39 | } 40 | 41 | func TestLongerTransferBigCwnd(t *testing.T) { 42 | // create dummy ccp 43 | ccp, err := ipc.SetupCcpListen(ipc.UNIX) 44 | if err != nil { 45 | t.Error(err) 46 | return 47 | } 48 | 49 | go dummyCcp(ccp) 50 | testTransfer(t, "40002", bytes.Repeat([]byte{'c'}, 20000), false) 51 | ccp.Close() 52 | } 53 | 54 | func dummyCcp(ccp *ipc.Ipc) { 55 | go func() { 56 | ch, err := ccp.ListenCreateMsg() 57 | if err != nil { 58 | log.Error(err) 59 | return 60 | } 61 | for m := range ch { 62 | log.WithFields(log.Fields{ 63 | "msg": m, 64 | }).Info("got msg") 65 | } 66 | }() 67 | 68 | go func() { 69 | ackCh, err := ccp.ListenMeasureMsg() 70 | if err != nil { 71 | log.Error(err) 72 | return 73 | } 74 | for ack := range ackCh { 75 | log.WithFields(log.Fields{ 76 | "msg": ack, 77 | }).Info("got msg") 78 | } 79 | }() 80 | } 81 | 82 | func testTransfer(t *testing.T, port string, data []byte, smallcwnd bool) { 83 | done := make(chan error) 84 | cleanup := make(chan *Sock) 85 | exit := make(chan interface{}) 86 | fmt.Println("starting receiver...") 87 | go receiver(port, data, done, cleanup, exit) 88 | <-time.After(time.Duration(100) * time.Millisecond) 89 | fmt.Println("starting sender...") 90 | go sender(port, data, done, cleanup, exit, smallcwnd) 91 | 92 | err := <-done 93 | if err != nil { 94 | t.Error(err) 95 | } 96 | 97 | s1 := <-cleanup 98 | s1.Close() 99 | fmt.Printf("closed %s\n", s1.name) 100 | for { 101 | select { 102 | case s2 := <-cleanup: 103 | s2.Close() 104 | return 105 | default: 106 | select { 107 | case exit <- struct{}{}: 108 | default: 109 | } 110 | } 111 | } 112 | } 113 | 114 | func sender(port string, data []byte, done chan error, cleanup chan *Sock, exit chan interface{}, smallcwnd bool) { 115 | sock, err := Socket("127.0.0.1", port, "SENDER") 116 | if err != nil { 117 | select { 118 | case done <- err: 119 | default: 120 | } 121 | return 122 | } 123 | 124 | if smallcwnd { 125 | sock.cwnd = 1 * PACKET_SIZE 126 | } else { 127 | sock.cwnd = 5 * PACKET_SIZE 128 | } 129 | 130 | sent, err := sock.Write(data) 131 | if err != nil { 132 | select { 133 | case done <- err: 134 | default: 135 | } 136 | } 137 | 138 | loop: 139 | for { 140 | select { 141 | case a := <-sent: 142 | log.WithFields(log.Fields{"acked": a, "tot": len(data)}).Debug("data acked") 143 | if a >= uint32(len(data)) { 144 | log.WithFields(log.Fields{"acked": a, "tot": len(data)}).Debug("stopping") 145 | select { 146 | case done <- nil: 147 | default: 148 | } 149 | break loop 150 | } 151 | case <-exit: 152 | break loop 153 | case <-time.After(3 * time.Second): 154 | select { 155 | case done <- fmt.Errorf("test timeout"): 156 | default: 157 | break loop 158 | } 159 | } 160 | } 161 | 162 | cleanup <- sock 163 | } 164 | 165 | func receiver(port string, expect []byte, done chan error, cleanup chan *Sock, exit chan interface{}) { 166 | sock, err := Socket("", port, "RCVR") 167 | if err != nil { 168 | done <- err 169 | return 170 | } 171 | 172 | var b bytes.Buffer 173 | buf := b.Bytes() 174 | rcvd := sock.Read(10) 175 | start := time.Now() 176 | loop: 177 | for { 178 | select { 179 | case r := <-rcvd: 180 | b.Write(r) 181 | log.WithFields(log.Fields{ 182 | "time": time.Since(start), 183 | "got": b.Len(), 184 | "tot": len(expect), 185 | }).Info("app receiver") 186 | if b.Len() >= len(expect) || time.Since(start) > time.Duration(5)*time.Second { 187 | log.WithFields(log.Fields{ 188 | "time": time.Since(start), 189 | "got": b.Len(), 190 | "tot": len(expect), 191 | }).Info("app receiver stopping") 192 | break loop 193 | } 194 | case <-exit: 195 | break loop 196 | case <-time.After(3 * time.Second): 197 | break loop 198 | } 199 | } 200 | 201 | err = nil 202 | if b.Len() != len(expect) { 203 | err = fmt.Errorf("received data doesn't match length: \n%v\n%v", len(buf), len(expect)) 204 | goto cl 205 | } 206 | 207 | if !bytes.Equal(b.Bytes(), expect) { 208 | err = fmt.Errorf("received data doesn't match: \n%v\n%v", len(buf), len(expect)) 209 | } 210 | 211 | cl: 212 | select { 213 | case done <- err: 214 | default: 215 | } 216 | 217 | cleanup <- sock 218 | } 219 | -------------------------------------------------------------------------------- /udpDataplane/tx.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/akshayknarayan/udp/packetops" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | func (sock *Sock) nextPacket() (*Packet, error) { 12 | sock.mux.Lock() 13 | defer sock.mux.Unlock() 14 | 15 | seq := sock.inFlight.getNextPkt(sock.nextSeqNo) 16 | if seq >= uint32(sock.writeBufPos) { 17 | //nothing more to write 18 | return nil, fmt.Errorf("nothing more to write") 19 | } 20 | 21 | var payl []byte 22 | if packetEnd := seq + PACKET_SIZE; packetEnd <= uint32(sock.writeBufPos) { 23 | payl = sock.writeBuf[seq:packetEnd] 24 | } else { 25 | payl = sock.writeBuf[seq:sock.writeBufPos] 26 | } 27 | 28 | ackNo, err := sock.rcvWindow.cumAck(sock.lastAck) 29 | if err != nil { 30 | ackNo = sock.lastAck 31 | log.WithFields(log.Fields{"name": sock.name, "side": "rcvWindow", "ackNo": ackNo}).Debug(err) 32 | } 33 | 34 | sock.lastAck = ackNo 35 | 36 | pkt := &Packet{ 37 | SeqNo: seq, 38 | AckNo: sock.lastAck, 39 | Flag: ACK, 40 | Length: uint16(len(payl)), 41 | Sack: sock.rcvWindow.getSack(sock.lastAck), 42 | Payload: payl, 43 | } 44 | 45 | if seq == sock.nextSeqNo { 46 | sock.inFlight.addPkt(time.Now(), pkt) 47 | sock.nextSeqNo += uint32(pkt.Length) 48 | } 49 | return pkt, nil 50 | } 51 | 52 | func (sock *Sock) nextAck() (*Packet, error) { 53 | sock.mux.Lock() 54 | defer sock.mux.Unlock() 55 | 56 | ackNo, err := sock.rcvWindow.cumAck(sock.lastAck) 57 | if err != nil { 58 | ackNo = sock.lastAck 59 | log.WithFields(log.Fields{"name": sock.name, "side": "rcvWindow", "ackNo": ackNo}).Debug(err) 60 | } 61 | 62 | sock.lastAck = ackNo 63 | 64 | return &Packet{ 65 | SeqNo: sock.nextSeqNo, 66 | AckNo: sock.lastAck, 67 | Flag: ACK, 68 | Length: 0, 69 | Sack: sock.rcvWindow.getSack(sock.lastAck), 70 | Payload: []byte{}, 71 | }, nil 72 | } 73 | 74 | func (sock *Sock) tx() { 75 | for { 76 | select { 77 | case <-sock.closed: 78 | log.WithFields(log.Fields{"where": "tx", "name": sock.name}).Debug("closed, exiting") 79 | return 80 | case <-sock.shouldTx: 81 | // got new ack 82 | // or got new data and want to send ack 83 | log.WithFields(log.Fields{ 84 | "name": sock.name, 85 | }).Debug("tx") 86 | case <-time.After(time.Duration(3) * time.Second): 87 | // timeout, assume entire window lost 88 | if len(sock.inFlight.order) > 0 { 89 | log.WithFields(log.Fields{ 90 | "name": sock.name, 91 | "sock.inFlight": sock.inFlight.order, 92 | }).Debug("timeout!") 93 | firstUnacked, err := sock.inFlight.start() 94 | if err != nil { 95 | break 96 | } 97 | 98 | sock.dupAckCnt = 0 99 | sock.inFlight.timeout() 100 | select { 101 | case sock.notifyDrops <- notifyDrop{ev: "timeout", lastAck: firstUnacked}: 102 | default: 103 | } 104 | } 105 | } 106 | 107 | sock.doTx() 108 | } 109 | } 110 | 111 | func (sock *Sock) doTx() { 112 | sent := false 113 | sock.mux.Lock() 114 | cwnd := sock.cwnd 115 | sock.mux.Unlock() 116 | for sock.inFlight.size() < cwnd { 117 | pkt, err := sock.nextPacket() 118 | if err != nil { 119 | log.WithFields(log.Fields{ 120 | "name": sock.name, 121 | }).Debug(err) 122 | break 123 | } 124 | 125 | sent = true 126 | 127 | packetops.SendPacket(sock.conn, pkt, 0) 128 | 129 | log.WithFields(log.Fields{ 130 | "name": sock.name, 131 | "bytesInFlight": sock.inFlight.size(), 132 | "seqNo": pkt.SeqNo, 133 | "ackNo": pkt.AckNo, 134 | "length": pkt.Length, 135 | "inFlight": sock.inFlight.getOrder(), 136 | "cwnd": cwnd, 137 | }).Debug("sent packet") 138 | } 139 | 140 | // send an ACK 141 | if !sent { 142 | pkt, err := sock.nextAck() 143 | if err != nil { 144 | return 145 | } 146 | 147 | packetops.SendPacket(sock.conn, pkt, 0) 148 | 149 | log.WithFields(log.Fields{ 150 | "name": sock.name, 151 | "ackNo": pkt.AckNo, 152 | }).Debug("sent ack") 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /udpDataplane/window.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type windowEntry struct { 12 | p *Packet 13 | t time.Time 14 | active bool 15 | } 16 | 17 | type window struct { 18 | mux sync.RWMutex 19 | pkts map[uint32]windowEntry 20 | order []uint32 21 | } 22 | 23 | func (w *window) getOrder() (ord []uint32) { 24 | w.mux.Lock() 25 | defer w.mux.Unlock() 26 | 27 | ord = make([]uint32, len(w.order)) 28 | copy(ord, w.order) 29 | return 30 | } 31 | 32 | func makeWindow() *window { 33 | return &window{ 34 | pkts: make(map[uint32]windowEntry), 35 | order: make([]uint32, 0), 36 | } 37 | } 38 | 39 | func insertIndex(list []uint32, it uint32) int { 40 | if len(list) == 0 { 41 | return len(list) 42 | } 43 | 44 | if it < list[0] { 45 | return 0 46 | } 47 | 48 | return 1 + insertIndex(list[1:], it) 49 | } 50 | 51 | // both sides 52 | func (w *window) addPkt(t time.Time, p *Packet) { 53 | w.mux.Lock() 54 | defer w.mux.Unlock() 55 | 56 | if e, ok := w.pkts[p.SeqNo]; ok { 57 | e.t = t 58 | w.pkts[p.SeqNo] = e 59 | return 60 | } 61 | 62 | ind := insertIndex(w.order, p.SeqNo) 63 | 64 | ent := windowEntry{ 65 | p: p, 66 | t: t, 67 | active: true, 68 | } 69 | w.pkts[p.SeqNo] = ent 70 | 71 | // insert into slice at index 72 | w.order = append(w.order, 0) 73 | copy(w.order[ind+1:], w.order[ind:]) 74 | w.order[ind] = p.SeqNo 75 | 76 | if len(w.order) != len(w.pkts) { 77 | log.WithFields(log.Fields{ 78 | "order": w.order, 79 | "pkts": w.pkts, 80 | }).Panic("window out of sync") 81 | } 82 | } 83 | 84 | func remove(ord []uint32, vals []uint32) (newOrd []uint32) { 85 | newOrd = make([]uint32, 0) 86 | for _, o := range ord { 87 | found := false 88 | for _, rem := range vals { 89 | if o == rem { 90 | found = true 91 | break 92 | } 93 | } 94 | 95 | if !found { 96 | newOrd = append(newOrd, o) 97 | } 98 | } 99 | 100 | return 101 | } 102 | 103 | // sender side 104 | // what cumulative ack has been received 105 | func (w *window) rcvdPkt( 106 | t time.Time, 107 | p *Packet, 108 | ) (seqNo uint32, rtt time.Duration, err error) { 109 | rtts := make([]time.Duration, 0) 110 | 111 | seqNo, err = w.start() 112 | if err != nil { 113 | return 114 | } 115 | 116 | w.mux.Lock() 117 | defer w.mux.Unlock() 118 | 119 | ind := 0 120 | for i, seq := range w.order { 121 | ind = i + 1 122 | if seq < p.AckNo { 123 | rtts = append(rtts, t.Sub(w.pkts[seq].t)) 124 | seqNo = seq + uint32(w.pkts[seq].p.Length) 125 | delete(w.pkts, seq) 126 | continue 127 | } 128 | ind -= 1 129 | break 130 | } 131 | 132 | w.order = w.order[ind:] 133 | 134 | // handle SACKs 135 | removeVals := make([]uint32, 0) 136 | for i, v := range p.Sack { 137 | if v { 138 | seq := p.AckNo + uint32(1460*(i+1)) 139 | e := w.pkts[seq] 140 | rtts = append(rtts, t.Sub(e.t)) 141 | delete(w.pkts, seq) 142 | removeVals = append(removeVals, seq) 143 | } 144 | } 145 | 146 | w.order = remove(w.order, removeVals) 147 | 148 | if len(w.order) != len(w.pkts) { 149 | log.WithFields(log.Fields{ 150 | "ind": ind, 151 | "order": w.order, 152 | "pkts": w.pkts, 153 | }).Panic("window out of sync") 154 | } 155 | 156 | rtt = time.Minute 157 | for _, r := range rtts { 158 | if r < rtt { 159 | rtt = r 160 | } 161 | } 162 | 163 | return 164 | } 165 | 166 | func (w *window) timeout() { 167 | w.mux.Lock() 168 | defer w.mux.Unlock() 169 | 170 | for i, e := range w.pkts { 171 | e.active = false 172 | w.pkts[i] = e 173 | } 174 | } 175 | 176 | func (w *window) drop(dropped uint32, p *Packet) { 177 | droppedSeqs := []uint32{dropped} 178 | for i, v := range p.Sack { 179 | if !v { 180 | seq := p.AckNo + uint32(1460*(i+1)) 181 | droppedSeqs = append(droppedSeqs, seq) 182 | } 183 | } 184 | 185 | w.mux.Lock() 186 | defer w.mux.Unlock() 187 | 188 | for _, seq := range droppedSeqs { 189 | if ent, ok := w.pkts[seq]; ok { 190 | ent.active = false 191 | w.pkts[seq] = ent 192 | } 193 | } 194 | } 195 | 196 | func (w *window) getNextPkt(newPacket uint32) (seq uint32) { 197 | w.mux.Lock() 198 | defer w.mux.Unlock() 199 | 200 | for _, s := range w.order { 201 | e, _ := w.pkts[s] 202 | if !e.active { 203 | e.active = true 204 | w.pkts[s] = e 205 | return s 206 | } 207 | } 208 | 209 | return newPacket 210 | } 211 | 212 | // window size in bytes 213 | func (w *window) size() uint32 { 214 | w.mux.RLock() 215 | defer w.mux.RUnlock() 216 | 217 | sz := uint32(0) 218 | for _, e := range w.pkts { 219 | if e.active { 220 | sz += uint32(e.p.Length) + uint32(10) 221 | } 222 | } 223 | 224 | return sz 225 | } 226 | 227 | func (w *window) start() (uint32, error) { 228 | w.mux.RLock() 229 | defer w.mux.RUnlock() 230 | 231 | if len(w.order) == 0 { 232 | return 0, fmt.Errorf("empty window") 233 | } 234 | 235 | return w.order[0], nil 236 | } 237 | 238 | // receiver side 239 | // what cumulative ack to send 240 | func (w *window) cumAck(start uint32) (uint32, error) { 241 | w.mux.Lock() 242 | defer w.mux.Unlock() 243 | 244 | if len(w.order) == 0 { 245 | return 0, fmt.Errorf("empty window") 246 | } 247 | 248 | cumAck := start 249 | for len(w.order) > 0 && cumAck == w.order[0] { 250 | w.order = w.order[1:] 251 | nextCumAck := uint32(w.pkts[cumAck].p.Length) 252 | delete(w.pkts, cumAck) 253 | cumAck += nextCumAck 254 | } 255 | 256 | if len(w.order) != len(w.pkts) { 257 | log.WithFields(log.Fields{ 258 | "order": w.order, 259 | "pkts": w.pkts, 260 | }).Panic("window out of sync") 261 | } 262 | 263 | return cumAck, nil 264 | } 265 | 266 | // receiver side 267 | // the SACK got-packet vector, starting from right after the cumulative ack 268 | // must call cumAck() before this to ensure cumulative ack is correct 269 | func (w *window) getSack(cumAck uint32) (sack []bool) { 270 | w.mux.RLock() 271 | defer w.mux.RUnlock() 272 | 273 | sack = make([]bool, 16) 274 | for _, seq := range w.order { 275 | sackInd := (seq - cumAck) / 1460 276 | if sackInd >= uint32(len(sack)) { 277 | break 278 | } 279 | 280 | sack[sackInd-1] = true // -1 because we know the cumulative ack hasn't been received 281 | } 282 | 283 | return 284 | } 285 | -------------------------------------------------------------------------------- /udpDataplane/window_test.go: -------------------------------------------------------------------------------- 1 | package udpDataplane 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | func TestAddWindow(t *testing.T) { 11 | w := makeWindow() 12 | 13 | for i := 0; i < 5; i++ { 14 | w.addPkt(time.Now(), &Packet{ 15 | SeqNo: uint32(i), 16 | AckNo: 0, 17 | Flag: ACK, 18 | Length: 0, 19 | Payload: []byte{}, 20 | }) 21 | 22 | log.WithFields(log.Fields{ 23 | "order": w.order, 24 | "pkts": w.pkts, 25 | "new": i, 26 | }).Info("addPkt") 27 | 28 | for i, seq := range w.order { 29 | if seq != uint32(i) { 30 | t.Errorf("expected seq %d, got seq %d in %v", i, seq, w) 31 | return 32 | } 33 | } 34 | } 35 | } 36 | 37 | func TestGotPacket(t *testing.T) { 38 | w := makeWindow() 39 | 40 | for i := 0; i < 6; i++ { 41 | w.addPkt(time.Now(), &Packet{ 42 | SeqNo: uint32(i), 43 | AckNo: 0, 44 | Flag: ACK, 45 | Length: 1, 46 | Payload: []byte{'a'}, 47 | }) 48 | 49 | log.WithFields(log.Fields{ 50 | "order": w.order, 51 | "pkts": w.pkts, 52 | "new": i, 53 | }).Info("addPkt") 54 | } 55 | 56 | s, err := w.start() 57 | if err != nil { 58 | t.Error(err) 59 | return 60 | } 61 | 62 | if s != 0 { 63 | t.Errorf("expected start 0, got %d", s) 64 | return 65 | } 66 | 67 | ack, err := w.cumAck(0) 68 | if err != nil { 69 | t.Error(err) 70 | return 71 | } 72 | 73 | if ack != 6 { 74 | t.Errorf("expected cum ack 6, got %d", ack) 75 | return 76 | } 77 | 78 | s, err = w.start() 79 | if err == nil { 80 | t.Errorf("expected error, got %v", s) 81 | return 82 | } 83 | } 84 | 85 | func TestRcvdPacket(t *testing.T) { 86 | w := makeWindow() 87 | 88 | for i := 0; i < 6; i++ { 89 | w.addPkt(time.Now(), &Packet{ 90 | SeqNo: uint32(i), 91 | AckNo: 0, 92 | Flag: ACK, 93 | Length: 1, 94 | Payload: []byte{'a'}, 95 | }) 96 | 97 | log.WithFields(log.Fields{ 98 | "order": w.order, 99 | "pkts": w.pkts, 100 | "new": i, 101 | }).Info("addPkt") 102 | } 103 | 104 | s, err := w.start() 105 | if err != nil { 106 | t.Error(err) 107 | return 108 | } 109 | 110 | if s != 0 { 111 | t.Errorf("expected start 0, got %d", s) 112 | return 113 | } 114 | 115 | cumAcked, _, err := w.rcvdPkt(time.Now(), &Packet{ 116 | SeqNo: 0, 117 | AckNo: 3, 118 | Flag: ACK, 119 | Length: 0, 120 | Payload: []byte{'a'}, 121 | }) 122 | if err != nil { 123 | t.Error(err) 124 | return 125 | } 126 | 127 | if cumAcked != 3 { 128 | t.Errorf("expected cum ack 6, got %d", cumAcked) 129 | return 130 | } 131 | 132 | s, err = w.start() 133 | if err != nil { 134 | t.Error(err) 135 | return 136 | } 137 | 138 | if s != 3 { 139 | t.Errorf("expected start 3, got %d", s) 140 | return 141 | } 142 | } 143 | 144 | func TestOutOfOrderReceive(t *testing.T) { 145 | w := makeWindow() 146 | 147 | for i := 1; i < 6; i++ { 148 | w.addPkt(time.Now(), &Packet{ 149 | SeqNo: uint32(i), 150 | AckNo: 0, 151 | Flag: ACK, 152 | Length: 1, 153 | Payload: []byte{'a'}, 154 | }) 155 | 156 | log.WithFields(log.Fields{ 157 | "order": w.order, 158 | "pkts": w.pkts, 159 | "new": i, 160 | }).Info("addPkt") 161 | } 162 | 163 | w.addPkt(time.Now(), &Packet{ 164 | SeqNo: uint32(0), 165 | AckNo: 0, 166 | Flag: ACK, 167 | Length: 1, 168 | Payload: []byte{'a'}, 169 | }) 170 | log.WithFields(log.Fields{ 171 | "order": w.order, 172 | "pkts": w.pkts, 173 | "new": 0, 174 | }).Info("addPkt") 175 | 176 | ack, err := w.cumAck(0) 177 | if err != nil { 178 | t.Error(err) 179 | return 180 | } 181 | 182 | log.WithFields(log.Fields{ 183 | "order": w.order, 184 | "pkts": w.pkts, 185 | }).Info("after cumAck") 186 | 187 | if ack != 6 { 188 | t.Errorf("expected cumAck 6, got %d", ack) 189 | return 190 | } 191 | 192 | s, err := w.start() 193 | if err == nil { 194 | t.Errorf("expected error, got %v", s) 195 | return 196 | } 197 | } 198 | 199 | func TestSendSack(t *testing.T) { 200 | w := makeWindow() 201 | 202 | for i := 0; i < 8*1460; i += 1460 { 203 | if i == 2*1460 || i == 5*1460 { 204 | continue 205 | } 206 | 207 | w.addPkt(time.Now(), &Packet{ 208 | SeqNo: uint32(i), 209 | AckNo: 0, 210 | Flag: ACK, 211 | Length: 1460, 212 | Payload: []byte{'a'}, 213 | }) 214 | 215 | log.WithFields(log.Fields{ 216 | "order": w.order, 217 | "pkts": w.pkts, 218 | "new": i, 219 | }).Info("addPkt") 220 | } 221 | 222 | ack, err := w.cumAck(0) 223 | if err != nil { 224 | t.Error(err) 225 | return 226 | } 227 | 228 | log.WithFields(log.Fields{ 229 | "order": w.order, 230 | "pkts": w.pkts, 231 | }).Info("after cumAck") 232 | 233 | if ack != 2*1460 { 234 | t.Errorf("expected cumAck 2920, got %d", ack) 235 | return 236 | } 237 | 238 | sack := w.getSack(ack) 239 | expectSack := []bool{true, true, false, true, true, false, false, false, false, false, false, false, false, false, false, false} 240 | if !equal(sack, expectSack) { 241 | t.Errorf("sack doesn't match:\nexpect %v\ngot %v", expectSack, sack) 242 | return 243 | } 244 | } 245 | 246 | func TestRecvSack(t *testing.T) { 247 | w := makeWindow() 248 | 249 | for i := 0; i < 10*1460; i += 1460 { 250 | w.addPkt(time.Now(), &Packet{ 251 | SeqNo: uint32(i), 252 | AckNo: 0, 253 | Flag: ACK, 254 | Length: 1460, 255 | Payload: []byte{'a'}, 256 | }) 257 | 258 | log.WithFields(log.Fields{ 259 | "order": w.order, 260 | "pkts": w.pkts, 261 | "new": i, 262 | }).Info("addPkt") 263 | } 264 | 265 | cumAcked, _, err := w.rcvdPkt(time.Now(), &Packet{ 266 | SeqNo: 0, 267 | AckNo: 2 * 1460, 268 | Flag: ACK, 269 | Length: 0, 270 | Sack: []bool{ 271 | true, // 3 = 4380 272 | false, // 4 = 5840 273 | true, // 5 = 7300 274 | true, // 6 = 8760 275 | true, // 7 = 10220 276 | false, // 8 = 11680 277 | false, // 9 = 13140 278 | true, // 10 = 14600 279 | false, // 11 = 16060 280 | false, // 12 = 17520 281 | false, // 13 = 18980 282 | false, // 14 = 20440 283 | false, // 15 = 21900 284 | false, // 16 = 23360 285 | false, // 17 = 24820 286 | false, // 18 = 26280 287 | }, 288 | Payload: []byte{}, 289 | }) 290 | if err != nil { 291 | t.Error(err) 292 | return 293 | } 294 | 295 | if cumAcked != 2*1460 { 296 | t.Errorf("wrong cumAck on sender\nexpected 2920\ngot %v", cumAcked) 297 | return 298 | } 299 | 300 | if len(w.order) != 4 { 301 | t.Errorf("wrong order\nexpected [2920 5840 11680 13140]\ngot %v", w.order) 302 | return 303 | } 304 | for i, v := range w.order { 305 | switch i { 306 | case 0: 307 | if v != 2920 { 308 | goto fail 309 | } 310 | case 1: 311 | if v != 5840 { 312 | goto fail 313 | } 314 | case 2: 315 | if v != 11680 { 316 | goto fail 317 | } 318 | case 3: 319 | if v != 13140 { 320 | goto fail 321 | } 322 | } 323 | } 324 | 325 | goto pass 326 | 327 | fail: 328 | t.Errorf("wrong order\nexpected [2920 5840 11680 13140]\ngot %v", w.order) 329 | pass: 330 | return 331 | 332 | } 333 | -------------------------------------------------------------------------------- /unixsocket/bench_test.go: -------------------------------------------------------------------------------- 1 | package unixsocket 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "net" 7 | "os" 8 | "testing" 9 | "time" 10 | 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | func init() { 15 | log.SetLevel(log.InfoLevel) 16 | log.SetFormatter(&log.JSONFormatter{ 17 | TimestampFormat: time.RFC3339Nano, 18 | }) 19 | } 20 | 21 | type TimeMsg struct { 22 | t time.Time 23 | } 24 | 25 | func (t TimeMsg) Serialize() ([]byte, error) { 26 | buf := new(bytes.Buffer) 27 | err := binary.Write(buf, binary.LittleEndian, t.t.UnixNano()) 28 | return buf.Bytes(), err 29 | } 30 | 31 | func BenchmarkUnixSocketRtt(b *testing.B) { 32 | reader, writer := setup() 33 | b.ResetTimer() 34 | for i := 0; i < b.N; i++ { 35 | go benchReader(reader) 36 | //benchWriter(writer) 37 | dur := benchWriter(writer) 38 | log.WithFields(log.Fields{ 39 | "rtt": dur, 40 | }).Info("unix socket IPC RTT") 41 | } 42 | } 43 | 44 | func setup() (reader net.Conn, writer *SocketIpc) { 45 | ready := make(chan interface{}) 46 | readerCh := make(chan net.Conn) 47 | os.RemoveAll("/tmp/ccp-test") 48 | go makeBenchReader(ready, readerCh) 49 | <-ready 50 | writer = makeBenchWriter() 51 | reader = <-readerCh 52 | return 53 | } 54 | 55 | func makeBenchReader(ready chan interface{}, ret chan net.Conn) { 56 | addr, err := net.ResolveUnixAddr("unixgram", "/tmp/ccp-test") 57 | if err != nil { 58 | log.WithFields(log.Fields{ 59 | "err": err, 60 | }).Warn("addr resolve error") 61 | return 62 | } 63 | 64 | in, err := net.ListenUnix("unix", addr) 65 | if err != nil { 66 | log.WithFields(log.Fields{ 67 | "err": err, 68 | }).Warn("listen error") 69 | return 70 | } 71 | 72 | ready <- struct{}{} 73 | 74 | conn, err := in.AcceptUnix() 75 | if err != nil { 76 | log.WithFields(log.Fields{ 77 | "err": err, 78 | }).Warn("accept error") 79 | return 80 | } 81 | 82 | ret <- conn 83 | } 84 | 85 | func makeBenchWriter() *SocketIpc { 86 | addr, err := net.ResolveUnixAddr("unix", "/tmp/ccp-test") 87 | if err != nil { 88 | log.WithFields(log.Fields{ 89 | "err": err, 90 | }).Warn("addr resolve error") 91 | return nil 92 | } 93 | 94 | out, err := net.DialUnix("unix", nil, addr) 95 | if err != nil { 96 | log.WithFields(log.Fields{ 97 | "err": err, 98 | }).Warn("dial error") 99 | return nil 100 | } 101 | 102 | return &SocketIpc{ 103 | in: nil, 104 | out: out, 105 | } 106 | } 107 | 108 | func benchReader(conn net.Conn) { 109 | buf := make([]byte, 1024) 110 | _, err := conn.Read(buf) 111 | if err != nil { 112 | log.WithFields(log.Fields{ 113 | "err": err, 114 | }).Warn("readfrom error") 115 | return 116 | } 117 | 118 | _, err = conn.Write(buf) 119 | if err != nil { 120 | log.WithFields(log.Fields{ 121 | "err": err, 122 | }).Warn("writeto error") 123 | return 124 | } 125 | } 126 | 127 | func benchWriter(s *SocketIpc) time.Duration { 128 | err := s.SendMsg(&TimeMsg{t: time.Now()}) 129 | if err != nil { 130 | log.WithFields(log.Fields{ 131 | "err": err, 132 | }).Warn("sendmsg error") 133 | return time.Hour 134 | } 135 | 136 | buf := make([]byte, 1024) 137 | _, err = s.out.Read(buf) 138 | if err != nil { 139 | log.WithFields(log.Fields{ 140 | "err": err, 141 | }).Warn("read echo error") 142 | return time.Hour 143 | } 144 | 145 | var ns int64 146 | binary.Read(bytes.NewBuffer(buf), binary.LittleEndian, &ns) 147 | t := time.Unix(0, ns) 148 | 149 | return time.Since(t) 150 | } 151 | -------------------------------------------------------------------------------- /unixsocket/socket.go: -------------------------------------------------------------------------------- 1 | package unixsocket 2 | 3 | import ( 4 | "net" 5 | "os" 6 | 7 | "ccp/ipcBackend" 8 | 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | type SocketIpc struct { 13 | in *net.UnixConn 14 | out *net.UnixConn 15 | 16 | openFiles []string 17 | 18 | listenCh chan []byte 19 | 20 | err error 21 | killed chan interface{} 22 | } 23 | 24 | func New() ipcbackend.Backend { 25 | return &SocketIpc{openFiles: make([]string, 0)} 26 | } 27 | 28 | func (s *SocketIpc) SetupListen(loc string, id uint32) ipcbackend.Backend { 29 | if s.err != nil { 30 | return s 31 | } 32 | 33 | var addr *net.UnixAddr 34 | fd, newOpen, err := ipcbackend.AddressForListen(loc, id) 35 | if err != nil { 36 | s.err = err 37 | return s 38 | } 39 | 40 | s.openFiles = append(s.openFiles, newOpen) 41 | 42 | addr, err = net.ResolveUnixAddr("unixgram", fd) 43 | if err != nil { 44 | s.err = err 45 | return s 46 | } 47 | 48 | so, err := net.ListenUnixgram("unixgram", addr) 49 | if err != nil { 50 | s.err = err 51 | return s 52 | } 53 | 54 | s.in = so 55 | s.killed = make(chan interface{}) 56 | s.listenCh = make(chan []byte) 57 | go s.listen() 58 | return s 59 | } 60 | 61 | func (s *SocketIpc) SetupSend(loc string, id uint32) ipcbackend.Backend { 62 | if s.err != nil { 63 | return s 64 | } 65 | 66 | fd := ipcbackend.AddressForSend(loc, id) 67 | addr, err := net.ResolveUnixAddr("unixgram", fd) 68 | if err != nil { 69 | s.err = err 70 | return s 71 | } 72 | 73 | out, err := net.DialUnix("unixgram", nil, addr) 74 | if err != nil { 75 | s.err = err 76 | return s 77 | } 78 | 79 | s.out = out 80 | return s 81 | } 82 | 83 | func (s *SocketIpc) SetupFinish() (ipcbackend.Backend, error) { 84 | if s.err != nil { 85 | log.WithFields(log.Fields{ 86 | "err": s.err, 87 | }).Error("error setting up IPC") 88 | return s, s.err 89 | } else { 90 | return s, nil 91 | } 92 | } 93 | 94 | func (s *SocketIpc) SendMsg(msg ipcbackend.Msg) error { 95 | buf, err := msg.Serialize() 96 | if err != nil { 97 | return err 98 | } 99 | 100 | _, err = s.out.Write(buf) 101 | if err != nil { 102 | return err 103 | } 104 | 105 | return nil 106 | } 107 | 108 | func (s *SocketIpc) Listen() chan []byte { 109 | msgCh := make(chan []byte) 110 | 111 | go func() { 112 | for { 113 | select { 114 | case <-s.killed: 115 | close(msgCh) 116 | return 117 | case buf := <-s.listenCh: 118 | msgCh <- buf 119 | } 120 | } 121 | }() 122 | 123 | return msgCh 124 | } 125 | 126 | func (s *SocketIpc) Close() error { 127 | close(s.killed) 128 | return nil 129 | } 130 | 131 | func (s *SocketIpc) listen() { 132 | buf := make([]byte, 2048) 133 | writePos := 0 134 | for { 135 | select { 136 | case _, ok := <-s.killed: 137 | if !ok { 138 | if s.in != nil { 139 | s.in.Close() 140 | } 141 | 142 | for _, f := range s.openFiles { 143 | os.RemoveAll(f) 144 | } 145 | 146 | close(s.listenCh) 147 | 148 | log.WithFields(log.Fields{ 149 | "where": "socketIpc.listenMsg.checkKilled", 150 | }).Info("killed, closing") 151 | return 152 | } 153 | default: 154 | } 155 | 156 | ring := append(buf[writePos:], buf[:writePos]...) 157 | n, err := s.in.Read(ring) 158 | if err != nil { 159 | log.WithFields(log.Fields{ 160 | "where": "socketIpc.listenMsg.read", 161 | }).Warn(err) 162 | continue 163 | } 164 | 165 | writePos = (writePos + n) % len(buf) 166 | s.listenCh <- ring[:n] 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /unixsocket/socket_test.go: -------------------------------------------------------------------------------- 1 | package unixsocket 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | // mock message implementing ipcbackend.Msg 12 | type MockMsg struct { 13 | b string 14 | } 15 | 16 | func (m MockMsg) Serialize() ([]byte, error) { 17 | return []byte(m.b), nil 18 | } 19 | 20 | func TestCommunication(t *testing.T) { 21 | rdone := make(chan error) 22 | wdone := make(chan error) 23 | ready := make(chan interface{}) 24 | go reader(ready, rdone) 25 | go writer(ready, wdone) 26 | 27 | done := 0 28 | var err error 29 | for { 30 | select { 31 | case err = <-rdone: 32 | case err = <-wdone: 33 | case <-time.After(time.Second): 34 | t.Errorf("timed out") 35 | return 36 | } 37 | if err != nil { 38 | t.Error(err) 39 | return 40 | } 41 | done++ 42 | if done >= 2 { 43 | break 44 | } 45 | } 46 | } 47 | 48 | func reader(ready chan interface{}, done chan error) { 49 | os.RemoveAll("/tmp/ccp-test") 50 | addrOut, err := net.ResolveUnixAddr("unixgram", "/tmp/ccp-test") 51 | if err != nil { 52 | done <- err 53 | return 54 | } 55 | 56 | in, err := net.ListenUnixgram("unixgram", addrOut) 57 | if err != nil { 58 | done <- err 59 | return 60 | } 61 | 62 | ready <- struct{}{} 63 | 64 | buf := make([]byte, 1024) 65 | _, err = in.Read(buf) 66 | if err != nil { 67 | done <- err 68 | return 69 | } 70 | 71 | if string(buf[:3]) != "foo" { 72 | done <- fmt.Errorf("wrong message\ngot (%s)\nexpected (foo)", buf) 73 | return 74 | } 75 | 76 | in.Close() 77 | os.Remove("/tmp/ccp-test") 78 | done <- nil 79 | } 80 | 81 | func writer(ready chan interface{}, done chan error) { 82 | addrOut, err := net.ResolveUnixAddr("unixgram", "/tmp/ccp-test") 83 | if err != nil { 84 | done <- err 85 | return 86 | } 87 | 88 | <-ready 89 | 90 | out, err := net.DialUnix("unixgram", nil, addrOut) 91 | if err != nil { 92 | done <- err 93 | return 94 | } 95 | 96 | s := &SocketIpc{ 97 | in: nil, 98 | out: out, 99 | } 100 | 101 | err = s.SendMsg(&MockMsg{b: "foo"}) 102 | if err != nil { 103 | done <- err 104 | return 105 | } 106 | 107 | done <- nil 108 | } 109 | -------------------------------------------------------------------------------- /vegas/vegas.go: -------------------------------------------------------------------------------- 1 | package vegas 2 | 3 | import ( 4 | "math" 5 | 6 | "ccp/ccpFlow" 7 | "ccp/ccpFlow/pattern" 8 | "ccp/ipc" 9 | 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // implement ccpFlow.Flow interface 14 | type Vegas struct { 15 | pktSize uint32 16 | initCwnd float32 17 | 18 | cwnd float32 19 | lastAck uint32 20 | 21 | sockid uint32 22 | ipc ipc.SendOnly 23 | baseRTT float32 24 | alpha float32 25 | beta float32 26 | } 27 | 28 | func (v *Vegas) Name() string { 29 | return "vegas" 30 | } 31 | 32 | func (v *Vegas) Create( 33 | socketid uint32, 34 | send ipc.SendOnly, 35 | pktsz uint32, 36 | startSeq uint32, 37 | startCwnd uint32, 38 | ) { 39 | v.sockid = socketid 40 | v.ipc = send 41 | v.pktSize = pktsz 42 | if startSeq == 0 { 43 | v.lastAck = startSeq 44 | } else { 45 | v.lastAck = startSeq - 1 46 | } 47 | v.initCwnd = float32(pktsz * 10) 48 | v.cwnd = float32(pktsz * startCwnd) 49 | v.baseRTT = 0 50 | v.alpha = 2 51 | v.beta = 4 52 | 53 | v.newPattern() 54 | } 55 | 56 | func (v *Vegas) GotMeasurement(m ccpFlow.Measurement) { 57 | // reordering of messsages 58 | // if within 10 packets, assume no integer overflow 59 | if m.Ack < v.lastAck && m.Ack > v.lastAck-v.pktSize*10 { 60 | return 61 | } 62 | 63 | // handle integer overflow / sequence wraparound 64 | var newBytesAcked uint64 65 | if m.Ack < v.lastAck { 66 | newBytesAcked = uint64(math.MaxUint32) + uint64(m.Ack) - uint64(v.lastAck) 67 | } else { 68 | newBytesAcked = uint64(m.Ack) - uint64(v.lastAck) 69 | } 70 | 71 | RTT := float32(m.Rtt.Seconds()) 72 | if v.baseRTT <= 0 || RTT < v.baseRTT { 73 | v.baseRTT = RTT 74 | } 75 | 76 | inQueue := (v.cwnd * (RTT - v.baseRTT)) / (RTT * float32(v.pktSize)) 77 | if inQueue <= v.alpha { 78 | v.cwnd += float32(v.pktSize) 79 | } else if inQueue >= v.beta { 80 | v.cwnd -= float32(v.pktSize) 81 | } 82 | 83 | v.newPattern() 84 | 85 | log.WithFields(log.Fields{ 86 | "gotAck": m.Ack, 87 | "currCwnd": v.cwnd, 88 | "currLastAck": v.lastAck, 89 | "newlyAcked": newBytesAcked, 90 | "InQueue": inQueue, 91 | "baseRTT": v.baseRTT, 92 | "loss": m.Loss, 93 | }).Info("[vegas] got ack") 94 | 95 | v.lastAck = m.Ack 96 | return 97 | } 98 | 99 | func (v *Vegas) Drop(ev ccpFlow.DropEvent) { 100 | switch ev { 101 | case ccpFlow.DupAck: 102 | v.cwnd -= float32(v.pktSize) 103 | case ccpFlow.Timeout: 104 | v.cwnd -= float32(v.pktSize) 105 | default: 106 | log.WithFields(log.Fields{ 107 | "event": ev, 108 | }).Warn("[vegas] unknown drop event type") 109 | return 110 | } 111 | 112 | log.WithFields(log.Fields{ 113 | "currCwnd": v.cwnd, 114 | "event": ev, 115 | }).Info("[vegas] drop") 116 | 117 | v.newPattern() 118 | } 119 | 120 | func (v *Vegas) newPattern() { 121 | staticPattern, err := pattern. 122 | NewPattern(). 123 | Cwnd(uint32(v.cwnd)). 124 | WaitRtts(0.5). 125 | Report(). 126 | Compile() 127 | if err != nil { 128 | log.WithFields(log.Fields{ 129 | "err": err, 130 | "cwnd": v.cwnd, 131 | }).Info("make cwnd msg failed") 132 | return 133 | } 134 | 135 | err = v.ipc.SendPatternMsg(v.sockid, staticPattern) 136 | if err != nil { 137 | log.WithFields(log.Fields{"cwnd": v.cwnd, "name": v.sockid}).Warn(err) 138 | } 139 | } 140 | 141 | func Init() { 142 | ccpFlow.Register("vegas", func() ccpFlow.Flow { 143 | return &Vegas{} 144 | }) 145 | } 146 | --------------------------------------------------------------------------------