├── .gitignore ├── AUTHORS ├── LICENSE ├── README.md ├── doc ├── .allow-devtools-edit ├── LICENSE ├── anchor.html ├── api.html ├── big.html ├── build.html ├── config.html ├── cross.html ├── debug.html ├── hello.html ├── img │ ├── arm.jpeg │ ├── arm.png │ ├── arm.xcf │ ├── build.png │ ├── build.svg │ ├── hello.png │ ├── hello.svg │ ├── spawn.png │ ├── spawn.svg │ ├── sumr.png │ ├── sumr.svg │ ├── trend.png │ ├── trend.svg │ ├── trending.jpg │ ├── trending.png │ ├── tumblr.png │ ├── vena.png │ ├── vena.svg │ ├── venash.png │ └── venash.svg ├── index.html ├── lang.html ├── legacy.html ├── license.html ├── skin │ ├── main.css │ └── shadow.css ├── spawn.html ├── start.html ├── sumr.html └── trend.html ├── src ├── circuit │ ├── _exp │ │ ├── 4dcat │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4jailtail │ │ │ ├── .gitignore │ │ │ ├── hard.go │ │ │ └── main.go │ │ ├── 4stat │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── exp.go │ │ ├── shuttr │ │ │ ├── cmd │ │ │ │ ├── shuttr-series-server │ │ │ │ │ ├── .gitignore │ │ │ │ │ ├── filter.go │ │ │ │ │ ├── firehose.go │ │ │ │ │ ├── forward.go │ │ │ │ │ ├── http.go │ │ │ │ │ └── main.go │ │ │ │ └── shuttr-union-server │ │ │ │ │ ├── .gitignore │ │ │ │ │ ├── forward.go │ │ │ │ │ ├── http.go │ │ │ │ │ └── main.go │ │ │ ├── config │ │ │ │ └── config.go │ │ │ ├── proto │ │ │ │ └── proto.go │ │ │ ├── series │ │ │ │ ├── server.go │ │ │ │ └── timeline.go │ │ │ ├── shard │ │ │ │ └── topo.go │ │ │ ├── union │ │ │ │ ├── dashboard.go │ │ │ │ └── server.go │ │ │ └── util │ │ │ │ └── util.go │ │ ├── telefile │ │ │ ├── circuit.config │ │ │ ├── cli │ │ │ │ ├── .gitignore │ │ │ │ └── main.go │ │ │ ├── env.sh │ │ │ └── srv │ │ │ │ └── srv.go │ │ └── teleport │ │ │ ├── plain │ │ │ ├── gob.go │ │ │ └── plain.go │ │ │ ├── tcp │ │ │ ├── _auto.go │ │ │ ├── _perm.go │ │ │ ├── conn.go │ │ │ ├── dialer.go │ │ │ ├── error.go │ │ │ ├── gob.go │ │ │ ├── link.go │ │ │ ├── listener.go │ │ │ └── transport_test.go │ │ │ └── transport.go │ ├── app │ │ ├── doc.go │ │ └── sumr │ │ │ ├── api │ │ │ ├── api.go │ │ │ ├── config.go │ │ │ ├── feature.go │ │ │ ├── http.go │ │ │ ├── mkcfg │ │ │ │ ├── .gitignore │ │ │ │ └── mkcfg.go │ │ │ ├── replenish.go │ │ │ ├── reqresp.go │ │ │ └── reqresp_test.go │ │ │ ├── block │ │ │ ├── block.go │ │ │ ├── code.go │ │ │ ├── code_test.go │ │ │ ├── disk.go │ │ │ ├── disk_test.go │ │ │ ├── file.go │ │ │ ├── file_test.go │ │ │ └── sketch.go │ │ │ ├── client │ │ │ └── client.go │ │ │ ├── server │ │ │ ├── mkcfg │ │ │ │ ├── .gitignore │ │ │ │ └── mkcfg.go │ │ │ ├── server.go │ │ │ ├── spawn.go │ │ │ ├── util.go │ │ │ └── x.go │ │ │ └── sumr.go │ ├── c │ │ ├── .gitignore │ │ ├── README │ │ ├── TODO │ │ ├── build.go │ │ ├── build_test.go │ │ ├── dep │ │ │ └── dep.go │ │ ├── doc.go │ │ ├── errors │ │ │ └── errors.go │ │ ├── interface.go │ │ ├── log.go │ │ ├── source │ │ │ ├── flush.go │ │ │ ├── gopath.go │ │ │ ├── jail.go │ │ │ ├── layout.go │ │ │ ├── pkg.go │ │ │ └── src.go │ │ ├── testdata │ │ │ └── src │ │ │ │ ├── a │ │ │ │ └── a.go │ │ │ │ └── b │ │ │ │ └── b.go │ │ ├── types │ │ │ ├── compile.go │ │ │ ├── link.go │ │ │ ├── sweep.go │ │ │ ├── table.go │ │ │ └── type.go │ │ └── util │ │ │ ├── gofix-imports.go │ │ │ ├── import.go │ │ │ └── walk.go │ ├── cmd │ │ ├── 4build │ │ │ ├── .gitignore │ │ │ ├── flags.go │ │ │ ├── main.go │ │ │ ├── repo.go │ │ │ └── util.go │ │ ├── 4clear-helper │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4clear │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4cpu │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4crossbuild │ │ │ ├── .gitignore │ │ │ ├── cross.go │ │ │ └── main.go │ │ ├── 4deploy │ │ │ ├── .gitignore │ │ │ ├── install.go │ │ │ └── main.go │ │ ├── 4dls │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4hardkill │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4issue │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4kill │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4ls │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4stk │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── 4top │ │ │ ├── .gitignore │ │ │ └── main.go │ │ └── doc.go │ ├── kit │ │ ├── debug │ │ │ ├── ctrlc │ │ │ │ └── init.go │ │ │ ├── http │ │ │ │ ├── http.go │ │ │ │ └── trace │ │ │ │ │ └── trace.go │ │ │ ├── kill │ │ │ │ └── init.go │ │ │ └── sigpanic.go │ │ ├── fmt │ │ │ ├── deep.go │ │ │ └── deep_test.go │ │ ├── fs │ │ │ ├── diskfs │ │ │ │ └── diskfs.go │ │ │ ├── fileinfo.go │ │ │ ├── fs.go │ │ │ ├── http.go │ │ │ └── zipfs │ │ │ │ └── zipfs.go │ │ ├── iomisc │ │ │ └── combine.go │ │ ├── join │ │ │ └── setthenget.go │ │ ├── kit.go │ │ ├── llrb │ │ │ ├── ORIGIN │ │ │ ├── avgvar.go │ │ │ ├── llrb-stats.go │ │ │ ├── llrb.go │ │ │ └── llrb_test.go │ │ ├── lockfile │ │ │ ├── lockfile.go │ │ │ ├── lockfile_test.go │ │ │ └── ¶testdir │ │ │ │ └── test.lock │ │ ├── posix │ │ │ ├── fwderr.go │ │ │ └── posix.go │ │ ├── sched │ │ │ ├── limiter │ │ │ │ ├── l.go │ │ │ │ └── l_test.go │ │ │ └── sched.go │ │ ├── stat │ │ │ ├── moment.go │ │ │ ├── sliding.go │ │ │ ├── sliding_test.go │ │ │ └── time.go │ │ ├── tele │ │ │ ├── doc.go │ │ │ ├── file │ │ │ │ ├── file.go │ │ │ │ └── fileinfo.go │ │ │ └── io │ │ │ │ └── io.go │ │ ├── waterfill │ │ │ ├── waterfill.go │ │ │ └── waterfill_test.go │ │ ├── xor │ │ │ ├── util.go │ │ │ ├── xor.go │ │ │ └── xor_test.go │ │ └── zookeeper │ │ │ └── zutil │ │ │ ├── create.go │ │ │ ├── util.go │ │ │ └── watch.go │ ├── load │ │ ├── config │ │ │ ├── build.go │ │ │ ├── init.go │ │ │ ├── install.go │ │ │ ├── parse_test.go │ │ │ ├── spark.go │ │ │ └── zookeeper.go │ │ ├── load.go │ │ └── worker │ │ │ └── worker.go │ ├── sys │ │ ├── acid │ │ │ ├── acid.go │ │ │ ├── behalf.go │ │ │ └── jail.go │ │ ├── lang │ │ │ ├── README │ │ │ ├── TODO │ │ │ ├── _ptrptr_test.go │ │ │ ├── call.go │ │ │ ├── dial.go │ │ │ ├── doc.go │ │ │ ├── error.go │ │ │ ├── exp.go │ │ │ ├── flat.go │ │ │ ├── freeze.go │ │ │ ├── freeze_test.go │ │ │ ├── func.go │ │ │ ├── imp.go │ │ │ ├── lang.go │ │ │ ├── msg.go │ │ │ ├── perm.go │ │ │ ├── prof │ │ │ │ ├── profile.go │ │ │ │ ├── stat.go │ │ │ │ └── stopwatch.go │ │ │ ├── ptrptr.go │ │ │ ├── readwriter.go │ │ │ ├── runtime.go │ │ │ ├── runtime_test.go │ │ │ ├── sandbox.go │ │ │ ├── srv.go │ │ │ ├── types │ │ │ │ ├── gob.go │ │ │ │ ├── register.go │ │ │ │ ├── type.go │ │ │ │ └── util.go │ │ │ ├── value.go │ │ │ ├── vexp.go │ │ │ ├── vimp.go │ │ │ ├── vrewrite.go │ │ │ └── vrewrite_test.go │ │ ├── sys.go │ │ ├── transport │ │ │ ├── TODO │ │ │ ├── addr.go │ │ │ ├── conn.go │ │ │ ├── error.go │ │ │ ├── link.go │ │ │ ├── msg.go │ │ │ ├── swap.go │ │ │ ├── transport.go │ │ │ ├── transport_test.go │ │ │ └── trigger.go │ │ ├── worker │ │ │ ├── daemonize.go │ │ │ ├── proc.go │ │ │ └── spawn.go │ │ ├── zanchorfs │ │ │ ├── dir.go │ │ │ ├── file.go │ │ │ ├── fs.go │ │ │ └── zookeeper.go │ │ ├── zdurablefs │ │ │ ├── dir.go │ │ │ ├── file.go │ │ │ ├── fs.go │ │ │ └── zookeeper.go │ │ └── zissuefs │ │ │ ├── sendmail_test.go │ │ │ ├── zissuefs.go │ │ │ └── zookeeper.go │ ├── test │ │ ├── test.go │ │ └── xgc │ │ │ ├── README │ │ │ ├── circuit.config │ │ │ ├── env.sh │ │ │ ├── main │ │ │ ├── .gitignore │ │ │ ├── kick │ │ │ └── main.go │ │ │ ├── worker │ │ │ └── worker.go │ │ │ └── xgc.go │ └── use │ │ ├── anchorfs │ │ ├── anchorfs.go │ │ └── bind.go │ │ ├── circuit │ │ ├── bind.go │ │ ├── error.go │ │ ├── id.go │ │ ├── lang.go │ │ ├── runtime.go │ │ └── transport.go │ │ ├── durablefs │ │ └── durable.go │ │ ├── issuefs │ │ └── issue.go │ │ ├── use.go │ │ └── worker │ │ └── worker.go └── tumblr │ ├── firehose │ ├── event.go │ ├── firehose.go │ ├── firehose_test.go │ ├── redial.go │ ├── speed_test.go │ └── util.go │ ├── hbase │ ├── reader.go │ ├── reader_test.go │ └── testdata │ │ └── records │ ├── kafka │ ├── TODO │ ├── client.go │ ├── client_test.go │ ├── doc.go │ ├── eco.go │ ├── eco_test.go │ ├── error.go │ ├── message.go │ ├── message_test.go │ ├── multifetch.dump │ ├── request.go │ ├── request_test.go │ ├── response.go │ ├── response_test.go │ ├── time.go │ └── wire.go │ ├── opentsdb │ ├── besteffort.go │ ├── besteffort_test.go │ ├── opentsdb.go │ └── opentsdb_test.go │ ├── redis │ ├── conn.go │ ├── redis.go │ └── redis_test.go │ └── scribe │ ├── besteffort.go │ ├── conn.go │ ├── conn_test.go │ ├── server.go │ ├── server_test.go │ └── thrift │ ├── doc.go │ ├── fb303 │ ├── fbservice.go │ └── ttypes.go │ └── scribe │ ├── scribe.go │ └── ttypes.go └── tutorials ├── hello ├── app.config └── src │ └── hello │ ├── cmd │ └── hello-spawn │ │ ├── .gitignore │ │ └── main.go │ ├── worker │ └── worker.go │ └── x │ └── hello.go ├── legacy ├── app.config └── src │ └── legacy │ ├── .gitignore │ └── main.go ├── sumr ├── api.config ├── app.config ├── server.config └── src │ └── sumr │ ├── cmd │ ├── sumr-spawn-api │ │ ├── .gitignore │ │ └── main.go │ └── sumr-spawn-shard │ │ ├── .gitignore │ │ └── main.go │ └── worker │ └── init.go ├── trend ├── app.config └── src │ └── trend │ ├── cmd │ └── trend-spawn │ │ ├── .gitignore │ │ └── main.go │ ├── config │ └── config.go │ ├── worker │ └── init.go │ └── x │ ├── start.go │ └── trending.go ├── trending └── src │ ├── config │ └── config.go │ └── trending │ └── _trending.go └── vena ├── app.config ├── front.config ├── src └── vena │ ├── client │ └── client.go │ ├── cmd │ ├── vena-front-start │ │ ├── .gitignore │ │ └── main.go │ └── vena-server-start │ │ ├── .gitignore │ │ └── main.go │ ├── config.go │ ├── front │ ├── config.go │ ├── front.go │ ├── mkcfg │ │ ├── .gitignore │ │ └── mkcfg.go │ ├── replenish.go │ └── tsdb.go │ ├── server │ ├── server.go │ ├── sketch.go │ └── spawn.go │ ├── util │ └── util.go │ ├── vena.go │ └── worker │ └── init.go └── vena.config /.gitignore: -------------------------------------------------------------------------------- 1 | /pkg 2 | /bin 3 | /gocircuit.org 4 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of circuit authors. 2 | # Names should be added to this file as 3 | # Name or Organization 4 | # Please keep the list sorted. 5 | 6 | 7 | # Initial author 8 | Petar Maymounkov 9 | -------------------------------------------------------------------------------- /doc/.allow-devtools-edit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/.allow-devtools-edit -------------------------------------------------------------------------------- /doc/img/arm.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/arm.jpeg -------------------------------------------------------------------------------- /doc/img/arm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/arm.png -------------------------------------------------------------------------------- /doc/img/arm.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/arm.xcf -------------------------------------------------------------------------------- /doc/img/build.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/build.png -------------------------------------------------------------------------------- /doc/img/hello.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/hello.png -------------------------------------------------------------------------------- /doc/img/spawn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/spawn.png -------------------------------------------------------------------------------- /doc/img/sumr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/sumr.png -------------------------------------------------------------------------------- /doc/img/trend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/trend.png -------------------------------------------------------------------------------- /doc/img/trending.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/trending.jpg -------------------------------------------------------------------------------- /doc/img/trending.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/trending.png -------------------------------------------------------------------------------- /doc/img/tumblr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/tumblr.png -------------------------------------------------------------------------------- /doc/img/vena.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/vena.png -------------------------------------------------------------------------------- /doc/img/venash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/doc/img/venash.png -------------------------------------------------------------------------------- /doc/legacy.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Go'Circuit: Circuitizing legacy applications 4 | 5 | 6 | 7 | 8 |
GO’CIRCUIT
By
9 | 10 |
11 |

Circuitizing legacy applications

12 | 13 |
14 | A pre-existing, or “legacy”, Go application can be easily run and managed by the circuit with no modification of its internal logic at all. 15 |
16 | 17 |

Integration in a line

18 | 19 |

For any pre-existing application, simply add 20 | 

import _ "circuit/load"
21 | to the main program package and go on to compile it as usual with go build. 22 | 23 |

Executing the compiled binary is no different than before the circuit was imported into it, except for the added requirement that the CIR environment points to an app configuration file that specifies a Zookeeper section. (The Build and Deploy sections are not needed.) 24 | 25 |

You can find a sample application in subdirectory tutorials/legacy. When executed, the program will sleep for a short while, giving you an opportunity to switch terminals and verify its presence using the 4ls tool. 26 | 27 |

28 | 29 | 30 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/circuit/_exp/4dcat/.gitignore: -------------------------------------------------------------------------------- 1 | /4dcat 2 | -------------------------------------------------------------------------------- /src/circuit/_exp/4dcat/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // 4dcat prints the contents of a file from the durable file system 16 | package main 17 | 18 | import ( 19 | _ "circuit/load" 20 | "circuit/use/durablefs" 21 | "flag" 22 | "os" 23 | ) 24 | 25 | func main() { 26 | flag.Parse() 27 | if len(flag.Args()) != 1 { 28 | println("Usage:", os.Args[0], " DurablePath") 29 | os.Exit(1) 30 | } 31 | _, err := durablefs.OpenFile(flag.Arg(0)) 32 | if err != nil { 33 | println(err.Error()) 34 | os.Exit(1) 35 | } 36 | 37 | // XXX: Need to be able to read unregistered types 38 | } 39 | -------------------------------------------------------------------------------- /src/circuit/_exp/4jailtail/.gitignore: -------------------------------------------------------------------------------- 1 | 4jailtail 2 | -------------------------------------------------------------------------------- /src/circuit/_exp/4jailtail/hard.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "circuit/load/config" 19 | "circuit/use/circuit" 20 | "io" 21 | "os" 22 | "os/exec" 23 | "path" 24 | ) 25 | 26 | func tailViaSSH(addr circuit.Addr, jailpath string) { 27 | 28 | abs := path.Join(config.Config.Install.JailDir(), addr.WorkerID().String(), jailpath) 29 | 30 | cmd := exec.Command("ssh", addr.Host(), "tail -f "+abs) 31 | 32 | stdout, err := cmd.StdoutPipe() 33 | if err != nil { 34 | println("Pipe problem:", err.Error()) 35 | os.Exit(1) 36 | } 37 | 38 | if err = cmd.Start(); err != nil { 39 | println("Exec problem:", err.Error()) 40 | os.Exit(1) 41 | } 42 | 43 | io.Copy(os.Stdout, stdout) 44 | } 45 | -------------------------------------------------------------------------------- /src/circuit/_exp/4stat/.gitignore: -------------------------------------------------------------------------------- 1 | /4stat 2 | -------------------------------------------------------------------------------- /src/circuit/_exp/4stat/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | 17 | 4stat locates a worker through the anchor file system and prints the result of its stats reporting endpoint. 18 | 19 | */ 20 | package main 21 | 22 | import ( 23 | _ "circuit/load" 24 | "circuit/use/anchorfs" 25 | "circuit/use/circuit" 26 | "fmt" 27 | "os" 28 | ) 29 | 30 | func main() { 31 | if len(os.Args) != 3 { 32 | println("Usage:", os.Args[0], "AnchorPath Service") 33 | os.Exit(1) 34 | } 35 | file, err := anchorfs.OpenFile(os.Args[1]) 36 | if err != nil { 37 | fmt.Fprintf(os.Stderr, "Problem opening (%s)", err) 38 | os.Exit(1) 39 | } 40 | x, err := circuit.TryDial(file.Owner(), "acid") 41 | if err != nil { 42 | fmt.Fprintf(os.Stderr, "Problem dialing 'acid' service (%s)", err) 43 | os.Exit(1) 44 | } 45 | 46 | defer func() { 47 | if p := recover(); p != nil { 48 | fmt.Fprintf(os.Stderr, "Worker disappeared during call (%#v)", p) 49 | os.Exit(1) 50 | } 51 | }() 52 | 53 | r := x.Call("OnBehalfCallStringer", os.Args[2], "Stat") 54 | fmt.Println(r[0].(string)) 55 | } 56 | -------------------------------------------------------------------------------- /src/circuit/_exp/exp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package exp contains experimental projects 16 | package exp 17 | -------------------------------------------------------------------------------- /src/circuit/_exp/shuttr/cmd/shuttr-series-server/.gitignore: -------------------------------------------------------------------------------- 1 | /shuttr-series-server 2 | testdb* 3 | -------------------------------------------------------------------------------- /src/circuit/_exp/shuttr/cmd/shuttr-series-server/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "io/ioutil" 19 | "strconv" 20 | ) 21 | 22 | type Filter []int64 23 | 24 | func ParseFilter(name string) (Filter, error) { 25 | raw, err := ioutil.ReadFile(name) 26 | if err != nil { 27 | return nil, err 28 | } 29 | tokens := strings.SplitTrim(string(raw), " \n\r\t") 30 | result := make(Filter, len(tokens)) 31 | for i, t := range tokens { 32 | id, err := strconv.ParseInt(t, 10, 64) 33 | if err != nil { 34 | return nil, err 35 | } 36 | result[i] = id 37 | } 38 | return result, nil 39 | } 40 | -------------------------------------------------------------------------------- /src/circuit/_exp/shuttr/cmd/shuttr-series-server/firehose.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "bytes" 19 | "circuit/exp/shuttr/proto" 20 | "fmt" 21 | "os/exec" 22 | "tumblr/firehose" 23 | ) 24 | 25 | func StreamFirehose(freq *firehose.Request) <-chan *createRequest { 26 | ch := make(chan *createRequest) 27 | go func() { 28 | conn := firehose.Redial(freq) 29 | for { 30 | q := filter(conn.Read()) 31 | if q == nil { 32 | continue 33 | } 34 | println(fmt.Sprintf("CREATE blogID=%d postID=%d", q.TimelineID, q.PostID)) 35 | ch <- &createRequest{ 36 | Forwarded: false, 37 | Post: q, 38 | ReturnResponse: func(err error) { 39 | if err != nil { 40 | println("Firehose->XCreatePost error:", err.Error()) 41 | return 42 | } 43 | }, 44 | } 45 | } 46 | }() 47 | return ch 48 | } 49 | 50 | func filter(e *firehose.Event) *proto.XCreatePost { 51 | if e.Activity != firehose.CreatePost { 52 | return nil 53 | } 54 | return &proto.XCreatePost{TimelineID: e.Post.BlogID, PostID: e.Post.ID} 55 | } 56 | 57 | func sendmail(recipient, subject, body string) error { 58 | cmd := exec.Command("sendmail", recipient) 59 | var w bytes.Buffer 60 | w.WriteString("Subject: ") 61 | w.WriteString(subject) 62 | w.WriteByte('\n') 63 | w.Write([]byte(body)) 64 | cmd.Stdin = &w 65 | _, err := cmd.CombinedOutput() 66 | return err 67 | } 68 | -------------------------------------------------------------------------------- /src/circuit/_exp/shuttr/cmd/shuttr-union-server/.gitignore: -------------------------------------------------------------------------------- 1 | /shuttr-union-server 2 | /testdb* 3 | -------------------------------------------------------------------------------- /src/circuit/_exp/shuttr/shard/topo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package shard 16 | 17 | import ( 18 | "circuit/exp/shuttr/x" 19 | "circuit/kit/xor" 20 | ) 21 | 22 | type Shard struct { 23 | Pivot xor.Key 24 | Addr x.Addr 25 | HTTP int 26 | } 27 | 28 | func (sh *Shard) Key() xor.Key { 29 | return sh.Pivot 30 | } 31 | 32 | type Topo struct { 33 | metric xor.Metric 34 | } 35 | 36 | func New() *Topo { 37 | return &Topo{} 38 | } 39 | 40 | func NewPopulate(shards []*Shard) *Topo { 41 | t := &Topo{} 42 | t.Populate(shards) 43 | return t 44 | } 45 | 46 | func (t *Topo) Populate(shards []*Shard) { 47 | t.metric.Clear() 48 | for _, sh := range shards { 49 | t.Add(sh) 50 | } 51 | } 52 | 53 | func (t *Topo) Add(shard *Shard) { 54 | t.metric.Add(shard) 55 | } 56 | 57 | func (t *Topo) Find(key xor.Key) *Shard { 58 | nearest := t.metric.Nearest(key, 1) 59 | if len(nearest) == 0 { 60 | return nil 61 | } 62 | return nearest[0].(*Shard) 63 | } 64 | 65 | func (t *Topo) ChooseKey() xor.Key { 66 | return t.metric.ChooseMinK(5) 67 | } 68 | -------------------------------------------------------------------------------- /src/circuit/_exp/shuttr/util/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "sync" 19 | "github.com/petar/levigo" 20 | ) 21 | 22 | type Server struct { 23 | slk sync.Mutex 24 | cache *levigo.Cache 25 | DB *levigo.DB 26 | ReadAndCache *levigo.ReadOptions 27 | WriteSync *levigo.WriteOptions 28 | WriteNoSync *levigo.WriteOptions 29 | } 30 | 31 | func (srv *Server) Init(dbDir string, cacheSize int) error { 32 | var err error 33 | opts := levigo.NewOptions() 34 | srv.cache = levigo.NewLRUCache(cacheSize) 35 | opts.SetCache(srv.cache) 36 | opts.SetCreateIfMissing(true) 37 | 38 | if srv.DB, err = levigo.Open(dbDir, opts); err != nil { 39 | srv.cache.Close() 40 | return err 41 | } 42 | 43 | srv.ReadAndCache = levigo.NewReadOptions() 44 | srv.ReadAndCache.SetFillCache(true) 45 | 46 | srv.WriteSync = levigo.NewWriteOptions() 47 | srv.WriteSync.SetSync(true) 48 | 49 | srv.WriteNoSync = levigo.NewWriteOptions() 50 | srv.WriteSync.SetSync(false) 51 | 52 | return nil 53 | } 54 | 55 | func (srv *Server) Close() error { 56 | srv.slk.Lock() 57 | defer srv.slk.Unlock() 58 | if srv.cache != nil { 59 | srv.cache.Close() 60 | srv.cache = nil 61 | } 62 | srv.ReadAndCache.Close() 63 | srv.ReadAndCache = nil 64 | srv.WriteSync.Close() 65 | srv.WriteSync = nil 66 | srv.WriteNoSync.Close() 67 | srv.WriteNoSync = nil 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /src/circuit/_exp/telefile/circuit.config: -------------------------------------------------------------------------------- 1 | { 2 | "Zookeeper": { 3 | "Workers": ["localhost:2181"], 4 | "RootDir": "/circuit-telefile" 5 | }, 6 | "Install": { 7 | "RootDir": "/Users/petar/_telefile/circuit", 8 | "LibPath": ".", 9 | "Binary": "4r-telefile" 10 | }, 11 | "Build": { 12 | "Host": "localhost", 13 | "Jail": "/Users/petar/_telefile/build", 14 | "AppRepo": "{rsync}/Users/petar/gocircuit", 15 | "AppSrc": "/", 16 | "Pkg": "circuit/exp/telefile/srv", 17 | "Show": true, 18 | "RebuildGo": false, 19 | "ZookeeperInclude": "/Users/petar/local/include/c-client-src", 20 | "ZookeeperLib": "/Users/petar/local/lib", 21 | "CircuitRepo": "{rsync}/Users/petar/gocircuit", 22 | "CircuitSrc": "/", 23 | "Tool": "/Users/petar/gocircuit/bin/4build", 24 | "ShipDir": "/Users/petar/_telefile/ship" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/circuit/_exp/telefile/cli/.gitignore: -------------------------------------------------------------------------------- 1 | /cli 2 | -------------------------------------------------------------------------------- /src/circuit/_exp/telefile/cli/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "circuit/exp/telefile/srv" 19 | "circuit/kit/tele/file" 20 | 21 | _ "circuit/load" 22 | "circuit/use/circuit" 23 | "io" 24 | "os" 25 | ) 26 | 27 | func main() { 28 | println("Starting") 29 | r, _, err := circuit.Spawn("localhost", []string{"/telefile"}, srv.App{}, "/tmp/telehelo") 30 | if err != nil { 31 | println("Oh oh", err.Error()) 32 | return 33 | } 34 | fcli := file.NewFileClient(r[0].(circuit.X)) 35 | defer func() { 36 | recover() 37 | }() 38 | io.Copy(os.Stdout, fcli) 39 | } 40 | -------------------------------------------------------------------------------- /src/circuit/_exp/telefile/env.sh: -------------------------------------------------------------------------------- 1 | export CIR=$HOME/gocircuit/src/circuit/exp/telefile/circuit.config 2 | -------------------------------------------------------------------------------- /src/circuit/_exp/telefile/srv/srv.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package srv 16 | 17 | import ( 18 | "circuit/kit/tele/file" 19 | "circuit/use/circuit" 20 | "os" 21 | "time" 22 | ) 23 | 24 | type App struct{} 25 | 26 | func init() { 27 | circuit.RegisterFunc(App{}) 28 | } 29 | 30 | func (App) Open(filepath string) circuit.X { 31 | f, err := os.Open(filepath) 32 | if err != nil { 33 | return nil 34 | } 35 | circuit.Daemonize(func() { time.Sleep(5 * time.Second) }) 36 | return circuit.Ref(file.NewFileServer(f)) 37 | } 38 | -------------------------------------------------------------------------------- /src/circuit/_exp/teleport/plain/gob.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package plain 16 | 17 | import ( 18 | x "circuit/exp/teleport" 19 | "encoding/gob" 20 | "io" 21 | ) 22 | 23 | type ReadWriter interface { 24 | Read() (interface{}, error) 25 | Write(interface{}) error 26 | } 27 | 28 | type Closer interface { 29 | Close() error 30 | } 31 | 32 | type ReadWriteCloser interface { 33 | ReadWriter 34 | Closer 35 | } 36 | 37 | // gobConn implements ReadWriteCloser on top of a io.ReadWriteCloser 38 | type gobConn struct { 39 | *gob.Encoder 40 | *gob.Decoder 41 | io.ReadWriteCloser 42 | } 43 | 44 | type gobMsg struct { 45 | Payload interface{} 46 | } 47 | 48 | func newGobConn(c io.ReadWriteCloser) *gobConn { 49 | return &gobConn{ 50 | Encoder: gob.NewEncoder(c), 51 | Decoder: gob.NewDecoder(c), 52 | ReadWriteCloser: c, 53 | } 54 | } 55 | 56 | func (g *gobConn) Read() (interface{}, error) { 57 | var msg gobMsg 58 | if err := g.Decode(&msg); err != nil { 59 | return nil, err 60 | } 61 | return msg.Payload, nil 62 | } 63 | 64 | func (g *gobConn) Write(v interface{}) error { 65 | var msg gobMsg = gobMsg{v} 66 | return g.Encode(&msg) 67 | } 68 | 69 | func (g *gobConn) RemoteAddr() x.Addr { 70 | return x.Addr("hello") 71 | } 72 | -------------------------------------------------------------------------------- /src/circuit/_exp/teleport/plain/plain.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package plain is a debug-only implementation of the teleport transport interface 16 | package plain 17 | 18 | import ( 19 | x "circuit/exp/teleport" 20 | "net" 21 | "time" 22 | ) 23 | 24 | // Listener 25 | type Listener struct { 26 | l net.Listener 27 | } 28 | 29 | func NewListener(addr x.Addr) *Listener { 30 | l, err := net.Listen("tcp", string(addr)) 31 | if err != nil { 32 | panic(err) 33 | } 34 | return &Listener{l} 35 | } 36 | 37 | func (l *Listener) Accept() x.Conn { 38 | c, err := l.l.Accept() 39 | if err != nil { 40 | panic(err) 41 | } 42 | return newGobConn(c) 43 | } 44 | 45 | // Dialer 46 | type Dialer struct{} 47 | 48 | func NewDialer() Dialer { 49 | return Dialer{} 50 | } 51 | 52 | func (Dialer) Dial(addr x.Addr) x.Conn { 53 | for { 54 | tcpaddr, err := net.ResolveTCPAddr("tcp", string(addr)) 55 | if err != nil { 56 | println("tcp resolve:", err.Error()) 57 | time.Sleep(time.Second) 58 | continue 59 | } 60 | c, err := net.DialTCP("tcp", nil, tcpaddr) 61 | if err != nil { 62 | println("tcp dial:", err.Error()) 63 | time.Sleep(time.Second) 64 | continue 65 | } 66 | return newGobConn(c) 67 | } 68 | panic("u") 69 | } 70 | -------------------------------------------------------------------------------- /src/circuit/_exp/teleport/tcp/dialer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tcp 16 | 17 | import ( 18 | x "circuit/exp/teleport" 19 | "sync" 20 | ) 21 | 22 | type Dialer struct { 23 | sync.Mutex 24 | open map[x.Addr]*link 25 | } 26 | 27 | func NewDialer() *Dialer { 28 | return &Dialer{ 29 | open: make(map[x.Addr]*link), 30 | } 31 | } 32 | 33 | func (t *Dialer) Dial(addr x.Addr) x.Conn { 34 | t.Lock() 35 | l, ok := t.open[addr] 36 | if !ok { 37 | l = newDialLink(addr) 38 | t.open[addr] = l 39 | } 40 | t.Unlock() 41 | return l.Dial() // link.Dial may block 42 | } 43 | -------------------------------------------------------------------------------- /src/circuit/_exp/teleport/tcp/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tcp 16 | 17 | import "errors" 18 | 19 | var ( 20 | ErrClosed = errors.New("already closed") 21 | ErrCollision = errors.New("conn id collision") 22 | ErrNotSupported = errors.New("not supported") 23 | ErrAuth = errors.New("authentication") 24 | ErrProto = errors.New("protocol") 25 | ) 26 | -------------------------------------------------------------------------------- /src/circuit/_exp/teleport/tcp/gob.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tcp 16 | 17 | import ( 18 | x "circuit/exp/teleport" 19 | "encoding/gob" 20 | "io" 21 | "net" 22 | ) 23 | 24 | type ReadWriter interface { 25 | Read() (interface{}, error) 26 | Write(interface{}) error 27 | } 28 | 29 | type Closer interface { 30 | Close() error 31 | } 32 | 33 | type ReadWriteCloser interface { 34 | ReadWriter 35 | Closer 36 | } 37 | 38 | // gobConn implements ReadWriteCloser on top of a io.ReadWriteCloser 39 | type gobConn struct { 40 | *gob.Encoder 41 | *gob.Decoder 42 | io.ReadWriteCloser 43 | } 44 | 45 | type gobMsg struct { 46 | Payload interface{} 47 | } 48 | 49 | func newGobConn(c io.ReadWriteCloser) *gobConn { 50 | return &gobConn{ 51 | Encoder: gob.NewEncoder(c), 52 | Decoder: gob.NewDecoder(c), 53 | ReadWriteCloser: c, 54 | } 55 | } 56 | 57 | func (g *gobConn) Read() (interface{}, error) { 58 | var msg gobMsg 59 | if err := g.Decode(&msg); err != nil { 60 | return nil, err 61 | } 62 | return msg.Payload, nil 63 | } 64 | 65 | func (g *gobConn) Write(v interface{}) error { 66 | var msg gobMsg = gobMsg{v} 67 | return g.Encode(&msg) 68 | } 69 | 70 | func mustDial(addr x.Addr) net.Conn { 71 | conn, err := net.Dial("tcp", string(addr)) 72 | if err != nil { 73 | panic(err) 74 | } 75 | return conn 76 | } 77 | -------------------------------------------------------------------------------- /src/circuit/_exp/teleport/tcp/transport_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tcp 16 | 17 | import ( 18 | x "circuit/exp/teleport" 19 | _ "circuit/kit/debug/http/trace" 20 | "net/http" 21 | "sync" 22 | "testing" 23 | ) 24 | 25 | func init() { 26 | go http.ListenAndServe(":1505", nil) 27 | } 28 | 29 | func TestTransport(t *testing.T) { 30 | const N = 100 31 | ch := make(chan int) 32 | d := NewDialer() 33 | laddr := x.Addr("localhost:9001") 34 | l := NewListener(":9001") 35 | 36 | go func() { 37 | for i := 0; i < N; i++ { 38 | c := l.Accept() 39 | v, err := c.Read() 40 | if err != nil { 41 | t.Errorf("read (%s)", err) 42 | } 43 | if v.(int) != 3 { 44 | t.Errorf("value") 45 | } 46 | c.Close() 47 | } 48 | ch <- 1 49 | }() 50 | 51 | var slk sync.Mutex 52 | sent := 0 53 | for i := 0; i < N; i++ { 54 | go func() { 55 | c := d.Dial(laddr) 56 | if err := c.Write(int(3)); err != nil { 57 | t.Errorf("write (%s)", err) 58 | } 59 | c.Close() 60 | slk.Lock() 61 | defer slk.Unlock() 62 | sent++ 63 | }() 64 | } 65 | <-ch 66 | } 67 | -------------------------------------------------------------------------------- /src/circuit/_exp/teleport/transport.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package teleport implements an experimental transport layer that can overcome network outages without affecting upstream clients 16 | package teleport 17 | 18 | import ( 19 | "strconv" 20 | "strings" 21 | ) 22 | 23 | type Addr string 24 | 25 | func (addr Addr) Port() int { 26 | i := strings.Index(string(addr), ":") 27 | if i < 0 { 28 | panic("endpoint address has no port") 29 | } 30 | port, err := strconv.Atoi(string(addr)[i+1:]) 31 | if err != nil { 32 | panic("endpoint address with invalid port") 33 | } 34 | return port 35 | } 36 | 37 | type Conn interface { 38 | Read() (interface{}, error) 39 | Write(interface{}) error 40 | Close() error 41 | RemoteAddr() Addr 42 | } 43 | 44 | type Listener interface { 45 | Accept() Conn 46 | } 47 | 48 | type Dialer interface { 49 | Dial(addr Addr) Conn 50 | } 51 | 52 | type Transport interface { 53 | Dialer 54 | Listener 55 | } 56 | -------------------------------------------------------------------------------- /src/circuit/app/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Miscellaneous circuit applications 16 | package app 17 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/api/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package api 16 | 17 | import "encoding/gob" 18 | 19 | func init() { 20 | gob.Register(&Config{}) 21 | gob.Register(&WorkerConfig{}) 22 | } 23 | 24 | // Config specifies a cluster of HTTP API servers 25 | type Config struct { 26 | Anchor string // Anchor for the sumr API workers 27 | ReadOnly bool // Reject requests resulting in change 28 | Workers []*WorkerConfig // Specification of service workers 29 | } 30 | 31 | // WorkerConfig specifies an individual API server 32 | type WorkerConfig struct { 33 | Host string // Host is the circuit hostname where the worker is to be deployed 34 | Port int // Port is the port number when the HTTP API server is to listen 35 | } 36 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/api/mkcfg/.gitignore: -------------------------------------------------------------------------------- 1 | mkcfg 2 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/api/mkcfg/mkcfg.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // mkcfg prints out an empty sumr api configuration 16 | package main 17 | 18 | import ( 19 | "circuit/app/sumr/api" 20 | "encoding/json" 21 | "fmt" 22 | ) 23 | 24 | func main() { 25 | raw, _ := json.MarshalIndent( 26 | &api.Config{ 27 | Anchor: "", 28 | Workers: []*api.WorkerConfig{ 29 | &api.WorkerConfig{ 30 | Host: "host3.datacenter.net", 31 | Port: 4000, 32 | }, 33 | &api.WorkerConfig{ 34 | Host: "host4.datacenter.net", 35 | Port: 4000, 36 | }, 37 | }, 38 | }, 39 | "", "\t", 40 | ) 41 | fmt.Printf("%s\n", raw) 42 | } 43 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/api/reqresp_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package api 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "testing" 21 | ) 22 | 23 | func TestReadBatch(t *testing.T) { 24 | const src = `{"f":{"a":"b"},"v":1}{"f":{"c":"d"}, "v":-1}` 25 | r := bytes.NewBufferString(src) 26 | req, err := readAddRequestBatch(Now(), r) 27 | if err != nil { 28 | t.Errorf("read batch (%s)", err) 29 | } 30 | if len(req) != 2 { 31 | t.Errorf("wrong number of read requests") 32 | } 33 | for _, r := range req { 34 | fmt.Printf("%T\n", r) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/block/code.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package block 16 | 17 | import ( 18 | "bytes" 19 | "circuit/app/sumr" 20 | "encoding/binary" 21 | "time" 22 | ) 23 | 24 | type add struct { 25 | UTime time.Time 26 | Key sumr.Key 27 | Value float64 28 | } 29 | 30 | type addOnDisk struct { 31 | UTime int64 32 | Key sumr.Key 33 | Value float64 34 | } 35 | 36 | // OPTIMIZE: Use a code object that uses the same underlying gob coder on each file 37 | 38 | func encodeAdd(a *add) []byte { 39 | var w bytes.Buffer 40 | if err := binary.Write(&w, binary.LittleEndian, &addOnDisk{a.UTime.UnixNano(), a.Key, a.Value}); err != nil { 41 | panic("sumr coder") 42 | } 43 | return w.Bytes() 44 | } 45 | 46 | func decodeAdd(p []byte) (*add, error) { 47 | r := bytes.NewBuffer(p) 48 | _a := &addOnDisk{} 49 | if err := binary.Read(r, binary.LittleEndian, _a); err != nil { 50 | return nil, err 51 | } 52 | return &add{time.Unix(0, _a.UTime), _a.Key, _a.Value}, nil 53 | } 54 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/block/code_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package block 16 | 17 | import ( 18 | "reflect" 19 | "testing" 20 | "time" 21 | ) 22 | 23 | func TestCode(t *testing.T) { 24 | a := &add{time.Now(), 1, 0.1} 25 | b := encodeAdd(a) 26 | println(len(b)) 27 | a2, err := decodeAdd(b) 28 | if err != nil { 29 | t.Errorf("decode add (%s)", err) 30 | } 31 | if !reflect.DeepEqual(a, a2) { 32 | t.Errorf("mismatch") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/block/sketch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package block 16 | 17 | import ( 18 | "circuit/app/sumr" 19 | "time" 20 | ) 21 | 22 | // Sketch is the type of the value stored within the key-value store. 23 | // This particular example implementation uses a sketch whose job is to 24 | // be a simple counter. Users will typically make their own copy of the sumr 25 | // service and substitute types like Sketch with ones suitable to their needs. 26 | type Sketch struct { 27 | UpdateTime time.Time // Application-level timestamp of the key 28 | Key sumr.Key 29 | Sum float64 30 | } 31 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/server/mkcfg/.gitignore: -------------------------------------------------------------------------------- 1 | mkcfg 2 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/server/mkcfg/mkcfg.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // mkcfg prints out an empty sumr shard servers configuration 16 | package main 17 | 18 | import ( 19 | "circuit/app/sumr/server" 20 | "encoding/json" 21 | "fmt" 22 | "time" 23 | ) 24 | 25 | func main() { 26 | raw, _ := json.MarshalIndent( 27 | &server.Config{ 28 | Workers: []*server.WorkerConfig{ 29 | &server.WorkerConfig{ 30 | Host: "host1.datacenter.net", 31 | DiskPath: "/tmp/sumr", 32 | Forget: time.Hour, 33 | }, 34 | &server.WorkerConfig{ 35 | Host: "host2.datacenter.net", 36 | DiskPath: "/tmp/sumr", 37 | Forget: time.Hour, 38 | }, 39 | }, 40 | }, 41 | "", "\t", 42 | ) 43 | fmt.Printf("%s\n", raw) 44 | } 45 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/server/x.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "circuit/use/circuit" 19 | "time" 20 | ) 21 | 22 | // Main wraps the worker function that starts a sumr shard server 23 | type main struct{} 24 | 25 | func init() { 26 | circuit.RegisterFunc(main{}) 27 | } 28 | 29 | // Main starts a sumr shard server 30 | // diskpath is a directory path on the local file system, where the function is executed, 31 | // where the shard will persist its data. 32 | func (main) Main(diskpath string, forgetafter time.Duration) (circuit.XPerm, error) { 33 | srv, err := New(diskpath, forgetafter) 34 | if err != nil { 35 | return nil, err 36 | } 37 | circuit.Daemonize(func() { <-(chan int)(nil) }) 38 | return circuit.PermRef(srv), nil 39 | } 40 | -------------------------------------------------------------------------------- /src/circuit/app/sumr/sumr.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package sumr implements a distributed persistent key-value store 16 | package sumr 17 | 18 | import "fmt" 19 | 20 | // Key is the type of keys used in the sumr database 21 | type Key int64 22 | 23 | // String returns the textual representation of this key 24 | func (k Key) String() string { 25 | return fmt.Sprintf("K%016x", int64(k)) 26 | } 27 | -------------------------------------------------------------------------------- /src/circuit/c/.gitignore: -------------------------------------------------------------------------------- 1 | /_tmp 2 | -------------------------------------------------------------------------------- /src/circuit/c/README: -------------------------------------------------------------------------------- 1 | FUNC 2 | 3 | = register all public types that have public methods 4 | restrict just to struct types (simplicity and safety) 5 | 6 | = register all public functions that return public types 7 | 8 | TYPE 9 | 10 | = public types (IN PROGRESS) 11 | that have >0 public methods 12 | register as circuit value types 13 | under all receiver type variations 14 | seen in its public methods 15 | 16 | = create boilerplate cross-client types 17 | for each registered value type 18 | two client types: 19 | LocalClient, when the origin is in the same process; and 20 | RemoteClient, otherwise 21 | one interface type: 22 | interface containing public methods of value type 23 | 24 | √ given multiple target packages, 25 | compute dependency packages 26 | parse types thru-out tree 27 | rewrite 28 | 29 | 30 | 31 | 32 | argument, or return value, kind: 33 | unsafe pointer 34 | panic 35 | struct 36 | pass by value, because linguistically the client may access its fields directly 37 | ptr 38 | return as local ptr to whatever the return rules are for the target type 39 | eventually, 40 | for target types struct, slice, array, map, builtin 41 | after function completes, 42 | check if target object was modified, and 43 | return the modifications back to the origin 44 | slice, array, map 45 | pass by value 46 | interface 47 | acceps a local compatible type or cross-pointer ??? 48 | chan 49 | n/i temporarily 50 | accept/return local or remote channel 51 | func 52 | n/i temporarily 53 | local func or wrapper that calls remote 54 | -------------------------------------------------------------------------------- /src/circuit/c/TODO: -------------------------------------------------------------------------------- 1 | 2 | To be able to parse Go's source tree: 3 | * recognize // +build ignore 4 | * recognize source file naming patterns xxx_os_arch.go, etc. 5 | -------------------------------------------------------------------------------- /src/circuit/c/build_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package c 16 | 17 | import ( 18 | "circuit/c/source" 19 | "os" 20 | "testing" 21 | ) 22 | 23 | var ( 24 | testLayout = source.NewLayout(os.Getenv("GOROOT"), source.GoPaths{"./testdata"}, "") 25 | ) 26 | 27 | func TestBuild(t *testing.T) { 28 | b, err := NewBuild(testLayout, "./_tmp") 29 | if err != nil { 30 | t.Fatalf("build (%s)", err) 31 | } 32 | if err := b.Build("b"); err != nil { 33 | t.Errorf("run (%s)", err) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/circuit/c/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package c implements a circuit compiler (IN DEVELOPMENT) 16 | package c 17 | -------------------------------------------------------------------------------- /src/circuit/c/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package errors implements error facilities for the circuit compiler 16 | package errors 17 | 18 | import ( 19 | "fmt" 20 | "go/token" 21 | ) 22 | 23 | type SourceError struct { 24 | FileSet *token.FileSet 25 | Pos token.Pos 26 | Msg string 27 | } 28 | 29 | func NewSource(fset *token.FileSet, pos token.Pos, fmts string, args ...interface{}) error { 30 | return &SourceError{ 31 | FileSet: fset, 32 | Pos: pos, 33 | Msg: fmt.Sprintf(fmts, args...), 34 | } 35 | } 36 | 37 | func (e *SourceError) Error() string { 38 | if e.Pos > 0 { 39 | pos := e.FileSet.Position(e.Pos) 40 | return fmt.Sprintf("%s • %s", pos, e.Msg) 41 | } 42 | return e.Msg 43 | } 44 | 45 | type StringError string 46 | 47 | func New(fmts string, args ...interface{}) error { 48 | return StringError(fmt.Sprintf(fmts, args...)) 49 | } 50 | 51 | func (e StringError) Error() string { 52 | return string(e) 53 | } 54 | 55 | var ( 56 | ErrNotFound = New("not found") 57 | ) 58 | -------------------------------------------------------------------------------- /src/circuit/c/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package c 16 | 17 | import ( 18 | "fmt" 19 | "go/token" 20 | "os" 21 | "sync" 22 | ) 23 | 24 | var ( 25 | llk sync.Mutex 26 | indent int 27 | ) 28 | 29 | func Indent() { 30 | llk.Lock() 31 | defer llk.Unlock() 32 | indent++ 33 | } 34 | 35 | func Unindent() { 36 | llk.Lock() 37 | defer llk.Unlock() 38 | indent-- 39 | } 40 | 41 | func Log(fmt_ string, arg_ ...interface{}) { 42 | for i := 0; i < indent; i++ { 43 | print(" ") 44 | } 45 | fmt.Fprintf(os.Stderr, fmt_, arg_...) 46 | println("") 47 | } 48 | 49 | func LogFileSet(fset *token.FileSet) { 50 | Log("FileSet:") 51 | fset.Iterate(func(f *token.File) bool { 52 | Log(" %s", f.Name()) 53 | return true 54 | }) 55 | } 56 | -------------------------------------------------------------------------------- /src/circuit/c/source/flush.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package source 16 | 17 | import ( 18 | "go/ast" 19 | "go/printer" 20 | "go/token" 21 | "os" 22 | "path" 23 | ) 24 | 25 | // Flush writes out all compiled and transformed packages to their location 26 | // inside the compilation jail 27 | func (s *Source) Flush() error { 28 | // For every Pkg 29 | for pkgPath, _ := range s.pkg { 30 | if err := s.FlushPkg(pkgPath); err != nil { 31 | return err 32 | } 33 | } 34 | return nil 35 | } 36 | 37 | func (s *Source) FlushPkg(pkgPath string) error { 38 | 39 | pkg := s.GetPkg(pkgPath) 40 | 41 | // For every ast.Package 42 | for _, pkgAST := range pkg.PkgAST { 43 | // For every ast.File 44 | for filePath, fileAST := range pkgAST.Files { 45 | _, fileName := path.Split(filePath) 46 | f, err := s.jail.CreateSourceFile(pkgPath, fileName) 47 | if err != nil { 48 | return err 49 | } 50 | if err := flushFile(f, pkg.FileSet, fileAST); err != nil { 51 | return err 52 | } 53 | } 54 | } 55 | return nil 56 | } 57 | 58 | func flushFile(f *os.File, fileSet *token.FileSet, file *ast.File) error { 59 | defer f.Close() 60 | return printer.Fprint(f, fileSet, file) 61 | } 62 | -------------------------------------------------------------------------------- /src/circuit/c/source/jail.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package source 16 | 17 | import ( 18 | "os" 19 | "path" 20 | ) 21 | 22 | type Jail struct { 23 | root string 24 | } 25 | 26 | func NewJail(root string) (*Jail, error) { 27 | j := &Jail{root} 28 | if err := j.mkdirs(); err != nil { 29 | return nil, err 30 | } 31 | return j, nil 32 | } 33 | 34 | func (j *Jail) mkdirs() error { 35 | if err := os.MkdirAll(path.Join(j.root, "src"), 0700); err != nil { 36 | return err 37 | } 38 | return nil 39 | } 40 | 41 | // AbsPkgPath returns the absolute local path of package pkg within the jail 42 | func (j *Jail) AbsPkgPath(pkgPath string) string { 43 | return path.Join(j.root, "src", pkgPath) 44 | } 45 | 46 | func (j *Jail) MakePkgDir(pkgPaths ...string) error { 47 | for _, pkgPath := range pkgPaths { 48 | if err := os.MkdirAll(j.AbsPkgPath(pkgPath), 0700); err != nil { 49 | return err 50 | } 51 | } 52 | return nil 53 | } 54 | 55 | func (j *Jail) CreateSourceFile(pkgPath, fileName string) (*os.File, error) { 56 | absPath := j.AbsPkgPath(pkgPath) 57 | if err := os.MkdirAll(absPath, 0770); err != nil { 58 | return nil, err 59 | } 60 | f, err := os.Create(path.Join(absPath, fileName)) 61 | if err != nil { 62 | return nil, err 63 | } 64 | return f, nil 65 | } 66 | -------------------------------------------------------------------------------- /src/circuit/c/testdata/src/a/a.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package a 16 | 17 | import ( 18 | "os" 19 | _ "strings" 20 | ) 21 | 22 | type T1 (int) 23 | 24 | func (t T1) P1() { 25 | println(t) 26 | } 27 | 28 | func (t T1) P1() { 29 | println(t) 30 | } 31 | 32 | func (t *T1) P2() { 33 | println(t) 34 | } 35 | 36 | type ( 37 | T2 struct { 38 | } 39 | 40 | T3 interface { 41 | } 42 | 43 | T4 []byte 44 | 45 | T5 [3]byte 46 | T6 *os.File 47 | ) 48 | -------------------------------------------------------------------------------- /src/circuit/c/testdata/src/b/b.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package b 16 | 17 | import _ "a" 18 | -------------------------------------------------------------------------------- /src/circuit/c/types/link.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | /* 18 | func (tt *Table) linkType(…) … { 19 | … 20 | switch q := spec.Type.(type) { 21 | // Built-in types or references to other types in this package 22 | case *ast.Ident: 23 | ? 24 | case *ast.ParenExpr: 25 | ? 26 | case *ast.SelectorExpr: 27 | ? 28 | case *ast.StarExpr: 29 | // r.Elem will be filled in during a follow up sweep of all types 30 | r.Kind = reflect.Ptr 31 | case *ast.ArrayType: 32 | XX // Slice or array kind? 33 | r.Kind = reflect.Array 34 | case *ast.ChanType: 35 | r.Kind = reflect.Chan 36 | case *ast.FuncType: 37 | r.Kind = reflect.Func 38 | case *ast.InterfaceType: 39 | r.Kind = reflect.Interface 40 | case *ast.MapType: 41 | r.Kind = reflect.Map 42 | case *ast.StructType: 43 | r.Kind = reflect.Struct 44 | default: 45 | return nil, errors.NewSource(fset, spec.Name.NamePos, "unexpected type definition") 46 | } 47 | 48 | return r 49 | } 50 | */ 51 | -------------------------------------------------------------------------------- /src/circuit/cmd/4build/.gitignore: -------------------------------------------------------------------------------- 1 | /4build 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4clear-helper/.gitignore: -------------------------------------------------------------------------------- 1 | /4clear-helper 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4clear-helper/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | 4clear-helper is used internally by 4clear. 17 | */ 18 | package main 19 | 20 | import ( 21 | "circuit/kit/lockfile" 22 | "circuit/use/circuit" 23 | "fmt" 24 | "os" 25 | "path" 26 | ) 27 | 28 | func main() { 29 | if len(os.Args) != 2 { 30 | println("Usage:", os.Args[0], "JailDir") 31 | os.Exit(1) 32 | } 33 | jailDir := os.Args[1] 34 | 35 | jail, err := os.Open(jailDir) 36 | if err != nil { 37 | fmt.Fprintf(os.Stderr, "Cannot open jail directory (%s)\n", err) 38 | os.Exit(1) 39 | } 40 | defer jail.Close() 41 | fifi, err := jail.Readdir(0) 42 | if err != nil { 43 | fmt.Fprintf(os.Stderr, "Cannot read jail directory (%s)\n", err) 44 | os.Exit(1) 45 | } 46 | 47 | for _, fi := range fifi { 48 | if !fi.IsDir() { 49 | continue 50 | } 51 | if _, err := circuit.ParseWorkerID(fi.Name()); err != nil { 52 | continue 53 | } 54 | 55 | workerJail := path.Join(jailDir, fi.Name()) 56 | println("Clearing", workerJail) 57 | l, err := lockfile.Create(path.Join(workerJail, "lock")) 58 | if err != nil { 59 | // This worker is alive; still holding lock; move on 60 | println(err.Error()) 61 | continue 62 | } 63 | l.Release() 64 | if err := os.RemoveAll(workerJail); err != nil { 65 | fmt.Fprintf(os.Stderr, "Cannot remove worker jail %s (%s)\n", workerJail, err) 66 | os.Exit(1) 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/circuit/cmd/4clear/.gitignore: -------------------------------------------------------------------------------- 1 | /4clear 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4cpu/.gitignore: -------------------------------------------------------------------------------- 1 | /4cpu 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4crossbuild/.gitignore: -------------------------------------------------------------------------------- 1 | /4crossbuild 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4crossbuild/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | 4crossbuild automates the process of cross-building a circuit application remotely. 17 | 18 | Invocation: 19 | 20 | % CIR=app.config 4crossbuild [-show] 21 | 22 | 4crossbuild considers the app configuration supplied and builds the circuit application 23 | on a remote build host, specified in app.config. 24 | 25 | When show is enabled, the standard error of all command execution involved in the remote build process are 26 | forwarded to local standard error. 27 | */ 28 | package main 29 | 30 | import ( 31 | "circuit/load/config" 32 | "flag" 33 | "os" 34 | ) 35 | 36 | var flagShow = flag.Bool("show", true, "Verbose mode") 37 | 38 | func main() { 39 | flag.Parse() 40 | c := config.Config.Build 41 | c.Binary = config.Config.Deploy.Worker 42 | if c == nil { 43 | println("Circuit build configuration not specified in environment") 44 | os.Exit(1) 45 | } 46 | println("Building circuit on", c.Host) 47 | c.Show = *flagShow 48 | if err := Build(c); err != nil { 49 | println(err.Error()) 50 | os.Exit(1) 51 | } 52 | println("Done.") 53 | } 54 | -------------------------------------------------------------------------------- /src/circuit/cmd/4deploy/.gitignore: -------------------------------------------------------------------------------- 1 | /4deploy 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4dls/.gitignore: -------------------------------------------------------------------------------- 1 | /4dls 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4hardkill/.gitignore: -------------------------------------------------------------------------------- 1 | /4hardkill 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4issue/.gitignore: -------------------------------------------------------------------------------- 1 | /4issue 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4kill/.gitignore: -------------------------------------------------------------------------------- 1 | /4kill 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4ls/.gitignore: -------------------------------------------------------------------------------- 1 | /4ls 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4stk/.gitignore: -------------------------------------------------------------------------------- 1 | /4stk 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/4stk/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | 17 | 4stk locates a worker through the anchor file system and prints its current stack trace. 18 | 19 | % CIR=app.config 4stk {AnchorFile} 20 | 21 | Print out the stack trace of the worker that owns the anchor file. 22 | 23 | */ 24 | package main 25 | 26 | import ( 27 | _ "circuit/load" 28 | "circuit/use/anchorfs" 29 | "circuit/use/circuit" 30 | "fmt" 31 | "log" 32 | "os" 33 | ) 34 | 35 | func main() { 36 | if len(os.Args) != 2 { 37 | println("Usage:", os.Args[0], "AnchorPath") 38 | os.Exit(1) 39 | } 40 | file, err := anchorfs.OpenFile(os.Args[1]) 41 | if err != nil { 42 | log.Printf("Problem opening (%s)", err) 43 | os.Exit(1) 44 | } 45 | x, err := circuit.TryDial(file.Owner(), "acid") 46 | if err != nil { 47 | log.Printf("Problem dialing acid service (%s)", err) 48 | os.Exit(1) 49 | } 50 | 51 | defer func() { 52 | if p := recover(); p != nil { 53 | log.Printf("Worker disappeared during call (%#v)", p) 54 | os.Exit(1) 55 | } 56 | }() 57 | 58 | retrn := x.Call("RuntimeProfile", "goroutine", 1) 59 | if err, ok := retrn[1].(error); ok && err != nil { 60 | log.Printf("Problem obtaining runtime profile (%s)", err) 61 | os.Exit(1) 62 | } 63 | fmt.Println(string(retrn[0].([]byte))) 64 | } 65 | -------------------------------------------------------------------------------- /src/circuit/cmd/4top/.gitignore: -------------------------------------------------------------------------------- 1 | /4top 2 | -------------------------------------------------------------------------------- /src/circuit/cmd/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package cmd implements command-line utilities for building and maintaining circuit applications 16 | package doc 17 | -------------------------------------------------------------------------------- /src/circuit/kit/debug/ctrlc/init.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package ctrlc has the side effect of installing a Ctrl-C signal handler that throws a panic 16 | package ctrlc 17 | 18 | import "circuit/kit/debug" 19 | 20 | func init() { 21 | debug.InstallCtrlCPanic() 22 | } 23 | -------------------------------------------------------------------------------- /src/circuit/kit/debug/http/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package http contains implementation for various HTTP reporting endpoint 16 | package http 17 | -------------------------------------------------------------------------------- /src/circuit/kit/debug/http/trace/trace.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package trace has the side effect of installing HTTP endpoints that report tracing information 16 | package trace 17 | 18 | import ( 19 | "net/http" 20 | "runtime/pprof" 21 | "strconv" 22 | ) 23 | 24 | func init() { 25 | http.HandleFunc("/_pprof", serveRuntimeProfile) 26 | http.HandleFunc("/_g", serveGoroutineProfile) 27 | http.HandleFunc("/_s", serveStackProfile) 28 | } 29 | 30 | func serveStackProfile(w http.ResponseWriter, r *http.Request) { 31 | prof := pprof.Lookup("goroutine") 32 | if prof == nil { 33 | http.Error(w, "unknown profile name", 400) 34 | return 35 | } 36 | prof.WriteTo(w, 2) 37 | } 38 | 39 | func serveGoroutineProfile(w http.ResponseWriter, r *http.Request) { 40 | prof := pprof.Lookup("goroutine") 41 | if prof == nil { 42 | http.Error(w, "unknown profile name", 400) 43 | return 44 | } 45 | prof.WriteTo(w, 1) 46 | } 47 | 48 | func serveRuntimeProfile(w http.ResponseWriter, r *http.Request) { 49 | name := r.URL.Query().Get("n") 50 | debug, err := strconv.Atoi(r.URL.Query().Get("d")) 51 | if err != nil { 52 | http.Error(w, "non-integer or missing debug flag", 400) 53 | return 54 | } 55 | 56 | prof := pprof.Lookup(name) 57 | if prof == nil { 58 | http.Error(w, "unknown profile name", 400) 59 | return 60 | } 61 | prof.WriteTo(w, debug) 62 | } 63 | -------------------------------------------------------------------------------- /src/circuit/kit/debug/kill/init.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package kill has the side effect of installing a KILL signal handler that throws a panic 16 | package kill 17 | 18 | import "circuit/kit/debug" 19 | 20 | func init() { 21 | debug.InstallKillPanic() 22 | } 23 | -------------------------------------------------------------------------------- /src/circuit/kit/fmt/deep_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fmt 16 | 17 | import ( 18 | "os" 19 | "testing" 20 | ) 21 | 22 | func TestDeep(t *testing.T) { 23 | s := []interface{}{"a", 2, "c"} 24 | Deep(os.Stdout, s) 25 | } 26 | -------------------------------------------------------------------------------- /src/circuit/kit/fs/fileinfo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fs 16 | 17 | import ( 18 | "os" 19 | "time" 20 | ) 21 | 22 | // FileInfo is a file meta-information structure. 23 | type FileInfo struct { 24 | 25 | // XName is the absolute name of the file. 26 | XName string 27 | 28 | // XSize is the file size in bytes. 29 | XSize int64 30 | 31 | // XMode is the file mode. 32 | XMode os.FileMode 33 | 34 | // XModTime is the time when the file was modified last. 35 | XModTime time.Time 36 | 37 | // XIsDir indicates if this file is a directory. 38 | XIsDir bool 39 | } 40 | 41 | // Name returns the absolute name of this file 42 | func (fi *FileInfo) Name() string { 43 | return fi.XName 44 | } 45 | 46 | // Size returns the file size in bytes 47 | func (fi *FileInfo) Size() int64 { 48 | return fi.XSize 49 | } 50 | 51 | // Mode returns this file's mode 52 | func (fi *FileInfo) Mode() os.FileMode { 53 | return fi.XMode 54 | } 55 | 56 | // ModTime returns the time when the file was modified last 57 | func (fi *FileInfo) ModTime() time.Time { 58 | return fi.XModTime 59 | } 60 | 61 | // IsDir returns true if this file is a directory 62 | func (fi *FileInfo) IsDir() bool { 63 | return fi.XIsDir 64 | } 65 | 66 | // Sys returns system-specific file annotations 67 | func (fi *FileInfo) Sys() interface{} { 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /src/circuit/kit/fs/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fs 16 | 17 | import "net/http" 18 | 19 | func HTTPFileSystem(fs FS) http.FileSystem { 20 | return httpFS{fs} 21 | } 22 | 23 | type httpFS struct { 24 | fs FS 25 | } 26 | 27 | func (httpfs httpFS) Open(name string) (http.File, error) { 28 | return httpfs.fs.Open(name) 29 | } 30 | -------------------------------------------------------------------------------- /src/circuit/kit/iomisc/combine.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package iomisc implements miscellaneous I/O facilities 16 | package iomisc 17 | 18 | import ( 19 | "io" 20 | "sync" 21 | ) 22 | 23 | type combinedReader struct { 24 | pipe *io.PipeReader 25 | wlk sync.Mutex 26 | closed int 27 | } 28 | 29 | // Combine returns an io.Reader that greedily reads from r1 and r2 in parallel 30 | func Combine(r1, r2 io.Reader) io.Reader { 31 | pr, pw := io.Pipe() 32 | c := &combinedReader{pipe: pr} 33 | go c.readTo(r1, pw) 34 | go c.readTo(r2, pw) 35 | return c 36 | } 37 | 38 | func (c *combinedReader) readTo(r io.Reader, w *io.PipeWriter) { 39 | p := make([]byte, 1e5) 40 | for { 41 | n, err := r.Read(p) 42 | if n > 0 { 43 | c.wlk.Lock() 44 | w.Write(p[:n]) 45 | c.wlk.Unlock() 46 | } 47 | if err != nil { 48 | c.wlk.Lock() 49 | defer c.wlk.Unlock() 50 | c.closed++ 51 | if c.closed == 2 { 52 | w.Close() 53 | } 54 | return 55 | } 56 | } 57 | } 58 | 59 | func (c *combinedReader) Read(p []byte) (int, error) { 60 | return c.pipe.Read(p) 61 | } 62 | -------------------------------------------------------------------------------- /src/circuit/kit/join/setthenget.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package join provides a mechanism for linking an implementation package to a declaration package 16 | package join 17 | 18 | import ( 19 | "sync" 20 | ) 21 | 22 | // SetThenGet is a synchronized interface value, which can be set once and read many times 23 | type SetThenGet struct { 24 | Name string 25 | lk sync.Mutex 26 | v interface{} 27 | } 28 | 29 | // Set sets the value to v 30 | func (j *SetThenGet) Set(v interface{}) { 31 | j.lk.Lock() 32 | defer j.lk.Unlock() 33 | if j.v != nil { 34 | panic(j.Name + " already set") 35 | } 36 | j.v = v 37 | } 38 | 39 | // Get returns this value 40 | func (j *SetThenGet) Get() interface{} { 41 | j.lk.Lock() 42 | defer j.lk.Unlock() 43 | if j.v == nil { 44 | panic(j.Name + " not set") 45 | } 46 | return j.v 47 | } 48 | -------------------------------------------------------------------------------- /src/circuit/kit/kit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package kit contains various utility packages useful with the circuit 16 | package kit 17 | -------------------------------------------------------------------------------- /src/circuit/kit/llrb/ORIGIN: -------------------------------------------------------------------------------- 1 | Code taken from: 2 | https://github.com/petar/GoLLRB 3 | 4 | Commit SHA of clone: 5 | a4de6ccdcb03ab78ab4b88e9d6a79cd58d1ec9bf 6 | -------------------------------------------------------------------------------- /src/circuit/kit/llrb/avgvar.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Copyright 2010 Petar Maymounkov. All rights reserved. 16 | // Use of this source code is governed by a BSD-style 17 | // license that can be found in the LICENSE file. 18 | 19 | package llrb 20 | 21 | import "math" 22 | 23 | // avgVar maintains the average and variance of a stream of numbers 24 | // in a space-efficient manner. 25 | type avgVar struct { 26 | count int64 27 | sum, sumsq float64 28 | } 29 | 30 | func (av *avgVar) Init() { 31 | av.count = 0 32 | av.sum = 0.0 33 | av.sumsq = 0.0 34 | } 35 | 36 | func (av *avgVar) Add(sample float64) { 37 | av.count++ 38 | av.sum += sample 39 | av.sumsq += sample * sample 40 | } 41 | 42 | func (av *avgVar) GetCount() int64 { return av.count } 43 | 44 | func (av *avgVar) GetAvg() float64 { return av.sum / float64(av.count) } 45 | 46 | func (av *avgVar) GetTotal() float64 { return av.sum } 47 | 48 | func (av *avgVar) GetVar() float64 { 49 | a := av.GetAvg() 50 | return av.sumsq/float64(av.count) - a*a 51 | } 52 | 53 | func (av *avgVar) GetStdDev() float64 { return math.Sqrt(av.GetVar()) } 54 | -------------------------------------------------------------------------------- /src/circuit/kit/llrb/llrb-stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Copyright 2010 Petar Maymounkov. All rights reserved. 16 | // Use of this source code is governed by a BSD-style 17 | // license that can be found in the LICENSE file. 18 | 19 | package llrb 20 | 21 | // GetHeight() returns an item in the tree with key @key, and it's height in the tree 22 | func (t *Tree) GetHeight(key Item) (result Item, depth int) { 23 | return t.getHeight(t.root, key) 24 | } 25 | 26 | func (t *Tree) getHeight(h *Node, item Item) (Item, int) { 27 | if h == nil { 28 | return nil, 0 29 | } 30 | if t.less(item, h.Item) { 31 | result, depth := t.getHeight(h.Left, item) 32 | return result, depth + 1 33 | } 34 | if t.less(h.Item, item) { 35 | result, depth := t.getHeight(h.Right, item) 36 | return result, depth + 1 37 | } 38 | return h.Item, 0 39 | } 40 | 41 | // HeightStats() returns the average and standard deviation of the height 42 | // of elements in the tree 43 | func (t *Tree) HeightStats() (avg, stddev float64) { 44 | av := &avgVar{} 45 | heightStats(t.root, 0, av) 46 | return av.GetAvg(), av.GetStdDev() 47 | } 48 | 49 | func heightStats(h *Node, d int, av *avgVar) { 50 | if h == nil { 51 | return 52 | } 53 | av.Add(float64(d)) 54 | if h.Left != nil { 55 | heightStats(h.Left, d+1, av) 56 | } 57 | if h.Right != nil { 58 | heightStats(h.Right, d+1, av) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/circuit/kit/lockfile/lockfile.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package os provides application-level utilities that are implemented using OS facilities (like lock files) 16 | package lockfile 17 | 18 | import ( 19 | "os" 20 | "syscall" 21 | ) 22 | 23 | // LockFile represents an exclusive, advisory OS file lock 24 | type LockFile struct { 25 | file *os.File 26 | } 27 | 28 | // Create creates a file with an exclusive, advisory lock 29 | func Create(name string) (*LockFile, error) { 30 | file, err := os.Create(name) 31 | if err != nil { 32 | return nil, err 33 | } 34 | err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) 35 | if err != nil { 36 | file.Close() 37 | return nil, err 38 | } 39 | return &LockFile{file}, nil 40 | } 41 | 42 | // Release releases the OS file lock and closes the respective file 43 | func (lf *LockFile) Release() error { 44 | if err := syscall.Flock(int(lf.file.Fd()), syscall.LOCK_UN); err != nil { 45 | lf.file.Close() 46 | lf.file = nil 47 | return err 48 | } 49 | if err := lf.file.Close(); err != nil { 50 | lf.file = nil 51 | return err 52 | } 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /src/circuit/kit/lockfile/lockfile_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lockfile 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestLockFile(t *testing.T) { 22 | const name = "/tmp/test.lock" 23 | lock, err := Create(name) 24 | if err != nil { 25 | t.Fatalf("create lock (%s)", err) 26 | } 27 | 28 | if _, err := Create(name); err == nil { 29 | t.Errorf("re-create lock should not succceed", err) 30 | } 31 | 32 | if err = lock.Release(); err != nil { 33 | t.Fatalf("release lock (%s)", err) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/circuit/kit/lockfile/¶testdir/test.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/src/circuit/kit/lockfile/¶testdir/test.lock -------------------------------------------------------------------------------- /src/circuit/kit/sched/limiter/l_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package limiter 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | ) 21 | 22 | func TestLimiter(t *testing.T) { 23 | l := New(2) 24 | for i := 0; i < 9; i++ { 25 | i_ := i 26 | l.Go(func() { 27 | println("{", i_) 28 | time.Sleep(time.Second) 29 | println("}", i_) 30 | }) 31 | } 32 | l.Wait() 33 | // TODO: Test that all routines open and close 34 | println("DONE") 35 | } 36 | -------------------------------------------------------------------------------- /src/circuit/kit/sched/sched.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package sched contains primitives for scheduling 16 | package sched 17 | -------------------------------------------------------------------------------- /src/circuit/kit/stat/sliding_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package stat 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | ) 21 | 22 | func TestCarousel(t *testing.T) { 23 | c := NewSlidingMoment(10, time.Second) 24 | 25 | now := time.Now().UnixNano() 26 | for i := 0; i < 20; i++ { 27 | slot := c.Slot(time.Unix(0, now-int64(i)*1e9)) 28 | if slot != nil { 29 | println("ok") 30 | slot.Add(1) 31 | } else { 32 | println("gh") 33 | } 34 | } 35 | for i := 0; i < 5; i++ { 36 | slot := c.Slot(time.Unix(0, now+int64(i)*1e9)) 37 | slot.Add(5) 38 | } 39 | slots, tlast := c.Slots() 40 | println("timelast", tlast.UnixNano()) 41 | for _, slot := range slots { 42 | println("slot", slot.Count(), slot.Max()) 43 | } 44 | println(c.Weight()) 45 | } 46 | -------------------------------------------------------------------------------- /src/circuit/kit/stat/time.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package stat 16 | 17 | import ( 18 | "time" 19 | ) 20 | 21 | // TimeSampler is a facility for collection stopwatch statistics over multiple experiments. 22 | // TimeSampler is not synchronized. Only one measurement can take place at a time. 23 | type TimeSampler struct { 24 | m Moment 25 | t0 *time.Time 26 | } 27 | 28 | // Init initializes the time sampler. 29 | func (x *TimeSampler) Init() { 30 | x.m.Init() 31 | x.t0 = nil 32 | } 33 | 34 | // Start initiates a new measurement. 35 | func (x *TimeSampler) Start() { 36 | if x.t0 != nil { 37 | panic("previous sample not completed") 38 | } 39 | t0 := time.Now() 40 | x.t0 = &t0 41 | } 42 | 43 | // Stop ends an experiment and records the elapsed time as a sample in an underlying moment sketch. 44 | func (x *TimeSampler) Stop() { 45 | t1 := time.Now() 46 | diff := t1.Sub(*x.t0) 47 | x.t0 = nil 48 | x.m.Add(float64(diff)) 49 | } 50 | 51 | // Moment returns the underlying moment sketch. 52 | func (x *TimeSampler) Moment() *Moment { 53 | return &x.m 54 | } 55 | 56 | // Average returns the average experiment time. 57 | func (x *TimeSampler) Average() float64 { 58 | return x.m.Average() 59 | } 60 | 61 | // StdDev returns the standard deviation across all experiments. 62 | func (x *TimeSampler) StdDev() float64 { 63 | return x.m.StdDev() 64 | } 65 | -------------------------------------------------------------------------------- /src/circuit/kit/tele/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package tele encloses utilities for exporting high-level objects like file systems, channels, etc. 16 | package tele 17 | -------------------------------------------------------------------------------- /src/circuit/kit/waterfill/waterfill_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package waterfill 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | type testBin int 23 | 24 | func (p *testBin) Add() { 25 | (*p)++ 26 | } 27 | 28 | func (p *testBin) Less(fb FillBin) bool { 29 | return *p < *(fb.(*testBin)) 30 | } 31 | 32 | func (p *testBin) String() string { 33 | return fmt.Sprintf("%02d", *p) 34 | } 35 | 36 | func TestFill(t *testing.T) { 37 | bin := make([]FillBin, 10) 38 | for i, _ := range bin { 39 | b := testBin(i * 2) 40 | bin[i] = &b 41 | } 42 | f := NewFill(bin) 43 | for i := 0; i < 30; i++ { 44 | println(f.String()) 45 | f.Add() 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/circuit/kit/xor/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package xor 16 | 17 | import "math/rand" 18 | 19 | // ChooseMinK chooses k random keys and returns the one which, if inserted into 20 | // the metric, would result in the shallowest position in the XOR tree. 21 | // In other words, it returns the most balanced choice. We recommend k equals 7. 22 | func (m *Metric) ChooseMinK(k int) Key { 23 | if m == nil { 24 | return Key(rand.Int63()) 25 | } 26 | var min_id Key 27 | var min_d int = 1000 28 | for k > 0 { 29 | // Note: The last bit is not really randomized here 30 | id := Key(rand.Int63()) 31 | d, err := m.Add(id) 32 | if err != nil { 33 | continue 34 | } 35 | m.Remove(id) 36 | if d < min_d { 37 | min_id = id 38 | min_d = d 39 | } 40 | k-- 41 | } 42 | return min_id 43 | } 44 | -------------------------------------------------------------------------------- /src/circuit/kit/xor/xor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package xor 16 | 17 | import ( 18 | "fmt" 19 | "math/rand" 20 | "testing" 21 | ) 22 | 23 | const K = 16 24 | 25 | func TestXOR(t *testing.T) { 26 | m := &Metric{} 27 | for i := 0; i < K; i++ { 28 | m.Add(Key(i)) 29 | } 30 | for piv := 0; piv < K; piv++ { 31 | nearest := m.Nearest(Key(piv), K/2) 32 | fmt.Println(Key(piv).ShortString(4)) 33 | for _, n := range nearest { 34 | fmt.Println(" ", n.Key().ShortString(4)) 35 | } 36 | } 37 | } 38 | 39 | const stressN = 1000000 40 | 41 | func TestStress(t *testing.T) { 42 | m := &Metric{} 43 | var h []Key 44 | for i := 0; i < stressN; i++ { 45 | id := Key(rand.Int63()) 46 | h = append(h, id) 47 | if _, err := m.Add(id); err != nil { 48 | t.Errorf("add (%s)", err) 49 | } 50 | } 51 | perm := rand.Perm(len(h)) 52 | for _, j := range perm { 53 | m.Remove(h[j]) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/circuit/kit/zookeeper/zutil/create.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package util implements commonly used Zookeeper patterns and recipes 16 | package zutil 17 | 18 | import ( 19 | zookeeper "github.com/petar/gozk" 20 | "errors" 21 | "path" 22 | "strings" 23 | ) 24 | 25 | // CreateRecursive creates the directory leafPath and any parent subdirectories if necessary 26 | func CreateRecursive(z *zookeeper.Conn, leafPath string, aclv []zookeeper.ACL) error { 27 | leafPath = path.Clean(leafPath) 28 | if len(leafPath) == 0 || leafPath[0] != '/' { 29 | return errors.New("zookeeper util path syntax") 30 | } 31 | parts := strings.Split(leafPath, "/") 32 | if len(parts) < 1 { 33 | return errors.New("creating zookeeper root") 34 | } 35 | prefix := "/" 36 | for i := 0; i < len(parts); i++ { 37 | prefix = path.Join(prefix, parts[i]) 38 | if _, err := z.Create(prefix, "", 0, aclv); err != nil && !IsNodeExists(err) { 39 | return err 40 | } 41 | } 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /src/circuit/load/config/parse_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "testing" 21 | ) 22 | 23 | // To run this test, you must set the CIR environment variable first 24 | func TestParse(t *testing.T) { 25 | fmt.Printf("CIR=%#v %#v %#v\n", Zookeeper, Install, Build) 26 | } 27 | -------------------------------------------------------------------------------- /src/circuit/load/config/spark.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import "circuit/use/circuit" 18 | 19 | // SparkConfig captures a few worker startup parameters that can be configured on each execution 20 | type SparkConfig struct { 21 | // ID is the ID of the worker instance 22 | ID circuit.WorkerID 23 | 24 | // BindAddr is the network address the worker will listen to for incoming connections 25 | BindAddr string 26 | 27 | // Host is the host name of the hosting machine 28 | Host string 29 | 30 | // Anchor is the set of anchor directories that the worker registers with 31 | Anchor []string 32 | } 33 | 34 | // DefaultSpark is the default configuration used for workers started from the command line, which 35 | // are often not intended to be contacted back from other workers 36 | var DefaultSpark = &SparkConfig{ 37 | ID: circuit.ChooseWorkerID(), 38 | BindAddr: "", // Don't accept incoming circuit calls from other workers 39 | Host: "", // " 40 | Anchor: []string{}, // Don't register within the anchor file system 41 | } 42 | -------------------------------------------------------------------------------- /src/circuit/load/worker/worker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Importing package worker has the side effect of turning your program into a circuit worker executable 16 | package worker 17 | 18 | import ( 19 | _ "circuit/kit/debug/kill" 20 | _ "circuit/load" 21 | ) 22 | 23 | func init() { 24 | // After package load installs and activates all circuit-related logic, 25 | // this function blocks forever, never allowing execution of main. 26 | <-(chan struct{})(nil) 27 | } 28 | -------------------------------------------------------------------------------- /src/circuit/sys/acid/behalf.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package acid 16 | 17 | import ( 18 | "circuit/use/circuit" 19 | "fmt" 20 | "reflect" 21 | ) 22 | 23 | type Stringer interface { 24 | String() string 25 | } 26 | 27 | func (s *Acid) OnBehalfCallStringer(service, proc string) (r string) { 28 | 29 | // If anything goes wrong, let's not panic the worker 30 | defer func() { 31 | if p := recover(); p != nil { 32 | r = fmt.Sprintf("Stat likely not supported:\n%#v", p) 33 | } 34 | }() 35 | 36 | // Obtain service object in this worker 37 | srv := circuit.DialSelf(service) 38 | if srv == nil { 39 | return "Service not available" 40 | } 41 | 42 | // Find Stat method in service receiver s 43 | sv := reflect.ValueOf(srv) 44 | out := sv.MethodByName(proc).Call(nil) 45 | if len(out) != 1 { 46 | return "Service's Stat method returns more than one value" 47 | } 48 | 49 | return out[0].Interface().(Stringer).String() 50 | } 51 | -------------------------------------------------------------------------------- /src/circuit/sys/acid/jail.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package acid 16 | 17 | import ( 18 | teleio "circuit/kit/tele/io" 19 | "circuit/load/config" 20 | "circuit/use/circuit" 21 | "io" 22 | "os/exec" 23 | "path" 24 | ) 25 | 26 | // JailTail opens a file within this worker's jail directory and prepares a 27 | // cross-circuit pointer to the open file 28 | func (a *Acid) JailTail(jailFile string) (circuit.X, error) { 29 | abs := path.Join(config.Config.Deploy.JailDir(), circuit.WorkerAddr().WorkerID().String(), jailFile) 30 | 31 | cmd := exec.Command("/bin/sh", "-c", "tail -f "+abs) 32 | 33 | stdout, err := cmd.StdoutPipe() 34 | if err != nil { 35 | return nil, circuit.FlattenError(err) 36 | } 37 | 38 | if err = cmd.Start(); err != nil { 39 | return nil, circuit.FlattenError(err) 40 | } 41 | 42 | return circuit.Ref(teleio.NewServer(&tailStdout{stdout, cmd})), nil 43 | } 44 | 45 | type tailStdout struct { 46 | io.ReadCloser 47 | cmd *exec.Cmd 48 | } 49 | 50 | func (*tailStdout) Write([]byte) (int, error) { 51 | panic("write not supported") 52 | } 53 | 54 | func (t *tailStdout) Close() error { 55 | println("CLOSING TAIL") 56 | err := t.ReadCloser.Close() 57 | t.cmd.Process.Kill() 58 | return circuit.FlattenError(err) 59 | } 60 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/README: -------------------------------------------------------------------------------- 1 | 2 | SUMMARY 3 | The language runtime converts linguistic operations 4 | (calling a remote value's method or forking a go 5 | routine remotely) into request-response interactions 6 | with remote runtimes. 7 | 8 | I/O: 9 | The ResponseWriter interfaces abstracts away an asynchronous 10 | request-response architecture with other opaquely 11 | addressable entities. 12 | 13 | ResponseWriter exposes communication over Go runtime interface{}s 14 | to the user. These values passed for these interfaces must 15 | be gob encodable. This design enables an implementation 16 | that allows for maintaining one encoder/decoder per remote 17 | runtime, thereby benefitting from better data compression 18 | over time. 19 | 20 | TIMEOUTS: 21 | The I/O infrastructure is responsible for timeing out request/response 22 | interactions if the destination is suspected dead. 23 | 24 | If a req/resp pair to a given destination timesout, the destination 25 | is considered ... dead for good? 26 | 27 | TYPES AND GOB: 28 | To ensure argument and return values are gob encodable, 29 | their types are registered with gob at the point where their 30 | receiver objects are registered with the circuit 31 | type system. 32 | 33 | FLATTENING AND UNFLATTENING 34 | Gob flattens values that are given as input to Encode or that are assigned to 35 | interface variables inside composite types. For functional arguments and 36 | return values, we can correct for the flattening since we keep the original 37 | type information. However we cannot do this for interface variables inside 38 | composite types passed as in or out of a function call. 39 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/TODO: -------------------------------------------------------------------------------- 1 | * optimize handling of passing X that points to a local object 2 | * also receiving X that points to a local object 3 | 4 | * write doc 5 | 6 | * monitor size of imp/exp tables and frequency of all runtime op types 7 | # exp perm handles 8 | # exp perm values 9 | # exp nonperm handles 10 | # exp nonperm values 11 | # exp importers 12 | # imp perm 13 | # imp nonperm 14 | # imp nonperm exporters 15 | # calls per ? 16 | 17 | * expose all the pprof stuff 18 | 19 | * add passing channels 20 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/_ptrptr_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | import ( 18 | "fmt" 19 | "runtime" 20 | "strconv" 21 | "testing" 22 | "time" 23 | ) 24 | 25 | func TestPtrPtr(t *testing.T) { 26 | l1 := NewSandbox() 27 | r1 := New(l1, &testBoot{"π1"}) 28 | 29 | l2 := NewSandbox() 30 | r2 := New(l2, &testBoot{"π2"}) 31 | 32 | p2, err := r1.TryDial(l2.Addr()) 33 | if err != nil { 34 | t.Fatalf("dial 1->2 (%s)", err) 35 | } 36 | 37 | p1, err := r2.TryDial(l1.Addr()) 38 | if err != nil { 39 | t.Fatalf("dial 2->1 (%s)", err) 40 | } 41 | 42 | if p1.Call("Name")[0].(string) != "π1" { 43 | t.Errorf("return val 1") 44 | } 45 | 46 | if p2.Call("Name")[0].(string) != "π2" { 47 | t.Errorf("return val 2") 48 | } 49 | p2.Call("ReturnNilMap") 50 | } 51 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/call.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | import ( 18 | "circuit/sys/lang/types" 19 | "fmt" 20 | "reflect" 21 | "runtime/debug" 22 | ) 23 | 24 | // call invokes the method of r encoded by f with respect to t, with arguments a 25 | func call(recv reflect.Value, t *types.TypeChar, id types.FuncID, arg []interface{}) (reply []interface{}, err error) { 26 | // Recover panic in user code and return it in error argument 27 | defer func() { 28 | p := recover() 29 | if p == nil { 30 | return 31 | } 32 | t := string(debug.Stack()) 33 | switch q := p.(type) { 34 | case error: 35 | err = NewError(q.Error() + "\n" + t) 36 | default: 37 | err = NewError(fmt.Sprintf("%#v\n%s", q, t)) 38 | } 39 | }() 40 | 41 | fn := t.Func[id] 42 | if fn == nil { 43 | return nil, NewError("no func") 44 | } 45 | av := make([]reflect.Value, 0, 1+len(arg)) 46 | av = append(av, recv) 47 | for _, a := range arg { 48 | av = append(av, reflect.ValueOf(a)) 49 | } 50 | rv := fn.Method.Func.Call(av) 51 | reply = make([]interface{}, len(rv)) 52 | for i, r := range rv { 53 | reply[i] = r.Interface() 54 | } 55 | return reply, nil 56 | } 57 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | /* 18 | 19 | FORKING A GO ROUTINE ON A REMOTE RUNTIME 20 | 21 | import . "circuit/use/circuit" 22 | 23 | type MyFunc struct{} 24 | func (MyFunc) AnyName(anyArg anyType) (anyReturn anyType) { 25 | ... 26 | } 27 | func init() { types.RegisterFunc(MyFunc{}) } 28 | 29 | func main() { 30 | Go(conn, MyFunc{}, a1) 31 | } 32 | 33 | */ 34 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | import ( 18 | "encoding/gob" 19 | "fmt" 20 | ) 21 | 22 | func init() { 23 | gob.Register(&errorString{}) 24 | } 25 | 26 | var ErrParse = NewError("parse") 27 | 28 | // NewError creates a simple text-based error that is serializable 29 | func NewError(fmt_ string, arg_ ...interface{}) error { 30 | return &errorString{fmt.Sprintf(fmt_, arg_...)} 31 | } 32 | 33 | type errorString struct { 34 | S string 35 | } 36 | 37 | func (e *errorString) Error() string { 38 | return e.S 39 | } 40 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/flat.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | import ( 18 | "reflect" 19 | ) 20 | 21 | func unflattenValue(v reflect.Value, t reflect.Type) reflect.Value { 22 | // When t is an Interface, we can't do much, since we don't know the 23 | // original (unflattened) type of the value placed in v, so we just nop it. 24 | if t.Kind() == reflect.Interface { 25 | return v 26 | } 27 | // v can be invalid, if it holds the nil value for pointer type 28 | if !v.IsValid() { 29 | return v 30 | } 31 | // Make sure v is indeed flat 32 | if v.Kind() == reflect.Ptr { 33 | panic("unflattening non-flat value") 34 | } 35 | // Add a *, one at a time 36 | for t.Kind() == reflect.Ptr { 37 | if v.CanAddr() { 38 | v = v.Addr() 39 | } else { 40 | pw := reflect.New(v.Type()) 41 | pw.Elem().Set(v) 42 | v = pw 43 | } 44 | t = t.Elem() 45 | } 46 | return v 47 | } 48 | 49 | func unflattenSlice(s []interface{}, t []reflect.Type) []interface{} { 50 | for i, v := range s { 51 | w := unflattenValue(reflect.ValueOf(v), t[i]) 52 | // If type is *T, v can be invalid (nil) before and after the unflatten call 53 | if w.IsValid() { 54 | s[i] = w.Interface() 55 | } 56 | } 57 | return s 58 | } 59 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/freeze.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | import ( 18 | "runtime/debug" 19 | ) 20 | 21 | func (r *Runtime) Export(val ...interface{}) interface{} { 22 | expHalt, _ := r.exportValues(val, nil) 23 | return &exportedMsg{ 24 | Value: expHalt, 25 | Stack: string(debug.Stack()), 26 | } 27 | } 28 | 29 | func (r *Runtime) Import(exported interface{}) ([]interface{}, string, error) { 30 | h, ok := exported.(*exportedMsg) 31 | if !ok { 32 | return nil, "", NewError("foreign saved message (msg=%T)", exported) 33 | } 34 | val, err := r.importValues(h.Value, nil, nil, false, nil) 35 | if err != nil { 36 | return nil, "", err 37 | } 38 | return val, h.Stack, nil 39 | } 40 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/freeze_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | func TestExportImport(t *testing.T) { 23 | r := New(NewSandbox()) 24 | x := r.Export(1, 2) 25 | fmt.Printf("x=%#v\n", x) 26 | v, s, err := r.Import(x) 27 | if err != nil { 28 | t.Errorf("import (%s)", err) 29 | } 30 | fmt.Printf("v=%#v, #s=%d\n", v, len(s)) 31 | } 32 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/perm.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | /* 18 | 19 | Ref PermRef exportRewrite importRewrite 20 | ----------------------------------------------------------------------------- 21 | T *ref *permref = = 22 | xptr = panic ptrPtrMsg n/a 23 | *ref = panic ptrMsg n/a 24 | xpermptr = = permPtrPtrMsg n/a 25 | *permref panic = permPtrMsg n/a 26 | ----------------------------------------------------------------------------- 27 | *ptrMsg n/a n/a n/a *ptr 28 | *ptrPtrMsg n/a n/a n/a *ptr 29 | *permPtrMsg n/a n/a n/a *permptr 30 | *permPtrPtrMsg n/a n/a n/a *permptr 31 | 32 | 33 | USER VS RUNTIME TYPES 34 | 35 | X ≈ xptr, *_ref, *_ptr, *ptrMsg, *ptrPtrMsg 36 | XPerm ≈ xpermptr, *_permref, *_permptr, *permPtrMsg, *permPtrPtrMsg 37 | 38 | */ 39 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/srv.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | import ( 18 | "sync" 19 | ) 20 | 21 | type srvTabl struct { 22 | sync.Mutex 23 | name map[string]interface{} 24 | } 25 | 26 | func (t *srvTabl) Init() *srvTabl { 27 | t.Lock() 28 | defer t.Unlock() 29 | t.name = make(map[string]interface{}) 30 | return t 31 | } 32 | 33 | func (t *srvTabl) Add(name string, receiver interface{}) { 34 | t.Lock() 35 | defer t.Unlock() 36 | if _, present := t.name[name]; present { 37 | panic("service already listening") 38 | } 39 | x := receiver 40 | t.name[name] = x 41 | } 42 | 43 | func (t *srvTabl) Get(name string) interface{} { 44 | t.Lock() 45 | defer t.Unlock() 46 | return t.name[name] 47 | } 48 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/types/gob.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "encoding/gob" 19 | "reflect" 20 | "time" 21 | ) 22 | 23 | // Register some common types. Repeated registration is ok. 24 | func init() { 25 | gob.Register(make(map[string]interface{})) 26 | gob.Register(make(map[string]string)) 27 | gob.Register(make(map[string]int)) 28 | gob.Register(make([]interface{}, 0)) 29 | gob.Register(time.Duration(0)) 30 | } 31 | 32 | // gobFlattenRegister registers the flattened type of t with gob 33 | // E.g. the flattened type of *T is T, of **T is T, etc. 34 | // Interface types cannot be registered. 35 | func gobFlattenRegister(t reflect.Type) { 36 | if t.Kind() == reflect.Interface { 37 | return 38 | } 39 | for t.Kind() == reflect.Ptr { 40 | t = t.Elem() 41 | } 42 | pz := reflect.New(t) 43 | gob.Register(pz.Elem().Interface()) 44 | } 45 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/types/register.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "strconv" 19 | ) 20 | 21 | var ( 22 | ValueTabl *TypeTabl = makeTypeTabl() // Type table for values 23 | FuncTabl *TypeTabl = makeTypeTabl() // Type table for functions 24 | ) 25 | 26 | // RegisterValue registers the type of x with the type table. 27 | // Types need to be registered before values can be imported. 28 | func RegisterValue(value interface{}) { 29 | ValueTabl.Add(makeType(value)) 30 | } 31 | 32 | // RegisterFunc ... 33 | func RegisterFunc(fn interface{}) { 34 | t := makeType(fn) 35 | if len(t.Func) != 1 { 36 | panic("fn type must have exactly one method: " + strconv.Itoa(len(t.Func))) 37 | } 38 | FuncTabl.Add(t) 39 | } 40 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/types/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "encoding/binary" 19 | "hash/fnv" 20 | ) 21 | 22 | func sliceStringID32(sign []string) int32 { 23 | h := fnv.New32a() 24 | for _, s := range sign { 25 | h.Write([]byte(s)) 26 | } 27 | return int32Bytes(h.Sum(nil)) 28 | } 29 | 30 | func sliceStringID64(sign []string) int64 { 31 | h := fnv.New64a() 32 | for _, s := range sign { 33 | h.Write([]byte(s)) 34 | } 35 | return int64Bytes(h.Sum(nil)) 36 | } 37 | 38 | func int64Bytes(p []byte) int64 { 39 | return int64(binary.BigEndian.Uint64(p)) 40 | } 41 | 42 | func int32Bytes(p []byte) int32 { 43 | return int32(binary.BigEndian.Uint32(p)) 44 | } 45 | -------------------------------------------------------------------------------- /src/circuit/sys/lang/vrewrite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lang 16 | 17 | import ( 18 | //"fmt" 19 | "reflect" 20 | "testing" 21 | ) 22 | 23 | type testStruct struct { 24 | a int 25 | B interface{} 26 | P interface{} 27 | Q *testStruct 28 | } 29 | 30 | type testReplacement struct{} 31 | 32 | func testRewrite(src, dst reflect.Value) bool { 33 | switch src.Interface().(type) { 34 | case *_ref: 35 | dst.Set(reflect.ValueOf(&testReplacement{})) 36 | return true 37 | } 38 | return false 39 | } 40 | 41 | func TestRewriteValue(t *testing.T) { 42 | sv := &testStruct{ 43 | a: 3, 44 | B: Ref(float64(1.1)), 45 | P: testStruct{B: int(2)}, 46 | Q: &testStruct{B: int(3)}, 47 | } 48 | /*xsv :=*/ rewriteInterface(testRewrite, sv) 49 | //fmt.Printf("%#v\n%#v\n", sv, xsv) 50 | // XXX: Add test 51 | } 52 | -------------------------------------------------------------------------------- /src/circuit/sys/sys.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package sys encloses the implementation of the circuit runtime 16 | package sys 17 | -------------------------------------------------------------------------------- /src/circuit/sys/transport/TODO: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tumblr/gocircuit/bbc21edaf84ef04e3c8cb9b0cd7a87b77152fe01/src/circuit/sys/transport/TODO -------------------------------------------------------------------------------- /src/circuit/sys/transport/addr.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package transport 16 | 17 | import ( 18 | "circuit/use/circuit" 19 | "encoding/gob" 20 | "net" 21 | "sync" 22 | ) 23 | 24 | // Addr maintains a single unique instance for each addr. 25 | // Addr object uniqueness is required by the circuit.Addr interface. 26 | type Addr struct { 27 | ID circuit.WorkerID 28 | PID int 29 | Addr *net.TCPAddr 30 | } 31 | 32 | func init() { 33 | gob.Register(&Addr{}) 34 | } 35 | 36 | func NewAddr(id circuit.WorkerID, pid int, hostport string) (circuit.Addr, error) { 37 | a, err := net.ResolveTCPAddr("tcp", hostport) 38 | if err != nil { 39 | return nil, err 40 | } 41 | return &Addr{ID: id, PID: pid, Addr: a}, nil 42 | } 43 | 44 | func (a *Addr) Host() string { 45 | return a.Addr.IP.String() 46 | } 47 | 48 | func (a *Addr) String() string { 49 | return a.ID.String() + "@" + a.Addr.String() 50 | } 51 | 52 | func (a *Addr) WorkerID() circuit.WorkerID { 53 | return a.ID 54 | } 55 | 56 | type addrTabl struct { 57 | lk sync.Mutex 58 | tabl map[circuit.WorkerID]*Addr 59 | } 60 | 61 | func makeAddrTabl() *addrTabl { 62 | return &addrTabl{tabl: make(map[circuit.WorkerID]*Addr)} 63 | } 64 | 65 | func (t *addrTabl) Normalize(addr *Addr) *Addr { 66 | t.lk.Lock() 67 | defer t.lk.Unlock() 68 | 69 | a, ok := t.tabl[addr.ID] 70 | if ok { 71 | return a 72 | } 73 | t.tabl[addr.ID] = addr 74 | return addr 75 | } 76 | -------------------------------------------------------------------------------- /src/circuit/sys/transport/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package transport 16 | 17 | import "circuit/use/circuit" 18 | 19 | var ( 20 | ErrEnd = circuit.NewError("end") 21 | ErrAlreadyClosed = circuit.NewError("already closed") 22 | errCollision = circuit.NewError("conn id collision") 23 | ErrNotSupported = circuit.NewError("not supported") 24 | ErrAuth = circuit.NewError("authentication") 25 | ) 26 | -------------------------------------------------------------------------------- /src/circuit/sys/transport/msg.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package transport 16 | 17 | import ( 18 | "circuit/use/circuit" 19 | "encoding/gob" 20 | ) 21 | 22 | func init() { 23 | gob.Register(&welcomeMsg{}) 24 | gob.Register(&openMsg{}) 25 | gob.Register(&connMsg{}) 26 | gob.Register(&linkMsg{}) 27 | } 28 | 29 | // linkMsg is the link-level message format between to endpoints. 30 | // The link level is responsible for ensuring reliable and ordered delivery in 31 | // the presence of network partitions and lost connections, assuming an 32 | // eventual successful reconnect. 33 | type linkMsg struct { 34 | SeqNo int64 // OPT: Use circular integer comparison and fewer bits 35 | AckNo int64 36 | Payload interface{} 37 | } 38 | 39 | type welcomeMsg struct { 40 | ID circuit.WorkerID // Runtime ID of sender 41 | PID int // Process ID of sender runtime 42 | } 43 | 44 | type openMsg struct { 45 | ID connID 46 | } 47 | 48 | type connMsg struct { 49 | ID connID 50 | Payload interface{} 51 | } 52 | -------------------------------------------------------------------------------- /src/circuit/sys/transport/transport_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package transport 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestTransport(t *testing.T) { 22 | ch := make(chan int) 23 | t1, t2 := New(1, ":9000"), New(2, ":9001") 24 | 25 | go func() { 26 | c := t2.Accept() 27 | v, err := c.Read() 28 | if err != nil { 29 | t.Errorf("read (%s)", err) 30 | } 31 | if v.(int) != 3 { 32 | t.Errorf("value") 33 | } 34 | ch <- 1 35 | }() 36 | 37 | c12, err := t1.Dial(t2.Addr()) 38 | if err != nil { 39 | t.Fatalf("dial, %s\n", err) 40 | } 41 | if err = c12.Write(int(3)); err != nil { 42 | t.Errorf("write (%s)", err) 43 | } 44 | <-ch 45 | } 46 | -------------------------------------------------------------------------------- /src/circuit/sys/transport/trigger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package transport 16 | 17 | import ( 18 | "sync" 19 | ) 20 | 21 | type Trigger struct { 22 | lk sync.Mutex 23 | engaged bool 24 | nwaiters int 25 | ch chan struct{} 26 | } 27 | 28 | func (t *Trigger) Lock() bool { 29 | t.lk.Lock() 30 | if t.ch == nil { 31 | t.ch = make(chan struct{}) 32 | } 33 | if t.engaged { 34 | t.nwaiters++ 35 | t.lk.Unlock() 36 | <-t.ch 37 | return false 38 | } 39 | t.engaged = true 40 | t.lk.Unlock() 41 | return true 42 | } 43 | 44 | func (t *Trigger) Unlock() { 45 | t.lk.Lock() 46 | defer t.lk.Unlock() 47 | if !t.engaged { 48 | panic("unlocking a non-engaged trigger") 49 | } 50 | for t.nwaiters > 0 { 51 | t.ch <- struct{}{} 52 | t.nwaiters-- 53 | } 54 | t.engaged = false 55 | } 56 | -------------------------------------------------------------------------------- /src/circuit/sys/worker/proc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package worker 16 | 17 | import ( 18 | "circuit/sys/transport" 19 | "circuit/use/circuit" 20 | "io" 21 | ) 22 | 23 | type Console struct { 24 | stdin io.WriteCloser 25 | stdout io.ReadCloser 26 | stderr io.ReadCloser 27 | } 28 | 29 | type Process struct { 30 | console Console 31 | addr *transport.Addr 32 | } 33 | 34 | func (p *Process) Addr() circuit.Addr { 35 | return p.addr 36 | } 37 | 38 | func (p *Process) Kill() error { 39 | return kill(p.addr) 40 | } 41 | 42 | func (p *Process) Stdin() io.WriteCloser { 43 | panic("ni") 44 | return p.console.stdin 45 | } 46 | 47 | func (p *Process) Stdout() io.ReadCloser { 48 | panic("ni") 49 | return p.console.stdout 50 | } 51 | 52 | func (p *Process) Stderr() io.ReadCloser { 53 | panic("ni") 54 | return p.console.stderr 55 | } 56 | -------------------------------------------------------------------------------- /src/circuit/sys/zanchorfs/file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package zanchorfs 16 | 17 | import ( 18 | "circuit/use/circuit" 19 | ) 20 | 21 | type File struct { 22 | owner circuit.Addr 23 | } 24 | 25 | func (f *File) Owner() circuit.Addr { 26 | return f.owner 27 | } 28 | -------------------------------------------------------------------------------- /src/circuit/sys/zanchorfs/zookeeper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package zanchorfs 16 | 17 | import ( 18 | zookeeper "github.com/petar/gozk" 19 | "circuit/kit/zookeeper/zutil" 20 | ) 21 | 22 | func Dial(zookeepers []string) *zookeeper.Conn { 23 | c, err := zutil.DialUntilReady(zutil.ZookeeperString(zookeepers)) 24 | if err != nil { 25 | panic(err) 26 | } 27 | return c 28 | } 29 | -------------------------------------------------------------------------------- /src/circuit/sys/zdurablefs/fs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package zdurablefs implements a durable file system using Apache Zookeeper 16 | package zdurablefs 17 | 18 | import ( 19 | zookeeper "github.com/petar/gozk" 20 | "circuit/use/circuit" 21 | "path" 22 | ) 23 | 24 | var ( 25 | ErrClosed = circuit.NewError("durable file system: closed") 26 | ) 27 | 28 | // FS implements a durable file system on top of Zookeeper 29 | type FS struct { 30 | conn *zookeeper.Conn 31 | zroot string 32 | } 33 | 34 | func New(conn *zookeeper.Conn, zroot string) *FS { 35 | return &FS{conn: conn, zroot: zroot} 36 | } 37 | 38 | func (fs *FS) Remove(fpath string) error { 39 | return fs.conn.Delete(path.Join(fs.zroot, fpath), -1) 40 | } 41 | -------------------------------------------------------------------------------- /src/circuit/sys/zdurablefs/zookeeper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package zdurablefs 16 | 17 | import ( 18 | zookeeper "github.com/petar/gozk" 19 | "circuit/kit/zookeeper/zutil" 20 | ) 21 | 22 | func Dial(zookeepers []string) *zookeeper.Conn { 23 | c, err := zutil.DialUntilReady(zutil.ZookeeperString(zookeepers)) 24 | if err != nil { 25 | panic(err) 26 | } 27 | return c 28 | } 29 | -------------------------------------------------------------------------------- /src/circuit/sys/zissuefs/sendmail_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package zissuefs 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestSendmail(t *testing.T) { 22 | if err := sendmail("user@test.com", "hi subject", "some body\n\naha\nx\n"); err != nil { 23 | t.Fatalf("sendmail (%s)", err) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/circuit/sys/zissuefs/zookeeper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package zissuefs 16 | 17 | import ( 18 | zookeeper "github.com/petar/gozk" 19 | "circuit/kit/zookeeper/zutil" 20 | ) 21 | 22 | func Dial(zookeepers []string) *zookeeper.Conn { 23 | c, err := zutil.DialUntilReady(zutil.ZookeeperString(zookeepers)) 24 | if err != nil { 25 | panic(err) 26 | } 27 | return c 28 | } 29 | -------------------------------------------------------------------------------- /src/circuit/test/test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package test contains end-to-end circuit test cases 16 | package test 17 | -------------------------------------------------------------------------------- /src/circuit/test/xgc/README: -------------------------------------------------------------------------------- 1 | XGC contains a circuit app for testing the cross-worker garbage collection. 2 | -------------------------------------------------------------------------------- /src/circuit/test/xgc/circuit.config: -------------------------------------------------------------------------------- 1 | { 2 | "Zookeeper": { 3 | "Workers": ["localhost:2181"], 4 | "RootDir": "/circuit-test" 5 | }, 6 | "Install": { 7 | "RootDir": "/tmp/_circuit_test", 8 | "LibPath": ".", 9 | "Binary": "4r-test" 10 | }, 11 | "Build": { 12 | "Host": "localhost", 13 | "Jail": "/tmp/_circuit_test/build", 14 | "AppRepo": "{rsync}/Users/petar/gocircuit", 15 | "AppSrc": "/", 16 | "Pkg": "circuit/test/xgc/worker", 17 | "RebuildGo": false, 18 | "ZookeeperInclude": "/Users/petar/local/include/c-client-src", 19 | "ZookeeperLib": "/Users/petar/local/lib", 20 | "CircuitRepo": "{rsync}/Users/petar/gocircuit", 21 | "CircuitSrc": "/", 22 | "Tool": "/Users/petar/gocircuit/bin/4build", 23 | "ShipDir": "/tmp/_circuit_test/ship" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/circuit/test/xgc/env.sh: -------------------------------------------------------------------------------- 1 | export CIR=$HOME/gocircuit/src/circuit/test/xgc/circuit.config 2 | -------------------------------------------------------------------------------- /src/circuit/test/xgc/main/.gitignore: -------------------------------------------------------------------------------- 1 | /main 2 | -------------------------------------------------------------------------------- /src/circuit/test/xgc/main/kick: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo localhost | 4clear && \ 3 | 4crossbuild -v && echo localhost | 4deploy && \ 4 | echo "--------------------------------" && go build && ./main \ 5 | && 4ls /... | grep 'xgc/R' 6 | -------------------------------------------------------------------------------- /src/circuit/test/xgc/main/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package main is the main executable for starting the circuit application 16 | package main 17 | 18 | import ( 19 | _ "circuit/kit/debug/ctrlc" 20 | _ "circuit/load" 21 | "circuit/test/xgc/worker" 22 | "circuit/use/circuit" 23 | "runtime" 24 | ) 25 | 26 | // TODO: Make sure finalizer called BECAUSE worker died or worker asked us to release handle 27 | 28 | func main() { 29 | ch := make(chan int) 30 | spark(ch) 31 | 32 | println("Waiting for finalizer call ...") 33 | // Force the garbage collector to collect 34 | go func() { 35 | for i := 0; i < 1e9; i++ { 36 | _ = make([]int, i) 37 | } 38 | }() 39 | <-ch 40 | println("Success") 41 | } 42 | 43 | func spark(ch chan int) { 44 | d := &worker.Dummy{} 45 | runtime.SetFinalizer(d, func(h *worker.Dummy) { 46 | println("finalizing dummy") 47 | close(ch) 48 | }) 49 | defer runtime.GC() 50 | 51 | // Test: 52 | // Spawn a worker and pass an x-pointer to it; 53 | // Worker proceeds to die right away; 54 | // Check that finalizer of local dummy called when local runtime notices remote is dead 55 | _, addr, err := circuit.Spawn("localhost", []string{"/xgc"}, worker.Start{}, circuit.Ref(d)) 56 | if err != nil { 57 | panic(err) 58 | } 59 | println(addr.String()) 60 | } 61 | -------------------------------------------------------------------------------- /src/circuit/test/xgc/worker/worker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package worker defines the worker function for this application 16 | package worker 17 | 18 | import ( 19 | "circuit/use/circuit" 20 | ) 21 | 22 | type Start struct{} 23 | 24 | func (Start) Main(dummy circuit.X) { 25 | } 26 | 27 | func init() { 28 | circuit.RegisterFunc(Start{}) 29 | } 30 | 31 | type Dummy struct{} 32 | 33 | func init() { circuit.RegisterValue(&Dummy{}) } 34 | 35 | func (*Dummy) Ping() {} 36 | -------------------------------------------------------------------------------- /src/circuit/test/xgc/xgc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package xgc implements a circuit application for testing cross-runtime garbage collection 16 | package xgc 17 | -------------------------------------------------------------------------------- /src/circuit/use/anchorfs/bind.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package anchorfs 16 | 17 | import ( 18 | "circuit/kit/join" 19 | "circuit/use/circuit" 20 | ) 21 | 22 | var link = join.SetThenGet{Name: "anchor file system"} 23 | 24 | // Bind is used internally to bind an implementation of this package to the public methods of this package 25 | func Bind(v interface{}) { 26 | link.Set(v) 27 | } 28 | 29 | func get() fs { 30 | return link.Get().(fs) 31 | } 32 | 33 | // CreateFile creates a new ephemeral file in the anchor directory anchor and saves the worker address addr in it. 34 | func CreateFile(anchor string, addr circuit.Addr) error { 35 | return get().CreateFile(anchor, addr) 36 | } 37 | 38 | // Created returns a slive of anchor directories within which this worker has created files with CreateFile. 39 | func Created() []string { 40 | return get().Created() 41 | } 42 | 43 | // OpenDir opens the anchor directory anchor 44 | func OpenDir(anchor string) (Dir, error) { 45 | return get().OpenDir(anchor) 46 | } 47 | 48 | // OpenFile opens the anchor file anchor 49 | func OpenFile(anchor string) (File, error) { 50 | return get().OpenFile(anchor) 51 | } 52 | -------------------------------------------------------------------------------- /src/circuit/use/circuit/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package circuit 16 | 17 | import ( 18 | "encoding/gob" 19 | "errors" 20 | "fmt" 21 | ) 22 | 23 | func init() { 24 | gob.Register(&errorString{}) 25 | gob.Register(errors.New("")) 26 | } 27 | 28 | // NewError creates a simple text-based error that is registered with package 29 | // encoding/gob and therefore can be used in places of error interfaces during 30 | // cross-calls. In contrast, note that due to the rules of gob encoding error objects 31 | // that are not explicitly registered with gob cannot be assigned to error interfaces 32 | // that are to be gob-serialized during a cross-call. 33 | func NewError(fmt_ string, arg_ ...interface{}) error { 34 | return &errorString{fmt.Sprintf(fmt_, arg_...)} 35 | } 36 | 37 | // FlattenError converts any error into a gob-serializable one that can be used in cross-calls. 38 | func FlattenError(err error) error { 39 | if err == nil { 40 | return nil 41 | } 42 | return NewError(err.Error()) 43 | } 44 | 45 | type errorString struct { 46 | S string 47 | } 48 | 49 | func (e *errorString) Error() string { 50 | return e.S 51 | } 52 | -------------------------------------------------------------------------------- /src/circuit/use/circuit/lang.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package circuit 16 | 17 | // X represents a cross-interface value. 18 | type X interface { 19 | 20 | // Addr returns the address of the runtime, hosting the object underlying the cross-interface value. 21 | Addr() Addr 22 | 23 | // Call invokes the method named proc of the actual object (possibly 24 | // living remotely) underlying the cross-interface. The invokation 25 | // arguments are take from in, and the returned values are placed in 26 | // the returned slice. 27 | // 28 | // Errors can only occur as a result of physical/external circumstances 29 | // that impede cross-worker communication. Such errors are returned in 30 | // the form of panics. 31 | Call(proc string, in ...interface{}) []interface{} 32 | 33 | // IsX is used internally. 34 | IsX() 35 | 36 | // String returns a human-readable representation of the cross-interface. 37 | String() string 38 | } 39 | 40 | // XPerm represents a permanent cross-interface value. 41 | type XPerm interface { 42 | 43 | // A permanent cross-interface can be used as a non-permanent one. 44 | X 45 | 46 | // IsPerm is used internally. 47 | IsXPerm() 48 | } 49 | 50 | // Func is a symbolic type that refers to circuit worker function types. 51 | // These are types with a singleton public method. 52 | type Func interface{} 53 | -------------------------------------------------------------------------------- /src/circuit/use/circuit/runtime.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package circuit 16 | 17 | type runtime interface { 18 | // Low-level 19 | WorkerAddr() Addr 20 | SetBoot(interface{}) 21 | Kill(Addr) error 22 | 23 | // Spawn mechanism 24 | Spawn(string, []string, Func, ...interface{}) ([]interface{}, Addr, error) 25 | Daemonize(func()) 26 | 27 | // Cross-services 28 | Dial(Addr, string) X 29 | DialSelf(string) interface{} 30 | TryDial(Addr, string) (X, error) 31 | Listen(string, interface{}) 32 | 33 | // Persistence of XPerm values 34 | Export(...interface{}) interface{} 35 | Import(interface{}) ([]interface{}, string, error) 36 | 37 | // Cross-interfaces 38 | Ref(interface{}) X 39 | PermRef(interface{}) XPerm 40 | } 41 | -------------------------------------------------------------------------------- /src/circuit/use/use.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package use contains the programming interfaces exposed by the circuit runtime 16 | package use 17 | -------------------------------------------------------------------------------- /src/circuit/use/worker/worker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package worker implements low-level routines for spawning and killing a worker process 16 | package worker 17 | 18 | import ( 19 | "circuit/kit/join" 20 | "circuit/use/circuit" 21 | ) 22 | 23 | // Spawn starts a new worker process on host and registers it under the given 24 | // anchors directories in the anchor file system. On success, Spawn returns 25 | // the address of the new work. Spawn is a low-level function. The spawned 26 | // worker will wait idle for further interaction. It is the caller's responsibility 27 | // to manage the lifespan of the newworker. 28 | func Spawn(host string, anchors ...string) (circuit.Addr, error) { 29 | return get().Spawn(host, anchors...) 30 | } 31 | 32 | // Kill kills the circuit worker with the given addr 33 | func Kill(addr circuit.Addr) error { 34 | return get().Kill(addr) 35 | } 36 | 37 | type commander interface { 38 | Spawn(string, ...string) (circuit.Addr, error) 39 | Kill(circuit.Addr) error 40 | } 41 | 42 | // Binding mechanism 43 | var link = join.SetThenGet{Name: "commander system"} 44 | 45 | // Bind is used internally to bind an implementation of this package to the public methods of this package 46 | func Bind(v interface{}) { 47 | link.Set(v.(commander)) 48 | } 49 | 50 | func get() commander { 51 | return link.Get().(commander) 52 | } 53 | -------------------------------------------------------------------------------- /src/tumblr/firehose/speed_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package firehose 16 | 17 | import ( 18 | "fmt" 19 | "sync" 20 | "testing" 21 | "time" 22 | ) 23 | 24 | func TestSpeed(t *testing.T) { 25 | freq := &Request{ 26 | HostPort: "", // Firehose host and port 27 | Username: "", // Your username 28 | Password: "", // Your password 29 | ApplicationID: "", // Your application ID 30 | ClientID: "", // Your client ID 31 | Offset: "", // Your offset 32 | } 33 | 34 | conns := make([]*Conn, 2) 35 | for i, _ := range conns { 36 | fmt.Printf("dialing %d\n", i) 37 | var err error 38 | conns[i], err = Dial(freq) 39 | if err != nil { 40 | t.Fatalf("dial (%s)", err) 41 | } 42 | } 43 | fmt.Printf("reading\n") 44 | 45 | var lk sync.Mutex 46 | var nread int64 47 | var t0 time.Time = time.Now() 48 | 49 | for _, conn := range conns { 50 | go func(conn *Conn) { 51 | for { 52 | if _, err := conn.Read(); err != nil { 53 | t.Errorf("read (%s)", err) 54 | continue 55 | } 56 | lk.Lock() 57 | nread++ 58 | k := nread 59 | lk.Unlock() 60 | if k%10 == 0 { 61 | fmt.Printf("%g read/sec\n", float64(k)/(float64(time.Now().Sub(t0))/1e9)) 62 | } 63 | } 64 | }(conn) 65 | } 66 | <-(chan int)(nil) 67 | } 68 | -------------------------------------------------------------------------------- /src/tumblr/firehose/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package firehose 16 | 17 | func getString(m map[string]interface{}, key string) string { 18 | s, _ := m[key].(string) 19 | return s 20 | } 21 | 22 | func getBool(m map[string]interface{}, key string) bool { 23 | v, present := m[key] 24 | if !present { 25 | return false 26 | } 27 | b, ok := v.(bool) 28 | if !ok { 29 | return false 30 | } 31 | return b 32 | } 33 | 34 | func getInt(m map[string]interface{}, key string) int { 35 | v, present := m[key] 36 | if !present { 37 | return 0 38 | } 39 | i, ok := v.(float64) 40 | if !ok { 41 | return 0 42 | } 43 | return int(i) 44 | } 45 | 46 | func getInt64(m map[string]interface{}, key string) (int64, error) { 47 | v, present := m[key] 48 | if !present { 49 | return 0, ErrMissing 50 | } 51 | i, ok := v.(float64) 52 | if !ok { 53 | return 0, ErrType 54 | } 55 | return int64(i), nil 56 | } 57 | 58 | func getMap(m map[string]interface{}, key string) map[string]interface{} { 59 | v, present := m[key] 60 | if !present { 61 | return nil 62 | } 63 | r, ok := v.(map[string]interface{}) 64 | if !ok { 65 | return nil 66 | } 67 | return r 68 | } 69 | 70 | func getSlice(m map[string]interface{}, key string) []interface{} { 71 | v, present := m[key] 72 | if !present { 73 | return nil 74 | } 75 | r, ok := v.([]interface{}) 76 | if !ok { 77 | return nil 78 | } 79 | return r 80 | } 81 | -------------------------------------------------------------------------------- /src/tumblr/hbase/reader_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package hbase 16 | 17 | import ( 18 | "io" 19 | "testing" 20 | ) 21 | 22 | type record struct { 23 | Field1 uint64 24 | Field2 uint64 25 | Field3 int64 26 | } 27 | 28 | func TestReader(t *testing.T) { 29 | r, err := OpenFile("testdata/records") 30 | if err != nil { 31 | t.Fatalf("open (%s)", err) 32 | } 33 | var v record 34 | for { 35 | err = r.Read(&v) 36 | if err == io.EOF { 37 | break 38 | } 39 | if err != nil { 40 | t.Fatalf("read (%s)", err) 41 | } 42 | println(v.Follower, v.Followee, v.Time) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/tumblr/kafka/TODO: -------------------------------------------------------------------------------- 1 | Short reads do not return an err 2 | Last read before an EOF may contain the EOF 3 | -------------------------------------------------------------------------------- /src/tumblr/kafka/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | Package kafka implements a client for Apache Kafka. 17 | It is implemented after the following specifications: 18 | 19 | https://cwiki.apache.org/confluence/display/KAFKA/Wire+Format 20 | https://cwiki.apache.org/confluence/display/KAFKA/Writing+a+Driver+for+Kafka 21 | 22 | The package exposes functionality at varying levels of detail. 23 | */ 24 | package kafka 25 | -------------------------------------------------------------------------------- /src/tumblr/kafka/eco_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kafka 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | func TestEcosystem(t *testing.T) { 23 | eco, err := NewEcosystem("127.0.0.1:2181") 24 | if err != nil { 25 | t.Fatalf("connect to Zookeeper (%s)", err) 26 | } 27 | brokers, err := eco.Brokers() 28 | if err != nil { 29 | t.Fatalf("get brokers (%s)", err) 30 | } 31 | for _, be := range brokers { 32 | fmt.Printf("%s\n", be) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/tumblr/kafka/message_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kafka 16 | 17 | import ( 18 | "bytes" 19 | "reflect" 20 | "testing" 21 | ) 22 | 23 | var ( 24 | testMessage = &Message{ 25 | Compression: NoCompression, 26 | Payload: []byte{1, 2, 3}, 27 | } 28 | ) 29 | 30 | func TestMessage(t *testing.T) { 31 | var w bytes.Buffer 32 | m0 := testMessage 33 | if err := m0.Write(&w); err != nil { 34 | t.Fatalf("message write (%s)", err) 35 | } 36 | r := bytes.NewBuffer(w.Bytes()) 37 | m1 := &Message{} 38 | _, err := m1.Read(r) 39 | if err != nil { 40 | t.Fatalf("message read (%s)", err) 41 | } 42 | if !reflect.DeepEqual(m0, m1) { 43 | t.Errorf("expecting %v got %v", m0, m1) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/tumblr/kafka/multifetch.dump: -------------------------------------------------------------------------------- 1 | ---------------------------------------------------------------- 2 | 0x0, 0x0, 0x0, 0x72, LENGTH OF THE REST 3 | =114 4 | 5 | 0x0, 0x0. ERROR CODE 6 | ---------------------------------------------------------------- 7 | 0x0, 0x0, 0x0, 0x2, LENGTH OF REST IN THIS BLOCK 8 | 9 | 0x0, 0x0. ERROR CODE 10 | ---------------------------------------------------------------- 11 | 0x0, 0x0, 0x0, 0x66, LENGTH OF REST IN THIS BLOCK 12 | =102 13 | 0x0, 0x0. ERROR CODE 14 | 15 | 0x0, 0x0, 0x0, 0x8, 0x0, 0x55, 0xbc, 0x80, 0x1d, 0x1, 0x2, 0x3, MSG (12 bytes) 16 | 0x0, 0x0, 0x0, 0x8, 0x0, 0x6c, 0x5c, 0x20, 0xbe, 0x4, 0x5, 0x6, " 17 | 0x0, 0x0, 0x0, 0x8, 0x0, 0x55, 0xbc, 0x80, 0x1d, 0x1, 0x2, 0x3, " 18 | 0x0, 0x0, 0x0, 0x8, 0x0, 0x6c, 0x5c, 0x20, 0xbe, 0x4, 0x5, 0x6, " 19 | 0x0, 0x0. 0x0, 0x8, 0x0, 0x55, 0xbc, 0x80, 0x1d, 0x1, 0x2, 0x3, " 20 | 0x0, 0x0, 0x0, 0x8, 0x0, 0x6c, 0x5c, 0x20, 0xbe, 0x4, 0x5, 0x6, " 21 | 0x0, 0x0, 0x0, 0x8, 0x0, 0x55, 0xbc, 0x80, 0x1d, 0x1, 0x2, 0x3, " 22 | 0x0, 0x0, 0x0, 0x8, 0x0, 0x6c, 0x5c, 0x20, 0xbe, 0x4, 0x5, 0x6, " 23 | 0x0, 0x0, 0x0, 0x8 PARTIAL MSG 24 | 100 bytes total in this block 25 | ---------------------------------------------------------------- 26 | -------------------------------------------------------------------------------- /src/tumblr/kafka/time.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kafka 16 | 17 | import ( 18 | "time" 19 | ) 20 | 21 | // Handy time constants for use in Kafka client invokations 22 | const ( 23 | Second = 1 24 | Minute = 60 * Second 25 | Hour = 60 * Minute 26 | Day = 24 * Hour 27 | Week = 7 * Day 28 | 29 | Earliest = -2 30 | Latest = -1 31 | ) 32 | 33 | // TimeToKafka converts t to the Kafka time format 34 | func TimeToKafka(t time.Time) int64 { 35 | return t.UnixNano() / 1e9 36 | } 37 | 38 | // Now returns the current time in the Kafka format 39 | func Now() int64 { 40 | return TimeToKafka(time.Now()) 41 | } 42 | -------------------------------------------------------------------------------- /src/tumblr/opentsdb/besteffort_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package opentsdb 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | ) 21 | 22 | func TestBestPut(t *testing.T) { 23 | conn, err := BestEffortDial("opentsdb.datacenter.net") 24 | if err != nil { 25 | t.Fatalf("dial (%s)", err) 26 | } 27 | time.Sleep(3 * time.Second) 28 | err = conn.Put("wiktor.is.in.love", 5, Tag{"host", "test.host"}) 29 | if err != nil { 30 | t.Errorf("put (%s)", err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/tumblr/opentsdb/opentsdb_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package opentsdb 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestPut(t *testing.T) { 22 | conn, err := Dial("opentsdb.datacenter.net") 23 | if err != nil { 24 | t.Fatalf("dial (%s)", err) 25 | } 26 | err = conn.Put("hello.world", 5, Tag{"host", "test.host"}) 27 | if err != nil { 28 | t.Errorf("put (%s)", err) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/tumblr/redis/redis_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package redis 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | func TestLowLevel(t *testing.T) { 23 | c, err := Dial("localhost:6300") 24 | if err != nil { 25 | fmt.Printf("err (%s)\n", err) 26 | return 27 | } 28 | err = c.WriteMultiBulk("get", "chris") 29 | if err != nil { 30 | fmt.Printf("err2 (%s)\n", err) 31 | return 32 | } 33 | resp, err := c.ReadResponse() 34 | if err != nil { 35 | fmt.Printf("read resp (%s)\n", err) 36 | return 37 | } 38 | fmt.Println(ResponseString(resp)) 39 | } 40 | 41 | func TestSetGet(t *testing.T) { 42 | c, err := Dial("test.datacenter.net:7000") 43 | if err != nil { 44 | fmt.Printf("err (%s)\n", err) 45 | return 46 | } 47 | if err = c.SetInt("oOOo", 345); err != nil { 48 | t.Fatalf("set (%s)", err) 49 | } 50 | i, err := c.GetInt("oOOo") 51 | if err != nil { 52 | t.Fatalf("get (%s)", err) 53 | } 54 | if i != 345 { 55 | t.Errorf("mismatch") 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/tumblr/scribe/conn_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scribe 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestConn(t *testing.T) { 22 | conn, err := Dial("devbox:1464") 23 | if err != nil { 24 | t.Fatalf("dial (%s)", err) 25 | } 26 | if err = conn.Emit([]Message{Message{"test-cat", "test-msg"}}...); err != nil { 27 | t.Errorf("emit (%s)", err) 28 | } 29 | if err = conn.Close(); err != nil { 30 | t.Errorf("close (%s)", err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/tumblr/scribe/thrift/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package thrift holds various Apache Thrift protocol definitions 16 | package thrift 17 | -------------------------------------------------------------------------------- /src/tumblr/scribe/thrift/fb303/ttypes.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* Autogenerated by Thrift Compiler (0.9.0-dev) 16 | * 17 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 18 | */ 19 | package fb303 20 | 21 | /** 22 | *Common status reporting mechanism across all services 23 | */ 24 | type FbStatus int 25 | 26 | const ( 27 | DEAD FbStatus = 0 28 | STARTING FbStatus = 1 29 | ALIVE FbStatus = 2 30 | STOPPING FbStatus = 3 31 | STOPPED FbStatus = 4 32 | WARNING FbStatus = 5 33 | ) 34 | 35 | func (p FbStatus) String() string { 36 | switch p { 37 | case DEAD: 38 | return "DEAD" 39 | case STARTING: 40 | return "STARTING" 41 | case ALIVE: 42 | return "ALIVE" 43 | case STOPPING: 44 | return "STOPPING" 45 | case STOPPED: 46 | return "STOPPED" 47 | case WARNING: 48 | return "WARNING" 49 | } 50 | return "" 51 | } 52 | 53 | func FromFbStatusString(s string) FbStatus { 54 | switch s { 55 | case "DEAD": 56 | return DEAD 57 | case "STARTING": 58 | return STARTING 59 | case "ALIVE": 60 | return ALIVE 61 | case "STOPPING": 62 | return STOPPING 63 | case "STOPPED": 64 | return STOPPED 65 | case "WARNING": 66 | return WARNING 67 | } 68 | return FbStatus(-10000) 69 | } 70 | 71 | func (p FbStatus) Value() int { 72 | return int(p) 73 | } 74 | 75 | func (p FbStatus) IsEnum() bool { 76 | return true 77 | } 78 | 79 | func init() { 80 | } 81 | -------------------------------------------------------------------------------- /tutorials/hello/app.config: -------------------------------------------------------------------------------- 1 | { 2 | "Zookeeper": { 3 | "Workers": ["localhost:2181"], 4 | "Dir": "/circuit/hello" 5 | }, 6 | "Deploy": { 7 | "Dir": "/Users/petar/.hello/deploy", 8 | "Worker": "4hello" 9 | }, 10 | "Build": { 11 | "Host": "localhost", 12 | "Jail": "/Users/petar/.hello/build", 13 | "ZookeeperInclude": "/Users/petar/gocircuit/misc/starter-kit-osx/zookeeper/include", 14 | "ZookeeperLib": "/Users/petar/gocircuit/misc/starter-kit-osx/zookeeper/lib", 15 | "Tool": "/Users/petar/gocircuit/bin/4build", 16 | "PrefixPath": "", 17 | 18 | "AppRepo": "{rsync}/Users/petar/gocircuit", 19 | "AppSrc": "/tutorials/hello", 20 | 21 | "GoRepo": "{rsync}/Users/petar/go", 22 | "RebuildGo": false, 23 | 24 | "CircuitRepo": "{rsync}/Users/petar/gocircuit", 25 | "CircuitSrc": "/", 26 | 27 | "WorkerPkg": "hello/worker", 28 | "CmdPkgs": ["hello/cmd/hello-spawn"], 29 | "ShipDir": "/Users/petar/.hello/ship" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tutorials/hello/src/hello/cmd/hello-spawn/.gitignore: -------------------------------------------------------------------------------- 1 | /hello-spawn 2 | -------------------------------------------------------------------------------- /tutorials/hello/src/hello/worker/worker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package main implements the executable for a circuit worker that has the hello logic linked in 16 | package main 17 | 18 | import ( 19 | // Package worker ensures that this executable will act as a circuit worker 20 | _ "circuit/load/worker" 21 | 22 | // Importing hello ensures that the hello tutorial's logic is linked into the worker executable 23 | _ "hello/x" 24 | ) 25 | 26 | // Main will never be executed. 27 | func main() {} 28 | -------------------------------------------------------------------------------- /tutorials/hello/src/hello/x/hello.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package x 16 | 17 | import ( 18 | "circuit/use/circuit" 19 | "fmt" 20 | "time" 21 | ) 22 | 23 | // App is a user-defined type, whose only public method will be registered with 24 | // the circuit as a function that can be spawned remotely. 25 | type App struct{} 26 | 27 | // Main is App's only public method. 28 | // The name of this method and its signature (arguments and their types and 29 | // return values and their types) are up to you. 30 | func (App) Main(suffix string) time.Time { 31 | circuit.Daemonize(func() { 32 | fmt.Printf("Waiting ...\n") 33 | time.Sleep(30 * time.Second) 34 | fmt.Printf("Hello %s\n", suffix) 35 | }) 36 | return time.Now() 37 | } 38 | 39 | // The circuit requires that all types, that hold only methods designated for 40 | // remote execution, be registered during package initialization, using RegisterFunc. 41 | func init() { circuit.RegisterFunc(App{}) } 42 | -------------------------------------------------------------------------------- /tutorials/legacy/app.config: -------------------------------------------------------------------------------- 1 | { 2 | "Zookeeper": { 3 | "Workers": ["localhost:2181"], 4 | "Dir": "/circuit/legacy" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /tutorials/legacy/src/legacy/.gitignore: -------------------------------------------------------------------------------- 1 | /legacy 2 | -------------------------------------------------------------------------------- /tutorials/legacy/src/legacy/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | _ "circuit/load" // Link the circuit into your legacy executable 19 | "time" 20 | ) 21 | 22 | func main() { 23 | println("Waiting...") 24 | time.Sleep(20 * time.Second) 25 | println("Done.") 26 | } 27 | -------------------------------------------------------------------------------- /tutorials/sumr/api.config: -------------------------------------------------------------------------------- 1 | { 2 | "Anchor": "/tutorial/sumr/api", 3 | "ReadOnly": false, 4 | "Workers": [ 5 | { 6 | "Host": "localhost", 7 | "Port": 43000 8 | }, 9 | { 10 | "Host": "localhost", 11 | "Port": 43001 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /tutorials/sumr/app.config: -------------------------------------------------------------------------------- 1 | { 2 | "Zookeeper": { 3 | "Workers": ["localhost:2181"], 4 | "Dir": "/circuit/sumr" 5 | }, 6 | "Deploy": { 7 | "Dir": "/Users/petar/.sumr/deploy", 8 | "Worker": "4sumr" 9 | }, 10 | "Build": { 11 | "Host": "localhost", 12 | "Jail": "/Users/petar/.sumr/build", 13 | "ZookeeperInclude": "/Users/petar/gocircuit/misc/starter-kit-osx/zookeeper/include", 14 | "ZookeeperLib": "/Users/petar/gocircuit/misc/starter-kit-osx/zookeeper/lib", 15 | "Tool": "/Users/petar/gocircuit/bin/4build", 16 | "PrefixPath": "", 17 | 18 | "AppRepo": "{rsync}/Users/petar/gocircuit", 19 | "AppSrc": "/tutorials/sumr", 20 | 21 | "GoRepo": "{rsync}/Users/petar/go", 22 | "RebuildGo": false, 23 | 24 | "CircuitRepo": "{rsync}/Users/petar/gocircuit", 25 | "CircuitSrc": "/", 26 | 27 | "WorkerPkg": "sumr/worker", 28 | "CmdPkgs": ["sumr/cmd/sumr-spawn-shard", "sumr/cmd/sumr-spawn-api"], 29 | "ShipDir": "/Users/petar/.sumr/ship" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tutorials/sumr/server.config: -------------------------------------------------------------------------------- 1 | { 2 | "Anchor": "/tutorial/sumr/shard", 3 | "Workers": [ 4 | { 5 | "Host": "localhost", 6 | "DiskPath": "/tmp/sumr1", 7 | "Forget": 3600000000000 8 | }, 9 | { 10 | "Host": "localhost", 11 | "DiskPath": "/tmp/sumr2", 12 | "Forget": 3600000000000 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /tutorials/sumr/src/sumr/cmd/sumr-spawn-api/.gitignore: -------------------------------------------------------------------------------- 1 | /sumr-spawn-api 2 | -------------------------------------------------------------------------------- /tutorials/sumr/src/sumr/cmd/sumr-spawn-api/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // sumr-spawn-api is a command-line executable that launches HTTP API frontends for a sumr database 16 | package main 17 | 18 | import ( 19 | "circuit/app/sumr/api" 20 | _ "circuit/load" 21 | "encoding/json" 22 | "flag" 23 | "fmt" 24 | "io/ioutil" 25 | "os" 26 | ) 27 | 28 | var ( 29 | flagAPI = flag.String("api", "", "sumr HTTP API configuration file name") 30 | flagDurable = flag.String("durable", "", "checkpoint durable file name") 31 | ) 32 | 33 | func main() { 34 | flag.Parse() 35 | 36 | if *flagAPI == "" || *flagDurable == "" { 37 | flag.Usage() 38 | os.Exit(1) 39 | } 40 | 41 | // Read HTTP API configuration 42 | raw, err := ioutil.ReadFile(*flagAPI) 43 | if err != nil { 44 | panic(err) 45 | } 46 | apiConfig := &api.Config{} 47 | if err = json.Unmarshal(raw, apiConfig); err != nil { 48 | panic(err) 49 | } 50 | 51 | // Start the sumr HTTP API endpoint workers 52 | println("Replenishing API servers") 53 | result := api.Replenish(*flagDurable, apiConfig) 54 | for i, rd := range result { 55 | if rd.Err != nil { 56 | fmt.Printf("––Problem API #%d err=%s\n", i, rd.Err) 57 | } else { 58 | fmt.Printf("––Started API #%d worker=%s\n", i, rd.Addr.WorkerID()) 59 | } 60 | } 61 | 62 | println("HTTP API frontends for sumr spawned successfully.") 63 | } 64 | -------------------------------------------------------------------------------- /tutorials/sumr/src/sumr/cmd/sumr-spawn-shard/.gitignore: -------------------------------------------------------------------------------- 1 | /sumr-spawn-shard 2 | -------------------------------------------------------------------------------- /tutorials/sumr/src/sumr/cmd/sumr-spawn-shard/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // sumr-spawn-shard is a command-line executable that launches a sumr database 16 | package main 17 | 18 | import ( 19 | "circuit/app/sumr/server" 20 | _ "circuit/load" 21 | "encoding/json" 22 | "flag" 23 | "fmt" 24 | "io/ioutil" 25 | "os" 26 | ) 27 | 28 | var ( 29 | flagSumr = flag.String("sumr", "", "sumr database configuration file name") 30 | flagDurable = flag.String("durable", "", "checkpoint durable file name") 31 | ) 32 | 33 | func main() { 34 | flag.Parse() 35 | 36 | if *flagSumr == "" || *flagDurable == "" { 37 | flag.Usage() 38 | os.Exit(1) 39 | } 40 | 41 | // Read sumr configuration 42 | raw, err := ioutil.ReadFile(*flagSumr) 43 | if err != nil { 44 | panic(err) 45 | } 46 | config := &server.Config{} 47 | if err = json.Unmarshal(raw, config); err != nil { 48 | panic(err) 49 | } 50 | 51 | // Start the sumr shard workers 52 | println("Starting shard servers") 53 | chk := server.Spawn(*flagDurable, config) 54 | for _, wchk := range chk.Workers { 55 | fmt.Printf("––Started shard: worker=%s key=%s\n", wchk.Addr.WorkerID(), wchk.ShardKey) 56 | } 57 | 58 | println("sumr shards spawned successfully.") 59 | } 60 | -------------------------------------------------------------------------------- /tutorials/sumr/src/sumr/worker/init.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package main implements the executable for a circuit worker with sumr capabilities 16 | package main 17 | 18 | import ( 19 | // Package worker ensures that this executable will act as a circuit worker 20 | _ "circuit/load/worker" 21 | 22 | // The sumr-specific packages ensure that the sumr logic is linked into this worker 23 | _ "circuit/app/sumr/api" 24 | _ "circuit/app/sumr/server" 25 | ) 26 | 27 | // Main will never be executed. 28 | func main() {} 29 | -------------------------------------------------------------------------------- /tutorials/trend/app.config: -------------------------------------------------------------------------------- 1 | { 2 | "Zookeeper": { 3 | "Workers": ["localhost:2181"], 4 | "Dir": "/circuit/trend" 5 | }, 6 | "Deploy": { 7 | "Dir": "/Users/petar/.trend/deploy", 8 | "Worker": "4trend" 9 | }, 10 | "Build": { 11 | "Host": "localhost", 12 | "Jail": "/Users/petar/.trend/build", 13 | "ZookeeperInclude": "/Users/petar/gocircuit/misc/starter-kit-osx/zookeeper/include", 14 | "ZookeeperLib": "/Users/petar/gocircuit/misc/starter-kit-osx/zookeeper/lib", 15 | "Tool": "/Users/petar/gocircuit/bin/4build", 16 | "PrefixPath": "", 17 | 18 | "AppRepo": "{rsync}/Users/petar/gocircuit", 19 | "AppSrc": "/tutorials/trend", 20 | 21 | "GoRepo": "{rsync}/Users/petar/go", 22 | "RebuildGo": false, 23 | 24 | "CircuitRepo": "{rsync}/Users/petar/gocircuit", 25 | "CircuitSrc": "/", 26 | 27 | "WorkerPkg": "trend/worker", 28 | "CmdPkgs": ["trend/cmd/trend-spawn"], 29 | "ShipDir": "/Users/petar/.trend/ship" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tutorials/trend/src/trend/cmd/trend-spawn/.gitignore: -------------------------------------------------------------------------------- 1 | trend-spawn 2 | -------------------------------------------------------------------------------- /tutorials/trend/src/trend/cmd/trend-spawn/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | _ "circuit/load" 19 | "trend/x" 20 | ) 21 | 22 | func main() { 23 | x.Main() 24 | } 25 | -------------------------------------------------------------------------------- /tutorials/trend/src/trend/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import ( 18 | "circuit/load/config" 19 | "strings" 20 | ) 21 | 22 | var ( 23 | MapperHost []string 24 | ReducerHost []string 25 | AggregatorHost string 26 | ) 27 | 28 | func init() { 29 | zk := config.Config.Zookeeper.Workers[0] 30 | switch { 31 | case strings.Index(zk, "datacenter") >= 0: 32 | MapperHost = []string{ 33 | "host0.datacenter.net", 34 | "host0.datacenter.net", 35 | "host0.datacenter.net", 36 | "host0.datacenter.net", 37 | "host0.datacenter.net", 38 | } 39 | ReducerHost = []string{ 40 | "host1.datacenter.net", 41 | "host1.datacenter.net", 42 | "host1.datacenter.net", 43 | "host1.datacenter.net", 44 | "host1.datacenter.net", 45 | } 46 | AggregatorHost = "host2.datacenter.net" 47 | 48 | case strings.Index(zk, "localhost") >= 0 || strings.Index(zk, "127.0.0.1") >= 0: 49 | MapperHost = []string{ 50 | "localhost", 51 | "localhost", 52 | "localhost", 53 | "localhost", 54 | "localhost", 55 | } 56 | ReducerHost = []string{ 57 | "localhost", 58 | "localhost", 59 | "localhost", 60 | "localhost", 61 | "localhost", 62 | } 63 | AggregatorHost = "localhost" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /tutorials/trend/src/trend/worker/init.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package main implements the executable for a circuit worker with sumr capabilities 16 | package main 17 | 18 | import ( 19 | // Package worker ensures that this executable will act as a circuit worker 20 | _ "circuit/load/worker" 21 | 22 | // The trend-specific packages ensure that the trend logic is linked into this worker 23 | _ "trend/x" 24 | ) 25 | 26 | // Main will never be executed. 27 | func main() {} 28 | -------------------------------------------------------------------------------- /tutorials/trend/src/trend/x/start.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package x 16 | 17 | import ( 18 | "circuit/use/circuit" 19 | "trend/config" 20 | ) 21 | 22 | func Main() { 23 | println("Kicking aggregator") 24 | _, addr, err := circuit.Spawn(config.AggregatorHost, []string{"/tutorial/aggregator"}, StartAggregator{}, "/tutorial/reducer") 25 | if err != nil { 26 | panic(err) 27 | } 28 | println(addr.String()) 29 | 30 | println("Kicking reducers") 31 | reducer := make([]circuit.X, len(config.ReducerHost)) 32 | for i, h := range config.ReducerHost { 33 | retrn, addr, err := circuit.Spawn(h, []string{"/tutorial/reducer"}, StartReducer{}) 34 | if err != nil { 35 | panic(err) 36 | } 37 | reducer[i] = retrn[0].(circuit.X) 38 | println(addr.String()) 39 | } 40 | 41 | println("Kicking mappers") 42 | for _, h := range config.MapperHost { 43 | _, addr, err := circuit.Spawn(h, []string{"/tutorial/mapper"}, StartMapper{}, testFirehose, reducer) 44 | if err != nil { 45 | panic(err) 46 | } 47 | println(addr.String()) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tutorials/trending/src/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import ( 18 | "circuit/load/config" 19 | "circuit/use/circuit" 20 | "strings" 21 | ) 22 | 23 | var ( 24 | MapperHost []circuit.Host 25 | ReducerHost []circuit.Host 26 | AggregatorHost circuit.Host 27 | ) 28 | 29 | func init() { 30 | zk := config.Config.Zookeeper.Workers[0] 31 | switch { 32 | case strings.Index(zk, "localhost") >= 0 || strings.Index(zk, "127.0.0.1") >= 0: 33 | MapperHost = []circuit.Host{ 34 | "localhost", 35 | "localhost", 36 | "localhost", 37 | "localhost", 38 | "localhost", 39 | } 40 | ReducerHost = []circuit.Host{ 41 | "localhost", 42 | "localhost", 43 | "localhost", 44 | "localhost", 45 | "localhost", 46 | } 47 | AggregatorHost = "localhost" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tutorials/vena/app.config: -------------------------------------------------------------------------------- 1 | { 2 | "Zookeeper": { 3 | "Workers": ["localhost:2181"], 4 | "Dir": "/circuit/vena" 5 | }, 6 | "Deploy": { 7 | "Dir": "/Users/petar/.vena/deploy", 8 | "Worker": "4vena" 9 | }, 10 | "Build": { 11 | "Host": "localhost", 12 | "Jail": "/Users/petar/.vena/build", 13 | "ZookeeperInclude": "/Users/petar/gocircuit/misc/starter-kit-osx/zookeeper/include", 14 | "ZookeeperLib": "/Users/petar/gocircuit/misc/starter-kit-osx/zookeeper/lib", 15 | "CFLAGS": "-I/Users/petar/aux/leveldb/include", 16 | "LDFLAGS": "-lstdc++ /Users/petar/aux/leveldb/lib/libleveldb.a", 17 | "Tool": "/Users/petar/gocircuit/bin/4build", 18 | "PrefixPath": "", 19 | 20 | "AppRepo": "{rsync}/Users/petar/gocircuit", 21 | "AppSrc": "/tutorials/vena", 22 | 23 | "GoRepo": "{rsync}/Users/petar/go", 24 | "RebuildGo": false, 25 | 26 | "CircuitRepo": "{rsync}/Users/petar/gocircuit", 27 | "CircuitSrc": "/", 28 | 29 | "WorkerPkg": "vena/worker", 30 | "CmdPkgs": ["vena/cmd/vena-front-start", "vena/cmd/vena-server-start"], 31 | "ShipDir": "/Users/petar/.vena/ship" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tutorials/vena/front.config: -------------------------------------------------------------------------------- 1 | { 2 | "Anchor": "/vena/front", 3 | "Workers": [ 4 | { 5 | "Host": "localhost", 6 | "HTTPPort": 50500, 7 | "TSDBPort": 50600 8 | }, 9 | { 10 | "Host": "localhost", 11 | "HTTPPort": 50501, 12 | "TSDBPort": 50601 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/cmd/vena-front-start/.gitignore: -------------------------------------------------------------------------------- 1 | /vena-front-start 2 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/cmd/vena-front-start/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // sumr-spawn-api is a command-line executable that launches HTTP API frontends for a sumr database 16 | package main 17 | 18 | import ( 19 | "circuit/app/sumr/api" 20 | _ "circuit/load" 21 | "encoding/json" 22 | "flag" 23 | "fmt" 24 | "io/ioutil" 25 | "os" 26 | ) 27 | 28 | var ( 29 | flagAPI = flag.String("api", "", "sumr HTTP API configuration file name") 30 | flagDurable = flag.String("durable", "", "checkpoint durable file name") 31 | ) 32 | 33 | func main() { 34 | flag.Parse() 35 | 36 | if *flagAPI == "" || *flagDurable == "" { 37 | flag.Usage() 38 | os.Exit(1) 39 | } 40 | 41 | // Read HTTP API configuration 42 | raw, err := ioutil.ReadFile(*flagAPI) 43 | if err != nil { 44 | panic(err) 45 | } 46 | apiConfig := &api.Config{} 47 | if err = json.Unmarshal(raw, apiConfig); err != nil { 48 | panic(err) 49 | } 50 | 51 | // Start the sumr HTTP API endpoint workers 52 | println("Replenishing API servers") 53 | result := api.Replenish(*flagDurable, apiConfig) 54 | for i, rd := range result { 55 | if rd.Err != nil { 56 | fmt.Printf("––Problem API #%d err=%s\n", i, rd.Err) 57 | } else { 58 | fmt.Printf("––Started API #%d worker=%s\n", i, rd.Addr.WorkerID()) 59 | } 60 | } 61 | 62 | println("HTTP API frontends for sumr spawned successfully.") 63 | } 64 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/cmd/vena-server-start/.gitignore: -------------------------------------------------------------------------------- 1 | /vena-server-start 2 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/cmd/vena-server-start/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | _ "circuit/load" 19 | "encoding/json" 20 | "flag" 21 | "io/ioutil" 22 | "os" 23 | "vena" 24 | "vena/server" 25 | ) 26 | 27 | var ( 28 | flagConfig = flag.String("config", "", "vena database configuration file name") 29 | ) 30 | 31 | func main() { 32 | flag.Parse() 33 | 34 | if *flagConfig == "" { 35 | flag.Usage() 36 | os.Exit(1) 37 | } 38 | 39 | raw, err := ioutil.ReadFile(*flagConfig) 40 | if err != nil { 41 | panic(err) 42 | } 43 | config := &vena.Config{} 44 | if err = json.Unmarshal(raw, config); err != nil { 45 | panic(err) 46 | } 47 | 48 | println("Starting VENA shard servers") 49 | server.Spawn(config) 50 | println("VENA servers started successfully.") 51 | } 52 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/config.go: -------------------------------------------------------------------------------- 1 | package vena 2 | 3 | import ( 4 | "circuit/kit/xor" 5 | "path" 6 | ) 7 | 8 | type Config struct { 9 | Anchor string // Root anchor for the shards 10 | Shard []*ShardConfig 11 | } 12 | 13 | type ShardConfig struct { 14 | Key xor.Key 15 | Host string 16 | Dir string 17 | Cache int 18 | } 19 | 20 | func (c *Config) ShardAnchor(key xor.Key) string { 21 | return path.Join(c.Anchor, key.String()) 22 | } 23 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/front/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package front 16 | 17 | import ( 18 | "encoding/gob" 19 | "fmt" 20 | "path" 21 | ) 22 | 23 | func init() { 24 | gob.Register(&Config{}) 25 | gob.Register(&WorkerConfig{}) 26 | } 27 | 28 | type Config struct { 29 | Anchor string // Anchor for the front workers 30 | Workers []*WorkerConfig 31 | } 32 | 33 | func (c *Config) WorkerAnchor(i int) string { 34 | return path.Join(c.Anchor, fmt.Sprintf("%s:(%d,%d)", c.Workers[i].Host, c.Workers[i].HTTPPort, c.Workers[i].TSDBPort)) 35 | } 36 | 37 | func (c *Config) Worker(i int) (*WorkerConfig, string) { 38 | return c.Workers[i], c.WorkerAnchor(i) 39 | } 40 | 41 | type WorkerConfig struct { 42 | Host string // Host is the circuit hostname where the worker is to be deployed 43 | HTTPPort int 44 | TSDBPort int 45 | } 46 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/front/front.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package front 16 | 17 | import ( 18 | "circuit/kit/sched/limiter" 19 | "circuit/use/circuit" 20 | "fmt" 21 | "vena" 22 | "vena/client" 23 | ) 24 | 25 | type Front struct { 26 | client *client.Client 27 | lmtr limiter.Limiter 28 | } 29 | 30 | func init() { 31 | circuit.RegisterValue(&Front{}) 32 | } 33 | 34 | func New(c *vena.Config, httpPort, tsdbPort int) *Front { 35 | var err error 36 | front := &Front{} 37 | front.client, err = client.New(c) 38 | if err != nil { 39 | panic(err) 40 | } 41 | front.lmtr.Init(200) 42 | listenTSDB(fmt.Sprintf(":%d", tsdbPort), front) 43 | return front 44 | } 45 | 46 | func (front *Front) Put(time vena.Time, metric string, tags map[string]string, value float64) { 47 | front.lmtr.Open() 48 | defer front.lmtr.Close() 49 | front.client.Put(time, metric, tags, value) 50 | } 51 | 52 | func (front *Front) DieDieDie() {} 53 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/front/mkcfg/.gitignore: -------------------------------------------------------------------------------- 1 | mkcfg 2 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/front/mkcfg/mkcfg.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // mkcfg prints out an empty sumr api configuration 16 | package main 17 | 18 | import ( 19 | "encoding/json" 20 | "fmt" 21 | "vena/front" 22 | ) 23 | 24 | func main() { 25 | raw, _ := json.MarshalIndent( 26 | &front.Config{ 27 | Anchor: "", 28 | Workers: []*front.WorkerConfig{ 29 | &front.WorkerConfig{ 30 | Host: "host3.datacenter.net", 31 | HTTPPort: 4001, 32 | TSDBPort: 4002, 33 | }, 34 | &front.WorkerConfig{ 35 | Host: "host4.datacenter.net", 36 | HTTPPort: 4001, 37 | TSDBPort: 4002, 38 | }, 39 | }, 40 | }, 41 | "", "\t", 42 | ) 43 | fmt.Printf("%s\n", raw) 44 | } 45 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/server/spawn.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "circuit/kit/sched/limiter" 19 | "circuit/use/circuit" 20 | "vena" 21 | ) 22 | 23 | func Spawn(config *vena.Config) { 24 | lmtr := limiter.New(20) 25 | for _, sh_ := range config.Shard { 26 | sh := sh_ 27 | lmtr.Go(func() { spawn(sh, config.ShardAnchor(sh.Key)) }) 28 | } 29 | lmtr.Wait() 30 | } 31 | 32 | func spawn(config *vena.ShardConfig, anchor string) { 33 | _, addr, err := circuit.Spawn(config.Host, []string{anchor}, start{}, config.Dir, config.Cache) 34 | if err != nil { 35 | panic(err) 36 | } 37 | println("Shard started", addr.String()) 38 | } 39 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/util/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "sync" 19 | "github.com/petar/levigo" 20 | ) 21 | 22 | type Server struct { 23 | slk sync.Mutex 24 | cache *levigo.Cache 25 | DB *levigo.DB 26 | ReadAndCache *levigo.ReadOptions 27 | WriteSync *levigo.WriteOptions 28 | WriteNoSync *levigo.WriteOptions 29 | } 30 | 31 | func (srv *Server) Init(dbDir string, cacheSize int) error { 32 | var err error 33 | opts := levigo.NewOptions() 34 | srv.cache = levigo.NewLRUCache(cacheSize) 35 | opts.SetCache(srv.cache) 36 | opts.SetCreateIfMissing(true) 37 | 38 | if srv.DB, err = levigo.Open(dbDir, opts); err != nil { 39 | srv.cache.Close() 40 | return err 41 | } 42 | 43 | srv.ReadAndCache = levigo.NewReadOptions() 44 | srv.ReadAndCache.SetFillCache(true) 45 | 46 | srv.WriteSync = levigo.NewWriteOptions() 47 | srv.WriteSync.SetSync(true) 48 | 49 | srv.WriteNoSync = levigo.NewWriteOptions() 50 | srv.WriteSync.SetSync(false) 51 | 52 | return nil 53 | } 54 | 55 | func (srv *Server) Close() error { 56 | srv.slk.Lock() 57 | defer srv.slk.Unlock() 58 | if srv.cache != nil { 59 | srv.cache.Close() 60 | srv.cache = nil 61 | } 62 | srv.ReadAndCache.Close() 63 | srv.ReadAndCache = nil 64 | srv.WriteSync.Close() 65 | srv.WriteSync = nil 66 | srv.WriteNoSync.Close() 67 | srv.WriteNoSync = nil 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /tutorials/vena/src/vena/worker/init.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Tumblr, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package main implements the executable for a circuit worker with Vena capabilities 16 | package main 17 | 18 | import ( 19 | // Package worker ensures that this executable will act as a circuit worker 20 | _ "circuit/load/worker" 21 | 22 | // The Vena-specific packages ensure that the Vena logic is linked into this worker 23 | _ "vena/front" 24 | _ "vena/server" 25 | ) 26 | 27 | // Main will never be executed. 28 | func main() {} 29 | -------------------------------------------------------------------------------- /tutorials/vena/vena.config: -------------------------------------------------------------------------------- 1 | { 2 | "Anchor": "/vena/server", 3 | "Shard": [ 4 | { 5 | "Key": 0, 6 | "Host": "localhost", 7 | "Dir": "/Users/petar/.vena/db/0", 8 | "Cache": 100e6 9 | }, 10 | { 11 | "Key": 1, 12 | "Host": "localhost", 13 | "Dir": "/Users/petar/.vena/db/1", 14 | "Cache": 100e6 15 | } 16 | ] 17 | } 18 | --------------------------------------------------------------------------------