├── .npmignore ├── Dockerfile ├── Dockerfile.test ├── .gitignore ├── resource_path.go ├── circle.yml ├── package.json ├── filter_test.go ├── resource_path_test.go ├── object.go ├── cache_path.go ├── api.go ├── CONTRIBUTING.md ├── README.md ├── cli.go ├── filter.go └── LICENSE /.npmignore: -------------------------------------------------------------------------------- 1 | *.go 2 | node_modules 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.5-onbuild 2 | 3 | ENTRYPOINT ["go-wrapper", "run"] 4 | CMD [] 5 | VOLUME /root 6 | -------------------------------------------------------------------------------- /Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM golang:1.5-onbuild 2 | 3 | WORKDIR /go/src/app/tests 4 | ENTRYPOINT ["go", "test"] 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | auto 2 | 3 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 4 | *.o 5 | *.a 6 | *.so 7 | 8 | # Folders 9 | _obj 10 | _test 11 | 12 | # Architecture specific extensions/prefixes 13 | *.[568vq] 14 | [568vq].out 15 | 16 | *.cgo1.go 17 | *.cgo2.c 18 | _cgo_defun.c 19 | _cgo_gotypes.go 20 | _cgo_export.* 21 | 22 | _testmain.go 23 | 24 | *.exe 25 | *.prof 26 | 27 | node_modules 28 | -------------------------------------------------------------------------------- /resource_path.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/url" 5 | ) 6 | 7 | func dereferencePath(reqPath string) (*url.URL, error) { 8 | path, err := url.Parse(reqPath) 9 | if err != nil { 10 | return nil, err 11 | } 12 | if path.Scheme == "" { 13 | path.Scheme = "https" 14 | } 15 | return path, nil 16 | } 17 | 18 | // Download resource at path & 19 | // validate resource matches declared object type definition. 20 | func dereferenceObj(path *url.URL) *Object { 21 | obj := new(Object) 22 | obj.path = path 23 | return obj 24 | } 25 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | test: 2 | pre: 3 | - go get github.com/axw/gocov/gocov 4 | - go get github.com/mattn/goveralls 5 | - if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi 6 | override: 7 | - go test -v -cover -race -coverprofile=$HOME/coverage.out 8 | - ./node_modules/.bin/istanbul cover jasmine 9 | post: 10 | - $HOME/.go_workspace/bin/goveralls -coverprofile=$HOME/coverage.out -service=circle-ci -repotoken=$COVERALLS_TOKEN 11 | - $HOME/.go_workspace/bin/goveralls -gocovdata=./coverage/coverage.json -service=circle-ci -repotoken=$COVERALLS_TOKEN 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "iopipe", 3 | "private": false, 4 | "version": "0.0.5", 5 | "description": "iopipe sdk", 6 | "author": "Eric Windisch", 7 | "files": [ 8 | "js" 9 | ], 10 | "keywords": [ 11 | "iopipe", 12 | "flow", 13 | "waterfall", 14 | "async", 15 | "functional" 16 | ], 17 | "dependencies": { 18 | "aws-sdk": "", 19 | "read-stream": "", 20 | "request": "" 21 | }, 22 | "main": "js/iopipe.js", 23 | "license": "Apache-2.0", 24 | "repository": { 25 | "type": "git", 26 | "url": "https://github.com/iopipe/iopipe" 27 | }, 28 | "bugs": "https://github.com/iopipe/iopipe/issues", 29 | "devDependencies": { 30 | "jasmine": "", 31 | "istanbul": "" 32 | }, 33 | "scripts": { 34 | "test": "./node_modules/.bin/istanbul cover jasmine" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /filter_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestMakeRunFilter(t *testing.T) { 8 | fun, err := makeFilter(`module.exports = function(x, cxt) { cxt.done("hello") }`) 9 | if err != nil { 10 | t.Error(err) 11 | } 12 | result, err := fun("") 13 | if result != "hello" { 14 | t.Error("You should have had me at hello") 15 | } 16 | } 17 | 18 | func TestMakeRunEchoFilter(t *testing.T) { 19 | fun, err := makeFilter(`module.exports = function(input, cxt) { cxt.done(input) }`) 20 | if err != nil { 21 | t.Error(err) 22 | } 23 | result, err := fun("echo") 24 | if result != "echo" { 25 | t.Error("Filter did not echo input.") 26 | } 27 | } 28 | 29 | func TestMakeRunFailsInvalidJSFilter(t *testing.T) { 30 | /* Note: this is not valid ECMAScript */ 31 | fun, err := makeFilter("(╯°□°）╯︵ ┻━┻") 32 | _, err = fun("") 33 | if err == nil { 34 | t.Error("One should not (╯°□°）╯︵ ┻━┻") 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /resource_path_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestDereferencePath(t *testing.T) { 8 | url, err := dereferencePath("https://www.iopipe.com/") 9 | if err != nil { 10 | t.Error(err) 11 | } 12 | if url.Scheme != "https" { 13 | t.Error("Expected https") 14 | } 15 | } 16 | 17 | func TestDereferencePathPlainString(t *testing.T) { 18 | url, err := dereferencePath("plainstring") 19 | if err != nil { 20 | t.Error(err) 21 | } 22 | if url.Scheme != "https" { 23 | t.Error("Expected https") 24 | } 25 | } 26 | 27 | func TestDereferencePathHTTP(t *testing.T) { 28 | url, err := dereferencePath("http://address") 29 | if err != nil { 30 | t.Error(err) 31 | } 32 | if url.Scheme != "http" { 33 | t.Error("Expected http") 34 | } 35 | } 36 | 37 | func TestDereferenceObj(t *testing.T) { 38 | url, err := dereferencePath("http://127.0.0.1") 39 | if err != nil { 40 | t.Error(err) 41 | } 42 | obj := dereferenceObj(url) 43 | if url != obj.path { 44 | t.Error("URL and url.path did not match") 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /object.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "strings" 6 | 7 | "io/ioutil" 8 | "net/http" 9 | "net/url" 10 | ) 11 | 12 | /******************************************************* 13 | Object Mapper 14 | *******************************************************/ 15 | type objectInterface struct { 16 | ClassID string `json:"classid"` 17 | Properties map[string]interface{} `json:"properties"` 18 | } 19 | 20 | type Object struct { 21 | path *url.URL 22 | } 23 | 24 | func (object *Object) read() string { 25 | path := object.path.String() 26 | object.path.String() 27 | res, err := http.Get(path) 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | body, err := ioutil.ReadAll(res.Body) 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | res.Body.Close() 36 | return string(body[:]) 37 | } 38 | 39 | func (object *Object) update(content string) string { 40 | path := object.path.String() 41 | reader := strings.NewReader(content) 42 | 43 | res, err := http.Post(path, "application/json", reader) 44 | defer res.Body.Close() 45 | if err != nil { 46 | log.Fatal(err) 47 | } 48 | 49 | body, err := ioutil.ReadAll(res.Body) 50 | if err != nil { 51 | log.Fatal(err) 52 | } 53 | return string(body[:]) 54 | } 55 | -------------------------------------------------------------------------------- /cache_path.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Sirupsen/logrus" 5 | 6 | "fmt" 7 | "os" 8 | "path" 9 | 10 | "crypto/sha256" 11 | "io/ioutil" 12 | "os/user" 13 | ) 14 | 15 | func getCachePath(name string) (string, error) { 16 | myuser, err := user.Current() 17 | if err != nil { 18 | return "", err 19 | } 20 | pathParts := []string{myuser.HomeDir, ".iopipe", "filter_cache", name} 21 | return path.Join(pathParts...), nil 22 | } 23 | 24 | func ensureCachePath() error { 25 | path, err := getCachePath("") 26 | if err != nil { 27 | return err 28 | } 29 | return os.MkdirAll(path, 0700) 30 | } 31 | 32 | func readFilterCache(name string) ([]byte, error) { 33 | var err error 34 | 35 | diskPath, err := getCachePath(name) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | /* Do we have this cached? */ 41 | if _, err = os.Stat(diskPath); err != nil { 42 | return nil, err 43 | } 44 | script, err := ioutil.ReadFile(diskPath) 45 | 46 | logrus.Debug("Read filter from cache:\n" + string(script[:])) 47 | return script[:], nil 48 | } 49 | 50 | func writeFilterCache(body []byte) (string, error) { 51 | var err error 52 | 53 | /* Verify digest */ 54 | chksum := sha256.Sum256(body[:]) 55 | id := fmt.Sprintf("%x", chksum) 56 | diskPath, err := getCachePath(id) 57 | if err != nil { 58 | return id, err 59 | } 60 | 61 | /* Write cache */ 62 | if err = ioutil.WriteFile(diskPath, body, 0600); err != nil { 63 | return id, err 64 | } 65 | return id, nil 66 | } 67 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Sirupsen/logrus" 5 | 6 | "fmt" 7 | "os" 8 | 9 | "io/ioutil" 10 | "net/url" 11 | ) 12 | 13 | func Exec(args ...string) (string, error) { 14 | //var err error 15 | var lastObj string 16 | var msg string 17 | 18 | for i := 0; i < len(args); i++ { 19 | arg := args[i] 20 | 21 | path, err := url.Parse(arg) 22 | if err != nil { 23 | return "", err 24 | } 25 | 26 | logrus.Info("pipe[arg]: " + arg) 27 | 28 | if path.Scheme == "http" || path.Scheme == "https" { 29 | argPath, err := dereferencePath(arg) 30 | if err != nil { 31 | return "", err 32 | } 33 | argObj := dereferenceObj(argPath) 34 | 35 | // If first argument, then we must GET, 36 | // note that this case follows the '-' so all 37 | // shell input will pipe into the POST. 38 | if i == 0 { 39 | msg = argObj.read() 40 | } else { 41 | msg = argObj.update(lastObj) 42 | } 43 | } else if arg == "-" { 44 | script, err := ioutil.ReadAll(os.Stdin) 45 | if err != nil { 46 | return "", err 47 | } 48 | if i == 0 { 49 | // If first argument, assume is generic bytes input. 50 | msg = string(script[:]) 51 | } else { 52 | // If not the first argument, then expect pipescript 53 | filter, err := makeFilter(string(script[:])) 54 | if err != nil { 55 | return "", err 56 | } 57 | if msg, err = filter(lastObj); err != nil { 58 | return "", err 59 | } 60 | } 61 | } else { 62 | filter, err := getFilter(arg) 63 | if err != nil { 64 | return "", err 65 | } 66 | // Search 67 | if i == 0 { 68 | msg, err = filter("") 69 | } else { 70 | msg, err = filter(lastObj) 71 | } 72 | if err != nil { 73 | return "", err 74 | } 75 | } 76 | logrus.Debug(fmt.Sprintf("pipe[%i][raw]: %s\n", i, msg)) 77 | 78 | lastObj = msg 79 | } 80 | return msg, nil 81 | } 82 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to IO PIPE 2 | 3 | General rules: 4 | 5 | * We welcome pull requests! 6 | * Collaborate and be friendly (and follow community guidelines) 7 | * Sign your code! 8 | 9 | It's not a rule, but it's also recommended to drop into #iopipe on 10 | irc.freenode.org. 11 | 12 | # Submit pull requests 13 | 14 | Simply fork our repository on GitHub, then use the GitHub pull request feature. 15 | 16 | For non-trival fixes, please also submit a GitHub issue. 17 | 18 | # Sign your pull requests! 19 | 20 | If you agree to the developer certificate: 21 | 22 | ``` 23 | Developer Certificate of Origin 24 | Version 1.1 25 | 26 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 27 | 660 York Street, Suite 102, 28 | San Francisco, CA 94110 USA 29 | 30 | Everyone is permitted to copy and distribute verbatim copies of this 31 | license document, but changing it is not allowed. 32 | 33 | Developer's Certificate of Origin 1.1 34 | 35 | By making a contribution to this project, I certify that: 36 | 37 | (a) The contribution was created in whole or in part by me and I 38 | have the right to submit it under the open source license 39 | indicated in the file; or 40 | 41 | (b) The contribution is based upon previous work that, to the best 42 | of my knowledge, is covered under an appropriate open source 43 | license and I have the right under that license to submit that 44 | work with modifications, whether created in whole or in part 45 | by me, under the same open source license (unless I am 46 | permitted to submit under a different license), as indicated 47 | in the file; or 48 | 49 | (c) The contribution was provided directly to me by some other 50 | person who certified (a), (b) or (c) and I have not modified 51 | it. 52 | 53 | (d) I understand and agree that this project and the contribution 54 | are public and that a record of the contribution (including all 55 | personal information I submit with it, including my sign-off) is 56 | maintained indefinitely and may be redistributed consistent with 57 | this project or the open source license(s) involved. 58 | ``` 59 | 60 | Then simply sign your commits using `git commit -s` which will add a 61 | line to your commit message similar to: 62 | 63 | Signed-off-by: John Doe 64 | 65 | # Community Guidelines 66 | 67 | * No harrassment or abuse. This includes disrespect or abuse based on religion, 68 | sex, gender, or culture. 69 | 70 | * Encourage and welcome diversity. 71 | 72 | Failure to adhere to these guidelines will result in punative measures. 73 | Based on the severity of an infraction, violations may result in 74 | anything from a warning, to a probation, or even ejection from our 75 | community. 76 | 77 | Contact abuse@iopipe.com to report abuse or appeal violations. 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | IOpipe 2 | --------------------------------------- 3 | 4 | [![Gitter](https://img.shields.io/gitter/room/nwjs/nw.js.svg?maxAge=2592000)](https://gitter.im/iopipe/iopipe) 5 | 6 | Apache 2.0 licensed. 7 | 8 | [IOpipe](https://github.com/iopipe/iopipe) is a toolkit for building 9 | and orchestrating event-driven and serverless applications. These 10 | apps may run locally or in the cloud via AWS Lambda, Google Cloud Functions, 11 | or Azure Functions. 12 | 13 | This repository contains a CLI and Go SDK. 14 | 15 | ### CLI 16 | 17 | Use the CLI to create and export npm modules, share code via a code 18 | registry, and provide runtime of magnetic kernels. 19 | 20 | Download the [latest binary release](https://github.com/iopipe/iopipe/releases) and chmod 755 the file. 21 | 22 | Building from source? See [Build & Install from source](#build--install-from-source). 23 | 24 | Alternatively, download & alias our Docker image: 25 | 26 | ```bash 27 | $ docker pull iopipe/iopipe:trunk 28 | $ docker run --name iopipe-data iopipe/iopipe:trunk 29 | $ eval $(echo "alias iopipe='docker run --rm --volumes-from iopipe-data iopipe/iopipe:trunk'" | tee -a ~/.bashrc) 30 | $ iopipe --help 31 | ``` 32 | 33 | OS-specific packages are forthcoming. 34 | 35 | ### Command-line Examples 36 | 37 | ```sh 38 | # Import a kernel and name it com.example.SomeScript 39 | $ iopipe import --name com.example.SomeScript - <<<'input' 40 | 41 | # List kernels 42 | $ iopipe list 43 | 44 | # Fetch response and process it with com.example.SomeScript 45 | $ iopipe --debug exec http://localhost/some-request com.example.SomeScript 46 | 47 | # Fetch response and convert it with SomeScript, sending the result to otherhost 48 | $ iopipe --debug exec http://localhost/some-request com.example.SomeScript \ 49 | http://otherhost/request 50 | 51 | # Fetch response and convert it with SomeScript, send that result to otherhost, 52 | # & converting the response with the script ResponseScript 53 | $ iopipe --debug exec http://localhost/some-request com.example.SomeScript \ 54 | http://otherhost/request some.example.ResponseScript 55 | 56 | # Export an NPM module: 57 | $ iopipe export --name my-module-name http://localhost/some-request com.example.SomeScript 58 | ``` 59 | 60 | --------------------------------------- 61 | Build & Install from source 62 | --------------------------------------- 63 | 64 | With a functioning golang 1.5 development environment: 65 | 66 | ```bash 67 | $ go build 68 | $ ./iopipe --help 69 | ``` 70 | 71 | Alternatively use Docker to build & deploy: 72 | 73 | ```bash 74 | $ docker build -t iopipe-dev . 75 | $ docker run --name iopipe-data iopipe-dev 76 | $ eval $(echo "alias iopipe='docker run --rm --volumes-from iopipe-data iopipe-dev'" | tee -a ~/.bashrc) 77 | $ iopipe --help 78 | ``` 79 | --------------------------------------- 80 | Security 81 | --------------------------------------- 82 | 83 | Kernels are executed in individual virtual machines 84 | whenever allowed by the executing environment. 85 | The definition of a virtual machine here is lax, 86 | such that it may describe a Javascript VM, 87 | a Linux container, or a hardware-assisted x86 88 | virtual machine. Users should exercise caution 89 | when running community created kernels. 90 | 91 | It is a project priority to make fetching, publishing, 92 | and execution of kernels secure for a 93 | production-ready 1.0.0 release. 94 | 95 | Modules are fetched and stored using sha256 hashes, 96 | providing an advantage over module-hosting mechanisms 97 | which are based simply on a name and version. Future 98 | versions of IOpipe will likely implement TUF for 99 | state-of-the-art software assurance. 100 | 101 | Contact security@iopipe.com for questions. 102 | 103 | --------------------------------------- 104 | LICENSE 105 | --------------------------------------- 106 | 107 | Apache 2.0 108 | -------------------------------------------------------------------------------- /cli.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Sirupsen/logrus" 5 | "github.com/codegangsta/cli" 6 | 7 | "log" 8 | "os" 9 | "path" 10 | 11 | "encoding/json" 12 | ) 13 | 14 | var debug bool = false 15 | var name string 16 | 17 | func main() { 18 | ensureCachePath() 19 | 20 | //var debug bool = false 21 | app := cli.NewApp() 22 | app.Name = "iopipe" 23 | app.Usage = "cross-API interoperability & data manager" 24 | app.Flags = []cli.Flag{ 25 | cli.BoolFlag{ 26 | Name: "debug", 27 | Usage: "enable debugging output", 28 | Destination: &debug, 29 | }, 30 | } 31 | app.Action = func(c *cli.Context) { 32 | if debug { 33 | logrus.SetLevel(logrus.DebugLevel) 34 | } 35 | } 36 | app.Commands = []cli.Command{ 37 | /*{ 38 | Name: "create", 39 | Usage: "Create pipescript in local index.", 40 | Action: cmdCreate, 41 | },*/ 42 | { 43 | Name: "remove", 44 | Usage: "Remove filter from local index.", 45 | Action: cmdRemove, 46 | }, 47 | { 48 | Name: "exec", 49 | Usage: "Pipe from to stdout", 50 | Action: cmdExec, 51 | }, 52 | { 53 | Name: "export", 54 | Usage: "Export will write a Javascript library for the given pipeline", 55 | Action: cmdExport, 56 | Flags: []cli.Flag{ 57 | cli.StringFlag{ 58 | Name: "name", 59 | Usage: "NPM package name", 60 | Destination: &name, 61 | }, 62 | }, 63 | }, 64 | { 65 | Name: "import", 66 | Usage: "Import will bring a filter in from a javascript file or STDIN for -", 67 | Action: cmdImport, 68 | Flags: []cli.Flag{ 69 | cli.StringFlag{ 70 | Name: "name", 71 | Usage: "Tag to name", 72 | Destination: &name, 73 | }, 74 | }, 75 | }, 76 | { 77 | Name: "list", 78 | Usage: "List local and subscribed pipes", 79 | Action: cmdList, 80 | }, 81 | { 82 | Name: "login", 83 | Usage: "Login to an endpoint", 84 | Action: func(c *cli.Context) { 85 | logrus.Debug("Logging in to ", c.Args().First()) 86 | }, 87 | }, 88 | /*{ 89 | Name: "publish", 90 | Usage: "Publish a pipescript or pipeline", 91 | Action: func(c *cli.Context) { 92 | logrus.Debug("Pushing ", c.Args().First()) 93 | }, 94 | }, 95 | { 96 | Name: "subscribe", 97 | Usage: "Add pipescript or pipeline to local index", 98 | Action: func(c *cli.Context) { 99 | logrus.Debug("Pulling ", c.Args().First()) 100 | }, 101 | },*/ 102 | { 103 | Name: "tag", 104 | Usage: "Tag a pipescript or pipeline to a name", 105 | Action: cmdTag, 106 | }, 107 | } 108 | app.Run(os.Args) 109 | } 110 | 111 | func cmdList(c *cli.Context) { 112 | list, err := listScripts() 113 | if err != nil { 114 | log.Fatal(err) 115 | } 116 | for _, name := range list { 117 | println(name) 118 | } 119 | } 120 | 121 | func cmdLogin(c *cli.Context) { 122 | } 123 | 124 | func cmdPublish(c *cli.Context) { 125 | if len(c.Args()) == 0 { 126 | log.Fatal("No pipeline specified for subscription.") 127 | } 128 | pipeline := c.Args()[0] 129 | 130 | publishPipeline(pipeline) 131 | } 132 | 133 | func cmdSubscribe(c *cli.Context) { 134 | if len(c.Args()) == 0 { 135 | log.Fatal("No pipeline specified for subscription.") 136 | } 137 | pipeline := c.Args()[0] 138 | 139 | subscribePipeline(pipeline) 140 | } 141 | 142 | func cmdCreate(c *cli.Context) { 143 | if len(c.Args()) == 0 { 144 | log.Fatal("No filters specified.") 145 | } 146 | pipeline := c.Args() 147 | 148 | plid, err := createPipeline(pipeline) 149 | if err != nil { 150 | log.Fatal(err) 151 | } 152 | 153 | if name != "" { 154 | err = tagPipeline(plid) 155 | if err != nil { 156 | log.Fatal(err) 157 | } 158 | } 159 | } 160 | 161 | func cmdRemove(c *cli.Context) { 162 | if len(c.Args()) == 0 { 163 | log.Fatal("No filters specified.") 164 | } 165 | filter := c.Args()[0] 166 | 167 | err := removeFilter(filter) 168 | if err != nil { 169 | log.Fatal(err) 170 | } 171 | } 172 | 173 | func cmdImport(c *cli.Context) { 174 | if len(c.Args()) == 0 { 175 | log.Fatal("No pipeline specified for import.") 176 | } 177 | file := c.Args()[0] 178 | id, err := importScript(file) 179 | if err != nil { 180 | log.Fatal(err) 181 | } 182 | if name != "" { 183 | if err = tagFilter(id, name); err != nil { 184 | log.Fatal(err) 185 | } 186 | } 187 | println(id) 188 | } 189 | 190 | func cmdExport(c *cli.Context) { 191 | if len(c.Args()) == 0 { 192 | log.Fatal("No pipeline specified for export.") 193 | } 194 | if name == "" { 195 | log.Fatal("Name not specified. Must pass required --name flag.") 196 | } 197 | exportScript(name, c.Args()...) 198 | } 199 | 200 | func cmdTag(c *cli.Context) { 201 | var err error 202 | 203 | id := c.Args()[0] 204 | name := c.Args()[1] 205 | if name == "" { 206 | log.Fatal("Invalid name") 207 | } 208 | if err = tagFilter(id, name); err != nil { 209 | log.Fatal(err) 210 | } 211 | } 212 | 213 | func execFilter(lastObj string, toObjType string) (msg string, err error) { 214 | var obj objectInterface 215 | if err = json.Unmarshal([]byte(lastObj), &obj); err != nil { 216 | return "", err 217 | } 218 | lastObjType := obj.ClassID 219 | 220 | logrus.Info("pipe[lastObjType/toObjType]: %s/%s\n", lastObjType, toObjType) 221 | 222 | filter, err := getFilter(path.Join(lastObjType, toObjType)) 223 | if err != nil { 224 | return "", err 225 | } 226 | return filter(lastObj) 227 | } 228 | 229 | // Handle the 'exec' CLI command. 230 | func cmdExec(c *cli.Context) { 231 | var msg string 232 | var err error 233 | if debug { 234 | logrus.SetLevel(logrus.DebugLevel) 235 | } 236 | if msg, err = Exec(c.Args()...); err != nil { 237 | log.Fatal(err) 238 | return 239 | } 240 | println(msg) 241 | } 242 | -------------------------------------------------------------------------------- /filter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Sirupsen/logrus" 5 | "github.com/robertkrimen/otto" 6 | 7 | "bufio" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "log" 12 | "os" 13 | "path" 14 | "strings" 15 | 16 | "crypto/sha256" 17 | "io/ioutil" 18 | "net/http" 19 | ) 20 | 21 | const FILTER_BASE string = "http://192.241.174.50/filters/" 22 | const REQUIREJS_URL string = "http://requirejs.org/docs/release/2.1.22/r.js" 23 | 24 | type filterTuple struct { 25 | fromObjType string 26 | toObjType string 27 | } 28 | 29 | func listScripts() ([]string, error) { 30 | var results []string 31 | 32 | path, err := getCachePath("") 33 | if err != nil { 34 | return nil, err 35 | } 36 | fi, err := ioutil.ReadDir(path) 37 | if err != nil { 38 | return nil, err 39 | } 40 | for _, file := range fi { 41 | results = append(results, file.Name()) 42 | } 43 | return results, nil 44 | } 45 | 46 | func exportScript(name string, args ...string) error { 47 | var basedir = path.Join("export", name) 48 | var err error 49 | if err = os.MkdirAll(basedir, 700); err != nil { 50 | return err 51 | } 52 | 53 | var scriptf = path.Join(basedir, "index.js") 54 | var packagef = path.Join(basedir, "package.json") 55 | var filters []string 56 | for _, filter := range args { 57 | filters = append(filters, fmt.Sprintf("\"%s\"", filter)) 58 | } 59 | var pipeline = strings.Join(filters, ",") 60 | 61 | var jscript = fmt.Sprintf( 62 | `/* Generated by 'iopipe export' 63 | var iopipe = require("iopipe") 64 | module.exports = function() { 65 | if (arguments.length > 2) { 66 | throw "Too many arguments" 67 | } 68 | if (arguments.length == 2) { 69 | callback = arguments[1] 70 | return iopipe.define(%s, callback)(arguments[0]) 71 | } 72 | if (arguments.length == 1) { 73 | callback = arguments[0] 74 | return iopipe.define(%s)(arguments[0]) 75 | } 76 | return iopipe.exec(%s) 77 | } 78 | `, pipeline, pipeline, pipeline) 79 | 80 | var jpackage = fmt.Sprintf( 81 | `{ 82 | "name": "%s", 83 | "version": "0.0.1", 84 | "description": "%s exported via iopipe", 85 | "author": "iopipe exporter", 86 | "dependencies": { 87 | "iopipe": "", 88 | "read-stream": "", 89 | "request": "" 90 | }, 91 | "main": "./index.js" 92 | } 93 | `, name, name) 94 | 95 | if err = ioutil.WriteFile(scriptf, []byte(jscript), 400); err != nil { 96 | return err 97 | } 98 | if err = ioutil.WriteFile(packagef, []byte(jpackage), 400); err != nil { 99 | return err 100 | } 101 | return nil 102 | } 103 | 104 | func makeFilter(script string) (func(input string) (string, error), error) { 105 | vm := otto.New() 106 | return makeFilterWithVM(vm, script) 107 | } 108 | 109 | func makeFilterWithVM(vm *otto.Otto, script string) (func(input string) (string, error), error) { 110 | var ( 111 | res *http.Response 112 | body []byte 113 | err error 114 | ) 115 | 116 | res, err = http.Get(REQUIREJS_URL) 117 | if err != nil { 118 | log.Fatal(err) 119 | } 120 | body, err = ioutil.ReadAll(res.Body) 121 | if err != nil { 122 | log.Fatal(err) 123 | } 124 | res.Body.Close() 125 | rjs := string(body[:]) 126 | 127 | return func(input string) (string, error) { 128 | logrus.Debug("Adding RequireJS") 129 | vm.Run(rjs) 130 | 131 | logrus.Debug("Executing script: " + script) 132 | _, err := vm.Run(` 133 | var module = { "exports": function() { } } 134 | `) 135 | if err != nil { 136 | return "", err 137 | } 138 | _, err = vm.Run(script) 139 | if err != nil { 140 | return "", err 141 | } 142 | event := make(chan otto.Value) 143 | 144 | oval, err := vm.ToValue(input) 145 | if err != nil { 146 | return "", err 147 | } 148 | jscontext := createContext(vm, event) 149 | 150 | go vm.Call("module.exports", nil, oval, jscontext) 151 | value := <-event 152 | return value.ToString() 153 | }, nil 154 | } 155 | 156 | func createContext(vm *otto.Otto, event chan otto.Value) otto.Value { 157 | var cbfunc = func(invar ...interface{}) { 158 | var lovar otto.Value 159 | if len(invar) > 0 { 160 | lovar, _ = vm.ToValue(invar[0]) 161 | } else { 162 | lovar = otto.NullValue() 163 | } 164 | event <- lovar 165 | close(event) 166 | } 167 | vm.Set("_iopipe_cb", cbfunc) 168 | vm.Run(`_iopipe_context = { "done": _iopipe_cb, 169 | "success": _iopipe_cb, 170 | "fail": _iopipe_cb }`) 171 | jscontext, _ := vm.Get("_iopipe_context") 172 | return jscontext 173 | } 174 | 175 | func fetchFilter(filterPath string) ([]byte, error) { 176 | var ( 177 | res *http.Response 178 | body []byte 179 | err error 180 | ) 181 | 182 | path := path.Join(FILTER_BASE, filterPath) 183 | res, err = http.Get(path) 184 | if err != nil { 185 | return nil, err 186 | } 187 | body, err = ioutil.ReadAll(res.Body) 188 | if err != nil { 189 | return nil, err 190 | } 191 | res.Body.Close() 192 | 193 | /* Verify digest */ 194 | chksum := sha256.Sum256(body[:]) 195 | if filterPath != string(chksum[:]) { 196 | return nil, errors.New("Checksum failure") 197 | } 198 | 199 | return body, nil 200 | } 201 | 202 | func importScript(file string) (string, error) { 203 | var err error 204 | var fH io.Reader 205 | 206 | if file == "-" { 207 | fH = os.Stdin 208 | } else { 209 | fH, err = os.Open(file) 210 | if err != nil { 211 | return "", err 212 | } 213 | } 214 | reader := bufio.NewReader(fH) 215 | body, err := ioutil.ReadAll(reader) 216 | if err != nil { 217 | return "", err 218 | } 219 | 220 | id, err := writeFilterCache(body[:]) 221 | if err != nil { 222 | return id, err 223 | } 224 | return id, nil 225 | } 226 | 227 | func getFilter(filterPath string) (func(input string) (string, error), error) { 228 | var script []byte 229 | var err error 230 | 231 | /* Do we have this cached? */ 232 | if script, err := readFilterCache(filterPath); err == nil { 233 | return makeFilter(string(script[:])) 234 | } else { 235 | return nil, err 236 | } 237 | 238 | /* If not, fetch */ 239 | if script, err = fetchFilter(filterPath); err != nil { 240 | return nil, err 241 | } 242 | if _, err = writeFilterCache(script); err != nil { 243 | return nil, err 244 | } 245 | 246 | return makeFilter(string(script[:])) 247 | } 248 | 249 | func getPipeline(filterPath string) (func(input string) (string, error), error) { 250 | var script []byte 251 | var err error 252 | 253 | diskPath, err := getCachePath("") 254 | if err != nil { 255 | return nil, err 256 | } 257 | 258 | /* Do we have this cached? */ 259 | if _, err := os.Stat(diskPath); err == nil { 260 | script, err = ioutil.ReadFile(diskPath) 261 | return makeFilter(string(script[:])) 262 | } 263 | 264 | /* If not, fetch */ 265 | if script, err = fetchFilter(filterPath); err != nil { 266 | return nil, err 267 | } 268 | /* Write cache */ 269 | if _, err = writeFilterCache(script); err != nil { 270 | return nil, err 271 | } 272 | 273 | return makeFilter(string(script[:])) 274 | } 275 | 276 | func findFilters(fromObjType string, toObjType string) (string, error) { 277 | var ( 278 | res *http.Response 279 | body []byte 280 | err error 281 | ) 282 | 283 | path := path.Join(FILTER_BASE, fromObjType, toObjType) 284 | res, err = http.Get(path) 285 | if err != nil { 286 | return "", err 287 | } 288 | body, err = ioutil.ReadAll(res.Body) 289 | if err != nil { 290 | return "", err 291 | } 292 | res.Body.Close() 293 | response := string(body[:]) 294 | return response, nil 295 | } 296 | 297 | func findPipelines(fromObjType string, toObjType string) (string, error) { 298 | var ( 299 | res *http.Response 300 | body []byte 301 | err error 302 | ) 303 | 304 | path := path.Join(FILTER_BASE, fromObjType, toObjType) 305 | res, err = http.Get(path) 306 | if err != nil { 307 | return "", err 308 | } 309 | body, err = ioutil.ReadAll(res.Body) 310 | if err != nil { 311 | return "", err 312 | } 313 | res.Body.Close() 314 | response := string(body[:]) 315 | return response, nil 316 | } 317 | 318 | func publishPipeline(pipelineName string) { 319 | } 320 | 321 | func subscribePipeline(pipelineName string) { 322 | } 323 | 324 | func createPipeline(pipeparts []string) (string, error) { 325 | return "", nil 326 | } 327 | 328 | func removeFilter(filterid string) error { 329 | path, err := getCachePath(filterid) 330 | if err != nil { 331 | return err 332 | } 333 | err = os.Remove(path) 334 | if err != nil { 335 | return err 336 | } 337 | return nil 338 | } 339 | 340 | func tagPipeline(pipeline string) error { 341 | return nil 342 | } 343 | 344 | func tagFilter(filterid string, name string) error { 345 | path, err := getCachePath(filterid) 346 | if err != nil { 347 | return err 348 | } 349 | destpath, err := getCachePath(name) 350 | if err != nil { 351 | return err 352 | } 353 | err = os.Symlink(path, destpath) 354 | return err 355 | } 356 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | https://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of yright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | Copyright 2015, Eric Windisch. IO PIPE 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | https://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | --------------------------------------------------------------------------------