├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── api.go ├── config.go ├── config.json.example ├── config └── bitrun.conf ├── container.go ├── exec.go ├── lang.go ├── languages.json.example ├── main.go ├── pool.go ├── request.go ├── run.go ├── throttler.go └── utils.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | bin/* 27 | languages.json 28 | config.json -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2016 Dan Sosedoff 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | FILENAME = bitrun-api 2 | TARGETS = darwin/amd64 linux/amd64 3 | 4 | build: 5 | go build -o ./bin/$(FILENAME) 6 | 7 | all: 8 | gox \ 9 | -osarch="$(TARGETS)" \ 10 | -output="./bin/$(FILENAME)_{{.OS}}_{{.Arch}}" 11 | 12 | setup: 13 | go get || true 14 | 15 | clean: 16 | rm -rf ./bin/* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BitRun API 2 | 3 | BitRun API is a project that provides ability to execute code snippets 4 | written in various languages (Ruby, Pythong, Node, PHP, Go, ...). Under the hood 5 | it used Docker to run the code and provides process and filesystem isolation. API 6 | service is written in Go and could be easily installed on the server and only 7 | requires Docker to run. 8 | 9 | ## Reference 10 | 11 | To execute the code you must provide filename and content. 12 | 13 | ``` 14 | POST https://bit.run/api/v1/run 15 | ``` 16 | 17 | Parameters: 18 | 19 | - `filename` - Name of the file to run. This is needed to determine the language. Required. 20 | - `content` - Code to execute. Required. 21 | 22 | Example: 23 | 24 | ```bash 25 | curl \ 26 | -i \ 27 | -X POST "https://bit.run/api/v1/run" \ 28 | -d "filename=test.rb&content=puts 'Hello World'" 29 | ``` 30 | 31 | Output: 32 | 33 | ``` 34 | HTTP/1.1 200 OK 35 | Content-Type: text/plain 36 | Content-Length: 13 37 | X-Run-Command: ruby test.rb 38 | X-Run-Duration: 261.752379ms 39 | X-Run-Exitcode: 0 40 | 41 | Hello World 42 | ``` 43 | 44 | If request is successful, API will respond with plaintext of the executed command. 45 | Extra meta data will be included in the headers: 46 | 47 | - `X-Run-Command` - full command that was executed 48 | - `X-Run-Duration` - how long it took to process the request (not to run the code) 49 | - `X-Run-Exitcode` - exit code of executed command 50 | 51 | Each run is limited by 10 seconds. If your code runs longer than 10s API will 52 | respond with 400 and provide error message: 53 | 54 | ```json 55 | { 56 | "error": "Operation timed out after 10s" 57 | } 58 | ``` 59 | 60 | ### Command override 61 | 62 | By default, bitrun will execute code snippet with a default command. For example, 63 | ruby snippet will use the following command: "ruby main.rb". To override the command, 64 | you can specify `command` parameter when making an API call: 65 | 66 | ```bash 67 | curl \ 68 | -i \ 69 | -X POST "https://bit.run/api/v1/run" \ 70 | -d "filename=test.rb&content=puts 'Hello World'&command=ruby -v" 71 | ``` 72 | 73 | Response: 74 | 75 | ``` 76 | HTTP/1.1 200 OK 77 | Content-Type: text/plain 78 | Content-Length: 59 79 | X-Run-Command: ruby test.rb 80 | X-Run-Duration: 126.436286ms 81 | X-Run-Exitcode: 0 82 | 83 | ruby 2.2.3p173 (2015-08-18 revision 51636) [x86_64-linux] 84 | ``` 85 | 86 | ### Supported languages 87 | 88 | To check which languages are currently supported, make a call: 89 | 90 | ``` 91 | GET https://bit.run/api/v1/config 92 | ``` 93 | 94 | Response will include supported languages along with commands used to run scripts: 95 | 96 | ```json 97 | { 98 | ".rb": { 99 | "image": "ruby:2.2", 100 | "command": "ruby %s", 101 | "format": "text/plain" 102 | }, 103 | ".py": { 104 | "image": "python:2.7", 105 | "command": "python %s", 106 | "format": "text/plain" 107 | }, 108 | ".php": { 109 | "image": "php:5.6", 110 | "command": "php %s", 111 | "format": "text/plain" 112 | } 113 | } 114 | ``` 115 | 116 | ## License 117 | 118 | MIT -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "strconv" 7 | "strings" 8 | 9 | docker "github.com/fsouza/go-dockerclient" 10 | gin "github.com/gin-gonic/gin" 11 | ) 12 | 13 | func errorResponse(status int, err error, c *gin.Context) { 14 | result := map[string]string{"error": err.Error()} 15 | c.JSON(status, result) 16 | } 17 | 18 | func performRun(run *Run) (*RunResult, error) { 19 | // Try to get a warmed-up container for the run 20 | if run.Request.Clean == false && pools[run.Request.Image] != nil { 21 | container, err := pools[run.Request.Image].Get() 22 | 23 | if err == nil { 24 | log.Println("got warmed-up container for image:", run.Request.Image, container.ID) 25 | result, err := run.StartExecWithTimeout(container) 26 | return result, err 27 | } 28 | } 29 | 30 | log.Println("setting up container for image:", run.Request.Image) 31 | if err := run.Setup(); err != nil { 32 | return nil, err 33 | } 34 | 35 | return run.StartWithTimeout() 36 | } 37 | 38 | func HandleRun(c *gin.Context) { 39 | req, err := ParseRequest(c.Request) 40 | if err != nil { 41 | errorResponse(400, err, c) 42 | return 43 | } 44 | 45 | config, exists := c.Get("config") 46 | if !exists { 47 | errorResponse(400, fmt.Errorf("Cant get config"), c) 48 | return 49 | } 50 | 51 | client, exists := c.Get("client") 52 | if !exists { 53 | errorResponse(400, fmt.Errorf("Cant get client"), c) 54 | return 55 | } 56 | 57 | run := NewRun(config.(*Config), client.(*docker.Client), req) 58 | defer run.Destroy() 59 | 60 | result, err := performRun(run) 61 | if err != nil { 62 | errorResponse(400, err, c) 63 | return 64 | } 65 | 66 | c.Header("X-Run-Command", req.Command) 67 | c.Header("X-Run-ExitCode", strconv.Itoa(result.ExitCode)) 68 | c.Header("X-Run-Duration", result.Duration) 69 | 70 | c.Data(200, req.Format, result.Output) 71 | } 72 | 73 | func HandleConfig(c *gin.Context) { 74 | c.JSON(200, Extensions) 75 | } 76 | 77 | func authMiddleware(config *Config) gin.HandlerFunc { 78 | return func(c *gin.Context) { 79 | if config.ApiToken != "" { 80 | token := c.Request.FormValue("api_token") 81 | 82 | if token != config.ApiToken { 83 | errorResponse(400, fmt.Errorf("Api token is invalid"), c) 84 | c.Abort() 85 | return 86 | } 87 | } 88 | 89 | c.Next() 90 | } 91 | } 92 | 93 | func throttleMiddleware(throttler *Throttler) gin.HandlerFunc { 94 | return func(c *gin.Context) { 95 | ip := strings.Split(c.Request.RemoteAddr, ":")[0] 96 | 97 | // Bypass throttling for whitelisted IPs 98 | if throttler.Whitelisted(ip) { 99 | c.Next() 100 | return 101 | } 102 | 103 | if err := throttler.Add(ip); err != nil { 104 | errorResponse(429, err, c) 105 | c.Abort() 106 | return 107 | } 108 | 109 | c.Next() 110 | throttler.Remove(ip) 111 | } 112 | } 113 | 114 | func corsMiddleware() gin.HandlerFunc { 115 | return func(c *gin.Context) { 116 | c.Header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") 117 | c.Header("Access-Control-Allow-Origin", "*") 118 | c.Header("Access-Control-Expose-Headers", "*") 119 | } 120 | } 121 | 122 | func RunApi(config *Config, client *docker.Client) { 123 | throttler := NewThrottler(config.ThrottleConcurrency, config.ThrottleQuota) 124 | throttler.SetWhitelist(config.ThrottleWhitelist) 125 | throttler.StartPeriodicFlush() 126 | 127 | gin.SetMode(gin.ReleaseMode) 128 | router := gin.Default() 129 | 130 | v1 := router.Group("/api/v1/") 131 | { 132 | v1.Use(authMiddleware(config)) 133 | v1.Use(corsMiddleware()) 134 | v1.Use(throttleMiddleware(throttler)) 135 | 136 | v1.Use(func(c *gin.Context) { 137 | c.Set("config", config) 138 | c.Set("client", client) 139 | }) 140 | 141 | v1.GET("/config", HandleConfig) 142 | v1.POST("/run", HandleRun) 143 | } 144 | 145 | fmt.Println("starting server on", config.Listen) 146 | router.Run(config.Listen) 147 | } 148 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | "time" 8 | ) 9 | 10 | type PoolConfig struct { 11 | Image string `json:"image"` 12 | Capacity int `json:"capacity"` 13 | Standby int `json:"standby"` 14 | } 15 | 16 | type Config struct { 17 | Listen string `json:"listen"` 18 | DockerHost string `json:"docker_host"` 19 | LanguagesPath string `json:"languages_path"` 20 | SharedPath string `json:"shared_path"` 21 | RunDuration time.Duration `json:"run_duration"` 22 | ThrottleQuota int `json:"throttle_quota"` 23 | ThrottleConcurrency int `json:"throttle_concurrency"` 24 | ThrottleWhitelist []string `json:"throttle_whitelist"` 25 | NetworkDisabled bool `json:"network_disabled"` 26 | MemoryLimit int64 `json:"memory_limit"` 27 | Pools []PoolConfig `json:"pools"` 28 | ApiToken string `json:"api_token"` 29 | FetchImages bool `json:"fetch_images"` 30 | Namespaces bool `json:"namespaces"` 31 | } 32 | 33 | func NewConfig() *Config { 34 | cfg := Config{ 35 | DockerHost: os.Getenv("DOCKER_HOST"), 36 | SharedPath: os.Getenv("SHARED_PATH"), 37 | } 38 | 39 | cfg.Listen = "127.0.0.1:5000" 40 | cfg.SharedPath = expandPath(cfg.SharedPath) 41 | cfg.RunDuration = time.Second * 10 42 | cfg.ThrottleQuota = 5 43 | cfg.ThrottleConcurrency = 1 44 | cfg.ThrottleWhitelist = []string{} 45 | cfg.NetworkDisabled = false 46 | cfg.MemoryLimit = 67108864 47 | cfg.Pools = []PoolConfig{} 48 | cfg.FetchImages = false 49 | cfg.Namespaces = false 50 | cfg.LanguagesPath = "./languages.json" 51 | 52 | return &cfg 53 | } 54 | 55 | func NewConfigFromFile(path string) (*Config, error) { 56 | data, err := ioutil.ReadFile(path) 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | config := Config{} 62 | 63 | err = json.Unmarshal(data, &config) 64 | 65 | if err == nil { 66 | config.SharedPath = expandPath(config.SharedPath) 67 | config.RunDuration = config.RunDuration * time.Second 68 | 69 | if config.Listen == "" { 70 | config.Listen = "127.0.0.1:5000" 71 | } 72 | } 73 | 74 | return &config, err 75 | } 76 | -------------------------------------------------------------------------------- /config.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "listen": "127.0.0.1:5000", 3 | "docker_host": "tcp://127.0.0.1:2375", 4 | "shared_path": "/tmp/bitrun_shared", 5 | "languages_path": "./languages.json", 6 | "run_duration": 10, 7 | "throttle_quota": 5, 8 | "throttle_concurrency": 1, 9 | "throttle_whitelist": [ 10 | "127.0.0.1" 11 | ], 12 | "network_disabled": false, 13 | "memory_limit": 67108864, 14 | "fetch_images": true, 15 | "pools": [ 16 | { "image": "bitrun/ruby:2.2", "capacity": 10 } 17 | ] 18 | } -------------------------------------------------------------------------------- /config/bitrun.conf: -------------------------------------------------------------------------------- 1 | [program:bitrun-api] 2 | environment=CONFIG=/etc/bitrun.json 3 | command=/usr/local/bin/bitrun-api 4 | numprocs=1 5 | directory=/tmp 6 | autostart=true 7 | stdout_logfile=/var/log/bitrun.stdout.log 8 | stderr_logfile=/var/log/bitrun.stderr.log -------------------------------------------------------------------------------- /container.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "time" 8 | 9 | docker "github.com/fsouza/go-dockerclient" 10 | ) 11 | 12 | func CreateContainer(client *docker.Client, config *Config, image string, standby int, env string) (*docker.Container, error) { 13 | id, _ := randomHex(20) 14 | volumePath := fmt.Sprintf("%s/%s", config.SharedPath, id) 15 | name := fmt.Sprintf("bitrun-%v", time.Now().UnixNano()) 16 | 17 | if err := os.Mkdir(volumePath, 0777); err != nil { 18 | return nil, err 19 | } 20 | 21 | opts := docker.CreateContainerOptions{ 22 | Name: name, 23 | HostConfig: &docker.HostConfig{ 24 | Binds: []string{ 25 | volumePath + ":/code", 26 | volumePath + ":/tmp", 27 | }, 28 | ReadonlyRootfs: true, 29 | Memory: config.MemoryLimit, 30 | MemorySwap: 0, 31 | }, 32 | Config: &docker.Config{ 33 | Hostname: "bitrun", 34 | Image: image, 35 | Labels: map[string]string{"id": id}, 36 | AttachStdout: false, 37 | AttachStderr: false, 38 | AttachStdin: false, 39 | Tty: false, 40 | NetworkDisabled: config.NetworkDisabled, 41 | WorkingDir: "/code", 42 | Cmd: []string{"sleep", fmt.Sprintf("%v", standby)}, 43 | Env: strings.Split(env, "\n"), 44 | }, 45 | } 46 | 47 | container, err := client.CreateContainer(opts) 48 | if err == nil { 49 | container.Config = opts.Config 50 | } 51 | 52 | return container, err 53 | } 54 | -------------------------------------------------------------------------------- /exec.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "strings" 8 | "time" 9 | 10 | docker "github.com/fsouza/go-dockerclient" 11 | ) 12 | 13 | func (run *Run) StartExec(container *docker.Container) (*RunResult, error) { 14 | run.Container = container 15 | run.VolumePath = fmt.Sprintf("%s/%s", run.Config.SharedPath, container.Config.Labels["id"]) 16 | fullPath := fmt.Sprintf("%s/%s", run.VolumePath, run.Request.Filename) 17 | 18 | if err := ioutil.WriteFile(fullPath, []byte(run.Request.Content), 0666); err != nil { 19 | return nil, err 20 | } 21 | 22 | ts := time.Now() 23 | 24 | exec, err := run.Client.CreateExec(docker.CreateExecOptions{ 25 | AttachStdout: true, 26 | AttachStderr: true, 27 | AttachStdin: true, 28 | Cmd: []string{"bash", "-c", run.Request.Command}, 29 | Container: container.ID, 30 | }) 31 | 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | buff := bytes.NewBuffer([]byte{}) 37 | stdin := strings.NewReader(run.Request.Input) 38 | 39 | execOpts := docker.StartExecOptions{ 40 | InputStream: stdin, 41 | OutputStream: buff, 42 | ErrorStream: buff, 43 | RawTerminal: false, 44 | } 45 | 46 | if err = run.Client.StartExec(exec.ID, execOpts); err != nil { 47 | return nil, err 48 | } 49 | 50 | result := RunResult{} 51 | 52 | execInfo, err := run.Client.InspectExec(exec.ID) 53 | if err == nil { 54 | result.ExitCode = execInfo.ExitCode 55 | } 56 | 57 | result.Duration = time.Now().Sub(ts).String() 58 | result.Output = buff.Bytes() 59 | 60 | return &result, nil 61 | } 62 | 63 | func (run *Run) StartExecWithTimeout(container *docker.Container) (*RunResult, error) { 64 | duration := run.Config.RunDuration 65 | timeout := time.After(duration) 66 | chDone := make(chan Done) 67 | 68 | go func() { 69 | res, err := run.StartExec(container) 70 | chDone <- Done{res, err} 71 | }() 72 | 73 | select { 74 | case done := <-chDone: 75 | return done.RunResult, done.error 76 | case <-timeout: 77 | return nil, fmt.Errorf("Operation timed out after %s", duration.String()) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /lang.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "path/filepath" 8 | "strings" 9 | ) 10 | 11 | type Language struct { 12 | Image string `json:"image"` 13 | Command string `json:"command"` 14 | Format string `json:"format"` 15 | } 16 | 17 | var Extensions map[string]Language 18 | 19 | func ValidLanguage(ext string) bool { 20 | for k, _ := range Extensions { 21 | if k == ext { 22 | return true 23 | } 24 | } 25 | 26 | return false 27 | } 28 | 29 | func GetLanguageConfig(filename string) (*Language, error) { 30 | ext := filepath.Ext(strings.ToLower(filename)) 31 | 32 | if !ValidLanguage(ext) { 33 | return nil, fmt.Errorf("Extension is not supported: %s", ext) 34 | } 35 | 36 | lang := Extensions[ext] 37 | return &lang, nil 38 | } 39 | 40 | func LoadLanguages(file string) error { 41 | data, err := ioutil.ReadFile(file) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | err = json.Unmarshal(data, &Extensions) 47 | 48 | for k, lang := range Extensions { 49 | if lang.Format == "" { 50 | lang.Format = "text/plain" 51 | } 52 | 53 | Extensions[k] = lang 54 | } 55 | 56 | return err 57 | } 58 | -------------------------------------------------------------------------------- /languages.json.example: -------------------------------------------------------------------------------- 1 | { 2 | ".rb": { 3 | "image": "bitrun/ruby:2.2", 4 | "command": "ruby %s" 5 | }, 6 | ".py": { 7 | "image": "python:2.7", 8 | "command": "python %s" 9 | }, 10 | ".js": { 11 | "image": "bitrun/node:4.1", 12 | "command": "node %s" 13 | }, 14 | ".go": { 15 | "image": "golang:1.5", 16 | "command": "go run %s" 17 | }, 18 | ".php": { 19 | "image": "php:5.6", 20 | "command": "php %s" 21 | }, 22 | ".coffee": { 23 | "image": "bitrun/node:4.1", 24 | "command": "coffee %s" 25 | }, 26 | ".exs": { 27 | "image": "trenpixster/elixir:latest", 28 | "command": "elixir %s" 29 | }, 30 | ".sh": { 31 | "image": "debian:jessie", 32 | "command": "bash %s" 33 | }, 34 | ".rs": { 35 | "image": "jimmycuadra/rust:latest", 36 | "command": "rustc -o main %s && ./main" 37 | }, 38 | ".c": { 39 | "image": "gcc:latest", 40 | "command": "cc -o main %s && ./main" 41 | }, 42 | ".lol": { 43 | "image": "bitrun/lci:0.10", 44 | "command": "lci ./%s" 45 | }, 46 | ".arnie": { 47 | "image": "sosedoff/arnoldc:latest", 48 | "command": "java -jar /arnoldc.jar %s && java main" 49 | }, 50 | ".bf": { 51 | "image": "sosedoff/brainfuck:latest", 52 | "command": "brainfuck %s" 53 | }, 54 | ".swift": { 55 | "image": "swiftdocker/swift:latest", 56 | "command": "swift %s" 57 | }, 58 | ".dart": { 59 | "image": "google/dart:latest", 60 | "command": "dart %s" 61 | }, 62 | ".lua": { 63 | "image": "bitrun/lua:latest", 64 | "command": "lua %s" 65 | } 66 | } -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "strings" 8 | 9 | docker "github.com/fsouza/go-dockerclient" 10 | ) 11 | 12 | const VERSION = "0.3.1" 13 | 14 | func requireEnvVar(name string) { 15 | if os.Getenv(name) == "" { 16 | err := fmt.Errorf("Please set %s environment variable", name) 17 | log.Fatalln(err) 18 | } 19 | } 20 | 21 | func getConfig() *Config { 22 | if os.Getenv("CONFIG") != "" { 23 | config, err := NewConfigFromFile(os.Getenv("CONFIG")) 24 | if err != nil { 25 | log.Fatalln(err) 26 | } 27 | 28 | return config 29 | } 30 | 31 | requireEnvVar("DOCKER_HOST") 32 | requireEnvVar("SHARED_PATH") 33 | 34 | return NewConfig() 35 | } 36 | 37 | func pullImage(name string, client *docker.Client) error { 38 | chunks := strings.Split(name, ":") 39 | 40 | imgName := chunks[0] 41 | imgTag := "latest" 42 | 43 | if len(chunks) == 2 { 44 | imgTag = chunks[1] 45 | } 46 | 47 | auth := docker.AuthConfiguration{} 48 | opts := docker.PullImageOptions{ 49 | Repository: imgName, 50 | Tag: imgTag, 51 | OutputStream: os.Stdout, 52 | } 53 | 54 | return client.PullImage(opts, auth) 55 | } 56 | 57 | func checkImages(client *docker.Client, config *Config) error { 58 | images, err := client.ListImages(docker.ListImagesOptions{}) 59 | if err != nil { 60 | log.Fatalln(err) 61 | } 62 | 63 | imagesWithTags := map[string]bool{} 64 | 65 | for _, image := range images { 66 | for _, tag := range image.RepoTags { 67 | imagesWithTags[tag] = true 68 | } 69 | } 70 | 71 | fmt.Println("checking images...") 72 | for _, lang := range Extensions { 73 | if imagesWithTags[lang.Image] == true { 74 | log.Printf("image %s exists", lang.Image) 75 | } else { 76 | if config.FetchImages { 77 | log.Println("pulling", lang.Image, "image...") 78 | err := pullImage(lang.Image, client) 79 | if err != nil { 80 | log.Fatalln(err) 81 | } 82 | } else { 83 | return fmt.Errorf("image %s does not exist", lang.Image) 84 | } 85 | } 86 | } 87 | 88 | return nil 89 | } 90 | 91 | func main() { 92 | log.Printf("bitrun api v%s\n", VERSION) 93 | 94 | config := getConfig() 95 | 96 | err := LoadLanguages(config.LanguagesPath) 97 | if err != nil { 98 | log.Fatalln(err) 99 | } 100 | 101 | client, err := docker.NewClient(config.DockerHost) 102 | if err != nil { 103 | log.Fatalln(err) 104 | } 105 | 106 | err = checkImages(client, config) 107 | if err != nil { 108 | log.Fatalln(err) 109 | } 110 | 111 | go RunPool(config, client) 112 | RunApi(config, client) 113 | } 114 | -------------------------------------------------------------------------------- /pool.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | "sync" 8 | "time" 9 | 10 | docker "github.com/fsouza/go-dockerclient" 11 | ) 12 | 13 | var pools map[string]*Pool 14 | 15 | type Pool struct { 16 | Config *Config 17 | Client *docker.Client 18 | Containers map[string]*docker.Container 19 | Image string 20 | Capacity int 21 | Standby int 22 | sync.Mutex 23 | } 24 | 25 | func findImage(client *docker.Client, image string) (*docker.APIImages, error) { 26 | images, err := client.ListImages(docker.ListImagesOptions{}) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | for _, img := range images { 32 | for _, t := range img.RepoTags { 33 | if t == image { 34 | return &img, nil 35 | } 36 | } 37 | } 38 | 39 | return nil, fmt.Errorf("invalid image:", image) 40 | } 41 | 42 | func NewPool(config *Config, client *docker.Client, image string, capacity int, standby int) (*Pool, error) { 43 | _, err := findImage(client, image) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | if standby <= 60 { 49 | standby = 86400 50 | } 51 | 52 | pool := &Pool{ 53 | Config: config, 54 | Client: client, 55 | Containers: map[string]*docker.Container{}, 56 | Image: image, 57 | Capacity: capacity, 58 | Standby: standby, 59 | } 60 | 61 | return pool, nil 62 | } 63 | 64 | func (pool *Pool) Exists(id string) bool { 65 | return pool.Containers[id] != nil 66 | } 67 | 68 | func (pool *Pool) Load() error { 69 | pool.Lock() 70 | defer pool.Unlock() 71 | 72 | containers, err := pool.Client.ListContainers(docker.ListContainersOptions{All: true}) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | for _, c := range containers { 78 | if c.Image == pool.Image && c.Labels["id"] != "" { 79 | pool.Containers[c.ID] = &docker.Container{ 80 | ID: c.ID, 81 | Config: &docker.Config{ 82 | Labels: c.Labels, 83 | }, 84 | } 85 | } 86 | } 87 | 88 | return nil 89 | } 90 | 91 | func (pool *Pool) Add() error { 92 | container, err := CreateContainer(pool.Client, pool.Config, pool.Image, pool.Standby, "") 93 | if err != nil { 94 | return err 95 | } 96 | 97 | if err = pool.Client.StartContainer(container.ID, nil); err != nil { 98 | return err 99 | } 100 | 101 | pool.Lock() 102 | defer pool.Unlock() 103 | 104 | pool.Containers[container.ID] = container 105 | return nil 106 | } 107 | 108 | func (pool *Pool) Fill() { 109 | num := pool.Capacity - len(pool.Containers) 110 | 111 | // Pool is full 112 | if num <= 0 { 113 | return 114 | } 115 | 116 | log.Printf("adding %v containers to %v pool, standby: %vs\n", num, pool.Image, pool.Standby) 117 | 118 | for i := 0; i < num; i++ { 119 | err := pool.Add() 120 | if err != nil { 121 | log.Println("error while adding to pool:", err) 122 | } 123 | } 124 | } 125 | 126 | func (pool *Pool) Monitor() { 127 | for { 128 | pool.Fill() 129 | time.Sleep(time.Second * 3) 130 | } 131 | } 132 | 133 | func (pool *Pool) Remove(id string) { 134 | pool.Lock() 135 | defer pool.Unlock() 136 | 137 | if pool.Containers[id] != nil { 138 | go destroyContainer(pool.Client, id) 139 | delete(pool.Containers, id) 140 | } 141 | } 142 | 143 | func (pool *Pool) Get() (*docker.Container, error) { 144 | pool.Lock() 145 | defer pool.Unlock() 146 | 147 | var container *docker.Container 148 | 149 | for _, v := range pool.Containers { 150 | container = v 151 | break 152 | } 153 | 154 | if container != nil { 155 | delete(pool.Containers, container.ID) 156 | return container, nil 157 | } 158 | 159 | return nil, fmt.Errorf("no contaienrs are available") 160 | } 161 | 162 | func RunPool(config *Config, client *docker.Client) { 163 | chEvents := make(chan *docker.APIEvents) 164 | pools = make(map[string]*Pool) 165 | 166 | // Setup docker event listener 167 | if err := client.AddEventListener(chEvents); err != nil { 168 | log.Fatalln(err) 169 | } 170 | 171 | go func() { 172 | for { 173 | event := <-chEvents 174 | if event == nil { 175 | continue 176 | } 177 | 178 | if event.Status == "die" { 179 | for _, pool := range pools { 180 | if pool.Exists(event.ID) { 181 | log.Println("pool's container got destroyed:", event.ID) 182 | pool.Remove(event.ID) 183 | } 184 | } 185 | } 186 | } 187 | }() 188 | 189 | for _, cfg := range config.Pools { 190 | if cfg.Capacity < 1 { 191 | continue 192 | } 193 | 194 | log.Println("initializing pool for:", cfg.Image) 195 | 196 | pool, err := NewPool(config, client, cfg.Image, cfg.Capacity, cfg.Standby) 197 | if err != nil { 198 | log.Fatalln(err) 199 | } 200 | 201 | err = pool.Load() 202 | if err != nil { 203 | log.Fatalln(err) 204 | } 205 | 206 | go pool.Monitor() 207 | pools[cfg.Image] = pool 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /request.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/sha1" 5 | "encoding/hex" 6 | "fmt" 7 | "net/http" 8 | "regexp" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type Request struct { 14 | Filename string 15 | Content string 16 | CacheKey string 17 | Command string 18 | Input string 19 | Image string 20 | Format string 21 | MemoryLimit int64 22 | NamespaceId string 23 | Env string 24 | Clean bool 25 | } 26 | 27 | var FilenameRegexp = regexp.MustCompile(`\A([a-z\d\-\_]+)\.[a-z]{1,12}\z`) 28 | 29 | func normalizeString(val string) string { 30 | return strings.ToLower(strings.TrimSpace(val)) 31 | } 32 | 33 | func parseInt(val string) int64 { 34 | if val == "" { 35 | return 0 36 | } 37 | 38 | result, err := strconv.Atoi(val) 39 | if err != nil { 40 | return 0 41 | } 42 | 43 | if result < 0 { 44 | result = 0 45 | } 46 | 47 | return int64(result) 48 | } 49 | 50 | func sha1Sum(input string) string { 51 | h := sha1.New() 52 | h.Write([]byte(input)) 53 | return hex.EncodeToString(h.Sum(nil)) 54 | } 55 | 56 | func ParseRequest(r *http.Request) (*Request, error) { 57 | req := Request{ 58 | Filename: normalizeString(r.FormValue("filename")), 59 | Command: normalizeString(r.FormValue("command")), 60 | Content: r.FormValue("content"), 61 | Input: r.FormValue("input"), 62 | Image: r.FormValue("image"), 63 | MemoryLimit: parseInt(r.FormValue("memory_limit")), 64 | NamespaceId: normalizeString(r.FormValue("namespace")), 65 | Env: strings.TrimSpace(r.FormValue("env")), 66 | Clean: false, 67 | } 68 | 69 | if r.FormValue("clean") == "1" { 70 | req.Clean = true 71 | } 72 | 73 | if req.Filename == "" { 74 | return nil, fmt.Errorf("Filename is required") 75 | } 76 | 77 | if !FilenameRegexp.Match([]byte(req.Filename)) { 78 | return nil, fmt.Errorf("Invalid filename") 79 | } 80 | 81 | if req.Content == "" { 82 | return nil, fmt.Errorf("Content is required") 83 | } 84 | 85 | lang, err := GetLanguageConfig(req.Filename) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | req.Format = lang.Format 91 | 92 | if req.Image == "" { 93 | req.Image = lang.Image 94 | } 95 | 96 | if req.Command == "" { 97 | req.Command = fmt.Sprintf(lang.Command, req.Filename) 98 | } 99 | 100 | // Calculate request cache key based on content, command and image 101 | req.CacheKey = sha1Sum(req.Content + req.Input + req.Command + req.Image) 102 | 103 | return &req, nil 104 | } 105 | -------------------------------------------------------------------------------- /run.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "time" 8 | 9 | docker "github.com/fsouza/go-dockerclient" 10 | ) 11 | 12 | type Run struct { 13 | Id string 14 | VolumePath string 15 | Config *Config 16 | Container *docker.Container 17 | Client *docker.Client 18 | Request *Request 19 | Done chan bool 20 | } 21 | 22 | type RunResult struct { 23 | ExitCode int `json:"exit_code"` 24 | Output []byte `json:"output"` 25 | Duration string `json:"-"` 26 | } 27 | 28 | type Done struct { 29 | *RunResult 30 | error 31 | } 32 | 33 | func NewRun(config *Config, client *docker.Client, req *Request) *Run { 34 | id, _ := randomHex(20) 35 | 36 | return &Run{ 37 | Id: id, 38 | Config: config, 39 | Client: client, 40 | VolumePath: fmt.Sprintf("%s/%s", config.SharedPath, id), 41 | Request: req, 42 | Done: make(chan bool), 43 | } 44 | } 45 | 46 | func (run *Run) Setup() error { 47 | container, err := CreateContainer(run.Client, run.Config, run.Request.Image, 60, run.Request.Env) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | volumePath := fmt.Sprintf("%s/%s", run.Config.SharedPath, container.Config.Labels["id"]) 53 | fullPath := fmt.Sprintf("%s/%s", volumePath, run.Request.Filename) 54 | 55 | if err := ioutil.WriteFile(fullPath, []byte(run.Request.Content), 0666); err != nil { 56 | return err 57 | } 58 | 59 | run.Container = container 60 | 61 | if err := run.Client.StartContainer(container.ID, nil); err != nil { 62 | return err 63 | } 64 | 65 | return nil 66 | } 67 | 68 | func (run *Run) Start() (*RunResult, error) { 69 | return run.StartExec(run.Container) 70 | } 71 | 72 | func (run *Run) StartWithTimeout() (*RunResult, error) { 73 | duration := run.Config.RunDuration 74 | timeout := time.After(duration) 75 | chDone := make(chan Done) 76 | 77 | go func() { 78 | res, err := run.Start() 79 | chDone <- Done{res, err} 80 | }() 81 | 82 | select { 83 | case done := <-chDone: 84 | return done.RunResult, done.error 85 | case <-timeout: 86 | return nil, fmt.Errorf("Operation timed out after %s", duration.String()) 87 | } 88 | } 89 | 90 | func (run *Run) Destroy() error { 91 | if run.Container != nil { 92 | destroyContainer(run.Client, run.Container.ID) 93 | } 94 | 95 | return os.RemoveAll(run.VolumePath) 96 | } 97 | 98 | func destroyContainer(client *docker.Client, id string) error { 99 | return client.RemoveContainer(docker.RemoveContainerOptions{ 100 | ID: id, 101 | RemoveVolumes: true, 102 | Force: true, 103 | }) 104 | } 105 | -------------------------------------------------------------------------------- /throttler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | type Throttler struct { 10 | Concurrency int 11 | Quota int 12 | Clients map[string]int 13 | Requests map[string]int 14 | Whitelist map[string]bool 15 | *sync.Mutex 16 | } 17 | 18 | func NewThrottler(concurrency int, quota int) *Throttler { 19 | return &Throttler{ 20 | Concurrency: concurrency, 21 | Quota: quota, 22 | Clients: make(map[string]int), 23 | Requests: make(map[string]int), 24 | Whitelist: make(map[string]bool), 25 | Mutex: &sync.Mutex{}, 26 | } 27 | } 28 | 29 | func (t *Throttler) StartPeriodicFlush() { 30 | go func() { 31 | for { 32 | t.Flush() 33 | time.Sleep(time.Second * 5) 34 | } 35 | }() 36 | } 37 | 38 | func (t *Throttler) Add(ip string) error { 39 | t.Lock() 40 | defer t.Unlock() 41 | 42 | t.Requests[ip]++ 43 | 44 | if t.Requests[ip] > t.Quota || t.Clients[ip] >= t.Concurrency { 45 | return fmt.Errorf("Too many requests") 46 | } 47 | 48 | t.Clients[ip]++ 49 | return nil 50 | } 51 | 52 | func (t *Throttler) Remove(ip string) { 53 | t.Lock() 54 | defer t.Unlock() 55 | 56 | t.Clients[ip]-- 57 | 58 | if t.Clients[ip] < 0 { 59 | t.Clients[ip] = 0 60 | } 61 | } 62 | 63 | func (t *Throttler) Flush() { 64 | t.Lock() 65 | defer t.Unlock() 66 | 67 | for k := range t.Clients { 68 | delete(t.Clients, k) 69 | } 70 | 71 | for k := range t.Requests { 72 | delete(t.Requests, k) 73 | } 74 | } 75 | 76 | func (t *Throttler) SetWhitelist(ips []string) { 77 | for _, ip := range ips { 78 | t.Whitelist[ip] = true 79 | } 80 | } 81 | 82 | func (t *Throttler) Whitelisted(ip string) bool { 83 | _, ok := t.Whitelist[ip] 84 | return ok 85 | } 86 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/hex" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func randomHex(n int) (string, error) { 11 | bytes := make([]byte, n) 12 | if _, err := rand.Read(bytes); err != nil { 13 | return "", err 14 | } 15 | return hex.EncodeToString(bytes), nil 16 | } 17 | 18 | func expandPath(path string) string { 19 | if strings.Index(path, "~") == 0 { 20 | path = strings.Replace(path, "~", os.Getenv("HOME"), 1) 21 | } 22 | 23 | return path 24 | } 25 | --------------------------------------------------------------------------------