├── CODEOWNERS ├── etc └── examples │ ├── python.py │ └── java.java ├── MAINTAINERS ├── Dockerfile.drivers ├── runtime ├── fixture │ ├── Dockerfile │ └── manifest.toml ├── image_formats_ostree.go ├── doc.go ├── image_formats.go ├── parse_image_name.go ├── common.go ├── config.go ├── common_test.go ├── image_test.go ├── config_test.go ├── image.go ├── container.go ├── container_test.go ├── storage_test.go ├── unpack.go ├── storage.go └── runtime.go ├── selinux ├── compile.sh └── bblfshd.te ├── cmd ├── bblfshctl │ ├── cmd │ │ ├── driver.go │ │ ├── parse.go │ │ ├── driver_remove.go │ │ ├── instances.go │ │ ├── base.go │ │ ├── status.go │ │ ├── driver_list.go │ │ └── driver_install.go │ └── main.go └── bblfshd │ └── main.go ├── .github └── workflows │ ├── enforce-labels.yaml │ ├── check.yaml │ ├── release_tags.yaml │ ├── release.yaml │ └── create_release.yaml ├── daemon ├── protocol │ ├── stringer.go │ ├── service_test.go │ ├── types.go │ ├── service.go │ └── generated.proto ├── language_test.go ├── language.go ├── errors.go ├── driver_test.go ├── daemon_test.go ├── service_test.go ├── common_test.go ├── metrics.go ├── driver.go ├── pool_test.go ├── daemon.go └── service.go ├── .gitignore ├── DCO ├── .travis.yml ├── Dockerfile ├── go.mod ├── rootless.md ├── Makefile ├── README.md └── bblfshd-seccomp.json /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @bblfsh/maintainers 2 | -------------------------------------------------------------------------------- /etc/examples/python.py: -------------------------------------------------------------------------------- 1 | print("Hello, World!") 2 | -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Denys Smirnov (@dennwc) 2 | -------------------------------------------------------------------------------- /Dockerfile.drivers: -------------------------------------------------------------------------------- 1 | ARG TAG 2 | FROM bblfsh/bblfshd:${TAG} 3 | 4 | RUN bblfshd install recommended -------------------------------------------------------------------------------- /runtime/fixture/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM busybox:latest 2 | 3 | ADD manifest.toml /opt/driver/etc/ 4 | -------------------------------------------------------------------------------- /runtime/image_formats_ostree.go: -------------------------------------------------------------------------------- 1 | // +build ostree 2 | 3 | package runtime 4 | 5 | import ( 6 | _ "github.com/containers/image/ostree" 7 | ) 8 | -------------------------------------------------------------------------------- /etc/examples/java.java: -------------------------------------------------------------------------------- 1 | public class HelloWorld { 2 | public static void main(String[] args) { 3 | System.out.println("Hello, World"); 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /selinux/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | checkmodule -M -m -o bblfshd.mod bblfshd.te && \ 4 | semodule_package -o bblfshd.pp -m bblfshd.mod && \ 5 | echo 'Module compiled, load it with "semodule -i bblfshd.pp"' 6 | -------------------------------------------------------------------------------- /runtime/fixture/manifest.toml: -------------------------------------------------------------------------------- 1 | language = "fixture" 2 | status = "beta" 3 | version = "42" 4 | build = 2015-10-21T04:29:00Z 5 | commit = "424242+" 6 | 7 | [runtime] 8 | os = "alpine" 9 | go_version = "1.8" 10 | native_version = ["3.6.2"] 11 | -------------------------------------------------------------------------------- /runtime/doc.go: -------------------------------------------------------------------------------- 1 | // Package runtime provide the runtime environment to execute the bblfsh drivers 2 | // 3 | // The runtime is based on libcontainer allowing to the runtime run the drivers 4 | // inside of a isolated lightweight container. 5 | package runtime 6 | -------------------------------------------------------------------------------- /selinux/bblfshd.te: -------------------------------------------------------------------------------- 1 | 2 | module bblfshd 1.0; 3 | 4 | require { 5 | type container_runtime_t; 6 | type spc_t; 7 | class fifo_file setattr; 8 | } 9 | 10 | #============= spc_t ============== 11 | allow spc_t container_runtime_t:fifo_file setattr; 12 | -------------------------------------------------------------------------------- /runtime/image_formats.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | _ "github.com/containers/image/directory" 5 | _ "github.com/containers/image/docker" 6 | _ "github.com/containers/image/docker/archive" 7 | _ "github.com/containers/image/docker/daemon" 8 | _ "github.com/containers/image/oci/layout" 9 | ) 10 | -------------------------------------------------------------------------------- /cmd/bblfshctl/cmd/driver.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | const ( 4 | DriverCommandDescription = "Manage drivers: install, remove and list" 5 | DriverCommandHelp = DriverCommandDescription 6 | ) 7 | 8 | type DriverCommand struct { 9 | ControlCommand 10 | } 11 | 12 | func (*DriverCommand) Execute([]string) error { 13 | return nil 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/enforce-labels.yaml: -------------------------------------------------------------------------------- 1 | name: Enforce PR labels 2 | 3 | on: 4 | pull_request: 5 | types: [labeled, unlabeled, opened, edited, synchronize] 6 | jobs: 7 | enforce-label: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: yogevbd/enforce-label-action@2.1.0 11 | with: 12 | REQUIRED_LABELS_ANY: "major,minor,patch" 13 | REQUIRED_LABELS_ANY_DESCRIPTION: "Select at least one label ['major','minor','patch']" 14 | BANNED_LABELS: "banned" -------------------------------------------------------------------------------- /daemon/protocol/stringer.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=Status -output stringer.go"; DO NOT EDIT. 2 | 3 | package protocol 4 | 5 | import "fmt" 6 | 7 | const _Status_name = "CreatedRunningPausingPausedStopped" 8 | 9 | var _Status_index = [...]uint8{0, 7, 14, 21, 27, 34} 10 | 11 | func (i Status) String() string { 12 | if i < 0 || i >= Status(len(_Status_index)-1) { 13 | return fmt.Sprintf("Status(%d)", i) 14 | } 15 | return _Status_name[_Status_index[i]:_Status_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | 3 | 4 | build 5 | coverage.txt 6 | /ostree 7 | 8 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 9 | *.o 10 | *.a 11 | *.so 12 | 13 | # Folders 14 | _obj 15 | _test 16 | .docsrv-resources 17 | 18 | # Architecture specific extensions/prefixes 19 | *.[568vq] 20 | [568vq].out 21 | 22 | *.cgo1.go 23 | *.cgo2.c 24 | _cgo_defun.c 25 | _cgo_gotypes.go 26 | _cgo_export.* 27 | 28 | _testmain.go 29 | 30 | *.exe 31 | *.test 32 | *.prof 33 | bblfshd 34 | bblfshctl 35 | -------------------------------------------------------------------------------- /.github/workflows/check.yaml: -------------------------------------------------------------------------------- 1 | name: Check 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | check: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Git Checkout 14 | uses: actions/checkout@v2 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.19 20 | 21 | - name: Test 22 | run: go test -v -coverprofile=coverage.txt -covermode=atomic ./... 23 | 24 | - name: Build 25 | run: go build -v ./... 26 | 27 | - name: coverage 28 | uses: codecov/codecov-action@v1 29 | with: 30 | token: ${{ secrets.CODECOV_TOKEN }} 31 | flags: unittests 32 | fail_ci_if_error: false # optional (default = false) 33 | verbose: false # optional (default = false) 34 | 35 | -------------------------------------------------------------------------------- /runtime/parse_image_name.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/containers/image/transports" 8 | "github.com/containers/image/types" 9 | "gopkg.in/src-d/go-errors.v1" 10 | ) 11 | 12 | var ErrInvalidImageName = errors.NewKind("invalid image name %q: %s") 13 | 14 | // ParseImageName converts a URL-like image name to a types.ImageReference. 15 | func ParseImageName(imgName string) (types.ImageReference, error) { 16 | // Copied from github.com/containers/image/transports/alltransports.go 17 | parts := strings.SplitN(imgName, ":", 2) 18 | if len(parts) != 2 { 19 | return nil, ErrInvalidImageName.New(imgName, "expected colon-separated transport:reference") 20 | } 21 | 22 | transport := transports.Get(parts[0]) 23 | if transport == nil { 24 | return nil, ErrInvalidImageName.New(imgName, fmt.Sprintf("unknown transport %q", parts[0])) 25 | } 26 | 27 | return transport.ParseReference(parts[1]) 28 | } 29 | -------------------------------------------------------------------------------- /daemon/language_test.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestGetLanguage(t *testing.T) { 10 | require := require.New(t) 11 | 12 | require.Equal("java", GetLanguage("foo.java", []byte(` 13 | package foo; 14 | `))) 15 | 16 | require.Equal("java", GetLanguage("", []byte(` 17 | // -*- java -*- 18 | package foo; 19 | import foo.bar; 20 | class Foo { 21 | public Foo() { } 22 | public int foo() { 23 | return 0; 24 | } 25 | } 26 | `))) 27 | 28 | require.Equal("cpp", GetLanguage("", []byte(` 29 | // -*- C++ -*- 30 | package foo; 31 | import foo.bar; 32 | class Foo { 33 | public Foo() { } 34 | public int foo() { 35 | return 0; 36 | } 37 | } 38 | `))) 39 | 40 | require.Equal("csharp", GetLanguage("", []byte(` 41 | // -*- C# -*- 42 | package foo; 43 | import foo.bar; 44 | class Foo { 45 | public Foo() { } 46 | public int foo() { 47 | return 0; 48 | } 49 | } 50 | `))) 51 | } 52 | -------------------------------------------------------------------------------- /runtime/common.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "fmt" 7 | "math/rand" 8 | "sync" 9 | "time" 10 | 11 | "github.com/oklog/ulid" 12 | ) 13 | 14 | type Digest []byte 15 | 16 | func NewDigest(s string) Digest { 17 | b, _ := hex.DecodeString(s) 18 | 19 | return Digest(b) 20 | } 21 | 22 | func ComputeDigest(input ...string) Digest { 23 | h := sha256.New() 24 | for _, i := range input { 25 | h.Write([]byte(i)) 26 | } 27 | 28 | return Digest(h.Sum(nil)) 29 | } 30 | 31 | func (d Digest) IsZero() bool { 32 | return len(d) == 0 33 | } 34 | 35 | func (d Digest) String() string { 36 | return fmt.Sprintf("%x", []byte(d)) 37 | } 38 | 39 | var randPool = &sync.Pool{ 40 | New: func() interface{} { 41 | return rand.NewSource(time.Now().UnixNano()) 42 | }, 43 | } 44 | 45 | // NewULID returns a new ULID, which is a lexically sortable UUID. 46 | func NewULID() ulid.ULID { 47 | entropy := randPool.Get().(rand.Source) 48 | id := ulid.MustNew(ulid.Timestamp(time.Now()), rand.New(entropy)) 49 | randPool.Put(entropy) 50 | 51 | return id 52 | } 53 | -------------------------------------------------------------------------------- /daemon/language.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/src-d/enry/v2" 8 | ) 9 | 10 | // GetLanguage detects the language of a file and returns it in a normalized 11 | // form. 12 | func GetLanguage(filename string, content []byte) string { 13 | totalEnryCalls.Add(1) 14 | defer prometheus.NewTimer(enryDetectLatency).ObserveDuration() 15 | 16 | lang := enry.GetLanguage(filename, content) 17 | if lang == enry.OtherLanguage { 18 | enryOtherResults.Add(1) 19 | return lang 20 | } 21 | enryLangResults.WithLabelValues(lang).Add(1) 22 | lang = normalize(lang) 23 | return lang 24 | } 25 | 26 | // normalize maps enry language names to the bblfsh ones. 27 | // TODO(bzz): remove this as soon as language aliases are supported in bblfsh 28 | // driver manifest. 29 | func normalize(languageName string) string { 30 | lang := strings.ToLower(languageName) 31 | lang = strings.Replace(lang, " ", "-", -1) 32 | lang = strings.Replace(lang, "+", "p", -1) 33 | lang = strings.Replace(lang, "#", "sharp", -1) 34 | return lang 35 | } 36 | -------------------------------------------------------------------------------- /runtime/config.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | 7 | "github.com/opencontainers/image-spec/specs-go/v1" 8 | ) 9 | 10 | const configExt = ".json" 11 | 12 | // ImageConfig describes some basic information about the image. 13 | type ImageConfig struct { 14 | // ImageRef is the original image reference used to retrieve the image. 15 | ImageRef string `json:"image_ref"` 16 | v1.Image 17 | } 18 | 19 | func WriteImageConfig(config *ImageConfig, path string) error { 20 | f, err := os.Create(path + configExt) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | var cerr error 26 | defer func() { cerr = f.Close() }() 27 | 28 | enc := json.NewEncoder(f) 29 | if err := enc.Encode(config); err != nil { 30 | return err 31 | } 32 | 33 | return cerr 34 | } 35 | 36 | func ReadImageConfig(path string) (*ImageConfig, error) { 37 | f, err := os.Open(path + configExt) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | var cerr error 43 | defer func() { cerr = f.Close() }() 44 | 45 | dec := json.NewDecoder(f) 46 | config := &ImageConfig{} 47 | if err := dec.Decode(config); err != nil { 48 | return nil, err 49 | } 50 | 51 | return config, cerr 52 | } 53 | -------------------------------------------------------------------------------- /DCO: -------------------------------------------------------------------------------- 1 | Developer's Certificate of Origin 1.1 2 | 3 | By making a contribution to this project, I certify that: 4 | 5 | (a) The contribution was created in whole or in part by me and I 6 | have the right to submit it under the open source license 7 | indicated in the file; or 8 | 9 | (b) The contribution is based upon previous work that, to the best 10 | of my knowledge, is covered under an appropriate open source 11 | license and I have the right under that license to submit that 12 | work with modifications, whether created in whole or in part 13 | by me, under the same open source license (unless I am 14 | permitted to submit under a different license), as indicated 15 | in the file; or 16 | 17 | (c) The contribution was provided directly to me by some other 18 | person who certified (a), (b) or (c) and I have not modified 19 | it. 20 | 21 | (d) I understand and agree that this project and the contribution 22 | are public and that a record of the contribution (including all 23 | personal information I submit with it, including my sign-off) is 24 | maintained indefinitely and may be redistributed consistent with 25 | this project or the open source license(s) involved. 26 | -------------------------------------------------------------------------------- /daemon/errors.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "github.com/bblfsh/bblfshd/daemon/protocol" 5 | "github.com/bblfsh/sdk/v3/driver" 6 | "gopkg.in/src-d/go-errors.v1" 7 | ) 8 | 9 | var ( 10 | // ErrUnexpected indicates unexpexted unrecoverable error condition. 11 | ErrUnexpected = errors.NewKind("unexpected error") 12 | // ErrRuntime indicates unrecoverable error condition at runtime. 13 | ErrRuntime = errors.NewKind("runtime failure") 14 | // ErrAlreadyInstalled indicates that a driver image was already installed 15 | // from the reference for the given language. 16 | ErrAlreadyInstalled = protocol.ErrAlreadyInstalled 17 | // ErrUnauthorized indicates that image registry access failed 18 | // and it either requires authentication or does not exist. 19 | ErrUnauthorized = errors.NewKind("unauthorized: authentication required to access %s (image: %s)") 20 | // ErrLanguageDetection indicates that language was not detected by Enry. 21 | ErrLanguageDetection = driver.ErrLanguageDetection 22 | // ErrUnknownEncoding is returned for parse requests with a file content in a non-UTF8 encoding. 23 | ErrUnknownEncoding = driver.ErrUnknownEncoding 24 | ) 25 | 26 | // ErrMissingDriver indicates that a driver image for the given language 27 | // can not be found. 28 | type ErrMissingDriver = driver.ErrMissingDriver 29 | -------------------------------------------------------------------------------- /cmd/bblfshctl/cmd/parse.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | bblfsh "github.com/bblfsh/go-client/v4" 8 | "github.com/bblfsh/sdk/v3/uast/uastyaml" 9 | ) 10 | 11 | const ( 12 | ParseCommandDescription = "Parse a file and prints the UAST or AST" 13 | ParseCommandHelp = ParseCommandDescription 14 | ) 15 | 16 | type ParseCommand struct { 17 | Args struct { 18 | File string `positional-arg-name:"filename" description:"file to parse"` 19 | } `positional-args:"yes"` 20 | 21 | Native bool `long:"native" description:"if native, the native AST will be returned"` 22 | UserCommand 23 | } 24 | 25 | func (c *ParseCommand) Execute(args []string) error { 26 | if c.Args.File == "" { 27 | return errors.New("file argument is mandatory") 28 | } 29 | if err := c.UserCommand.Execute(nil); err != nil { 30 | return err 31 | } 32 | 33 | cli, err := bblfsh.NewClientWithConnection(c.conn) 34 | if err != nil { 35 | return err 36 | } 37 | defer cli.Close() 38 | 39 | req := cli.NewParseRequest().ReadFile(c.Args.File) 40 | if c.Native { 41 | req = req.Mode(bblfsh.Native) 42 | } 43 | 44 | ast, _, err := req.UAST() 45 | if err != nil { 46 | return err 47 | } 48 | data, err := uastyaml.Marshal(ast) 49 | if err != nil { 50 | return err 51 | } 52 | fmt.Println(string(data)) 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - '1.16.x' 5 | 6 | go_import_path: github.com/bblfsh/bblfshd 7 | 8 | env: 9 | - GO111MODULE=on 10 | 11 | sudo: required 12 | services: 13 | - docker 14 | 15 | before_install: 16 | - sudo apt-get -qq update 17 | - sudo apt-get -qq install btrfs-tools libdevmapper-dev libgpgme11-dev libapparmor-dev libseccomp-dev 18 | - sudo apt-get -qq install autoconf automake bison e2fslibs-dev libfuse-dev libtool liblzma-dev gettext 19 | - OSTREE_VERSION=v2017.9 20 | - git clone https://github.com/ostreedev/ostree ${TRAVIS_BUILD_DIR}/ostree 21 | - pushd ${TRAVIS_BUILD_DIR}/ostree 22 | - git checkout $OSTREE_VERSION 23 | - ./autogen.sh --prefix=/usr/local 24 | - make all 25 | - sudo make install 26 | - popd 27 | - make dependencies 28 | 29 | # there's a workaround here because there will be no tags in cron https://github.com/travis-ci/travis-ci/issues/8146 30 | script: 31 | - make test-coverage 32 | - | 33 | if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then 34 | if CRON_TAG="$( git describe --exact-match "$(git rev-parse HEAD)" --tags 2>/dev/null )"; then 35 | make push-drivers 36 | fi 37 | fi 38 | 39 | after_success: 40 | - bash <(curl -s https://codecov.io/bash) 41 | 42 | before_deploy: 43 | - make push 44 | - make packages 45 | - make push-drivers 46 | 47 | deploy: 48 | provider: releases 49 | api_key: 50 | secure: $GITHUB_TOKEN 51 | file_glob: true 52 | file: ./build/*.tar.gz 53 | skip_cleanup: true 54 | on: 55 | tags: true 56 | -------------------------------------------------------------------------------- /runtime/common_test.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/containers/image/types" 9 | "gopkg.in/bblfsh/sdk.v1/manifest" 10 | "gopkg.in/bblfsh/sdk.v1/sdk/driver" 11 | ) 12 | 13 | const FixtureReference = "docker-daemon:bblfsh/bblfshd:fixture" 14 | 15 | func init() { 16 | Bootstrap() 17 | } 18 | 19 | func IfNetworking(t *testing.T) { 20 | if len(os.Getenv("TEST_NETWORKING")) != 0 { 21 | return 22 | } 23 | 24 | t.Skip("skipping network test use TEST_NETWORKING to run this test") 25 | } 26 | 27 | type FixtureDriverImage struct { 28 | N string 29 | M *manifest.Manifest 30 | } 31 | 32 | func (d *FixtureDriverImage) Name() string { 33 | return d.N 34 | } 35 | 36 | func (d *FixtureDriverImage) Digest() (Digest, error) { 37 | return ComputeDigest(d.N), nil 38 | } 39 | 40 | func (d *FixtureDriverImage) Inspect() (*types.ImageInspectInfo, error) { 41 | return nil, nil 42 | } 43 | 44 | func (d *FixtureDriverImage) WriteTo(path string) error { 45 | if err := os.MkdirAll(path, 0755); err != nil { 46 | return err 47 | } 48 | 49 | if err := WriteImageConfig(&ImageConfig{ImageRef: d.N}, path); err != nil { 50 | return err 51 | } 52 | 53 | if d.M == nil { 54 | return nil 55 | } 56 | 57 | m := filepath.Join(path, driver.ManifestLocation) 58 | if err := os.MkdirAll(filepath.Dir(m), 0777); err != nil { 59 | return err 60 | } 61 | 62 | w, err := os.Create(m) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | defer w.Close() 68 | return d.M.Encode(w) 69 | } 70 | -------------------------------------------------------------------------------- /runtime/image_test.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestDriverImageName(t *testing.T) { 13 | require := require.New(t) 14 | 15 | d, err := NewDriverImage("docker://busybox:latest") 16 | 17 | require.NoError(err) 18 | require.Equal("busybox:latest", d.Name()) 19 | } 20 | 21 | func TestDriverImageDigest(t *testing.T) { 22 | require := require.New(t) 23 | IfNetworking(t) 24 | 25 | d, err := NewDriverImage("docker://smolav/busybox-test-image:latest") 26 | require.NoError(err) 27 | 28 | h, err := d.Digest() 29 | require.NoError(err) 30 | require.Equal("116d67f147f35850964c8c98231a3316623cb1de4d6fa29a12587b8882e69c4c", h.String()) 31 | } 32 | 33 | func TestDriverImageInspect(t *testing.T) { 34 | require := require.New(t) 35 | IfNetworking(t) 36 | 37 | d, err := NewDriverImage("docker://busybox:latest") 38 | require.NoError(err) 39 | 40 | i, err := d.Inspect() 41 | require.NoError(err) 42 | require.Equal("linux", i.Os) 43 | } 44 | 45 | func TestDriverImageWriteTo(t *testing.T) { 46 | require := require.New(t) 47 | IfNetworking(t) 48 | 49 | dir, err := ioutil.TempDir("", "core-driver-writeto") 50 | require.NoError(err) 51 | defer os.RemoveAll(dir) 52 | 53 | d, err := NewDriverImage("docker://busybox:latest") 54 | require.NoError(err) 55 | 56 | err = d.WriteTo(dir) 57 | require.NoError(err) 58 | 59 | _, err = os.Stat(filepath.Join(dir, "bin/busybox")) 60 | require.NoError(err) 61 | } 62 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Base builder image with all libraries installed, including the source of the project 2 | FROM golang:1.12 as builder 3 | 4 | RUN apt-get update && \ 5 | apt-get install -y --no-install-recommends \ 6 | libostree-dev \ 7 | libglib2.0-dev \ 8 | btrfs-tools \ 9 | && apt-get clean 10 | 11 | ENV GOPATH=/go 12 | ENV GO111MODULE=on 13 | WORKDIR /go/src/github.com/bblfsh/bblfshd 14 | 15 | ADD . . 16 | 17 | 18 | # Actual build image that compiles bblfshd and bblfshctl 19 | FROM builder as binbuild 20 | 21 | RUN mkdir /build 22 | 23 | ARG BBLFSHD_VERSION=dev 24 | ARG BBLFSHD_BUILD=unknown 25 | 26 | ENV GO_LDFLAGS="-X 'main.version=${BBLFSHD_VERSION}' -X 'main.build=${BBLFSHD_BUILD}'" 27 | 28 | RUN go build -tags ostree --ldflags "${GO_LDFLAGS}" -o /build/bblfshd ./cmd/bblfshd/ 29 | RUN go build --ldflags "${GO_LDFLAGS}" -o /build/bblfshctl ./cmd/bblfshctl/ 30 | 31 | 32 | # Final image for bblfshd 33 | FROM debian:stretch-slim 34 | 35 | RUN apt-get update && \ 36 | apt-get install -y --no-install-recommends --no-install-suggests \ 37 | ca-certificates \ 38 | libostree-1-1 \ 39 | && apt-get clean 40 | 41 | ENV TINI_VERSION v0.18.0 42 | ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini 43 | RUN chmod +x /tini 44 | 45 | COPY --from=binbuild build /opt/bblfsh/bin/ 46 | ADD etc /opt/bblfsh/etc/ 47 | ENV PATH="/opt/bblfsh/bin:${PATH}" 48 | 49 | # Run bblfshd under Tini (see https://github.com/krallin/tini/issues/8 for details) 50 | ENTRYPOINT ["/tini", "--", "bblfshd"] 51 | 52 | -------------------------------------------------------------------------------- /runtime/config_test.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/opencontainers/image-spec/specs-go/v1" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestWriteImageConfig(t *testing.T) { 14 | require := require.New(t) 15 | 16 | dir, err := ioutil.TempDir("", "") 17 | require.NoError(err) 18 | defer os.RemoveAll(dir) 19 | 20 | dir = filepath.Join(dir, "foo") 21 | err = WriteImageConfig(&ImageConfig{ 22 | ImageRef: "foo", 23 | Image: v1.Image{ 24 | Author: "foo", 25 | OS: "bar", 26 | }, 27 | }, dir) 28 | require.NoError(err) 29 | 30 | b, err := ioutil.ReadFile(dir + ".json") 31 | require.NoError(err) 32 | require.Equal("{\"image_ref\":\"foo\",\"author\":\"foo\",\"architecture\":\"\",\"os\":\"bar\",\"config\":{},\"rootfs\":{\"type\":\"\",\"diff_ids\":null}}\n", string(b)) 33 | } 34 | 35 | func TestReadImageConfig(t *testing.T) { 36 | require := require.New(t) 37 | 38 | dir, err := ioutil.TempDir("", "") 39 | require.NoError(err) 40 | defer os.RemoveAll(dir) 41 | dir = filepath.Join(dir, "qux") 42 | 43 | content := "{\"image_ref\":\"foo\",\"author\":\"foo\",\"architecture\":\"\",\"os\":\"bar\",\"config\":{},\"rootfs\":{\"type\":\"\",\"diff_ids\":null}}\n" 44 | err = ioutil.WriteFile(dir+".json", []byte(content), 0644) 45 | require.NoError(err) 46 | 47 | config, err := ReadImageConfig(dir) 48 | require.NoError(err) 49 | 50 | require.Equal(&ImageConfig{ 51 | ImageRef: "foo", 52 | Image: v1.Image{ 53 | Author: "foo", 54 | OS: "bar", 55 | }, 56 | }, config) 57 | 58 | } 59 | -------------------------------------------------------------------------------- /cmd/bblfshctl/cmd/driver_remove.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/bblfsh/bblfshd/daemon/protocol" 9 | ) 10 | 11 | const ( 12 | DriverRemoveCommandDescription = "Removes the driver for the specified language" 13 | DriverRemoveCommandHelp = DriverRemoveCommandDescription 14 | ) 15 | 16 | type DriverRemoveCommand struct { 17 | Args struct { 18 | Language string `positional-arg-name:"language" description:"language supported by the driver"` 19 | } `positional-args:"yes"` 20 | 21 | All bool `long:"all" description:"removes all the installed drivers"` 22 | 23 | DriverCommand 24 | } 25 | 26 | func (c *DriverRemoveCommand) Execute(args []string) error { 27 | ctx := context.Background() 28 | if err := c.ControlCommand.Execute(nil); err != nil { 29 | return err 30 | } 31 | 32 | langs := []string{c.Args.Language} 33 | if c.All { 34 | r, err := c.srv.DriverStates(ctx, &protocol.DriverStatesRequest{}) 35 | if err != nil || len(r.Errors) > 0 { 36 | for _, e := range r.Errors { 37 | fmt.Fprintf(os.Stderr, "Error, %s\n", e) 38 | } 39 | 40 | return err 41 | } 42 | 43 | langs = make([]string, len(r.State)) 44 | for i, s := range r.State { 45 | langs[i] = s.Language 46 | } 47 | } 48 | 49 | for _, lang := range langs { 50 | r, err := c.srv.RemoveDriver(ctx, &protocol.RemoveDriverRequest{Language: lang}) 51 | if err != nil { 52 | return err 53 | } else if len(r.Errors) != 0 { 54 | for _, e := range r.Errors { 55 | fmt.Fprintf(os.Stderr, "Error, %s\n", e) 56 | } 57 | return fmt.Errorf("driver remove failed: %v", r.Errors) 58 | } 59 | } 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /cmd/bblfshctl/cmd/instances.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | "github.com/bblfsh/bblfshd/daemon/protocol" 11 | 12 | "github.com/docker/go-units" 13 | "github.com/olekukonko/tablewriter" 14 | ) 15 | 16 | const ( 17 | InstancesCommandDescription = "List the driver instances running on the daemon" 18 | InstancesCommandHelp = InstancesCommandDescription 19 | ) 20 | 21 | type InstancesCommand struct { 22 | ControlCommand 23 | } 24 | 25 | func (c *InstancesCommand) Execute(args []string) error { 26 | if err := c.ControlCommand.Execute(nil); err != nil { 27 | return err 28 | } 29 | 30 | r, err := c.srv.DriverInstanceStates(context.Background(), &protocol.DriverInstanceStatesRequest{}) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | if err == nil && len(r.Errors) == 0 { 36 | instancesStatusToText(r) 37 | return nil 38 | } 39 | 40 | return err 41 | } 42 | 43 | func instancesStatusToText(r *protocol.DriverInstanceStatesResponse) { 44 | table := tablewriter.NewWriter(os.Stdout) 45 | table.SetHeader([]string{"Instance ID", "Driver", "Status", "Created", "PIDs"}) 46 | table.SetAlignment(tablewriter.ALIGN_LEFT) 47 | 48 | for _, s := range r.State { 49 | var pids []string 50 | for _, pid := range s.Processes { 51 | pids = append(pids, fmt.Sprintf("%d", pid)) 52 | } 53 | 54 | line := fmt.Sprintf("%s\t%s\t%s\t%s\t%s", 55 | s.ID[:10], s.Image, 56 | s.Status, 57 | units.HumanDuration(time.Since(s.Created)), 58 | strings.Join(pids, ","), 59 | ) 60 | 61 | table.Append(strings.Split(line, "\t")) 62 | } 63 | 64 | table.Render() 65 | fmt.Printf("Response time %s\n", r.Elapsed) 66 | } 67 | -------------------------------------------------------------------------------- /cmd/bblfshctl/cmd/base.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "time" 8 | 9 | "github.com/bblfsh/bblfshd/daemon/protocol" 10 | 11 | "google.golang.org/grpc" 12 | sdk "gopkg.in/bblfsh/sdk.v1/protocol" 13 | ) 14 | 15 | type ControlCommand struct { 16 | Network string `long:"ctl-network" default:"unix" description:"control server network type"` 17 | Address string `long:"ctl-address" default:"/var/run/bblfshctl.sock" description:"control server address to connect"` 18 | 19 | conn *grpc.ClientConn 20 | srv protocol.ProtocolServiceClient 21 | } 22 | 23 | func (c *ControlCommand) Execute(args []string) error { 24 | var err error 25 | c.conn, err = dialGRPC(c.Network, c.Address) 26 | c.srv = protocol.NewProtocolServiceClient(c.conn) 27 | return err 28 | } 29 | 30 | type UserCommand struct { 31 | Network string `long:"endpoint" default:"tcp" description:"server network type"` 32 | Address string `long:"address" default:"localhost:9432" description:"server address to connect"` 33 | 34 | conn *grpc.ClientConn 35 | srv sdk.ProtocolServiceClient 36 | } 37 | 38 | func (c *UserCommand) Execute(args []string) error { 39 | var err error 40 | c.conn, err = dialGRPC(c.Network, c.Address) 41 | c.srv = sdk.NewProtocolServiceClient(c.conn) 42 | return err 43 | } 44 | 45 | func dialGRPC(network, address string) (*grpc.ClientConn, error) { 46 | conn, err := grpc.Dial(address, 47 | grpc.WithDialer(func(addr string, t time.Duration) (net.Conn, error) { 48 | return net.DialTimeout(network, address, t) 49 | }), 50 | grpc.WithBlock(), 51 | grpc.WithTimeout(5*time.Second), 52 | grpc.WithInsecure(), 53 | ) 54 | if err == context.DeadlineExceeded { 55 | return nil, fmt.Errorf("failed to connect to %s (%s): timeout", address, network) 56 | } 57 | return conn, err 58 | } 59 | -------------------------------------------------------------------------------- /daemon/protocol/service_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | 8 | "github.com/docker/distribution/registry/api/errcode" 9 | "github.com/stretchr/testify/mock" 10 | "github.com/stretchr/testify/require" 11 | "google.golang.org/grpc/codes" 12 | "google.golang.org/grpc/status" 13 | ) 14 | 15 | type mockedServiceAnyOtherErr struct { 16 | mock.Mock 17 | Service 18 | } 19 | 20 | func (s *mockedServiceAnyOtherErr) InstallDriver(language string, image string, update bool) error { 21 | return errcode.Errors{errors.New("any other error")} 22 | } 23 | 24 | func TestServiceMockDaemon_InstallDriverFails(t *testing.T) { 25 | require := require.New(t) 26 | //given 27 | s := new(mockedServiceAnyOtherErr) 28 | ps := &protocolServiceServer{s} 29 | 30 | //when 31 | res, err := ps.InstallDriver(context.Background(), &InstallDriverRequest{}) 32 | 33 | //then 34 | require.Nil(res) 35 | require.Error(err) 36 | } 37 | 38 | type mockedServiceUnauthorizedErr struct { 39 | mock.Mock 40 | Service 41 | } 42 | 43 | func (s *mockedServiceUnauthorizedErr) InstallDriver(language string, image string, update bool) error { 44 | return errcode.Errors{ 45 | errcode.Error{Code: errcode.ErrorCodeDenied}, 46 | errcode.Error{Code: errcode.ErrorCodeUnauthorized}, 47 | } 48 | } 49 | 50 | func TestServiceMockDaemon_InstallNonexistentDriver(t *testing.T) { 51 | require := require.New(t) 52 | //given 53 | s := new(mockedServiceUnauthorizedErr) 54 | ps := &protocolServiceServer{s} 55 | 56 | //when 57 | res, err := ps.InstallDriver(context.Background(), &InstallDriverRequest{}) 58 | 59 | //then 60 | require.Nil(res) 61 | require.Error(err) 62 | 63 | st, ok := status.FromError(err) 64 | require.True(ok) 65 | 66 | require.Equal(codes.Unauthenticated, st.Code()) 67 | } 68 | -------------------------------------------------------------------------------- /cmd/bblfshctl/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/bblfsh/bblfshd/cmd/bblfshctl/cmd" 8 | 9 | "github.com/jessevdk/go-flags" 10 | ) 11 | 12 | var ( 13 | version = "undefined" 14 | build = "undefined" 15 | ) 16 | 17 | func main() { 18 | parser := flags.NewNamedParser("bblfshctl", flags.Default) 19 | parser.AddCommand("status", 20 | cmd.StatusCommandDescription, cmd.StatusCommandHelp, 21 | &cmd.StatusCommand{}, 22 | ) 23 | 24 | parser.AddCommand("instances", 25 | cmd.InstancesCommandDescription, cmd.InstancesCommandHelp, 26 | &cmd.InstancesCommand{}, 27 | ) 28 | 29 | parser.AddCommand("parse", 30 | cmd.ParseCommandDescription, cmd.ParseCommandHelp, 31 | &cmd.ParseCommand{}, 32 | ) 33 | 34 | c, _ := parser.AddCommand("driver", 35 | cmd.DriverCommandDescription, cmd.DriverCommandHelp, 36 | // go-flags won't propagate DriverCommand flags to sub-commands, 37 | // so we will expect all flags to be passed to the sub-command itself (c1 c2 --x), 38 | // and not to the parent (c1 --x c2) 39 | &struct{}{}, 40 | ) 41 | 42 | c.AddCommand("list", 43 | cmd.DriverListCommandDescription, cmd.DriverListCommandHelp, 44 | &cmd.DriverListCommand{}, 45 | ) 46 | 47 | c.AddCommand("install", 48 | cmd.DriverInstallCommandDescription, cmd.DriverInstallCommandHelp, 49 | &cmd.DriverInstallCommand{}, 50 | ) 51 | 52 | c.AddCommand("remove", 53 | cmd.DriverRemoveCommandDescription, cmd.DriverRemoveCommandHelp, 54 | &cmd.DriverRemoveCommand{}, 55 | ) 56 | 57 | if _, err := parser.Parse(); err != nil { 58 | if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp { 59 | os.Exit(0) 60 | } else { 61 | fmt.Println() 62 | parser.WriteHelp(os.Stdout) 63 | fmt.Printf("\nBuild information\n commit: %s\n date: %s\n", version, build) 64 | os.Exit(1) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /cmd/bblfshctl/cmd/status.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/bblfsh/bblfshd/daemon/protocol" 10 | 11 | "github.com/olekukonko/tablewriter" 12 | ) 13 | 14 | const ( 15 | StatusCommandDescription = "List all the pools of driver instances running" 16 | StatusCommandHelp = StatusCommandDescription + "\n\n" + 17 | "The drivers are started on-demand based on the load of the server, \n" + 18 | "this driver instances are organized in pools by language.\n\n" + 19 | "This command prints a list of the pools running on the daemon, with \n" + 20 | "the number of requests success and failed, the number of instances \n" + 21 | "current and desired, the number of request waiting to be handle and \n" + 22 | "the drivers existed with with a non-zero code." 23 | ) 24 | 25 | type StatusCommand struct { 26 | ControlCommand 27 | } 28 | 29 | func (c *StatusCommand) Execute(args []string) error { 30 | if err := c.ControlCommand.Execute(nil); err != nil { 31 | return err 32 | } 33 | 34 | r, err := c.srv.DriverPoolStates(context.Background(), &protocol.DriverPoolStatesRequest{}) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | if err == nil && len(r.Errors) == 0 { 40 | daemonStatusToText(r) 41 | return nil 42 | } 43 | 44 | return err 45 | } 46 | 47 | func daemonStatusToText(r *protocol.DriverPoolStatesResponse) { 48 | table := tablewriter.NewWriter(os.Stdout) 49 | table.SetHeader([]string{"Language", "Success/Failed", "State/Desired", "Waiting", "Exited"}) 50 | table.SetAlignment(tablewriter.ALIGN_LEFT) 51 | 52 | for language, s := range r.State { 53 | line := fmt.Sprintf("%s\t%d/%d\t%d/%d\t%d\t%d", language, 54 | s.Success, s.Errors, 55 | s.Running, s.Wanted, s.Waiting, s.Exited, 56 | ) 57 | table.Append(strings.Split(line, "\t")) 58 | } 59 | 60 | table.Render() 61 | fmt.Printf("Response time %s\n", r.Elapsed) 62 | } 63 | -------------------------------------------------------------------------------- /cmd/bblfshctl/cmd/driver_list.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | "github.com/bblfsh/bblfshd/daemon/protocol" 11 | 12 | "github.com/docker/go-units" 13 | "github.com/olekukonko/tablewriter" 14 | ) 15 | 16 | const ( 17 | DriverListCommandDescription = "List the installed drivers for each language" 18 | DriverListCommandHelp = DriverListCommandDescription 19 | ) 20 | 21 | type DriverListCommand struct { 22 | DriverCommand 23 | } 24 | 25 | func (c *DriverListCommand) Execute(args []string) error { 26 | if err := c.ControlCommand.Execute(nil); err != nil { 27 | return err 28 | } 29 | 30 | r, err := c.srv.DriverStates(context.Background(), &protocol.DriverStatesRequest{}) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | if err == nil && len(r.Errors) == 0 { 36 | driverStatusToText(r) 37 | return nil 38 | } 39 | 40 | printErrors(r.Errors) 41 | return err 42 | } 43 | 44 | func printErrors(errors []string) { 45 | if len(errors) != 0 { 46 | fmt.Println("Errors:") 47 | for _, err := range errors { 48 | fmt.Printf("\t- %s\n", err) 49 | } 50 | } 51 | } 52 | 53 | func driverStatusToText(r *protocol.DriverStatesResponse) { 54 | table := tablewriter.NewWriter(os.Stdout) 55 | table.SetHeader([]string{"Language", "Image", "Version", "Status", "Created", "Go", "Native"}) 56 | table.SetAlignment(tablewriter.ALIGN_LEFT) 57 | 58 | for _, s := range r.State { 59 | var native []string 60 | for _, v := range s.NativeVersion { 61 | native = append(native, fmt.Sprintf("%s", v)) 62 | } 63 | 64 | line := fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t%s\t%s", 65 | s.Language, s.Reference, s.Version, 66 | s.Status, units.HumanDuration(time.Since(s.Build)), 67 | s.GoVersion, strings.Join(native, ","), 68 | ) 69 | table.Append(strings.Split(line, "\t")) 70 | } 71 | 72 | table.Render() 73 | fmt.Printf("Response time %s\n", r.Elapsed) 74 | } 75 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/bblfsh/bblfshd 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 // indirect 7 | github.com/bblfsh/go-client/v4 v4.0.1 8 | github.com/bblfsh/sdk/v3 v3.3.1 9 | github.com/briandowns/spinner v0.0.0-20170614154858-48dbb65d7bd5 10 | github.com/cenkalti/backoff v2.1.1+incompatible 11 | github.com/containers/image v3.0.0+incompatible 12 | github.com/containers/storage v1.28.1 // indirect 13 | github.com/docker/distribution v2.8.2+incompatible 14 | github.com/docker/docker v20.10.24+incompatible // indirect 15 | github.com/docker/docker-credential-helpers v0.6.0 // indirect 16 | github.com/docker/go-units v0.4.0 17 | github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect 18 | github.com/gogo/protobuf v1.3.2 19 | github.com/golang/protobuf v1.5.0 20 | github.com/google/go-github v17.0.0+incompatible // indirect 21 | github.com/jessevdk/go-flags v1.4.0 22 | github.com/klauspost/compress v1.16.0 // indirect 23 | github.com/morikuni/aec v1.0.0 // indirect 24 | github.com/oklog/ulid v1.3.1 25 | github.com/olekukonko/tablewriter v0.0.0-20170925234030-a7a4c189eb47 26 | github.com/opencontainers/image-spec v1.0.2 27 | github.com/opencontainers/runc v1.1.5 28 | github.com/opentracing/opentracing-go v1.1.0 29 | github.com/ostreedev/ostree-go v0.0.0-20170727130318-80ab7dbb8986 // indirect 30 | github.com/pkg/errors v0.9.1 31 | github.com/prometheus/client_golang v1.11.1 32 | github.com/prometheus/common v0.26.0 33 | github.com/src-d/enry/v2 v2.0.0 34 | github.com/stretchr/testify v1.7.0 35 | github.com/uber/jaeger-client-go v2.16.0+incompatible 36 | github.com/uber/jaeger-lib v2.0.0+incompatible // indirect 37 | github.com/ulikunitz/xz v0.5.11 // indirect 38 | github.com/vbatts/tar-split v0.11.2 // indirect 39 | golang.org/x/net v0.7.0 40 | google.golang.org/grpc v1.30.0 41 | gopkg.in/bblfsh/sdk.v1 v1.17.0 42 | gopkg.in/src-d/go-errors.v1 v1.0.0 43 | gopkg.in/src-d/go-log.v1 v1.0.2 44 | ) 45 | -------------------------------------------------------------------------------- /.github/workflows/release_tags.yaml: -------------------------------------------------------------------------------- 1 | name: Release docker image 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v3 14 | with: 15 | fetch-depth: 0 16 | 17 | - name: Configure Git 18 | run: | 19 | git config user.name "$GITHUB_ACTOR" 20 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 21 | - name: Set up QEMU 22 | uses: docker/setup-qemu-action@v2 23 | - name: Set up Docker Buildx 24 | uses: docker/setup-buildx-action@v2 25 | - name: Login to DockerHub 26 | uses: docker/login-action@v2 27 | with: 28 | username: ${{ secrets.DOCKERHUB_USERNAME }} 29 | password: ${{ secrets.DOCKERHUB_TOKEN }} 30 | 31 | - name: Build variables 32 | id: build_var 33 | run: | 34 | echo ::set-output name=SOURCE_NAME::${GITHUB_REF#refs/*/} 35 | echo ::set-output name=SOURCE_BRANCH::${GITHUB_REF#refs/heads/} 36 | echo ::set-output name=SOURCE_TAG::${GITHUB_REF#refs/tags/} 37 | echo ::set-output name=BUILD_DATE::$(date -u +"%Y-%m-%dT%H:%M:%SZ") 38 | echo ::set-output name=PROJECT_URL::${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY} 39 | echo ::set-output name=VCS_REF::$GITHUB_SHA 40 | 41 | - name: Build and push Docker images 42 | id: docker_build 43 | uses: docker/build-push-action@v4 44 | with: 45 | push: true 46 | build-args: | 47 | VERSION=${{ steps.build_var.outputs.SOURCE_TAG }} 48 | VCS_REF=${{ steps.build_var.outputs.VCS_REF }} 49 | BUILD_DATE=${{ steps.build_var.outputs.BUILD_DATE }} 50 | PROJECT_URL=${{ steps.build_var.outputs.PROJECT_URL }} 51 | tags: | 52 | w6dio/bblfshd:${{ steps.build_var.outputs.SOURCE_TAG }} 53 | 54 | - name: Image digest 55 | run: echo ${{ steps.docker_build.outputs.digest }} 56 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release latest docker image 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v3 14 | with: 15 | fetch-depth: 0 16 | 17 | - name: Configure Git 18 | run: | 19 | git config user.name "$GITHUB_ACTOR" 20 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 21 | 22 | - name: Set up QEMU 23 | uses: docker/setup-qemu-action@v2 24 | 25 | - name: Set up Docker Buildx 26 | uses: docker/setup-buildx-action@v2 27 | 28 | - name: Login to DockerHub 29 | uses: docker/login-action@v2 30 | with: 31 | username: ${{ secrets.DOCKERHUB_USERNAME }} 32 | password: ${{ secrets.DOCKERHUB_TOKEN }} 33 | 34 | 35 | - name: Build variables 36 | id: build_var 37 | run: | 38 | echo ::set-output name=SOURCE_NAME::${GITHUB_REF#refs/*/} 39 | echo ::set-output name=SOURCE_BRANCH::${GITHUB_REF#refs/heads/} 40 | echo ::set-output name=SOURCE_TAG::${GITHUB_REF#refs/tags/} 41 | echo ::set-output name=BUILD_DATE::$(date -u +"%Y-%m-%dT%H:%M:%SZ") 42 | echo ::set-output name=PROJECT_URL::${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY} 43 | echo ::set-output name=VCS_REF::$GITHUB_SHA 44 | 45 | - name: Build and push Docker images 46 | id: docker_build 47 | uses: docker/build-push-action@v4 48 | with: 49 | push: true 50 | build-args: | 51 | VERSION=${{ steps.build_var.outputs.SOURCE_BRANCH }} 52 | VCS_REF=${{ steps.build_var.outputs.VCS_REF }} 53 | BUILD_DATE=${{ steps.build_var.outputs.BUILD_DATE }} 54 | PROJECT_URL=${{ steps.build_var.outputs.PROJECT_URL }} 55 | tags: | 56 | w6dio/bblfshd:latest 57 | 58 | - name: Image digest 59 | run: echo ${{ steps.docker_build.outputs.digest }} 60 | 61 | -------------------------------------------------------------------------------- /runtime/image.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/containers/image/image" 7 | "github.com/containers/image/types" 8 | ) 9 | 10 | // DriverImage represents a docker image of a driver 11 | type DriverImage interface { 12 | Name() string 13 | Digest() (Digest, error) 14 | Inspect() (*types.ImageInspectInfo, error) 15 | WriteTo(path string) error 16 | } 17 | 18 | type driverImage struct { 19 | imageRef string 20 | ref types.ImageReference 21 | } 22 | 23 | // NewDriverImage returns a new DriverImage from an image reference. 24 | // For Docker use `docker://bblfsh/rust-driver:latest`. 25 | func NewDriverImage(imageRef string) (DriverImage, error) { 26 | ref, err := ParseImageName(imageRef) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | return &driverImage{ 32 | imageRef: imageRef, 33 | ref: ref, 34 | }, nil 35 | } 36 | 37 | // Name returns the name of the driver image based on the image reference. 38 | func (d *driverImage) Name() string { 39 | return strings.TrimPrefix(d.ref.StringWithinTransport(), "//") 40 | } 41 | 42 | // Digest computes a digest based on the image layers. 43 | func (d *driverImage) Digest() (Digest, error) { 44 | img, err := d.image() 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | defer img.Close() 50 | i, err := img.Inspect() 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | return ComputeDigest(i.Layers...), nil 56 | } 57 | 58 | func (d *driverImage) Inspect() (*types.ImageInspectInfo, error) { 59 | img, err := d.image() 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | defer img.Close() 65 | return img.Inspect() 66 | } 67 | 68 | // WriteTo writes the image to disk at the given path. 69 | func (d *driverImage) WriteTo(path string) error { 70 | img, err := d.image() 71 | if err != nil { 72 | return err 73 | } 74 | 75 | defer img.Close() 76 | if err := UnpackImage(img, path); err != nil { 77 | return err 78 | } 79 | 80 | config, err := img.OCIConfig() 81 | if err != nil { 82 | return err 83 | } 84 | 85 | return WriteImageConfig(&ImageConfig{ 86 | Image: *config, 87 | ImageRef: d.imageRef, 88 | }, path) 89 | } 90 | 91 | func (d *driverImage) image() (types.Image, error) { 92 | raw, err := d.ref.NewImageSource(nil) 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | unparsedImage := image.UnparsedFromSource(raw) 98 | return image.FromUnparsedImage(unparsedImage) 99 | } 100 | -------------------------------------------------------------------------------- /.github/workflows/create_release.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | types: closed 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | if: github.event.pull_request.merged 9 | steps: 10 | - uses: actions/checkout@v3 11 | - name: Tag and prepare release 12 | id: tag_and_prepare_release 13 | uses: K-Phoen/semver-release-action@master 14 | with: 15 | release_branch: main 16 | env: 17 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 18 | - name: Upload release notes 19 | if: steps.tag_and_prepare_release.outputs.tag 20 | uses: Roang-zero1/github-create-release-action@v3 21 | with: 22 | created_tag: ${{ steps.tag_and_prepare_release.outputs.tag }} 23 | env: 24 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 25 | - name: Configure Git 26 | run: | 27 | git config user.name "$GITHUB_ACTOR" 28 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 29 | - name: Set up QEMU 30 | uses: docker/setup-qemu-action@v2 31 | - name: Set up Docker Buildx 32 | uses: docker/setup-buildx-action@v2 33 | - name: Login to DockerHub 34 | uses: docker/login-action@v2 35 | with: 36 | username: ${{ secrets.DOCKERHUB_USERNAME }} 37 | password: ${{ secrets.DOCKERHUB_TOKEN }} 38 | 39 | - name: Build variables 40 | id: build_var 41 | run: | 42 | echo ::set-output name=SOURCE_NAME::${GITHUB_REF#refs/*/} 43 | echo ::set-output name=SOURCE_BRANCH::${GITHUB_REF#refs/heads/} 44 | echo ::set-output name=SOURCE_TAG::${GITHUB_REF#refs/tags/} 45 | echo ::set-output name=BUILD_DATE::$(date -u +"%Y-%m-%dT%H:%M:%SZ") 46 | echo ::set-output name=PROJECT_URL::${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY} 47 | echo ::set-output name=VCS_REF::$GITHUB_SHA 48 | 49 | - name: Build and push Docker images 50 | id: docker_build 51 | uses: docker/build-push-action@v4 52 | with: 53 | push: true 54 | build-args: | 55 | VERSION=${{ steps.tag_and_prepare_release.outputs.tag }} 56 | VCS_REF=${{ steps.build_var.outputs.VCS_REF }} 57 | BUILD_DATE=${{ steps.build_var.outputs.BUILD_DATE }} 58 | PROJECT_URL=${{ steps.build_var.outputs.PROJECT_URL }} 59 | tags: | 60 | w6dio/bblfshd:${{ steps.tag_and_prepare_release.outputs.tag }} 61 | 62 | - name: Image digest 63 | run: echo ${{ steps.docker_build.outputs.digest }} 64 | -------------------------------------------------------------------------------- /rootless.md: -------------------------------------------------------------------------------- 1 | # Rootless 2 | 3 | ## Requirements 4 | 5 | Being able to run the rootless containers may require `sudo` access in some OSs to enable unprivileged containers 6 | support (Arch or Debian for example), as documented, for example, in the 7 | [buildah documentation](https://wiki.archlinux.org/index.php/Buildah#Enable_support_to_build_unprivileged_containers) 8 | or in the [usernetes documentation](https://github.com/rootless-containers/usernetes#distribution-specific-hint): 9 | 10 | ```sh 11 | # Only for the current session 12 | sudo sysctl kernel.unprivileged_userns_clone=1 13 | ``` 14 | 15 | ```sh 16 | # Enable the permission permanently 17 | echo "kernel.unprivileged_userns_clone=1" >> /etc/sysctl.conf 18 | sudo sysctl -p 19 | ``` 20 | 21 | ## Run bblfshd in non-privileged mode 22 | 23 | As documented in the [Docker docs](https://docs.docker.com/engine/security/seccomp/), 24 | the default security profile disables commands such as `unshare`, `mount` or `sethostname` inside containers 25 | (which are needed for example to spawn containers inside `bblfshd` and also to give a `Hostname` to each 26 | container to be an identified driver). Also, as documented in 27 | [libcontainer#1658](https://github.com/opencontainers/runc/issues/1658), 28 | there is a known bug with rootless containers inside another non-root container and the `/proc` mount / masking. 29 | Adding a volume `-v /proc:/newproc` would solve that problem. 30 | 31 | Therefore to run `bblfshd` in non privileged mode, this would suffice: 32 | 33 | 34 | ```sh 35 | docker run --name bblfshd \ 36 | -p 9432:9432 \ 37 | -v /var/lib/bblfshd:/var/lib/bblfshd \ 38 | -v /proc:/newproc \ 39 | --security-opt seccomp=unconfined \ 40 | bblfshd 41 | ``` 42 | 43 | A better (and recommended) confinement configuration, would be: 44 | 45 | ```sh 46 | docker run --name bblfshd \ 47 | -p 9432:9432 \ 48 | -v /var/lib/bblfshd:/var/lib/bblfshd \ 49 | -v /proc:/newproc \ 50 | --security-opt seccomp=./bblfshd-seccomp.json \ 51 | bblfshd 52 | ``` 53 | 54 | [`./bblfshd-seccomp.json`](./bblfshd-seccomp.json) file is a modification of 55 | [`default.json`](https://github.com/moby/moby/blob/master/profiles/seccomp/default.json) from Docker which allows 56 | the following syscalls inside `bblfshd` container: `mount, unshare, pivot_root, keyctl, umount2, sethostname`. 57 | 58 | ## Known bugs 59 | 60 | Running `bblfshd` in rootless mode, you may see, in `bblfshd` logs, warning messages such as: 61 | 62 | ``` 63 | level=warning msg="no such directory for freezer.state" 64 | ``` 65 | 66 | They do not have further repercussions and are due to the inability of a rootless container to manage `cgroups` 67 | for containers created inside them. 68 | 69 | 70 | -------------------------------------------------------------------------------- /daemon/driver_test.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "github.com/bblfsh/bblfshd/daemon/protocol" 11 | "github.com/bblfsh/bblfshd/runtime" 12 | 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func init() { 17 | runtime.Bootstrap() 18 | } 19 | 20 | func TestNewDriver(t *testing.T) { 21 | require := require.New(t) 22 | 23 | run, image, path := NewRuntime(t) 24 | defer os.RemoveAll(path) 25 | 26 | _, err := run.InstallDriver(image, false) 27 | require.NoError(err) 28 | 29 | i, err := NewDriverInstance(run, "foo", image, &Options{ 30 | LogLevel: "debug", 31 | LogFormat: "text", 32 | }) 33 | 34 | require.NoError(err) 35 | 36 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) 37 | defer cancel() 38 | 39 | err = i.Start(ctx) 40 | require.NoError(err) 41 | 42 | time.Sleep(50 * time.Millisecond) 43 | 44 | err = i.Stop() 45 | require.NoError(err) 46 | } 47 | 48 | func TestDriverInstance_State(t *testing.T) { 49 | require := require.New(t) 50 | 51 | run, image, path := NewRuntime(t) 52 | defer os.RemoveAll(path) 53 | 54 | i, err := NewDriverInstance(run, "foo", image, &Options{ 55 | LogLevel: "debug", 56 | LogFormat: "text", 57 | }) 58 | 59 | require.NoError(err) 60 | 61 | state, err := i.State() 62 | require.NoError(err) 63 | require.Equal(protocol.Stopped, state.Status) 64 | require.Len(state.Processes, 0) 65 | require.True(state.Created.IsZero()) 66 | 67 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) 68 | defer cancel() 69 | 70 | err = i.Start(ctx) 71 | require.NoError(err) 72 | defer func() { 73 | err = i.Stop() 74 | require.NoError(err) 75 | }() 76 | 77 | time.Sleep(50 * time.Millisecond) 78 | 79 | state, err = i.State() 80 | require.NoError(err) 81 | require.Equal(protocol.Running, state.Status) 82 | require.Len(state.Processes, 2) 83 | require.False(state.Created.IsZero()) 84 | } 85 | 86 | func NewRuntime(t *testing.T) (*runtime.Runtime, runtime.DriverImage, string) { 87 | IfNetworking(t) 88 | 89 | require := require.New(t) 90 | 91 | dir, err := ioutil.TempDir(os.TempDir(), "bblfsh-runtime") 92 | require.NoError(err) 93 | 94 | run := runtime.NewRuntime(dir) 95 | err = run.Init() 96 | require.NoError(err) 97 | 98 | image, err := runtime.NewDriverImage("docker://bblfsh/python-driver:experimental") 99 | require.NoError(err) 100 | 101 | _, err = run.InstallDriver(image, false) 102 | require.NoError(err) 103 | 104 | return run, image, dir 105 | } 106 | 107 | func IfNetworking(t *testing.T) { 108 | if len(os.Getenv("TEST_NETWORKING")) != 0 { 109 | return 110 | } 111 | 112 | t.Skip("skipping network test use TEST_NETWORKING to run this test") 113 | } 114 | -------------------------------------------------------------------------------- /daemon/protocol/types.go: -------------------------------------------------------------------------------- 1 | //go:generate proteus -f $GOPATH/src/ -p github.com/bblfsh/bblfshd/daemon/protocol --verbose 2 | //go:generate stringer -type=Status -output stringer.go 3 | 4 | package protocol 5 | 6 | import ( 7 | "time" 8 | ) 9 | 10 | // Status is the status of a driver instance. 11 | //proteus:generate 12 | type Status int 13 | 14 | const ( 15 | // Created the container exists but has not been run yet. 16 | Created Status = iota 17 | // Running the container exists and is running. 18 | Running 19 | // Pausing the container exists, it is in the process of being paused. 20 | Pausing 21 | // Paused the container exists, but all its processes are paused. 22 | Paused 23 | // Stopped the container does not have a created or running process. 24 | Stopped 25 | ) 26 | 27 | //proteus:generate 28 | type DriverPoolState struct { 29 | // Instances number of driver instances wanted. 30 | Wanted int `json:"wanted"` 31 | // Running number of driver instances running. 32 | Running int `json:"running"` 33 | // Waiting number of request waiting for a request be executed. 34 | Waiting int `json:"waiting"` 35 | // Success number of requests executed successfully. 36 | Success int `json:"success"` 37 | // Errors number of errors trying to process a request. 38 | Errors int `json:"errors"` 39 | // Exited number of drivers exited unexpectedly. 40 | Exited int `json:"exited"` 41 | } 42 | 43 | //proteus:generate 44 | type DriverInstanceState struct { 45 | // ID of the container executing the driver. 46 | ID string `json:"id"` 47 | // Image used by the container. 48 | Image string 49 | // Status current status of the driver. 50 | Status Status `json:"status"` 51 | // Create when the driver instances was created. 52 | Created time.Time `json:"created"` 53 | // Processes are the pids of the processes running inside of the container. 54 | Processes []int `json:"processes"` 55 | } 56 | 57 | //proteus:generate 58 | type DriverImageState struct { 59 | // Referene is the image reference from where retrieved. 60 | Reference string `json:"reference"` 61 | // This fields are from manifest.Manifest, due to some limitation of 62 | // proteus, can't be used directly. 63 | // Language of the driver. 64 | Language string `json:"language"` 65 | // Version of the driver. 66 | Version string `json:"version,omitempty"` 67 | // Build time at the compilation of the image. 68 | Build time.Time `json:"build,omitempty"` 69 | // Status is the development status of the driver (alpha, beta, etc) 70 | Status string `json:"status"` 71 | // OS is the linux distribution running on the driver container. 72 | // 73 | // Deprecated: see GoVersion and NativeVersion 74 | OS string `json:"os"` 75 | // Native version is the version of the compiler/interpreter being use in the 76 | // native side of the driver. 77 | NativeVersion []string `json:"native_version"` 78 | // Go version of the go runtime being use in the driver. 79 | GoVersion string `json:"go_version"` 80 | } 81 | -------------------------------------------------------------------------------- /runtime/container.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package runtime 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | 9 | "github.com/opencontainers/runc/libcontainer" 10 | "github.com/opencontainers/runc/libcontainer/configs" 11 | ) 12 | 13 | // Process defines the process to be executed inside of a container. 14 | type Process libcontainer.Process 15 | 16 | // Container represent a container created from a driver image. 17 | type Container interface { 18 | // Returns the ID of the container 19 | ID() string 20 | // Returns the current status of the container. 21 | Status() (libcontainer.Status, error) 22 | // State returns the current container's state information. 23 | State() (*libcontainer.State, error) 24 | // Returns the PIDs inside this container. The PIDs are in the namespace of the calling process. 25 | Processes() ([]int, error) 26 | // Signal sends the provided signal code to the running process in the container. 27 | Signal(sig os.Signal) error 28 | // Returns the current config of the container. 29 | Config() configs.Config 30 | Command 31 | } 32 | 33 | // Command represents the main command of a container. 34 | type Command interface { 35 | // Run starts the specified command and waits for it to complete. 36 | Run() error 37 | // Start starts the specified command but does not wait for it to complete. 38 | // The Wait method will return the exit code and release associated 39 | // resources once the command exits. 40 | Start() error 41 | // Wait waits for the command to exit. It must have been started by Start. 42 | Wait() error 43 | // Stop kills the container. 44 | Stop() error 45 | } 46 | 47 | func newContainer(c libcontainer.Container, p *Process, config *ImageConfig) Container { 48 | cp := libcontainer.Process(*p) 49 | return &container{ 50 | Container: c, 51 | process: &cp, 52 | config: config, 53 | } 54 | } 55 | 56 | type container struct { 57 | libcontainer.Container 58 | process *libcontainer.Process 59 | config *ImageConfig 60 | } 61 | 62 | func (c *container) Start() error { 63 | env := make([]string, len(c.config.Config.Env)) 64 | copy(env, c.config.Config.Env) 65 | c.process.Env = append(env, c.process.Env...) 66 | 67 | if err := c.Container.Run(c.process); err != nil { 68 | _ = c.Container.Destroy() 69 | return err 70 | } 71 | 72 | return nil 73 | } 74 | 75 | func (c *container) Wait() error { 76 | _, err := c.process.Wait() 77 | return err 78 | } 79 | 80 | func (c *container) Run() error { 81 | if err := c.Start(); err != nil { 82 | return err 83 | } 84 | 85 | return c.Wait() 86 | } 87 | 88 | func (c *container) Stop() error { 89 | // Running bblfshd as a rootless container requires to use 90 | // SIGKILL instead of SIGTERM or SIGINT to kill the process. 91 | // Otherwise it ignores the order 92 | if err := c.process.Signal(syscall.SIGKILL); err != nil { 93 | return err 94 | } 95 | // kills all the remaining processes 96 | if err := c.Signal(syscall.SIGKILL); err != nil { 97 | return err 98 | } 99 | return c.Destroy() 100 | } 101 | 102 | func (c *container) Signal(sig os.Signal) error { 103 | return c.Container.Signal(sig, true) 104 | 105 | } 106 | -------------------------------------------------------------------------------- /daemon/daemon_test.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "net" 8 | "os" 9 | "reflect" 10 | "sync" 11 | "testing" 12 | "time" 13 | 14 | "github.com/stretchr/testify/require" 15 | "google.golang.org/grpc" 16 | "gopkg.in/bblfsh/sdk.v1/protocol" 17 | 18 | "github.com/bblfsh/bblfshd/runtime" 19 | ) 20 | 21 | // actual date format used in bblfshd is different 22 | const testBuildDate = "2019-01-28T16:49:06+01:00" 23 | 24 | func TestDaemonState(t *testing.T) { 25 | require := require.New(t) 26 | 27 | s, tmp := buildMockedDaemon(t) 28 | defer os.RemoveAll(tmp) 29 | 30 | pool := s.Current() 31 | require.Len(pool, 1) 32 | require.NotNil(pool["python"]) 33 | } 34 | 35 | func TestDaemon_InstallDriver(t *testing.T) { 36 | require := require.New(t) 37 | 38 | s, tmp := buildMockedDaemon(t) 39 | defer os.RemoveAll(tmp) 40 | 41 | err := s.InstallDriver("go", "docker://bblfsh/go-driver:latest", false) 42 | require.Nil(err) 43 | err = s.InstallDriver("go", "docker://bblfsh/go-driver:latest", false) 44 | require.True(ErrAlreadyInstalled.Is(err)) 45 | err = s.InstallDriver("go", "docker://bblfsh/go-driver:latest", true) 46 | require.Nil(err) 47 | } 48 | 49 | func TestDaemon_InstallNonexistentDriver(t *testing.T) { 50 | require := require.New(t) 51 | s, tmp := buildMockedDaemon(t) 52 | defer os.RemoveAll(tmp) 53 | 54 | err := s.InstallDriver("", "docker://list", false) 55 | require.Error(err, "An error was expected") 56 | require.Equal("errcode.Errors", reflect.TypeOf(err).String()) 57 | } 58 | 59 | func TestDaemonParse_MockedDriverParallelClients(t *testing.T) { 60 | require := require.New(t) 61 | 62 | d, tmp := buildMockedDaemon(t) 63 | defer os.RemoveAll(tmp) 64 | 65 | lis, err := net.Listen("tcp", "localhost:0") 66 | require.NoError(err) 67 | go d.UserServer.Serve(lis) 68 | defer func() { 69 | err = d.Stop() 70 | require.NoError(err) 71 | }() 72 | 73 | var wg sync.WaitGroup 74 | for i := 0; i < 20; i++ { 75 | i := i 76 | wg.Add(1) 77 | 78 | go func() { 79 | defer wg.Done() 80 | 81 | conn, err := grpc.Dial(lis.Addr().String(), 82 | grpc.WithBlock(), 83 | grpc.WithInsecure(), 84 | grpc.WithTimeout(2*time.Second), 85 | ) 86 | require.NoError(err) 87 | defer conn.Close() 88 | 89 | client := protocol.NewProtocolServiceClient(conn) 90 | var iwg sync.WaitGroup 91 | for j := 0; j < 50; j++ { 92 | iwg.Add(1) 93 | j := j 94 | go func() { 95 | defer iwg.Done() 96 | content := fmt.Sprintf("# -*- python -*-\nimport foo%d_%d", i, j) 97 | resp, err := client.Parse(context.TODO(), &protocol.ParseRequest{Content: content}) 98 | require.NoError(err) 99 | require.Equal(protocol.Ok, resp.Status, "%s: %v", resp.Status, resp.Errors) 100 | require.Equal(content, resp.UAST.Token) 101 | }() 102 | } 103 | iwg.Wait() 104 | 105 | err = conn.Close() 106 | require.NoError(err) 107 | }() 108 | } 109 | 110 | wg.Wait() 111 | } 112 | 113 | func buildMockedDaemon(t *testing.T, images ...runtime.DriverImage) (*Daemon, string) { 114 | require := require.New(t) 115 | 116 | dir, err := ioutil.TempDir(os.TempDir(), "bblfsh-runtime") 117 | require.NoError(err) 118 | 119 | r := runtime.NewRuntime(dir) 120 | err = r.Init() 121 | require.NoError(err) 122 | 123 | if images != nil { 124 | for _, image := range images { 125 | status, err := r.InstallDriver(image, false) 126 | require.NotNil(status) 127 | require.NoError(err) 128 | } 129 | } 130 | 131 | parsedBuild, err := time.Parse(time.RFC3339, testBuildDate) 132 | d := NewDaemon("foo", parsedBuild, r) 133 | 134 | dp := NewDriverPool(func(ctx context.Context) (Driver, error) { 135 | return newEchoDriver(), nil 136 | }) 137 | 138 | err = dp.Start(context.Background()) 139 | require.NoError(err) 140 | 141 | d.pool["python"] = dp 142 | 143 | return d, dir 144 | } 145 | -------------------------------------------------------------------------------- /daemon/service_test.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | "gopkg.in/bblfsh/sdk.v1/protocol" 11 | ) 12 | 13 | func TestHash(t *testing.T) { 14 | const data = "abc\n" 15 | require.Equal(t, "03cfd743661f07975fa2f1220c5194cbaff48451", hashSHA1(data)) 16 | require.Equal(t, "8baef1b4abc478178b004d62031cf7fe6db6f903", hashGit(data)) 17 | } 18 | 19 | func TestServiceParse(t *testing.T) { 20 | require := require.New(t) 21 | 22 | d, tmp := buildMockedDaemon(t) 23 | defer os.RemoveAll(tmp) 24 | 25 | s := NewService(d) 26 | resp := s.Parse(&protocol.ParseRequest{Filename: "foo.py", Content: "foo"}) 27 | require.Len(resp.Errors, 0) 28 | require.Equal("foo", resp.UAST.Token) 29 | require.Equal("python", resp.Language) 30 | require.True(resp.Elapsed.Nanoseconds() > 0) 31 | } 32 | 33 | // TODO(dennwc): Add test cases for V2 34 | func TestServiceParseV1(t *testing.T) { 35 | require := require.New(t) 36 | 37 | d, tmp := buildMockedDaemon(t) 38 | defer os.RemoveAll(tmp) 39 | 40 | s := NewService(d) 41 | req := &protocol.ParseRequest{Filename: "foo.py", Content: "foo"} 42 | lang, dp, err := s.selectPool(context.TODO(), req.Language, req.Content, req.Filename) 43 | require.NoError(err) 44 | require.Equal("python", lang) 45 | 46 | resp := &protocol.ParseResponse{} 47 | err = dp.Execute(func(ctx context.Context, driver Driver) error { 48 | ctx, cancel := context.WithCancel(ctx) 49 | cancel() // simulate context.Done 50 | 51 | // because we have a parseKillDelay ultimately we'll get response without errors 52 | resp, err = parseV1(ctx, dp, driver, req) 53 | return err 54 | }, req.Timeout) 55 | 56 | require.NoError(err) 57 | require.Len(resp.Errors, 0) 58 | require.Equal("foo", resp.UAST.Token) 59 | } 60 | 61 | func TestServiceNativeParse(t *testing.T) { 62 | require := require.New(t) 63 | 64 | d, tmp := buildMockedDaemon(t) 65 | defer os.RemoveAll(tmp) 66 | 67 | s := NewService(d) 68 | resp := s.NativeParse(&protocol.NativeParseRequest{Filename: "foo.py", Content: "foo"}) 69 | require.Len(resp.Errors, 0) 70 | require.Equal(resp.AST, "foo") 71 | require.Equal(resp.Language, "python") 72 | require.True(resp.Elapsed.Nanoseconds() > 0) 73 | } 74 | 75 | func TestServiceVersion(t *testing.T) { 76 | require := require.New(t) 77 | 78 | d, tmp := buildMockedDaemon(t) 79 | defer os.RemoveAll(tmp) 80 | 81 | s := NewService(d) 82 | resp := s.Version(&protocol.VersionRequest{}) 83 | require.Len(resp.Errors, 0) 84 | require.Equal(resp.Version, "foo") 85 | 86 | bdate, err := time.Parse(time.RFC3339, testBuildDate) 87 | require.NoError(err) 88 | require.Equal(resp.Build, bdate) 89 | } 90 | 91 | func TestControlServiceDriverPoolStates(t *testing.T) { 92 | require := require.New(t) 93 | 94 | d, tmp := buildMockedDaemon(t) 95 | defer os.RemoveAll(tmp) 96 | 97 | s := NewControlService(d) 98 | state := s.DriverPoolStates() 99 | require.Len(state, 1) 100 | require.Equal(state["python"].Running, 1) 101 | } 102 | 103 | func TestControlServiceDriverInstanceStates(t *testing.T) { 104 | require := require.New(t) 105 | 106 | d, tmp := buildMockedDaemon(t) 107 | defer os.RemoveAll(tmp) 108 | 109 | s := NewControlService(d) 110 | state, err := s.DriverInstanceStates() 111 | require.NoError(err) 112 | require.Len(state, 1) 113 | } 114 | 115 | func TestService_SupportedLanguages(t *testing.T) { 116 | require := require.New(t) 117 | 118 | d, tmp := buildMockedDaemon(t, newMockDriverImage("language-1"), newMockDriverImage("language-2")) 119 | defer os.RemoveAll(tmp) 120 | 121 | s := NewService(d) 122 | languages := s.SupportedLanguages(&protocol.SupportedLanguagesRequest{}) 123 | require.Len(languages.Errors, 0) 124 | require.Len(languages.Languages, 2) 125 | 126 | supportedLanguages := make([]string, 2) 127 | for i, lang := range languages.Languages { 128 | supportedLanguages[i] = lang.Name 129 | } 130 | 131 | require.Contains(supportedLanguages, "language-1") 132 | require.Contains(supportedLanguages, "language-2") 133 | } 134 | -------------------------------------------------------------------------------- /runtime/container_test.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | "github.com/stretchr/testify/suite" 12 | ) 13 | 14 | type ContainerSuite struct { 15 | suite.Suite 16 | RuntimePath string 17 | Runtime *Runtime 18 | Image DriverImage 19 | } 20 | 21 | func TestContainerSuite(t *testing.T) { 22 | suite.Run(t, new(ContainerSuite)) 23 | } 24 | 25 | func (s *ContainerSuite) SetupSuite() { 26 | require := require.New(s.T()) 27 | 28 | tmpDir, err := ioutil.TempDir(os.TempDir(), "bblfsh-runtime") 29 | require.NoError(err) 30 | s.RuntimePath = tmpDir 31 | 32 | rt := NewRuntime(s.RuntimePath) 33 | s.Runtime = rt 34 | err = rt.Init() 35 | require.NoError(err) 36 | 37 | d, err := NewDriverImage(FixtureReference) 38 | require.NoError(err) 39 | s.Image = d 40 | 41 | _, err = rt.InstallDriver(d, false) 42 | require.NoError(err) 43 | } 44 | 45 | func (s *ContainerSuite) TearDownSuite() { 46 | require := require.New(s.T()) 47 | require.NoError(os.RemoveAll(s.RuntimePath)) 48 | } 49 | 50 | func (s *ContainerSuite) TestContainer_Run() { 51 | require := require.New(s.T()) 52 | 53 | p := &Process{ 54 | Args: []string{"/bin/ls"}, 55 | Stdout: os.Stdout, 56 | Init: true, 57 | } 58 | 59 | c, err := s.Runtime.Container("run", s.Image, p, nil) 60 | require.NoError(err) 61 | 62 | err = c.Run() 63 | require.NoError(err) 64 | } 65 | 66 | func (s *ContainerSuite) TestContainer_StartStopStart() { 67 | require := require.New(s.T()) 68 | p := &Process{ 69 | Args: []string{"/bin/sleep", "5m"}, 70 | Stdout: os.Stdout, 71 | Init: true, 72 | } 73 | 74 | c, err := s.Runtime.Container("1", s.Image, p, nil) 75 | require.NoError(err) 76 | 77 | err = c.Start() 78 | require.NoError(err) 79 | 80 | time.Sleep(100 * time.Millisecond) 81 | err = c.Stop() 82 | require.NoError(err) 83 | 84 | p = &Process{ 85 | Args: []string{"/bin/sleep", "5m"}, 86 | Stdout: os.Stdout, 87 | Init: true, 88 | } 89 | 90 | c, err = s.Runtime.Container("2", s.Image, p, nil) 91 | require.NoError(err) 92 | 93 | err = c.Start() 94 | require.NoError(err) 95 | time.Sleep(100 * time.Millisecond) 96 | 97 | err = c.Stop() 98 | require.NoError(err) 99 | } 100 | 101 | func (s *ContainerSuite) TestContainer_StartWait() { 102 | require := require.New(s.T()) 103 | 104 | out := bytes.NewBuffer(nil) 105 | 106 | p := &Process{ 107 | Args: []string{"/bin/ls"}, 108 | Stdout: out, 109 | Init: true, 110 | } 111 | 112 | c, err := s.Runtime.Container("wait", s.Image, p, nil) 113 | require.NoError(err) 114 | 115 | err = c.Start() 116 | require.NoError(err) 117 | 118 | err = c.Wait() 119 | require.NoError(err) 120 | 121 | require.Equal("bin\ndev\netc\nhome\nopt\nproc\nroot\ntmp\nusr\nvar\n", out.String()) 122 | } 123 | 124 | func (s *ContainerSuite) TestContainer_StartWaitExit1() { 125 | require := require.New(s.T()) 126 | 127 | out := bytes.NewBuffer(nil) 128 | 129 | p := &Process{ 130 | Args: []string{"/bin/false"}, 131 | Stdout: out, 132 | Init: true, 133 | } 134 | 135 | c, err := s.Runtime.Container("wait-exit", s.Image, p, nil) 136 | require.NoError(err) 137 | 138 | err = c.Start() 139 | require.NoError(err) 140 | 141 | err = c.Wait() 142 | require.Error(err) 143 | 144 | require.Equal("", out.String()) 145 | } 146 | 147 | func (s *ContainerSuite) TestContainer_StartFailure() { 148 | require := require.New(s.T()) 149 | 150 | out := bytes.NewBuffer(nil) 151 | 152 | p := &Process{ 153 | Args: []string{"/bin/non-existent"}, 154 | Stdout: out, 155 | Init: true, 156 | } 157 | 158 | c, err := s.Runtime.Container("start-failure", s.Image, p, nil) 159 | require.NoError(err) 160 | 161 | err = c.Start() 162 | require.Error(err) 163 | } 164 | 165 | func (s *ContainerSuite) TestContainer_Env() { 166 | require := require.New(s.T()) 167 | 168 | out := bytes.NewBuffer(nil) 169 | 170 | p := &Process{ 171 | Args: []string{"/bin/env"}, 172 | Stdout: out, 173 | Init: true, 174 | } 175 | 176 | c, err := s.Runtime.Container("env", s.Image, p, nil) 177 | require.NoError(err) 178 | 179 | err = c.Run() 180 | require.NoError(err) 181 | require.Equal("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\nHOME=/root\n", out.String()) 182 | } 183 | -------------------------------------------------------------------------------- /runtime/storage_test.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | "gopkg.in/bblfsh/sdk.v1/manifest" 11 | ) 12 | 13 | func TestStorageInstall(t *testing.T) { 14 | require := require.New(t) 15 | 16 | dir, err := ioutil.TempDir("", "runtime-storage-install") 17 | require.NoError(err) 18 | defer os.RemoveAll(dir) 19 | 20 | d := &FixtureDriverImage{"//foo", &manifest.Manifest{Language: "Go"}} 21 | 22 | s := newStorage(filepath.Join(dir, "images"), filepath.Join(dir, "tmp")) 23 | m, err := s.Install(d, false) 24 | require.NoError(err) 25 | require.NotNil(m) 26 | require.Equal(m.Manifest.Language, "Go") 27 | } 28 | 29 | func TestStorageStatus(t *testing.T) { 30 | require := require.New(t) 31 | 32 | dir, err := ioutil.TempDir("", "runtime-storage-install") 33 | require.NoError(err) 34 | defer os.RemoveAll(dir) 35 | 36 | d := &FixtureDriverImage{"//foo", &manifest.Manifest{Language: "Go"}} 37 | 38 | s := newStorage(filepath.Join(dir, "images"), filepath.Join(dir, "tmp")) 39 | _, err = s.Install(d, false) 40 | require.NoError(err) 41 | 42 | status, err := s.Status(d) 43 | require.NoError(err) 44 | require.False(status.Digest.IsZero()) 45 | require.Equal("Go", status.Manifest.Language) 46 | require.Equal("//foo", status.Reference) 47 | } 48 | 49 | func TestStorageStatus_Dirty(t *testing.T) { 50 | require := require.New(t) 51 | 52 | dir, err := ioutil.TempDir("", "runtime-storage-status") 53 | require.NoError(err) 54 | defer os.RemoveAll(dir) 55 | 56 | d := &FixtureDriverImage{"//foo", &manifest.Manifest{Language: "Go"}} 57 | 58 | s := newStorage(filepath.Join(dir, "images"), filepath.Join(dir, "tmp")) 59 | _, err = s.Install(d, false) 60 | require.NoError(err) 61 | 62 | err = os.MkdirAll(filepath.Join(dir, "images", 63 | ComputeDigest("//foo").String(), 64 | ComputeDigest("bar").String(), 65 | ), 0777) 66 | require.NoError(err) 67 | 68 | di, err := s.Status(d) 69 | require.True(ErrDirtyDriverStorage.Is(err)) 70 | require.Nil(di) 71 | } 72 | 73 | func TestStorageStatus_NotInstalled(t *testing.T) { 74 | require := require.New(t) 75 | 76 | dir, err := ioutil.TempDir("", "runtime-storage-status-empty") 77 | require.NoError(err) 78 | defer os.RemoveAll(dir) 79 | 80 | d, err := NewDriverImage("docker://busybox:latest") 81 | require.NoError(err) 82 | 83 | s := newStorage(filepath.Join(dir, "images"), filepath.Join(dir, "tmp")) 84 | di, err := s.Status(d) 85 | require.True(ErrDriverNotInstalled.Is(err)) 86 | require.Nil(di) 87 | } 88 | 89 | func TestStorageRemove(t *testing.T) { 90 | require := require.New(t) 91 | 92 | dir, err := ioutil.TempDir("", "runtime-storage-remove") 93 | require.NoError(err) 94 | defer os.RemoveAll(dir) 95 | 96 | d := &FixtureDriverImage{"//foo", &manifest.Manifest{}} 97 | 98 | s := newStorage(filepath.Join(dir, "images"), filepath.Join(dir, "tmp")) 99 | _, err = s.Install(d, false) 100 | require.NoError(err) 101 | 102 | err = s.Remove(d) 103 | require.NoError(err) 104 | 105 | status, err := s.Status(d) 106 | require.True(ErrDriverNotInstalled.Is(err)) 107 | require.Nil(status) 108 | } 109 | 110 | func TestStorageRemove_Empty(t *testing.T) { 111 | require := require.New(t) 112 | 113 | dir, err := ioutil.TempDir("", "runtime-storage-remove-empty") 114 | require.NoError(err) 115 | defer os.RemoveAll(dir) 116 | 117 | d, err := NewDriverImage("docker://busybox:latest") 118 | require.NoError(err) 119 | 120 | s := newStorage(filepath.Join(dir, "images"), filepath.Join(dir, "tmp")) 121 | err = s.Remove(d) 122 | require.True(ErrDriverNotInstalled.Is(err)) 123 | } 124 | 125 | func TestStorageList(t *testing.T) { 126 | require := require.New(t) 127 | 128 | dir, err := ioutil.TempDir("", "runtime-storage-list") 129 | require.NoError(err) 130 | defer os.RemoveAll(dir) 131 | 132 | s := newStorage(filepath.Join(dir, "images"), filepath.Join(dir, "tmp")) 133 | 134 | _, err = s.Install(&FixtureDriverImage{"//foo", &manifest.Manifest{}}, false) 135 | require.NoError(err) 136 | 137 | _, err = s.Install(&FixtureDriverImage{"//bar/bar", &manifest.Manifest{}}, false) 138 | require.NoError(err) 139 | 140 | list, err := s.List() 141 | require.NoError(err) 142 | require.Len(list, 2) 143 | 144 | for _, status := range list { 145 | require.False(status.Digest.IsZero()) 146 | require.True(len(status.Reference) > 0) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /daemon/common_test.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "encoding/hex" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/bblfsh/bblfshd/daemon/protocol" 11 | "github.com/bblfsh/bblfshd/runtime" 12 | 13 | protocol2 "github.com/bblfsh/sdk/v3/protocol" 14 | "github.com/containers/image/types" 15 | oldctx "golang.org/x/net/context" 16 | "google.golang.org/grpc" 17 | "gopkg.in/bblfsh/sdk.v1/manifest" 18 | protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" 19 | "gopkg.in/bblfsh/sdk.v1/sdk/driver" 20 | "gopkg.in/bblfsh/sdk.v1/uast" 21 | ) 22 | 23 | type mockDriver struct { 24 | CalledClose int 25 | MockID string 26 | MockStatus protocol.Status 27 | } 28 | 29 | func newMockDriver(ctx context.Context) (Driver, error) { 30 | return &mockDriver{ 31 | MockID: runtime.NewULID().String(), 32 | MockStatus: protocol.Running, 33 | }, nil 34 | } 35 | 36 | func (d *mockDriver) ID() string { 37 | return d.MockID 38 | } 39 | 40 | func (d *mockDriver) Service() protocol1.ProtocolServiceClient { 41 | return nil 42 | } 43 | 44 | func (d *mockDriver) ServiceV2() protocol2.DriverClient { 45 | return nil 46 | } 47 | 48 | func (d *mockDriver) Start(ctx context.Context) error { 49 | return nil 50 | } 51 | 52 | func (d *mockDriver) Status() (protocol.Status, error) { 53 | return d.MockStatus, nil 54 | } 55 | 56 | func (d *mockDriver) State() (*protocol.DriverInstanceState, error) { 57 | return nil, nil 58 | } 59 | 60 | func (d *mockDriver) Stop() error { 61 | d.CalledClose++ 62 | return nil 63 | } 64 | 65 | func newEchoDriver() *echoDriver { 66 | d, _ := newMockDriver(context.Background()) 67 | return &echoDriver{ 68 | Driver: d, 69 | } 70 | } 71 | 72 | type echoDriver struct { 73 | Driver 74 | } 75 | 76 | func (d *echoDriver) NativeParse( 77 | _ oldctx.Context, in *protocol1.NativeParseRequest, opts ...grpc.CallOption) (*protocol1.NativeParseResponse, error) { 78 | return &protocol1.NativeParseResponse{ 79 | AST: in.Content, 80 | }, nil 81 | } 82 | 83 | func (d *echoDriver) Parse( 84 | _ oldctx.Context, in *protocol1.ParseRequest, opts ...grpc.CallOption) (*protocol1.ParseResponse, error) { 85 | return &protocol1.ParseResponse{ 86 | UAST: &uast.Node{ 87 | Token: in.Content, 88 | }, 89 | }, nil 90 | } 91 | 92 | func (d *echoDriver) Version( 93 | _ oldctx.Context, in *protocol1.VersionRequest, opts ...grpc.CallOption) (*protocol1.VersionResponse, error) { 94 | return &protocol1.VersionResponse{}, nil 95 | } 96 | 97 | func (d *echoDriver) SupportedLanguages( 98 | _ oldctx.Context, in *protocol1.SupportedLanguagesRequest, opts ...grpc.CallOption) (*protocol1.SupportedLanguagesResponse, error) { 99 | drivers := []protocol1.DriverManifest{protocol1.DriverManifest{Name: "Python"}} 100 | return &protocol1.SupportedLanguagesResponse{Languages: drivers}, nil 101 | } 102 | 103 | func (d *echoDriver) Service() protocol1.ProtocolServiceClient { 104 | return d 105 | } 106 | 107 | func newMockDriverImage(lang string) runtime.DriverImage { 108 | return &mockDriverImage{lang: lang} 109 | } 110 | 111 | type mockDriverImage struct { 112 | lang string 113 | } 114 | 115 | func (d *mockDriverImage) Name() string { 116 | return d.lang 117 | } 118 | 119 | func (d *mockDriverImage) Digest() (runtime.Digest, error) { 120 | return runtime.NewDigest(hex.EncodeToString([]byte(d.Name()))), nil 121 | } 122 | 123 | func (d *mockDriverImage) Inspect() (*types.ImageInspectInfo, error) { 124 | return &types.ImageInspectInfo{}, nil 125 | } 126 | 127 | func (d *mockDriverImage) WriteTo(path string) error { 128 | if err := writeManifest(d.Name(), path); err != nil { 129 | return err 130 | } 131 | 132 | return writeImageConfig(d.Name(), path) 133 | } 134 | 135 | func writeManifest(language, path string) error { 136 | manifest := manifest.Manifest{ 137 | Name: language, 138 | } 139 | 140 | manifestPath := filepath.Join(path, driver.ManifestLocation) 141 | manifestBaseDir := filepath.Dir(manifestPath) 142 | if err := os.MkdirAll(manifestBaseDir, 0755); err != nil { 143 | return err 144 | } 145 | 146 | manifestFile, err := os.Create(manifestPath) 147 | if err != nil { 148 | return err 149 | } 150 | defer manifestFile.Close() 151 | 152 | return manifest.Encode(manifestFile) 153 | } 154 | 155 | func writeImageConfig(language, path string) error { 156 | return runtime.WriteImageConfig(&runtime.ImageConfig{ 157 | ImageRef: fmt.Sprintf("%s-driver", language), 158 | }, path) 159 | } 160 | -------------------------------------------------------------------------------- /daemon/metrics.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | // Enry metrics 9 | var ( 10 | totalEnryCalls = promauto.NewCounter(prometheus.CounterOpts{ 11 | Name: "bblfshd_enry_total", 12 | Help: "The total number of calls to Enry to detect the language", 13 | }) 14 | enryLangResults = promauto.NewCounterVec(prometheus.CounterOpts{ 15 | Name: "bblfshd_enry_langs", 16 | Help: "The total number of Enry results for each programming language", 17 | }, []string{"lang"}) 18 | enryOtherResults = enryLangResults.WithLabelValues("other") 19 | enryDetectLatency = promauto.NewHistogram(prometheus.HistogramOpts{ 20 | Name: "bblfshd_enry_seconds", 21 | Help: "Time spent for detecting the language (seconds)", 22 | }) 23 | ) 24 | 25 | var ( 26 | driverLabelNames = []string{"lang", "image"} 27 | ) 28 | 29 | // Driver Control API metrics 30 | var ( 31 | driverInstallCalls = promauto.NewCounter(prometheus.CounterOpts{ 32 | Name: "bblfshd_driver_install_total", 33 | Help: "The total number of calls to install a driver", 34 | }) 35 | driverRemoveCalls = promauto.NewCounter(prometheus.CounterOpts{ 36 | Name: "bblfshd_driver_remove_total", 37 | Help: "The total number of calls to remove a driver", 38 | }) 39 | ) 40 | 41 | // Public API metrics 42 | // TODO(dennwc): add parse timeout metric 43 | var ( 44 | parseCalls = promauto.NewCounterVec(prometheus.CounterOpts{ 45 | Name: "bblfshd_parse_total", 46 | Help: "The total number of parse requests", 47 | }, []string{"vers"}) 48 | parseErrors = promauto.NewCounterVec(prometheus.CounterOpts{ 49 | Name: "bblfshd_parse_errors", 50 | Help: "The total number of failed parse requests", 51 | }, []string{"vers"}) 52 | parseLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ 53 | Name: "bblfshd_parse_seconds", 54 | Help: "Time spent on parse requests (seconds)", 55 | }, []string{"vers"}) 56 | parseContentSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ 57 | Name: "bblfshd_parse_bytes", 58 | Help: "Size of parsed files", 59 | }, []string{"vers"}) 60 | 61 | versionCalls = promauto.NewCounterVec(prometheus.CounterOpts{ 62 | Name: "bblfshd_version_total", 63 | Help: "The total number of version requests", 64 | }, []string{"vers"}) 65 | languagesCalls = promauto.NewCounterVec(prometheus.CounterOpts{ 66 | Name: "bblfshd_languages_total", 67 | Help: "The total number of supported languages requests", 68 | }, []string{"vers"}) 69 | 70 | versionCallsV1 = versionCalls.WithLabelValues("v1") 71 | languagesCallsV1 = languagesCalls.WithLabelValues("v1") 72 | parseCallsV1 = parseCalls.WithLabelValues("v1") 73 | parseErrorsV1 = parseErrors.WithLabelValues("v1") 74 | parseLatencyV1 = parseLatency.WithLabelValues("v1") 75 | parseContentSizeV1 = parseContentSize.WithLabelValues("v1") 76 | 77 | versionCallsV2 = versionCalls.WithLabelValues("v2") 78 | languagesCallsV2 = languagesCalls.WithLabelValues("v2") 79 | parseCallsV2 = parseCalls.WithLabelValues("v2") 80 | parseErrorsV2 = parseErrors.WithLabelValues("v2") 81 | parseLatencyV2 = parseLatency.WithLabelValues("v2") 82 | parseContentSizeV2 = parseContentSize.WithLabelValues("v2") 83 | ) 84 | 85 | // Scaling metrics 86 | var ( 87 | driversSpawned = promauto.NewCounterVec(prometheus.CounterOpts{ 88 | Name: "bblfshd_driver_spawn", 89 | Help: "The total number of driver spawn requests", 90 | }, driverLabelNames) 91 | driversSpawnErrors = promauto.NewCounterVec(prometheus.CounterOpts{ 92 | Name: "bblfshd_driver_spawn_errors", 93 | Help: "The total number of errors for driver spawn requests", 94 | }, driverLabelNames) 95 | driversKilled = promauto.NewCounterVec(prometheus.CounterOpts{ 96 | Name: "bblfshd_driver_kill", 97 | Help: "The total number of driver kill requests", 98 | }, driverLabelNames) 99 | 100 | driversRunning = promauto.NewGaugeVec(prometheus.GaugeOpts{ 101 | Name: "bblfshd_driver_scaling_total", 102 | Help: "The total number of drivers running", 103 | }, driverLabelNames) 104 | driversIdle = promauto.NewGaugeVec(prometheus.GaugeOpts{ 105 | Name: "bblfshd_driver_scaling_idle", 106 | Help: "The total number of idle drivers", 107 | }, driverLabelNames) 108 | driversRequests = promauto.NewGaugeVec(prometheus.GaugeOpts{ 109 | Name: "bblfshd_driver_scaling_load", 110 | Help: "The total number of requests waiting for a driver", 111 | }, driverLabelNames) 112 | driversTarget = promauto.NewGaugeVec(prometheus.GaugeOpts{ 113 | Name: "bblfshd_driver_scaling_target", 114 | Help: "The target number of drivers instances", 115 | }, driverLabelNames) 116 | ) 117 | -------------------------------------------------------------------------------- /daemon/protocol/service.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/docker/distribution/registry/api/errcode" 8 | xcontext "golang.org/x/net/context" 9 | "google.golang.org/grpc" 10 | "google.golang.org/grpc/codes" 11 | "google.golang.org/grpc/status" 12 | "gopkg.in/bblfsh/sdk.v1/protocol" 13 | "gopkg.in/src-d/go-errors.v1" 14 | ) 15 | 16 | var ( 17 | ErrAlreadyInstalled = errors.NewKind("driver already installed: %s (image reference: %s)") 18 | ) 19 | 20 | type Service interface { 21 | InstallDriver(language string, image string, update bool) error 22 | RemoveDriver(language string) error 23 | DriverStates() ([]*DriverImageState, error) 24 | DriverPoolStates() map[string]*DriverPoolState 25 | DriverInstanceStates() ([]*DriverInstanceState, error) 26 | } 27 | 28 | func RegisterService(srv *grpc.Server, s Service) { 29 | RegisterProtocolServiceServer(srv, &protocolServiceServer{s}) 30 | } 31 | 32 | type protocolServiceServer struct { 33 | s Service 34 | } 35 | 36 | type Response protocol.Response 37 | 38 | type DriverInstanceStatesResponse struct { 39 | protocol.Response 40 | // State represent the state of each driver instance in the daemon. 41 | State []*DriverInstanceState 42 | } 43 | 44 | func (s *protocolServiceServer) DriverInstanceStates(ctx xcontext.Context, _ *DriverInstanceStatesRequest) (*DriverInstanceStatesResponse, error) { 45 | resp := &DriverInstanceStatesResponse{} 46 | start := time.Now() 47 | defer func() { 48 | resp.Elapsed = time.Since(start) 49 | }() 50 | 51 | var err error 52 | resp.State, err = s.s.DriverInstanceStates() 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | return resp, nil 58 | } 59 | 60 | type DriverPoolStatesResponse struct { 61 | protocol.Response 62 | // State represent the state of each pool in the daemon. 63 | State map[string]*DriverPoolState 64 | } 65 | 66 | func (s *protocolServiceServer) DriverPoolStates(ctx xcontext.Context, _ *DriverPoolStatesRequest) (*DriverPoolStatesResponse, error) { 67 | resp := &DriverPoolStatesResponse{} 68 | start := time.Now() 69 | defer func() { 70 | resp.Elapsed = time.Since(start) 71 | }() 72 | 73 | resp.State = s.s.DriverPoolStates() 74 | return resp, nil 75 | } 76 | 77 | type DriverStatesResponse struct { 78 | protocol.Response 79 | // State represent the state of each driver in the storage. 80 | State []*DriverImageState 81 | } 82 | 83 | func (s *protocolServiceServer) DriverStates(ctx xcontext.Context, in *DriverStatesRequest) (*DriverStatesResponse, error) { 84 | resp := &DriverStatesResponse{} 85 | start := time.Now() 86 | defer func() { 87 | resp.Elapsed = time.Since(start) 88 | }() 89 | 90 | var err error 91 | resp.State, err = s.s.DriverStates() 92 | if err != nil { 93 | return nil, err 94 | } 95 | return resp, nil 96 | } 97 | 98 | type InstallDriverRequest struct { 99 | // Language supported by the driver being installed. 100 | Language string 101 | // ImageReference is the name of the image to be installed in the following 102 | // format: `transport:[//]name[:tag]`. The default value for tag is `latest` 103 | ImageReference string 104 | // Update indicates whether an image should be updated. When set to false, 105 | // the installation fails if the image already exists. 106 | Update bool 107 | } 108 | 109 | func (s *protocolServiceServer) InstallDriver(ctx xcontext.Context, req *InstallDriverRequest) (*Response, error) { 110 | resp := &Response{} 111 | start := time.Now() 112 | defer func() { 113 | resp.Elapsed = time.Since(start) 114 | }() 115 | 116 | err := s.s.InstallDriver( 117 | strings.ToLower(req.Language), 118 | req.ImageReference, 119 | req.Update, 120 | ) 121 | 122 | if ErrAlreadyInstalled.Is(err) { 123 | return nil, status.New(codes.AlreadyExists, err.Error()).Err() 124 | } else if errs, ok := err.(errcode.Errors); ok { //docker err codes analysis 125 | for _, erro := range errs { 126 | if errc, ok := erro.(errcode.Error); ok && errc.ErrorCode() == errcode.ErrorCodeUnauthorized { 127 | return nil, status.New(codes.Unauthenticated, err.Error()).Err() 128 | } 129 | } 130 | } 131 | if err != nil { 132 | return nil, err 133 | } 134 | return resp, nil 135 | } 136 | 137 | type RemoveDriverRequest struct { 138 | // Language supported by the driver to be deleted. 139 | Language string 140 | } 141 | 142 | func (s *protocolServiceServer) RemoveDriver(ctx xcontext.Context, req *RemoveDriverRequest) (result *Response, err error) { 143 | resp := &Response{} 144 | start := time.Now() 145 | defer func() { 146 | resp.Elapsed = time.Since(start) 147 | }() 148 | 149 | if err := s.s.RemoveDriver(strings.ToLower(req.Language)); err != nil { 150 | return nil, err 151 | } 152 | return resp, nil 153 | } 154 | -------------------------------------------------------------------------------- /daemon/protocol/generated.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package github.com.bblfsh.server.daemon.protocol; 3 | 4 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 5 | import "google/protobuf/timestamp.proto"; 6 | import "google/protobuf/duration.proto"; 7 | 8 | option (gogoproto.protosizer_all) = true; 9 | option (gogoproto.sizer_all) = false; 10 | option go_package = "protocol"; 11 | 12 | message DriverImageState { 13 | option (gogoproto.goproto_getters) = false; 14 | option (gogoproto.typedecl) = false; 15 | string reference = 1; 16 | string language = 2; 17 | string version = 3; 18 | google.protobuf.Timestamp build = 4 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; 19 | string status = 5; 20 | string os = 6 [(gogoproto.customname) = "OS"]; 21 | repeated string native_version = 7; 22 | string go_version = 8; 23 | } 24 | 25 | message DriverInstanceState { 26 | option (gogoproto.goproto_getters) = false; 27 | option (gogoproto.typedecl) = false; 28 | string id = 1 [(gogoproto.customname) = "ID"]; 29 | string image = 2; 30 | github.com.bblfsh.server.daemon.protocol.Status status = 3; 31 | google.protobuf.Timestamp created = 4 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; 32 | repeated int64 processes = 5 [(gogoproto.casttype) = "int"]; 33 | } 34 | 35 | message DriverInstanceStatesResponse { 36 | option (gogoproto.goproto_getters) = false; 37 | option (gogoproto.typedecl) = false; 38 | repeated string errors = 1; 39 | google.protobuf.Duration elapsed = 2 [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; 40 | repeated github.com.bblfsh.server.daemon.protocol.DriverInstanceState state = 3; 41 | } 42 | 43 | message DriverPoolState { 44 | option (gogoproto.goproto_getters) = false; 45 | option (gogoproto.typedecl) = false; 46 | int64 wanted = 1 [(gogoproto.casttype) = "int"]; 47 | int64 running = 2 [(gogoproto.casttype) = "int"]; 48 | int64 waiting = 3 [(gogoproto.casttype) = "int"]; 49 | int64 success = 4 [(gogoproto.casttype) = "int"]; 50 | int64 errors = 5 [(gogoproto.casttype) = "int"]; 51 | int64 exited = 6 [(gogoproto.casttype) = "int"]; 52 | } 53 | 54 | message DriverPoolStatesResponse { 55 | option (gogoproto.goproto_getters) = false; 56 | option (gogoproto.typedecl) = false; 57 | repeated string errors = 1; 58 | google.protobuf.Duration elapsed = 2 [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; 59 | map state = 3; 60 | } 61 | 62 | message DriverStatesResponse { 63 | option (gogoproto.goproto_getters) = false; 64 | option (gogoproto.typedecl) = false; 65 | repeated string errors = 1; 66 | google.protobuf.Duration elapsed = 2 [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; 67 | repeated github.com.bblfsh.server.daemon.protocol.DriverImageState state = 3; 68 | } 69 | 70 | message InstallDriverRequest { 71 | option (gogoproto.goproto_getters) = false; 72 | option (gogoproto.typedecl) = false; 73 | string language = 1; 74 | string image_reference = 2; 75 | bool update = 3; 76 | } 77 | 78 | message RemoveDriverRequest { 79 | option (gogoproto.goproto_getters) = false; 80 | option (gogoproto.typedecl) = false; 81 | string language = 1; 82 | } 83 | 84 | message Response { 85 | option (gogoproto.goproto_getters) = false; 86 | option (gogoproto.typedecl) = false; 87 | repeated string errors = 1; 88 | google.protobuf.Duration elapsed = 2 [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; 89 | } 90 | 91 | message DriverInstanceStatesRequest { 92 | } 93 | 94 | message DriverPoolStatesRequest { 95 | } 96 | 97 | message DriverStatesRequest { 98 | } 99 | 100 | // Status is the status of a driver instance. 101 | enum Status { 102 | option (gogoproto.enumdecl) = false; 103 | option (gogoproto.goproto_enum_prefix) = false; 104 | option (gogoproto.goproto_enum_stringer) = false; 105 | // Created the container exists but has not been run yet. 106 | CREATED = 0 [(gogoproto.enumvalue_customname) = "Created"]; 107 | // Running the container exists and is running. 108 | RUNNING = 1 [(gogoproto.enumvalue_customname) = "Running"]; 109 | // Pausing the container exists, it is in the process of being paused. 110 | PAUSING = 2 [(gogoproto.enumvalue_customname) = "Pausing"]; 111 | // Paused the container exists, but all its processes are paused. 112 | PAUSED = 3 [(gogoproto.enumvalue_customname) = "Paused"]; 113 | // Stopped the container does not have a created or running process. 114 | STOPPED = 4 [(gogoproto.enumvalue_customname) = "Stopped"]; 115 | } 116 | 117 | service ProtocolService { 118 | rpc DriverInstanceStates (github.com.bblfsh.server.daemon.protocol.DriverInstanceStatesRequest) returns (github.com.bblfsh.server.daemon.protocol.DriverInstanceStatesResponse); 119 | rpc DriverPoolStates (github.com.bblfsh.server.daemon.protocol.DriverPoolStatesRequest) returns (github.com.bblfsh.server.daemon.protocol.DriverPoolStatesResponse); 120 | rpc DriverStates (github.com.bblfsh.server.daemon.protocol.DriverStatesRequest) returns (github.com.bblfsh.server.daemon.protocol.DriverStatesResponse); 121 | rpc InstallDriver (github.com.bblfsh.server.daemon.protocol.InstallDriverRequest) returns (github.com.bblfsh.server.daemon.protocol.Response); 122 | rpc RemoveDriver (github.com.bblfsh.server.daemon.protocol.RemoveDriverRequest) returns (github.com.bblfsh.server.daemon.protocol.Response); 123 | } 124 | 125 | -------------------------------------------------------------------------------- /runtime/unpack.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "archive/tar" 5 | "compress/gzip" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "time" 12 | 13 | "github.com/containers/image/types" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | func UnpackImage(src types.Image, target string) error { 18 | ref := src.Reference() 19 | unpackLayer, err := getLayerUnpacker(ref) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | raw, err := ref.NewImageSource(nil) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | for _, layer := range src.LayerInfos() { 30 | rc, _, err := raw.GetBlob(layer) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | if err := unpackLayer(target, rc); err != nil { 36 | return err 37 | } 38 | } 39 | 40 | return nil 41 | } 42 | 43 | func getLayerUnpacker(ref types.ImageReference) (func(string, io.Reader) error, error) { 44 | transport := ref.Transport().Name() 45 | switch transport { 46 | case "docker-daemon": 47 | return untar, nil 48 | case "docker": 49 | return untarGzip, nil 50 | default: 51 | return nil, fmt.Errorf("unsupported transport: %s", transport) 52 | } 53 | } 54 | 55 | func untarGzip(dest string, r io.Reader) error { 56 | gz, err := gzip.NewReader(r) 57 | if err != nil { 58 | return errors.Wrap(err, "error creating gzip reader") 59 | } 60 | defer gz.Close() 61 | 62 | return untar(dest, gz) 63 | } 64 | 65 | func untar(dest string, r io.Reader) error { 66 | entries := make(map[string]bool) 67 | var dirs []*tar.Header 68 | tr := tar.NewReader(r) 69 | loop: 70 | for { 71 | hdr, err := tr.Next() 72 | switch err { 73 | case io.EOF: 74 | break loop 75 | case nil: 76 | // success, continue below 77 | default: 78 | return errors.Wrapf(err, "error advancing tar stream") 79 | } 80 | 81 | hdr.Name = filepath.Clean(hdr.Name) 82 | if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { 83 | // Not the root directory, ensure that the parent directory exists 84 | parent := filepath.Dir(hdr.Name) 85 | parentPath := filepath.Join(dest, parent) 86 | if _, err2 := os.Lstat(parentPath); err2 != nil && os.IsNotExist(err2) { 87 | if err3 := os.MkdirAll(parentPath, 0755); err3 != nil { 88 | return err3 89 | } 90 | } 91 | } 92 | path := filepath.Join(dest, hdr.Name) 93 | if entries[path] { 94 | return fmt.Errorf("duplicate entry for %s", path) 95 | } 96 | entries[path] = true 97 | rel, err := filepath.Rel(dest, path) 98 | if err != nil { 99 | return err 100 | } 101 | info := hdr.FileInfo() 102 | if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { 103 | return fmt.Errorf("%q is outside of %q", hdr.Name, dest) 104 | } 105 | 106 | if strings.HasPrefix(info.Name(), ".wh.") { 107 | path = strings.Replace(path, ".wh.", "", 1) 108 | 109 | if err := os.RemoveAll(path); err != nil { 110 | return errors.Wrap(err, "unable to delete whiteout path") 111 | } 112 | 113 | continue loop 114 | } 115 | 116 | switch hdr.Typeflag { 117 | case tar.TypeDir: 118 | if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { 119 | if err2 := os.MkdirAll(path, info.Mode()); err2 != nil { 120 | return errors.Wrap(err2, "error creating directory") 121 | } 122 | } 123 | 124 | case tar.TypeReg, tar.TypeRegA: 125 | f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, info.Mode()) 126 | if err != nil { 127 | return errors.Wrap(err, "unable to open file") 128 | } 129 | 130 | if _, err := io.Copy(f, tr); err != nil { 131 | f.Close() 132 | return errors.Wrap(err, "unable to copy") 133 | } 134 | f.Close() 135 | 136 | case tar.TypeLink: 137 | target := filepath.Join(dest, hdr.Linkname) 138 | 139 | trueTarget, err := filepath.EvalSymlinks(target) 140 | if err != nil { 141 | return err 142 | } 143 | if !strings.HasPrefix(trueTarget, dest) { 144 | return fmt.Errorf("hardlink %q -> %q outside destination", target, hdr.Linkname) 145 | } 146 | 147 | if !strings.HasPrefix(target, dest) { 148 | return fmt.Errorf("invalid hardlink %q -> %q", target, hdr.Linkname) 149 | } 150 | 151 | if err := os.Link(target, path); err != nil { 152 | return err 153 | } 154 | 155 | case tar.TypeSymlink: 156 | target := filepath.Join(filepath.Dir(path), hdr.Linkname) 157 | 158 | trueTarget, err := filepath.EvalSymlinks(target) 159 | if err != nil { 160 | return err 161 | } 162 | if !strings.HasPrefix(trueTarget, dest) { 163 | return fmt.Errorf("hardlink %q -> %q outside destination", target, hdr.Linkname) 164 | } 165 | if !strings.HasPrefix(target, dest) { 166 | return fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname) 167 | } 168 | 169 | err := os.Symlink(hdr.Linkname, path) 170 | if err != nil { 171 | if os.IsExist(err) { 172 | if err := os.Remove(path); err != nil { 173 | return err 174 | } 175 | 176 | if err := os.Symlink(hdr.Linkname, path); err != nil { 177 | return err 178 | } 179 | } 180 | } 181 | 182 | case tar.TypeXGlobalHeader: 183 | return nil 184 | } 185 | // Directory mtimes must be handled at the end to avoid further 186 | // file creation in them to modify the directory mtime 187 | if hdr.Typeflag == tar.TypeDir { 188 | dirs = append(dirs, hdr) 189 | } 190 | } 191 | for _, hdr := range dirs { 192 | path := filepath.Join(dest, hdr.Name) 193 | 194 | finfo := hdr.FileInfo() 195 | // I believe the old version was using time.Now().UTC() to overcome an 196 | // invalid error from chtimes.....but here we lose hdr.AccessTime like this... 197 | if err := os.Chtimes(path, time.Now().UTC(), finfo.ModTime()); err != nil { 198 | return errors.Wrap(err, "error changing time") 199 | } 200 | } 201 | return nil 202 | } 203 | -------------------------------------------------------------------------------- /runtime/storage.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/bblfsh/sdk/v3/driver" 9 | "github.com/bblfsh/sdk/v3/driver/manifest" 10 | "gopkg.in/src-d/go-errors.v1" 11 | ) 12 | 13 | var ( 14 | ErrDirtyDriverStorage = errors.NewKind("dirty driver storage") 15 | ErrDriverNotInstalled = errors.NewKind("driver not installed") 16 | ErrMalformedDriver = errors.NewKind("malformed driver, missing manifest.toml") 17 | ) 18 | 19 | // storage represents the DriverImage storage, taking care of filesystem 20 | // image operations, such as install, update, remove, etc. 21 | type storage struct { 22 | path string 23 | temp string 24 | } 25 | 26 | func newStorage(path, temp string) *storage { 27 | return &storage{path: path, temp: temp} 28 | } 29 | 30 | // Install installs a DriverImage extracting his content to the filesystem, 31 | // only one version per image can be stored, update is required to overwrite a 32 | // previous image if already exists otherwise, Install fails if an previous 33 | // image already exists. 34 | func (s *storage) Install(d DriverImage, update bool) (*DriverImageStatus, error) { 35 | current, err := s.RootFS(d) 36 | if err != nil && !ErrDriverNotInstalled.Is(err) { 37 | return nil, err 38 | } 39 | 40 | exists := current != "" 41 | if exists && !update { 42 | return nil, nil 43 | } 44 | 45 | di, err := d.Digest() 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | if exists { 51 | if err := s.Remove(d); err != nil { 52 | return nil, err 53 | } 54 | } 55 | 56 | tmp, err := s.tempPath() 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | if err := d.WriteTo(tmp); err != nil { 62 | return nil, err 63 | } 64 | 65 | m, err := newDriverImageStatus(tmp) 66 | if err != nil { 67 | if os.IsNotExist(err) { 68 | return nil, ErrMalformedDriver.New() 69 | } 70 | 71 | return nil, err 72 | } 73 | return m, s.moveImage(tmp, d, di) 74 | } 75 | 76 | func (s *storage) tempPath() (string, error) { 77 | if err := os.MkdirAll(s.temp, 0755); err != nil { 78 | return "", err 79 | } 80 | 81 | return ioutil.TempDir(s.temp, "image") 82 | } 83 | 84 | func (s *storage) moveImage(source string, d DriverImage, di Digest) error { 85 | root := s.rootFSPath(d, di) 86 | dir := filepath.Dir(root) 87 | if err := os.MkdirAll(dir, 0755); err != nil { 88 | return err 89 | } 90 | 91 | if err := os.Rename(source+configExt, root+configExt); err != nil { 92 | return err 93 | } 94 | 95 | return os.Rename(source, root) 96 | } 97 | 98 | // RootFS returns the path in the host filesystem to an installed image. 99 | func (s *storage) RootFS(d DriverImage) (string, error) { 100 | return s.rootFSFromBase(s.basePath(d)) 101 | } 102 | 103 | func (s *storage) rootFSFromBase(path string) (string, error) { 104 | dirs, err := getDirs(path) 105 | if err != nil { 106 | return "", err 107 | } 108 | 109 | switch len(dirs) { 110 | case 1: 111 | return dirs[0], nil 112 | case 0: 113 | return "", ErrDriverNotInstalled.New() 114 | default: 115 | return "", ErrDirtyDriverStorage.New() 116 | } 117 | } 118 | 119 | // Status returns the current status in the storage for a given DriverImage, nil 120 | // is returned if the image is not installed. 121 | func (s *storage) Status(d DriverImage) (*DriverImageStatus, error) { 122 | path, err := s.RootFS(d) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | return newDriverImageStatus(path) 128 | } 129 | 130 | // Remove removes a given DriverImage from the filesystem. 131 | func (s *storage) Remove(d DriverImage) error { 132 | path, err := s.RootFS(d) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | if err := os.RemoveAll(path + configExt); err != nil { 138 | return err 139 | } 140 | 141 | return os.RemoveAll(path) 142 | } 143 | 144 | // List lists all the driver images installed on disk. 145 | func (s *storage) List() ([]*DriverImageStatus, error) { 146 | config, err := filepath.Glob(filepath.Join(s.path, "*/*"+configExt)) 147 | if err != nil { 148 | return nil, err 149 | } 150 | 151 | var list []*DriverImageStatus 152 | for _, c := range config { 153 | root := c[:len(c)-len(configExt)] 154 | status, err := newDriverImageStatus(root) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | list = append(list, status) 160 | } 161 | 162 | return list, nil 163 | } 164 | 165 | func (s *storage) rootFSPath(d DriverImage, di Digest) string { 166 | return filepath.Join(s.basePath(d), di.String()) 167 | } 168 | 169 | func (s *storage) basePath(d DriverImage) string { 170 | return filepath.Join(s.path, ComputeDigest(d.Name()).String()) 171 | } 172 | 173 | func (s *storage) basePathExists(d DriverImage) (bool, error) { 174 | path := s.basePath(d) 175 | _, err := os.Stat(path) 176 | if err == nil { 177 | return true, nil 178 | } 179 | 180 | if os.IsNotExist(err) { 181 | return false, nil 182 | } 183 | 184 | return false, err 185 | } 186 | 187 | func getDirs(path string) ([]string, error) { 188 | files, err := ioutil.ReadDir(path) 189 | if err != nil { 190 | if os.IsNotExist(err) { 191 | return nil, nil 192 | } 193 | 194 | return nil, err 195 | } 196 | 197 | var dirs []string 198 | for _, f := range files { 199 | if !f.IsDir() { 200 | continue 201 | } 202 | 203 | dirs = append(dirs, filepath.Join(path, f.Name())) 204 | } 205 | 206 | return dirs, nil 207 | } 208 | 209 | func newDriverImageStatus(path string) (*DriverImageStatus, error) { 210 | manifest, err := manifest.Load(filepath.Join(path, driver.ManifestLocation)) 211 | if err != nil { 212 | return nil, err 213 | } 214 | 215 | config, err := ReadImageConfig(path) 216 | if err != nil { 217 | return nil, err 218 | } 219 | 220 | _, digest := filepath.Split(path) 221 | return &DriverImageStatus{ 222 | Reference: config.ImageRef, 223 | Digest: NewDigest(digest), 224 | Manifest: manifest, 225 | }, nil 226 | } 227 | 228 | // DriverImageStatus represents the status of an installed driver image on disk. 229 | type DriverImageStatus struct { 230 | Reference string 231 | Digest Digest 232 | Manifest *manifest.Manifest 233 | } 234 | -------------------------------------------------------------------------------- /daemon/driver.go: -------------------------------------------------------------------------------- 1 | // +build linux,cgo 2 | 3 | package daemon 4 | 5 | import ( 6 | "context" 7 | "encoding/json" 8 | "fmt" 9 | "net" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "syscall" 14 | "time" 15 | 16 | "github.com/bblfsh/bblfshd/daemon/protocol" 17 | "github.com/bblfsh/bblfshd/runtime" 18 | 19 | protocol2 "github.com/bblfsh/sdk/v3/protocol" 20 | "github.com/opencontainers/runc/libcontainer/configs" 21 | "google.golang.org/grpc" 22 | protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" 23 | ) 24 | 25 | type Driver interface { 26 | ID() string 27 | Start(ctx context.Context) error 28 | Stop() error 29 | Status() (protocol.Status, error) 30 | State() (*protocol.DriverInstanceState, error) 31 | Service() protocol1.ProtocolServiceClient 32 | ServiceV2() protocol2.DriverClient 33 | } 34 | 35 | // DriverInstance represents an instance of a driver. 36 | type DriverInstance struct { 37 | Language string 38 | Process *runtime.Process 39 | Container runtime.Container 40 | Image runtime.DriverImage 41 | 42 | ctx context.Context 43 | conn *grpc.ClientConn 44 | srv1 protocol1.ProtocolServiceClient 45 | srv2 protocol2.DriverClient 46 | tmp string 47 | } 48 | 49 | const ( 50 | DriverBinary = "/opt/driver/bin/driver" 51 | GRPCSocket = "rpc.sock" 52 | TmpPathPattern = "/tmp/%s" 53 | ) 54 | 55 | type Options struct { 56 | LogLevel string 57 | LogFormat string 58 | Env []string 59 | } 60 | 61 | // NewDriverInstance represents a running Driver in the runtime. Its holds the 62 | // container and the connection to the internal grpc server. 63 | func NewDriverInstance(r *runtime.Runtime, lang string, i runtime.DriverImage, o *Options) (*DriverInstance, error) { 64 | id := strings.ToLower(runtime.NewULID().String()) 65 | p := &runtime.Process{ 66 | Args: []string{ 67 | DriverBinary, 68 | "--log-level", o.LogLevel, 69 | "--log-format", o.LogFormat, 70 | "--log-fields", logFields(id, lang), 71 | "--network", "unix", 72 | "--address", fmt.Sprintf(TmpPathPattern, GRPCSocket), 73 | }, 74 | Env: o.Env, 75 | Stdout: os.Stdout, 76 | Stderr: os.Stderr, 77 | Init: true, 78 | } 79 | 80 | tmp := filepath.Join(r.Root, fmt.Sprintf(TmpPathPattern, id)) 81 | 82 | f := func(containerID string) *configs.Config { 83 | cfg := runtime.ContainerConfigFactory(containerID) 84 | cfg.Mounts = append(cfg.Mounts, &configs.Mount{ 85 | Source: tmp, 86 | Destination: "/tmp/", 87 | Device: "bind", 88 | Flags: syscall.MS_BIND | syscall.MS_REC | syscall.MS_NOSUID, 89 | PremountCmds: []configs.Command{ 90 | {Path: "mkdir", Args: []string{"-p", tmp}}, 91 | }, 92 | }) 93 | 94 | return cfg 95 | } 96 | 97 | c, err := r.Container(id, i, p, f) 98 | if err != nil { 99 | return nil, err 100 | } 101 | 102 | return &DriverInstance{ 103 | Language: lang, 104 | Process: p, 105 | Container: c, 106 | Image: i, 107 | 108 | ctx: context.Background(), 109 | tmp: tmp, 110 | }, nil 111 | } 112 | 113 | // ID returns the container id. 114 | func (i *DriverInstance) ID() string { 115 | return i.Container.ID() 116 | } 117 | 118 | // Start starts a container and connects to it. 119 | func (i *DriverInstance) Start(ctx context.Context) error { 120 | if err := i.Container.Start(); err != nil { 121 | return err 122 | } 123 | 124 | if err := i.dial(ctx); err != nil { 125 | _ = i.Container.Stop() 126 | return err 127 | } 128 | 129 | if err := i.loadVersion(); err != nil { 130 | return err 131 | } 132 | 133 | return nil 134 | } 135 | 136 | func (i *DriverInstance) dial(ctx context.Context) error { 137 | addr := filepath.Join(i.tmp, GRPCSocket) 138 | 139 | opts := []grpc.DialOption{ 140 | grpc.WithDialer(func(addr string, t time.Duration) (net.Conn, error) { 141 | return net.DialTimeout("unix", addr, t) 142 | }), 143 | // always wait for the connection to become active 144 | grpc.WithBlock(), 145 | // we want to know sooner rather than later 146 | // TODO(dennwc): sometimes the initialization of the container takes >5 sec 147 | // meaning that the time between Container.Start and the actual 148 | // execution of a Go server (not the native driver) takes this long 149 | grpc.WithBackoffMaxDelay(time.Second), 150 | grpc.WithInsecure(), 151 | } 152 | opts = append(opts, protocol2.DialOptions()...) 153 | conn, err := grpc.DialContext(ctx, addr, opts...) 154 | if err != nil { 155 | return err 156 | } 157 | 158 | i.conn = conn 159 | i.srv1 = protocol1.NewProtocolServiceClient(conn) 160 | i.srv2 = protocol2.NewDriverClient(conn) 161 | return err 162 | } 163 | 164 | func (i *DriverInstance) loadVersion() error { 165 | _, err := i.srv1.Version(context.Background(), &protocol1.VersionRequest{}) 166 | if err != nil { 167 | return err 168 | } 169 | 170 | return nil 171 | } 172 | 173 | // Status returns the current status of the container. 174 | func (i *DriverInstance) Status() (protocol.Status, error) { 175 | s, err := i.Container.Status() 176 | return protocol.Status(s), err 177 | } 178 | 179 | // State returns the current state of the driver instance. 180 | func (i *DriverInstance) State() (*protocol.DriverInstanceState, error) { 181 | status, err := i.Status() 182 | if err != nil { 183 | return nil, err 184 | } 185 | 186 | pid, err := i.Container.Processes() 187 | if err != nil { 188 | return nil, err 189 | } 190 | 191 | state, err := i.Container.State() 192 | if err != nil { 193 | return nil, err 194 | } 195 | 196 | return &protocol.DriverInstanceState{ 197 | ID: i.ID(), 198 | Image: i.Image.Name(), 199 | Status: status, 200 | Processes: pid, 201 | Created: state.Created, 202 | }, nil 203 | } 204 | 205 | // Stop stops the inner running container. 206 | func (i *DriverInstance) Stop() error { 207 | var first error 208 | if i.Container != nil { 209 | if err := i.Container.Stop(); err != nil && first == nil { 210 | first = err 211 | } 212 | } 213 | if i.conn != nil { 214 | if err := i.conn.Close(); err != nil && first == nil { 215 | first = err 216 | } 217 | } 218 | return first 219 | } 220 | 221 | // Service returns the client using the grpc connection. 222 | func (i *DriverInstance) Service() protocol1.ProtocolServiceClient { 223 | return i.srv1 224 | } 225 | 226 | // ServiceV2 returns the client using the grpc connection. 227 | func (i *DriverInstance) ServiceV2() protocol2.DriverClient { 228 | return i.srv2 229 | } 230 | 231 | func logFields(containerID, language string) string { 232 | js, _ := json.Marshal(map[string]string{ 233 | "id": containerID, 234 | "language": language, 235 | }) 236 | 237 | return string(js) 238 | } 239 | -------------------------------------------------------------------------------- /runtime/runtime.go: -------------------------------------------------------------------------------- 1 | // +build linux,cgo 2 | 3 | package runtime 4 | 5 | import ( 6 | "os" 7 | "path/filepath" 8 | "runtime" 9 | "strings" 10 | "syscall" 11 | 12 | "github.com/opencontainers/runc/libcontainer" 13 | "github.com/opencontainers/runc/libcontainer/configs" 14 | _ "github.com/opencontainers/runc/libcontainer/nsenter" 15 | ) 16 | 17 | const ( 18 | storagePath = "images" 19 | containersPath = "containers" 20 | temporalPath = "tmp" 21 | ) 22 | 23 | type ConfigFactory func(containerID string) *configs.Config 24 | 25 | type Runtime struct { 26 | ContainerConfigFactory ConfigFactory 27 | Root string 28 | 29 | s *storage 30 | f libcontainer.Factory 31 | } 32 | 33 | // NewRuntime create a new runtime using as storage the given path. 34 | func NewRuntime(path string) *Runtime { 35 | return &Runtime{ 36 | ContainerConfigFactory: ContainerConfigFactory, 37 | 38 | Root: path, 39 | s: newStorage( 40 | filepath.Join(path, storagePath), 41 | filepath.Join(path, temporalPath), 42 | ), 43 | } 44 | } 45 | 46 | // Init initialize the runtime. 47 | func (r *Runtime) Init() error { 48 | var err error 49 | r.f, err = libcontainer.New( 50 | filepath.Join(r.Root, containersPath), 51 | libcontainer.RootlessCgroupfs, 52 | ) 53 | 54 | return err 55 | } 56 | 57 | // InstallDriver installs a DriverImage extracting his content to the storage, 58 | // only one version per image can be stored, update is required to overwrite a 59 | // previous image if already exists otherwise, Install fails if an previous 60 | // image already exists. 61 | func (r *Runtime) InstallDriver(d DriverImage, update bool) (*DriverImageStatus, error) { 62 | return r.s.Install(d, update) 63 | } 64 | 65 | // RemoveDriver removes a given DriverImage from the image storage. 66 | func (r *Runtime) RemoveDriver(d DriverImage) error { 67 | return r.s.Remove(d) 68 | } 69 | 70 | // ListDrivers lists all the driver images installed on the storage. 71 | func (r *Runtime) ListDrivers() ([]*DriverImageStatus, error) { 72 | return r.s.List() 73 | } 74 | 75 | // Container returns a container for the given DriverImage and Process 76 | func (r *Runtime) Container(id string, d DriverImage, p *Process, f ConfigFactory) (Container, error) { 77 | if f == nil { 78 | f = r.ContainerConfigFactory 79 | } 80 | 81 | cfg := f(id) 82 | 83 | var err error 84 | cfg.Rootfs, err = r.s.RootFS(d) 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | imgConfig, err := ReadImageConfig(cfg.Rootfs) 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | c, err := r.f.Create(id, cfg) 95 | if err != nil { 96 | return nil, err 97 | } 98 | 99 | return newContainer(c, p, imgConfig), nil 100 | } 101 | 102 | // ContainerConfigFactory is the default container config factory, is returns a 103 | // config.Config, with the default setup. 104 | func ContainerConfigFactory(containerID string) *configs.Config { 105 | defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV 106 | 107 | return &configs.Config{ 108 | RootlessEUID: true, 109 | RootlessCgroups: true, 110 | Namespaces: configs.Namespaces([]configs.Namespace{ 111 | {Type: configs.NEWNS}, 112 | {Type: configs.NEWUTS}, 113 | {Type: configs.NEWIPC}, 114 | {Type: configs.NEWPID}, 115 | {Type: configs.NEWUSER}, 116 | }), 117 | UidMappings: []configs.IDMap{ 118 | {ContainerID: 0, HostID: os.Getuid(), Size: 1}, 119 | }, 120 | GidMappings: []configs.IDMap{ 121 | {ContainerID: 0, HostID: os.Getgid(), Size: 1}, 122 | }, 123 | Cgroups: &configs.Cgroup{ 124 | Name: containerID, 125 | Parent: "system", 126 | Resources: &configs.Resources{ 127 | MemorySwappiness: nil, 128 | AllowAllDevices: nil, 129 | AllowedDevices: configs.DefaultSimpleDevices, 130 | }, 131 | }, 132 | MaskPaths: []string{ 133 | "/proc/kcore", 134 | "/sys/firmware", 135 | }, 136 | ReadonlyPaths: []string{ 137 | "/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus", 138 | }, 139 | Devices: configs.DefaultSimpleDevices, 140 | Hostname: containerID, 141 | Mounts: []*configs.Mount{ 142 | { 143 | Source: "proc", 144 | Destination: "/proc", 145 | Device: "proc", 146 | Flags: defaultMountFlags, 147 | }, 148 | { 149 | Source: "tmpfs", 150 | Destination: "/dev", 151 | Device: "tmpfs", 152 | Flags: syscall.MS_NOSUID | syscall.MS_STRICTATIME, 153 | Data: "mode=755", 154 | }, 155 | { 156 | Source: "devpts", 157 | Destination: "/dev/pts", 158 | Device: "devpts", 159 | Flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, 160 | Data: "newinstance,ptmxmode=0666,mode=0620", 161 | }, 162 | { 163 | Source: "mqueue", 164 | Destination: "/dev/mqueue", 165 | Device: "mqueue", 166 | Flags: defaultMountFlags, 167 | }, 168 | { 169 | Source: "/etc/localtime", 170 | Destination: "/etc/localtime", 171 | Device: "bind", 172 | Flags: syscall.MS_BIND | syscall.MS_RDONLY, 173 | }, 174 | }, 175 | Rlimits: []configs.Rlimit{ 176 | { 177 | Type: syscall.RLIMIT_NOFILE, 178 | Hard: uint64(1025), 179 | Soft: uint64(1025), 180 | }, 181 | }, 182 | } 183 | } 184 | 185 | // Bootstrap perform the init process of a container. This function should be 186 | // called at the init function of the application. 187 | // 188 | // Because containers are spawned in a two step process you will need a binary 189 | // that will be executed as the init process for the container. In libcontainer, 190 | // we use the current binary (/proc/self/exe) to be executed as the init 191 | // process, and use arg "init", we call the first step process "bootstrap", so 192 | // you always need a "init" function as the entry of "bootstrap". 193 | // 194 | // In addition to the go init function the early stage bootstrap is handled by 195 | // importing nsenter. 196 | // 197 | // https://github.com/opencontainers/runc/blob/master/libcontainer/README.md 198 | func Bootstrap() { 199 | if len(os.Args) > 1 && os.Args[1] == "init" { 200 | runtime.GOMAXPROCS(1) 201 | runtime.LockOSThread() 202 | factory, _ := libcontainer.New("") 203 | if err := factory.StartInitialization(); err != nil { 204 | if strings.Contains(err.Error(), "permission denied") { 205 | panic("error bootstraping container " + 206 | "(hint: if SELinux is enabled, compile and load the policy module " + 207 | "in this repo's selinux/ directory): " + err.Error()) 208 | } else { 209 | panic(err) 210 | } 211 | } 212 | panic("--this line should have never been executed, congratulations--") 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: clean test build 2 | 3 | # Docsrv: configure the languages whose api-doc can be auto generated 4 | LANGUAGES = "go" 5 | # Docs: do not edit this 6 | DOCS_REPOSITORY := https://github.com/src-d/docs 7 | SHARED_PATH ?= $(shell pwd)/.docsrv-resources 8 | DOCS_PATH ?= $(SHARED_PATH)/.docs 9 | $(DOCS_PATH)/Makefile.inc: 10 | git clone --quiet --depth 1 $(DOCS_REPOSITORY) $(DOCS_PATH); 11 | -include $(DOCS_PATH)/Makefile.inc 12 | 13 | # Package configuration 14 | PROJECT = bblfshd 15 | GO_PKG = github.com/bblfsh/$(PROJECT) 16 | COMMANDS = bblfshd bblfshctl 17 | DEPENDENCIES = \ 18 | golang.org/x/tools/cmd/cover 19 | NOVENDOR_PACKAGES := $(shell go list ./... | grep -v '/vendor/') 20 | 21 | # Environment 22 | BASE_PATH := $(shell pwd) 23 | VENDOR_PATH := $(BASE_PATH)/vendor 24 | BUILD_PATH := $(BASE_PATH)/build 25 | CMD_PATH := $(BASE_PATH)/cmd 26 | 27 | # Build information 28 | BUILD ?= $(shell date +%FT%H:%M:%S%z) 29 | GIT_COMMIT=$(shell git rev-parse HEAD | cut -c1-7) 30 | GIT_DIRTY=$(shell test -n "`git status --porcelain`" && echo "-dirty" || true) 31 | DEV_PREFIX := dev 32 | VERSION ?= $(DEV_PREFIX)-$(GIT_COMMIT)$(GIT_DIRTY) 33 | TAG_DATE=$(shell date +%F) 34 | 35 | 36 | # Go parameters 37 | GO_CMD = go 38 | GO_BUILD = GO111MODULE=on $(GO_CMD) build 39 | GO_GET = GO111MODULE=on $(GO_CMD) get -v 40 | GO_TEST = GO111MODULE=on $(GO_CMD) test -v 41 | 42 | # Packages content 43 | PKG_OS_bblfshctl = darwin linux windows 44 | PKG_OS_bblfshd = linux 45 | PKG_ARCH = amd64 46 | 47 | # Coverage 48 | COVERAGE_REPORT = coverage.txt 49 | COVERAGE_PROFILE = profile.out 50 | COVERAGE_MODE = atomic 51 | 52 | ifneq ($(origin TRAVIS_TAG), undefined) 53 | VERSION := $(TRAVIS_TAG) 54 | endif 55 | 56 | ifneq ($(origin CRON_TAG), undefined) 57 | VERSION := $(CRON_TAG) 58 | endif 59 | 60 | # Build 61 | LDFLAGS = -X main.version=$(VERSION) -X main.build=$(BUILD) 62 | 63 | # Docker 64 | DOCKER_CMD = docker 65 | DOCKER_BUILD = $(DOCKER_CMD) build --build-arg BBLFSHD_VERSION=$(VERSION) --build-arg BBLFSHD_BUILD="$(BUILD)" 66 | DOCKER_RUN = $(DOCKER_CMD) run --rm 67 | DOCKER_BUILD_IMAGE = bblfshd-build 68 | DOCKER_TAG ?= $(DOCKER_CMD) tag 69 | DOCKER_PUSH ?= $(DOCKER_CMD) push 70 | DOCKER_PULL ?= $(DOCKER_CMD) pull 71 | 72 | # escape_docker_tag escape colon char to allow use a docker tag as rule 73 | define escape_docker_tag 74 | $(subst :,--,$(1)) 75 | endef 76 | 77 | # unescape_docker_tag an escaped docker tag to be use in a docker command 78 | define unescape_docker_tag 79 | $(subst --,:,$(1)) 80 | endef 81 | 82 | # if we are not in master, and it's not a tag the push is disabled 83 | ifneq ($(TRAVIS_BRANCH), master) 84 | ifeq ($(TRAVIS_TAG), ) 85 | pushdisabled = "push disabled for non-master branches" 86 | endif 87 | endif 88 | 89 | # if this is a pull request, the push is disabled 90 | ifneq ($(TRAVIS_PULL_REQUEST), false) 91 | pushdisabled = "push disabled for pull-requests" 92 | endif 93 | 94 | DOCKER_IMAGE ?= bblfsh/bblfshd 95 | DOCKER_IMAGE_VERSIONED ?= $(call escape_docker_tag,$(DOCKER_IMAGE):$(VERSION)) 96 | DOCKER_IMAGE_FIXTURE ?= $(DOCKER_IMAGE):fixture 97 | 98 | # Rules 99 | dependencies: build-fixture 100 | 101 | docker-build: 102 | docker build --target=builder -t $(DOCKER_BUILD_IMAGE) . 103 | 104 | test: dependencies docker-build 105 | $(DOCKER_RUN) --privileged -v /var/run/docker.sock:/var/run/docker.sock -v $(GOPATH)/src/$(GO_PKG):/go/src/$(GO_PKG) -e TEST_NETWORKING=1 $(DOCKER_BUILD_IMAGE) $(GO_TEST) ./... 106 | 107 | test-coverage: dependencies docker-build 108 | mkdir -p reports 109 | $(DOCKER_RUN) --privileged -v /var/run/docker.sock:/var/run/docker.sock -v $(GOPATH)/src/$(GO_PKG)/reports:/go/src/$(GO_PKG)/reports $(DOCKER_BUILD_IMAGE) make test-coverage-internal 110 | mv ./reports/* ./ && rm -rf reports 111 | 112 | test-coverage-internal: 113 | export TEST_NETWORKING=1; \ 114 | echo "" > reports/$(COVERAGE_REPORT); \ 115 | for dir in $(NOVENDOR_PACKAGES); do \ 116 | $(GO_TEST) $$dir -coverprofile=reports/$(COVERAGE_PROFILE) -covermode=$(COVERAGE_MODE); \ 117 | if [ $$? != 0 ]; then \ 118 | exit 2; \ 119 | fi; \ 120 | if [ -f reports/$(COVERAGE_PROFILE) ]; then \ 121 | cat reports/$(COVERAGE_PROFILE) >> reports/$(COVERAGE_REPORT); \ 122 | rm reports/$(COVERAGE_PROFILE); \ 123 | fi; \ 124 | done; 125 | 126 | build: dependencies docker-build 127 | $(DOCKER_BUILD) -t $(call unescape_docker_tag,$(DOCKER_IMAGE_VERSIONED)) . 128 | 129 | build-fixture: 130 | cd $(BASE_PATH)/runtime/fixture/; \ 131 | docker build -t $(DOCKER_IMAGE_FIXTURE) . 132 | 133 | 134 | build-drivers: 135 | ifeq ($(TRAVIS_EVENT_TYPE), cron) 136 | $(DOCKER_PULL) $(DOCKER_IMAGE):$(VERSION) 137 | else 138 | make build 139 | endif 140 | echo $(DOCKER_IMAGE):$(VERSION) 141 | docker build -f Dockerfile.drivers --build-arg TAG="$(VERSION)" -t "$(DOCKER_IMAGE):$(VERSION)-drivers-$(TAG_DATE)" . 142 | 143 | clean: 144 | rm -rf $(BUILD_PATH); \ 145 | $(GO_CLEAN) . 146 | 147 | push: build 148 | $(if $(pushdisabled),$(error $(pushdisabled))) 149 | 150 | @if [ "$$DOCKER_USERNAME" != "" ]; then \ 151 | $(DOCKER_CMD) login -u="$$DOCKER_USERNAME" -p="$$DOCKER_PASSWORD"; \ 152 | fi; 153 | 154 | $(DOCKER_PUSH) $(call unescape_docker_tag,$(DOCKER_IMAGE_VERSIONED)) 155 | @if [ "$$TRAVIS_TAG" != "" ]; then \ 156 | $(DOCKER_TAG) $(call unescape_docker_tag,$(DOCKER_IMAGE_VERSIONED)) \ 157 | $(call unescape_docker_tag,$(DOCKER_IMAGE)):latest; \ 158 | $(DOCKER_PUSH) $(call unescape_docker_tag,$(DOCKER_IMAGE):latest); \ 159 | fi; 160 | 161 | push-drivers: build-drivers 162 | $(if $(pushdisabled),$(error $(pushdisabled))) 163 | 164 | @if [ "$$DOCKER_USERNAME" != "" ]; then \ 165 | $(DOCKER_CMD) login -u="$$DOCKER_USERNAME" -p="$$DOCKER_PASSWORD"; \ 166 | fi; 167 | 168 | $(DOCKER_PUSH) "$(call unescape_docker_tag,$(DOCKER_IMAGE_VERSIONED))-drivers-$(TAG_DATE)" 169 | @if [ "$$TRAVIS_TAG" != "" ]; then \ 170 | $(DOCKER_TAG) "$(call unescape_docker_tag,$(DOCKER_IMAGE_VERSIONED))-drivers-$(TAG_DATE)" \ 171 | $(call unescape_docker_tag,$(DOCKER_IMAGE)):latest-drivers; \ 172 | $(DOCKER_PUSH) $(call unescape_docker_tag,$(DOCKER_IMAGE):latest-drivers); \ 173 | fi; 174 | 175 | packages: dependencies docker-build 176 | $(DOCKER_RUN) -v $(BUILD_PATH):/go/src/$(GO_PKG)/build \ 177 | -e TRAVIS_BRANCH=$(TRAVIS_BRANCH) \ 178 | -e TRAVIS_TAG=$(TRAVIS_TAG) \ 179 | -e TRAVIS_PULL_REQUEST=$(TRAVIS_PULL_REQUEST) \ 180 | $(DOCKER_BUILD_IMAGE) make packages-internal 181 | 182 | packages-internal: $(COMMANDS) 183 | 184 | $(COMMANDS): 185 | for arch in $(PKG_ARCH); do \ 186 | for os in $(PKG_OS_$@); do \ 187 | mkdir -p $(BUILD_PATH)/$@_$${os}_$${arch}; \ 188 | echo ; \ 189 | echo "$${os} - $@"; \ 190 | GOOS=$${os} GOARCH=$${arch} $(GO_BUILD) $$([ $${os} = "linux" ] && echo -tags ostree) \ 191 | --ldflags "$(LDFLAGS)" \ 192 | -o "$(BUILD_PATH)/$@_$${os}_$${arch}/$@" \ 193 | $(CMD_PATH)/$@/main.go; \ 194 | cd $(BUILD_PATH); \ 195 | tar -cvzf $@_$(VERSION)_$${os}_$${arch}.tar.gz $@_$${os}_$${arch}/; \ 196 | cd $(BASE_PATH); \ 197 | done; \ 198 | done; \ 199 | -------------------------------------------------------------------------------- /daemon/pool_test.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "runtime" 8 | "sync" 9 | "testing" 10 | "time" 11 | 12 | "github.com/bblfsh/bblfshd/daemon/protocol" 13 | 14 | "github.com/stretchr/testify/assert" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | func TestDriverPoolClose_StartNoopClose(t *testing.T) { 19 | require := require.New(t) 20 | dp := NewDriverPool(newMockDriver) 21 | 22 | ctx := context.Background() 23 | err := dp.Start(ctx) 24 | require.NoError(err) 25 | 26 | err = dp.Stop() 27 | require.NoError(err) 28 | 29 | err = dp.Stop() 30 | require.True(ErrPoolClosed.Is(err), "%v", err) 31 | 32 | err = dp.ExecuteCtx(ctx, func(ctx context.Context, d Driver) error { 33 | return errors.New("should not happen") 34 | }) 35 | require.True(ErrPoolClosed.Is(err), "%v", err) 36 | } 37 | 38 | func TestDriverPoolCurrent(t *testing.T) { 39 | require := require.New(t) 40 | 41 | dp := NewDriverPool(newMockDriver) 42 | 43 | err := dp.Start(context.Background()) 44 | require.NoError(err) 45 | 46 | require.Len(dp.Current(), 1) 47 | 48 | err = dp.Stop() 49 | require.NoError(err) 50 | } 51 | 52 | func TestDriverPoolExecute_Timeout(t *testing.T) { 53 | require := require.New(t) 54 | 55 | dp := NewDriverPool(func(ctx context.Context) (Driver, error) { 56 | select { 57 | case <-ctx.Done(): 58 | return nil, ctx.Err() 59 | case <-time.After(time.Millisecond): 60 | } 61 | return newMockDriver(ctx) 62 | }) 63 | 64 | err := dp.Start(context.Background()) 65 | require.NoError(err) 66 | defer dp.Stop() 67 | 68 | ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) 69 | defer cancel() 70 | 71 | err = dp.ExecuteCtx(ctx, func(ctx context.Context, d Driver) error { 72 | return errors.New("should not happen") 73 | }) 74 | require.True(err == context.DeadlineExceeded) 75 | } 76 | 77 | func TestDriverPoolState(t *testing.T) { 78 | require := require.New(t) 79 | assert := assert.New(t) 80 | 81 | dp := NewDriverPool(newMockDriver) 82 | 83 | err := dp.Start(context.Background()) 84 | require.NoError(err) 85 | assert.Equal(1, dp.State().Wanted) 86 | assert.Equal(1, dp.State().Running) 87 | 88 | err = dp.Stop() 89 | require.NoError(err) 90 | assert.Equal(0, dp.State().Wanted) 91 | assert.Equal(0, dp.State().Running) 92 | 93 | } 94 | 95 | func TestDiverPoolStart_FailingDriver(t *testing.T) { 96 | require := require.New(t) 97 | 98 | dp := NewDriverPool(func(ctx context.Context) (Driver, error) { 99 | return nil, fmt.Errorf("driver error") 100 | }) 101 | 102 | err := dp.Start(context.Background()) 103 | require.EqualError(err, "driver error") 104 | err = dp.Stop() 105 | require.True(ErrPoolClosed.Is(err)) 106 | } 107 | 108 | func TestDriverPoolExecute_Recovery(t *testing.T) { 109 | require := require.New(t) 110 | 111 | var called int 112 | dp := NewDriverPool(func(ctx context.Context) (Driver, error) { 113 | called++ 114 | return newMockDriver(ctx) 115 | }) 116 | 117 | ctx := context.Background() 118 | 119 | err := dp.Start(ctx) 120 | require.NoError(err) 121 | 122 | for i := 0; i < 100; i++ { 123 | err := dp.ExecuteCtx(ctx, func(_ context.Context, d Driver) error { 124 | require.NotNil(d) 125 | 126 | if i%10 == 9 { 127 | d.(*mockDriver).MockStatus = protocol.Stopped 128 | } 129 | 130 | return nil 131 | }) 132 | 133 | require.Nil(err) 134 | if i%10 != 9 { 135 | require.Len(dp.Current(), 1) 136 | } 137 | } 138 | 139 | err = dp.Stop() 140 | require.NoError(err) 141 | require.Equal(dp.State().Success, 100) 142 | require.Equal(dp.State().Exited, 10) 143 | require.Equal(dp.State().Wanted, 0) 144 | } 145 | 146 | func TestDriverPoolExecute_Sequential(t *testing.T) { 147 | require := require.New(t) 148 | 149 | dp := NewDriverPool(newMockDriver) 150 | 151 | ctx := context.Background() 152 | 153 | err := dp.Start(ctx) 154 | require.NoError(err) 155 | 156 | for i := 0; i < 100; i++ { 157 | err := dp.ExecuteCtx(ctx, func(_ context.Context, d Driver) error { 158 | require.NotNil(d) 159 | return nil 160 | }) 161 | 162 | require.Nil(err) 163 | require.Equal(dp.State().Running, 1) 164 | } 165 | 166 | err = dp.Stop() 167 | require.NoError(err) 168 | } 169 | 170 | func TestDriverPoolExecute_Parallel(t *testing.T) { 171 | require := require.New(t) 172 | 173 | oldWindow := policyDefaultWindow 174 | defer func() { 175 | policyDefaultWindow = oldWindow 176 | }() 177 | policyDefaultWindow = time.Second / 2 178 | 179 | dp := NewDriverPool(newMockDriver) 180 | 181 | ctx := context.Background() 182 | 183 | err := dp.Start(ctx) 184 | require.NoError(err) 185 | 186 | var wg sync.WaitGroup 187 | for i := 0; i < 100; i++ { 188 | wg.Add(1) 189 | go func() { 190 | defer wg.Done() 191 | err := dp.ExecuteCtx(ctx, func(_ context.Context, _ Driver) error { 192 | select { 193 | case <-ctx.Done(): 194 | return ctx.Err() 195 | case <-time.After(policyDefaultTick / 2): 196 | } 197 | return nil 198 | }) 199 | 200 | require.Nil(err) 201 | require.True(len(dp.Current()) >= 1) 202 | }() 203 | } 204 | 205 | wg.Wait() 206 | require.Len(dp.Current(), runtime.NumCPU()) 207 | 208 | // need approximately two full windows, times the inverse downscale factor 209 | time.Sleep(policyDefaultWindow * defaultPolicyTargetWindow * time.Duration(1/policyDefaultDownscale)) 210 | require.Equal(1, dp.State().Running) 211 | 212 | err = dp.Stop() 213 | require.NoError(err) 214 | } 215 | 216 | type mockScalingPolicy struct { 217 | Total, Idle, Load int 218 | Result int 219 | } 220 | 221 | func (p *mockScalingPolicy) Scale(total, idle, load int) int { 222 | p.Total = total 223 | p.Idle = idle 224 | p.Load = load 225 | return p.Result 226 | } 227 | 228 | func TestMinMax(t *testing.T) { 229 | require := require.New(t) 230 | 231 | m := &mockScalingPolicy{} 232 | p := MinMax(5, 10, m) 233 | m.Result = 1 234 | require.Equal(5, p.Scale(1, 0, 1)) 235 | m.Result = 5 236 | require.Equal(5, p.Scale(1, 0, 1)) 237 | m.Result = 7 238 | require.Equal(7, p.Scale(1, 0, 1)) 239 | m.Result = 10 240 | require.Equal(10, p.Scale(1, 0, 1)) 241 | m.Result = 11 242 | require.Equal(10, p.Scale(1, 0, 1)) 243 | } 244 | 245 | func TestMovingAverage(t *testing.T) { 246 | require := require.New(t) 247 | 248 | m := &mockScalingPolicy{} 249 | p := MovingAverage(1, m) 250 | p.Scale(1, 0, 2) 251 | require.Equal(1, m.Total) 252 | require.Equal(2, m.Load) 253 | p.Scale(1, 0, 50) 254 | require.Equal(1, m.Total) 255 | require.Equal(50, m.Load) 256 | 257 | p = MovingAverage(2, m) 258 | p.Scale(1, 0, 1) 259 | require.Equal(1, m.Load) 260 | p.Scale(1, 0, 3) 261 | require.Equal(2, m.Load) 262 | p.Scale(1, 0, 7) 263 | require.Equal(5, m.Load) 264 | 265 | p = MovingAverage(100, m) 266 | for i := 0; i < 100; i++ { 267 | p.Scale(1, 0, 200) 268 | require.Equal(200, m.Load) 269 | } 270 | 271 | for i := 0; i < 50; i++ { 272 | p.Scale(1, 0, 100) 273 | } 274 | require.Equal(150, m.Load) 275 | } 276 | 277 | func TestAIMD(t *testing.T) { 278 | require := require.New(t) 279 | 280 | p := AIMD(1, 0.5) 281 | 282 | require.Equal(1, p.Scale(0, 0, 0)) 283 | require.Equal(1, p.Scale(1, 0, 0)) 284 | require.Equal(1, p.Scale(1, 1, 0)) 285 | 286 | require.Equal(1, p.Scale(0, 0, 1)) 287 | require.Equal(2, p.Scale(1, 0, 1)) 288 | require.Equal(2, p.Scale(1, 1, 2)) 289 | 290 | require.Equal(1, p.Scale(1, 1, 0)) 291 | require.Equal(1, p.Scale(2, 2, 1)) 292 | require.Equal(2, p.Scale(2, 2, 2)) 293 | } 294 | -------------------------------------------------------------------------------- /cmd/bblfshd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | "os" 10 | "os/signal" 11 | "sync" 12 | "syscall" 13 | "time" 14 | 15 | _ "net/http/pprof" 16 | 17 | "gopkg.in/src-d/go-log.v1" 18 | 19 | "github.com/bblfsh/bblfshd/daemon" 20 | "github.com/bblfsh/bblfshd/runtime" 21 | 22 | cmdutil "github.com/bblfsh/sdk/v3/cmd" 23 | "github.com/bblfsh/sdk/v3/driver/manifest/discovery" 24 | "github.com/prometheus/client_golang/prometheus" 25 | "github.com/prometheus/client_golang/prometheus/promhttp" 26 | pversion "github.com/prometheus/common/version" 27 | jaegercfg "github.com/uber/jaeger-client-go/config" 28 | ) 29 | 30 | const ( 31 | undefined = "undefined" 32 | buildDateFormat = "2006-01-02T15:04:05-0700" 33 | ) 34 | 35 | var ( 36 | version = undefined 37 | build = undefined 38 | 39 | network *string 40 | address *string 41 | storage *string 42 | transport *string 43 | maxMessageSize *int 44 | 45 | ctl struct { 46 | network *string 47 | address *string 48 | } 49 | 50 | logcfg struct { 51 | level *string 52 | format *string 53 | fields *string 54 | } 55 | 56 | pprof struct { 57 | enabled *bool 58 | address *string 59 | } 60 | metrics struct { 61 | address *string 62 | } 63 | cmd *flag.FlagSet 64 | 65 | usrListener net.Listener 66 | ctlListener net.Listener 67 | ) 68 | 69 | func init() { 70 | pversion.Version = version 71 | pversion.BuildDate = build 72 | prometheus.MustRegister(pversion.NewCollector("bblfshd")) 73 | 74 | cmd = flag.NewFlagSet("bblfshd", flag.ExitOnError) 75 | network = cmd.String("network", "tcp", "network type: tcp, tcp4, tcp6, unix or unixpacket.") 76 | address = cmd.String("address", "0.0.0.0:9432", "address to listen.") 77 | storage = cmd.String("storage", "/var/lib/bblfshd", "path where all the runtime information is stored.") 78 | transport = cmd.String("transport", "docker", "default transport to fetch driver images: docker or docker-daemon)") 79 | maxMessageSize = cmdutil.FlagMaxGRPCMsgSizeMB(cmd) 80 | 81 | ctl.network = cmd.String("ctl-network", "unix", "control server network type: tcp, tcp4, tcp6, unix or unixpacket.") 82 | ctl.address = cmd.String("ctl-address", "/var/run/bblfshctl.sock", "control server address to listen.") 83 | 84 | logLevel := os.Getenv("LOG_LEVEL") 85 | if logLevel == "" { 86 | logLevel = "info" 87 | } 88 | logcfg.level = cmd.String("log-level", logLevel, "log level: panic, fatal, error, warning, info, debug.") 89 | logcfg.format = cmd.String("log-format", "text", "format of the logs: text or json.") 90 | logcfg.fields = cmd.String("log-fields", "", "extra fields to add to every log line in json format.") 91 | 92 | pprof.enabled = cmd.Bool("profiler", false, "run profiler http endpoint (pprof).") 93 | pprof.address = cmd.String("profiler-address", ":6060", "profiler address to listen on.") 94 | metrics.address = cmd.String("metrics-address", ":2112", "metrics address to listen on.") 95 | cmd.Parse(os.Args[1:]) 96 | 97 | buildLogger() 98 | runtime.Bootstrap() 99 | } 100 | 101 | func driverImage(id string) string { 102 | return fmt.Sprintf("docker://bblfsh/%s-driver:latest", id) 103 | } 104 | 105 | func installRecommended(d *daemon.Daemon) error { 106 | ctx := context.Background() 107 | list, err := discovery.OfficialDrivers(ctx, &discovery.Options{ 108 | NoMaintainers: true, 109 | }) 110 | if err != nil { 111 | return err 112 | } 113 | for _, dr := range list { 114 | if !dr.IsRecommended() { 115 | continue 116 | } 117 | image := driverImage(dr.Language) 118 | log.Infof("installing driver for %s (%s)", dr.Language, image) 119 | err = d.InstallDriver(dr.Language, image, false) 120 | if err != nil { 121 | return err 122 | } 123 | } 124 | return nil 125 | } 126 | 127 | func main() { 128 | log.Infof("bblfshd version: %s (build: %s)", version, build) 129 | 130 | if *pprof.enabled { 131 | log.Infof("running pprof on %s", *pprof.address) 132 | go func() { 133 | if err := http.ListenAndServe(*pprof.address, nil); err != nil { 134 | log.Errorf(err, "cannot start pprof") 135 | } 136 | }() 137 | } 138 | if *metrics.address != "" { 139 | log.Infof("running metrics on %s", *metrics.address) 140 | go func() { 141 | if err := http.ListenAndServe(*metrics.address, promhttp.Handler()); err != nil { 142 | log.Errorf(err, "cannot start metrics") 143 | } 144 | }() 145 | } 146 | 147 | if os.Getenv("JAEGER_AGENT_HOST") != "" { 148 | c, err := jaegercfg.FromEnv() 149 | if err != nil { 150 | log.Errorf(err, "error configuring tracer") 151 | os.Exit(1) 152 | } 153 | closer, err := c.InitGlobalTracer("bblfshd") 154 | if err != nil { 155 | log.Errorf(err, "error configuring tracer") 156 | os.Exit(1) 157 | } 158 | defer closer.Close() 159 | } 160 | 161 | r := buildRuntime() 162 | grpcOpts, err := cmdutil.GRPCSizeOptions(*maxMessageSize) 163 | if err != nil { 164 | log.Errorf(err, "cannot get gRPC server options\n") 165 | os.Exit(1) 166 | } 167 | 168 | parsedBuild, err := time.Parse(buildDateFormat, build) 169 | if err != nil { 170 | if build == undefined { 171 | parsedBuild = time.Now() 172 | log.Infof("using start time instead in this dev build: %s", 173 | parsedBuild.Format(buildDateFormat)) 174 | } else { 175 | log.Errorf(err, "wrong date format for this build") 176 | os.Exit(1) 177 | } 178 | } 179 | d := daemon.NewDaemon(version, parsedBuild, r, grpcOpts...) 180 | if args := cmd.Args(); len(args) == 2 && args[0] == "install" && args[1] == "recommended" { 181 | err := installRecommended(d) 182 | if err != nil { 183 | log.Errorf(err, "error listing drivers") 184 | os.Exit(1) 185 | } 186 | return 187 | } 188 | 189 | var wg sync.WaitGroup 190 | wg.Add(2) 191 | go func() { 192 | defer wg.Done() 193 | listenUser(d) 194 | }() 195 | go func() { 196 | defer wg.Done() 197 | listenControl(d) 198 | }() 199 | handleGracefullyShutdown(d) 200 | wg.Wait() 201 | } 202 | 203 | func listenUser(d *daemon.Daemon) { 204 | var err error 205 | usrListener, err = net.Listen(*network, *address) 206 | if err != nil { 207 | log.Errorf(err, "error creating listener") 208 | os.Exit(1) 209 | } 210 | 211 | allowAnyoneInUnixSocket(*network, *address) 212 | log.Infof("server listening in %s (%s)", *address, *network) 213 | if err = d.UserServer.Serve(usrListener); err != nil { 214 | log.Errorf(err, "error starting server") 215 | os.Exit(1) 216 | } 217 | } 218 | 219 | func listenControl(d *daemon.Daemon) { 220 | var err error 221 | if *ctl.network == "unix" { 222 | // Remove returns an error if file does not exists 223 | // if it returns nil, we know the file existed, so bblfshd might have crashed 224 | if err := os.Remove(*ctl.address); err == nil { 225 | log.Warningf("control socket %s (%s) already exists", *ctl.address, *ctl.network) 226 | } 227 | } 228 | ctlListener, err = net.Listen(*ctl.network, *ctl.address) 229 | if err != nil { 230 | log.Errorf(err, "error creating control listener") 231 | os.Exit(1) 232 | } 233 | 234 | allowAnyoneInUnixSocket(*ctl.network, *ctl.address) 235 | log.Infof("control server listening in %s (%s)", *ctl.address, *ctl.network) 236 | if err = d.ControlServer.Serve(ctlListener); err != nil { 237 | log.Errorf(err, "error starting control server") 238 | os.Exit(1) 239 | } 240 | } 241 | 242 | func allowAnyoneInUnixSocket(network, address string) { 243 | if network != "unix" { 244 | return 245 | } 246 | 247 | if err := os.Chmod(address, 0777); err != nil { 248 | log.Errorf(err, "error changing permissions to socket %q", address) 249 | os.Exit(1) 250 | } 251 | } 252 | 253 | func buildLogger() { 254 | log.New(nil) 255 | 256 | f := log.DefaultFactory 257 | f.Level = *logcfg.level 258 | f.Format = *logcfg.format 259 | f.Fields = *logcfg.fields 260 | if err := f.ApplyToLogrus(); err != nil { 261 | log.Errorf(err, "invalid logger configuration") 262 | os.Exit(1) 263 | } 264 | } 265 | 266 | func buildRuntime() *runtime.Runtime { 267 | log.Infof("initializing runtime at %s", *storage) 268 | 269 | r := runtime.NewRuntime(*storage) 270 | if err := r.Init(); err != nil { 271 | log.Errorf(err, "error initializing runtime") 272 | os.Exit(1) 273 | } 274 | 275 | return r 276 | } 277 | 278 | func handleGracefullyShutdown(d *daemon.Daemon) { 279 | var gracefulStop = make(chan os.Signal) 280 | signal.Notify(gracefulStop, syscall.SIGTERM) 281 | signal.Notify(gracefulStop, syscall.SIGINT) 282 | go waitForStop(gracefulStop, d) 283 | } 284 | 285 | func waitForStop(ch <-chan os.Signal, d *daemon.Daemon) { 286 | sig := <-ch 287 | log.Warningf("signal received %+v", sig) 288 | log.Warningf("stopping server") 289 | if err := d.Stop(); err != nil { 290 | log.Errorf(err, "error stopping server") 291 | } 292 | 293 | for _, l := range []net.Listener{ctlListener, usrListener} { 294 | if err := l.Close(); err != nil { 295 | log.Errorf(err, "error closing listener") 296 | } 297 | } 298 | 299 | os.Exit(0) 300 | } 301 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bblfshd [![Build Status](https://travis-ci.org/bblfsh/bblfshd.svg?branch=master)](https://travis-ci.org/bblfsh/bblfshd) [![codecov](https://codecov.io/gh/bblfsh/bblfshd/branch/master/graph/badge.svg)](https://codecov.io/gh/bblfsh/bblfshd) [![license](https://img.shields.io/badge/license-GPL--3.0-blue.svg)](https://github.com/bblfsh/bblfshd/blob/master/LICENSE) [![GitHub release](https://img.shields.io/github/release/bblfsh/bblfshd.svg)](https://github.com/bblfsh/bblfshd/releases) 2 | 3 | This repository contains bblfsh daemon (*bblfshd*), which includes the 4 | runtime that runs the driver in *containers* and the bblfshctl, a cli tool used 5 | to control the installed drivers and query the status of the daemon. 6 | 7 | Drivers are implemented as Docker images, each having their own repository in the 8 | [`bblfsh` organization](https://github.com/search?q=topic%3Adriver+org%3Abblfsh&type=Repositories) 9 | on GitHub. For more information, see [bblfsh SDK documentation](https://doc.bblf.sh/writing-a-driver/babelfish-sdk.html). 10 | 11 | ## Getting Started 12 | 13 | See the [Getting Started](https://doc.bblf.sh/using-babelfish/getting-started.html) guide. 14 | 15 | ### Quick start 16 | 17 | This project is now part of [source{d} Engine](https://sourced.tech/engine), 18 | which provides the simplest way to get started with a single command. 19 | Visit [sourced.tech/engine](https://sourced.tech/engine) for more information. 20 | 21 | #### Rootless mode 22 | 23 | The recommended way to run *bblfshd* by itself is using Docker: 24 | 25 | ```sh 26 | docker run -d --name bblfshd \ 27 | -p 9432:9432 \ 28 | -v /var/lib/bblfshd:/var/lib/bblfshd \ 29 | -v /proc:/newproc \ 30 | --security-opt seccomp=./bblfshd-seccomp.json \ 31 | bblfsh/bblfshd 32 | ``` 33 | 34 | On macOS, use this command instead to use a Docker volume: 35 | 36 | ```sh 37 | docker run -d --name bblfshd \ 38 | -p 9432:9432 \ 39 | -v bblfsh-storage:/var/lib/bblfshd bblfsh/bblfshd \ 40 | -v /proc:/newproc \ 41 | --security-opt seccomp=./bblfshd-seccomp.json \ 42 | bblfsh/bblfshd 43 | ``` 44 | 45 | 46 | To understand the flags `-v /proc:/newproc` and `--security-opt seccomp=./bblfshd-seccomp.json`, 47 | where [`bblfshd-seccomp.json`](./bblfshd-seccomp.json) is a file present in this repo, and check 48 | further requirements, please refer to [rootless.md](./rootless.md). `bblfshd` is based on 49 | [container technology](https://github.com/opencontainers/runc/tree/master/libcontainer) 50 | and interacts with the kernel at a low level. It exposes a gRPC server at the port `9432` by default 51 | which is used by the [clients](https://github.com/search?q=topic%3Aclient+org%3Abblfsh&type=Repositories) 52 | to interact with the server. Also, we mount the path `/var/lib/bblfshd/` where 53 | all the driver images and container instances will be stored. 54 | 55 | #### Privileged mode 56 | 57 | We advise against it, but if you prefer to run `bblfshd` in `privileged` mode to skip configuration steps of 58 | [rootless.md](rootless.md), you could do, in Linux: 59 | 60 | ```sh 61 | docker run -d --name bblfshd --privileged -p 9432:9432 -v /var/lib/bblfshd:/var/lib/bblfshd bblfsh/bblfshd 62 | ``` 63 | 64 | or macOs: 65 | 66 | ```sh 67 | docker run -d --name bblfshd --privileged -p 9432:9432 -v bblfsh-storage:/var/lib/bblfshd bblfsh/bblfshd 68 | ``` 69 | 70 | #### Install drivers 71 | 72 | Now you need to install the driver images into the daemon, you can install 73 | the official images just running the command: 74 | 75 | ```sh 76 | docker exec -it bblfshd bblfshctl driver install --all 77 | ``` 78 | 79 | You can check the installed versions by executing: 80 | ``` 81 | docker exec -it bblfshd bblfshctl driver list 82 | ``` 83 | 84 | ``` 85 | +----------+-------------------------------+---------+--------+---------+-----+-------------+ 86 | | LANGUAGE | IMAGE | VERSION | STATUS | CREATED | GO | NATIVE | 87 | +----------+-------------------------------+---------+--------+---------+-----+-------------+ 88 | | python | //bblfsh/python-driver:latest | v1.1.5 | beta | 4 days | 1.8 | 3.6.2 | 89 | | java | //bblfsh/java-driver:latest | v1.1.0 | alpha | 6 days | 1.8 | 8.131.11-r2 | 90 | +----------+-------------------------------+---------+--------+---------+-----+-------------+ 91 | ``` 92 | 93 | To test the driver you can execute a parse request to the server with the `bblfshctl parse` command, 94 | and an example contained in the Docker image: 95 | 96 | ```sh 97 | docker exec -it bblfshd bblfshctl parse /opt/bblfsh/etc/examples/python.py 98 | ``` 99 | 100 | ## SELinux 101 | 102 | If your system has SELinux enabled (which is the default in Fedora, Red Hat, CentOS 103 | and many others) you'll need to compile and load a policy module before running the 104 | bblfshd Docker image or running driver containers will fail with a `permission 105 | denied` message in the logs. 106 | 107 | To do this, run these commands from the project root: 108 | 109 | ```bash 110 | cd selinux/ 111 | sh compile.sh 112 | semodule -i bblfshd.pp 113 | ``` 114 | 115 | If you were already running an instance of *bblfshd*, you will need to delete the 116 | container (`docker rm -f bblfshd`) and run it again (`docker run...`). 117 | 118 | Once the module has been loaded with `semodule` the change should persist even 119 | if you reboot. If you want to permanently remove this module run `semodule -d bblfshd`. 120 | 121 | Alternatively, you could set SELinux to permissive module with: 122 | 123 | ``` 124 | echo 1 > /sys/fs/selinux/enforce 125 | ``` 126 | 127 | (doing this on production systems which usually have SELinux enabled by default 128 | should be strongly discouraged). 129 | 130 | ## Development 131 | 132 | If you wish to work on *bblfshd* , you'll first need [Go](http://www.golang.org) 133 | installed on your machine (version 1.11+ is *required*) and [Docker](https://docs.docker.com/engine/installation/). 134 | Docker is used to build and run tests in an isolated environment. 135 | 136 | For local development of *bblfshd*, first make sure Go is properly installed and 137 | that a [GOPATH](http://golang.org/doc/code.html#GOPATH) has been set. You will 138 | also need to add `$GOPATH/bin` to your `$PATH`. 139 | 140 | Next, using [Git](https://git-scm.com/), clone this repository into 141 | `$GOPATH/src/github.com/bblfsh/bblfshd`. All the necessary dependencies are 142 | automatically installed, so you just need to type `make`. This will compile the 143 | code and then run the tests. If this exits with exit status 0, then everything 144 | is working! 145 | 146 | 147 | ### Dependencies 148 | 149 | Ensure you have [`ostree`](https://github.com/ostreedev/ostree) and development libraries for `ostree` installed. 150 | 151 | You can install from your distribution pack manager as follow, or built [from source](https://github.com/ostreedev/ostree) (more on that [here](https://ostree.readthedocs.io/en/latest/#building)). 152 | 153 | Debian, Ubuntu, and related distributions: 154 | ``` 155 | $ apt-get install libostree-dev 156 | ``` 157 | 158 | Fedora, CentOS, RHEL, and related distributions: 159 | ```bash 160 | $ yum install -y ostree-devel 161 | ``` 162 | 163 | Arch and related distributions: 164 | 165 | ```bash 166 | $ pacman -S ostree 167 | ``` 168 | 169 | ### Building From Source 170 | 171 | Build with: 172 | 173 | ``` 174 | $ make build 175 | ``` 176 | 177 | ### Running Tests 178 | 179 | Run tests with: 180 | 181 | ``` 182 | $ make test 183 | ``` 184 | 185 | ### Environment variables 186 | 187 | - `BBLFSHD_MAX_DRIVER_INSTANCES` - maximal number of driver instances for each language. 188 | Default to a number of CPUs. 189 | 190 | - `BBLFSHD_MIN_DRIVER_INSTANCES` - minimal number of driver instances that will be run 191 | for each language. Default to 1. 192 | 193 | ### Enable tracing 194 | 195 | Bblfshd supports [OpenTracing](https://opentracing.io/) that can be used to profile request on a high level or trace 196 | individual requests to bblfshd and/or language drivers. 197 | 198 | To enable it, you can use [Jaeger](https://www.jaegertracing.io/docs/1.8/getting-started/). 199 | The easiest way is to start all-in-one Jaeger image: 200 | ``` 201 | docker run -d --name jaeger \ 202 | -e COLLECTOR_ZIPKIN_HTTP_PORT=9411 \ 203 | -p 5775:5775/udp \ 204 | -p 6831:6831/udp \ 205 | -p 6832:6832/udp \ 206 | -p 5778:5778 \ 207 | -p 16686:16686 \ 208 | -p 14268:14268 \ 209 | -p 9411:9411 \ 210 | jaegertracing/all-in-one:1.8 211 | ``` 212 | 213 | For Docker installation of bblfshd add the following flags: 214 | ``` 215 | --link jaeger:jaeger -e JAEGER_AGENT_HOST=jaeger -e JAEGER_AGENT_PORT=6831 -e JAEGER_SAMPLER_TYPE=const -e JAEGER_SAMPLER_PARAM=1 216 | ``` 217 | 218 | For bblfshd running locally, set following environment variables: 219 | ``` 220 | JAEGER_AGENT_HOST=localhost JAEGER_AGENT_PORT=6831 JAEGER_SAMPLER_TYPE=const JAEGER_SAMPLER_PARAM=1 221 | ``` 222 | 223 | Run few requests, and check traces at http://localhost:16686. 224 | 225 | For enabling tracing in production, consult [Jaeger documentation](https://www.jaegertracing.io/docs/1.8). 226 | 227 | ## License 228 | 229 | GPLv3, see [LICENSE](LICENSE) 230 | 231 | -------------------------------------------------------------------------------- /daemon/daemon.go: -------------------------------------------------------------------------------- 1 | // +build linux,cgo 2 | 3 | package daemon 4 | 5 | import ( 6 | "context" 7 | "os" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "google.golang.org/grpc" 13 | "google.golang.org/grpc/keepalive" 14 | 15 | "github.com/opentracing/opentracing-go" 16 | "gopkg.in/src-d/go-log.v1" 17 | 18 | "github.com/bblfsh/bblfshd/daemon/protocol" 19 | "github.com/bblfsh/bblfshd/runtime" 20 | 21 | "github.com/bblfsh/sdk/v3/driver" 22 | "github.com/bblfsh/sdk/v3/driver/manifest" 23 | protocol2 "github.com/bblfsh/sdk/v3/protocol" 24 | protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" 25 | ) 26 | 27 | const ( 28 | // keepaliveMinTime is the minimum amount of time a client should wait before sending 29 | // a keepalive ping. 30 | keepaliveMinTime = 1 * time.Minute 31 | 32 | // keepalivePingWithoutStream is a boolean flag. 33 | // If true, server allows keepalive pings even when there are no active 34 | // streams(RPCs). If false, and client sends ping when there are no active 35 | // streams, server will send GOAWAY and close the connection. 36 | keepalivePingWithoutStream = true 37 | ) 38 | 39 | // Daemon is a Babelfish server. 40 | type Daemon struct { 41 | UserServer *grpc.Server 42 | ControlServer *grpc.Server 43 | 44 | version string 45 | build time.Time 46 | runtime *runtime.Runtime 47 | driverEnv []string 48 | 49 | mu sync.RWMutex 50 | pool map[string]*DriverPool // language ID → driver pool 51 | aliases map[string]string // alias → language ID 52 | } 53 | 54 | // NewDaemon creates a new server based on the runtime with the given version. 55 | func NewDaemon(version string, build time.Time, r *runtime.Runtime, opts ...grpc.ServerOption) *Daemon { 56 | commonOpt := append(protocol2.ServerOptions(), 57 | // EnforcementPolicy is used to set keepalive enforcement policy on the 58 | // server-side. Server will close connection with a client that violates this 59 | // policy. 60 | grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ 61 | MinTime: keepaliveMinTime, 62 | PermitWithoutStream: keepalivePingWithoutStream, 63 | }), 64 | ) 65 | opts = append(commonOpt, opts...) 66 | 67 | d := &Daemon{ 68 | version: version, 69 | build: build, 70 | runtime: r, 71 | pool: make(map[string]*DriverPool), 72 | aliases: make(map[string]string), 73 | UserServer: grpc.NewServer(opts...), 74 | ControlServer: grpc.NewServer(commonOpt...), 75 | } 76 | registerGRPC(d) 77 | // pass tracing options to each driver 78 | for _, env := range os.Environ() { 79 | if !strings.HasPrefix(env, "JAEGER_") { 80 | continue 81 | } 82 | const traceHost = "JAEGER_AGENT_HOST=" 83 | if strings.HasPrefix(traceHost, "JAEGER_") { 84 | // drivers cannot use Docker DNS as bblfshd does, 85 | // so we need to remap an address manually 86 | if addr := os.Getenv("JAEGER_PORT_6831_UDP_ADDR"); addr != "" { 87 | env = traceHost + addr 88 | } 89 | } 90 | d.driverEnv = append(d.driverEnv, env) 91 | } 92 | return d 93 | } 94 | 95 | func registerGRPC(d *Daemon) { 96 | protocol1.DefaultService = NewService(d) 97 | protocol1.RegisterProtocolServiceServer(d.UserServer, protocol1.NewProtocolServiceServer()) 98 | 99 | s2 := NewServiceV2(d) 100 | protocol2.RegisterDriverServer(d.UserServer, s2) 101 | protocol2.RegisterDriverHostServer(d.UserServer, s2) 102 | protocol.RegisterService(d.ControlServer, NewControlService(d)) 103 | } 104 | 105 | func (d *Daemon) InstallDriver(language string, image string, update bool) error { 106 | driverInstallCalls.Add(1) 107 | 108 | img, err := runtime.NewDriverImage(image) 109 | if err != nil { 110 | return ErrRuntime.Wrap(err) 111 | } 112 | if language == "" { 113 | info, err := img.Inspect() 114 | if err != nil { 115 | return err 116 | } 117 | if lang, ok := info.Labels["bblfsh.language"]; ok { 118 | language = lang 119 | } else { 120 | return ErrLanguageDetection.New() 121 | } 122 | } 123 | 124 | s, _, err := d.getDriverImage(context.TODO(), language) 125 | if err != nil && !driver.IsMissingDriver(err) { 126 | return ErrRuntime.Wrap(err) 127 | } 128 | if err == nil { 129 | if !update { 130 | return ErrAlreadyInstalled.Wrap(err, language, image) 131 | } 132 | // TODO: the old driver should be removed only after a successful install of the new one 133 | if err := d.runtime.RemoveDriver(s); err != nil { 134 | return err 135 | } 136 | } 137 | 138 | _, err = d.runtime.InstallDriver(img, update) 139 | if err != nil { 140 | return err 141 | } 142 | 143 | log.Infof("driver %s installed %q", language, img.Name()) 144 | return nil 145 | } 146 | 147 | func (d *Daemon) RemoveDriver(language string) error { 148 | driverRemoveCalls.Add(1) 149 | 150 | img, _, err := d.getDriverImage(context.TODO(), language) 151 | if err != nil { 152 | return ErrRuntime.Wrap(err) 153 | } 154 | 155 | if err := d.runtime.RemoveDriver(img); err != nil { 156 | return err 157 | } 158 | if err := d.removePool(language); err != nil { 159 | return err 160 | } 161 | 162 | log.Infof("driver %s removed %q", language, img.Name()) 163 | return err 164 | } 165 | 166 | func (d *Daemon) DriverPool(ctx context.Context, language string) (*DriverPool, error) { 167 | language = strings.ToLower(language) 168 | d.mu.RLock() 169 | if l, ok := d.aliases[language]; ok { 170 | language = l 171 | } 172 | dp, ok := d.pool[language] 173 | d.mu.RUnlock() 174 | if ok { 175 | return dp, nil 176 | } 177 | 178 | d.mu.Lock() 179 | defer d.mu.Unlock() 180 | if l, ok := d.aliases[language]; ok { 181 | language = l 182 | } 183 | dp, ok = d.pool[language] 184 | if ok { 185 | return dp, nil 186 | } 187 | 188 | image, m, err := d.getDriverImage(ctx, language) 189 | if err != nil { 190 | return nil, ErrRuntime.Wrap(err) 191 | } 192 | 193 | return d.newDriverPool(ctx, m.Language, m.Aliases, image) 194 | } 195 | 196 | func driverWithLang(lang string, list []*runtime.DriverImageStatus) *runtime.DriverImageStatus { 197 | lang = strings.ToLower(lang) 198 | for _, d := range list { 199 | m := d.Manifest 200 | if strings.ToLower(m.Language) == lang { 201 | return d 202 | } 203 | for _, l := range m.Aliases { 204 | if strings.ToLower(l) == lang { 205 | return d 206 | } 207 | } 208 | } 209 | return nil 210 | } 211 | 212 | func (d *Daemon) getDriverImage(rctx context.Context, language string) (runtime.DriverImage, *manifest.Manifest, error) { 213 | sp, _ := opentracing.StartSpanFromContext(rctx, "bblfshd.runtime.ListDrivers") 214 | defer sp.Finish() 215 | 216 | list, err := d.runtime.ListDrivers() 217 | if err != nil { 218 | return nil, nil, err 219 | } 220 | dr := driverWithLang(language, list) 221 | if dr == nil { 222 | return nil, nil, &ErrMissingDriver{language} 223 | } 224 | img, err := runtime.NewDriverImage(dr.Reference) 225 | return img, dr.Manifest, err 226 | } 227 | 228 | // newDriverPool, instance a new driver pool for the given language and image 229 | // and should be called under a lock. 230 | func (d *Daemon) newDriverPool(rctx context.Context, language string, aliases []string, image runtime.DriverImage) (*DriverPool, error) { 231 | sp, ctx := opentracing.StartSpanFromContext(rctx, "bblfshd.pool.newDriverPool") 232 | defer sp.Finish() 233 | 234 | imageName := image.Name() 235 | labels := []string{language, imageName} 236 | 237 | dp := NewDriverPool(func(rctx context.Context) (Driver, error) { 238 | sp, ctx := opentracing.StartSpanFromContext(rctx, "bblfshd.pool.driverFactory") 239 | defer sp.Finish() 240 | 241 | log.Debugf("spawning driver instance %q ...", imageName) 242 | 243 | opts := d.getDriverInstanceOptions() 244 | driver, err := NewDriverInstance(d.runtime, language, image, opts) 245 | if err != nil { 246 | return nil, err 247 | } 248 | 249 | if err := driver.Start(ctx); err != nil { 250 | return nil, err 251 | } 252 | 253 | log.Infof("new driver instance started %s (%s)", imageName, driver.Container.ID()) 254 | return driver, nil 255 | }) 256 | dp.SetLabels(labels) 257 | 258 | if err := dp.Start(ctx); err != nil { 259 | return nil, err 260 | } 261 | 262 | d.pool[language] = dp 263 | for _, l := range aliases { 264 | log.Debugf("language alias: %s = %s", language, l) 265 | d.aliases[strings.ToLower(l)] = language 266 | } 267 | return dp, nil 268 | } 269 | 270 | func (d *Daemon) removePool(language string) error { 271 | d.mu.Lock() 272 | defer d.mu.Unlock() 273 | dp, ok := d.pool[language] 274 | if !ok { 275 | return nil 276 | } 277 | if err := dp.Stop(); err != nil && !ErrPoolClosed.Is(err) { 278 | return err 279 | } 280 | delete(d.pool, language) 281 | return nil 282 | } 283 | 284 | // Current returns the current list of driver pools. 285 | func (d *Daemon) Current() map[string]*DriverPool { 286 | d.mu.RLock() 287 | defer d.mu.RUnlock() 288 | 289 | out := make(map[string]*DriverPool, len(d.pool)) 290 | for k, pool := range d.pool { 291 | out[k] = pool 292 | } 293 | 294 | return out 295 | } 296 | 297 | // Stop stops all the pools and containers. 298 | func (d *Daemon) Stop() error { 299 | d.mu.Lock() 300 | defer d.mu.Unlock() 301 | var err error 302 | for _, dp := range d.pool { 303 | if cerr := dp.Stop(); cerr != nil && err != nil { 304 | err = cerr 305 | } 306 | } 307 | 308 | return err 309 | } 310 | 311 | func (d *Daemon) getDriverInstanceOptions() *Options { 312 | opts := &Options{Env: d.driverEnv} 313 | opts.LogLevel = log.DefaultLevel 314 | opts.LogFormat = "text" 315 | 316 | if log.DefaultFormat == log.JSONFormat { 317 | opts.LogFormat = "json" 318 | } 319 | 320 | return opts 321 | } 322 | 323 | func newResponseFromError(err error) protocol1.Response { 324 | return protocol1.Response{ 325 | Status: protocol1.Fatal, 326 | Errors: []string{err.Error()}, 327 | } 328 | } 329 | -------------------------------------------------------------------------------- /cmd/bblfshctl/cmd/driver_install.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "sort" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | "github.com/bblfsh/bblfshd/daemon" 14 | "google.golang.org/grpc/codes" 15 | "google.golang.org/grpc/status" 16 | 17 | "github.com/bblfsh/bblfshd/daemon/protocol" 18 | "github.com/bblfsh/sdk/v3/driver/manifest/discovery" 19 | 20 | "github.com/briandowns/spinner" 21 | ) 22 | 23 | var ( 24 | // DefaultTransport is the default transport used when is missing on the 25 | // image reference. 26 | DefaultTransport = "docker://" 27 | 28 | SupportedTransports = map[string]bool{ 29 | "docker": true, 30 | "docker-daemon": true, 31 | } 32 | ) 33 | 34 | var ( 35 | drivers struct { 36 | sync.Once 37 | List []discovery.Driver 38 | } 39 | ) 40 | 41 | func getOfficialDrivers() ([]discovery.Driver, error) { 42 | var err error 43 | drivers.Do(func() { 44 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 45 | defer cancel() 46 | 47 | drivers.List, err = discovery.OfficialDrivers(ctx, &discovery.Options{ 48 | NoMaintainers: true, 49 | }) 50 | if err != nil { 51 | fmt.Fprintf(os.Stderr, "Error, %s\n", err) 52 | } 53 | }) 54 | return drivers.List, err 55 | } 56 | 57 | func driverImage(id string) string { 58 | return fmt.Sprintf("docker://bblfsh/%s-driver:latest", id) 59 | } 60 | 61 | // allDrivers returns the list of all the official bblfsh drivers that are usable. 62 | func allDrivers() (map[string]string, error) { 63 | list, err := getOfficialDrivers() 64 | if err != nil { 65 | return nil, err 66 | } 67 | m := make(map[string]string, len(list)) 68 | for _, d := range list { 69 | if d.InDevelopment() { 70 | continue 71 | } 72 | m[d.Language] = driverImage(d.Language) 73 | } 74 | return m, nil 75 | } 76 | 77 | // recommendedDrivers returns the list of drivers in beta state or better. 78 | func recommendedDrivers() (map[string]string, error) { 79 | list, err := getOfficialDrivers() 80 | if err != nil { 81 | return nil, err 82 | } else if len(list) == 0 { 83 | return nil, errors.New("official drivers list is empty; try updating bblfshd") 84 | } 85 | m := make(map[string]string, len(list)) 86 | for _, d := range list { 87 | if !d.IsRecommended() { 88 | continue 89 | } 90 | m[d.Language] = driverImage(d.Language) 91 | } 92 | if len(m) == 0 { 93 | return nil, errors.New("recommended drivers list is empty; try updating bblfshd") 94 | } 95 | return m, nil 96 | } 97 | 98 | const ( 99 | DriverInstallCommandDescription = "Installs a new driver for a given language" 100 | DriverInstallCommandHelp = DriverInstallCommandDescription + "\n\n" + 101 | "Using `--all` all the official bblfsh driver are install in the \n" + 102 | "daemon. Using `--recommended` will only install the recommended, \n" + 103 | "more developed. Using `language` and `image` positional arguments \n" + 104 | "one single driver can be installed or updated.\n\n" + 105 | "Image reference format should be `[transport]name[:tag]`.\n" + 106 | "Defaults are 'docker://' for transport and 'latest' for tag." 107 | ) 108 | 109 | type DriverInstallCommand struct { 110 | Args struct { 111 | Language string `positional-arg-name:"language" description:"language supported by the driver (optional)"` 112 | ImageReference string `positional-arg-name:"image" description:"driver's image reference"` 113 | } `positional-args:"yes"` 114 | 115 | Update bool `long:"update" description:"replace the current image for the language if any"` 116 | All bool `long:"all" description:"installs all the official drivers"` 117 | Recommended bool `long:"recommended" description:"install the recommended official drivers"` 118 | Force bool `short:"f" long:"force" description:"ignore already installed errors"` 119 | 120 | DriverCommand 121 | } 122 | 123 | func (c *DriverInstallCommand) Execute(args []string) error { 124 | if err := c.Validate(); err != nil { 125 | return err 126 | } 127 | 128 | if err := c.ControlCommand.Execute(nil); err != nil { 129 | return err 130 | } 131 | 132 | if !c.All && !c.Recommended { 133 | if c.Args.ImageReference == "" { 134 | // TODO: go-flags does not support optional arguments in first positions 135 | c.Args.Language, c.Args.ImageReference = "", c.Args.Language 136 | } 137 | return c.installDrivers([]driverRef{{Lang: c.Args.Language, Ref: c.Args.ImageReference}}) 138 | } 139 | 140 | var ( 141 | m map[string]string 142 | err error 143 | ) 144 | if c.Recommended { 145 | m, err = recommendedDrivers() 146 | } else { 147 | m, err = allDrivers() 148 | } 149 | if err != nil { 150 | return err 151 | } 152 | 153 | list := make([]driverRef, 0, len(m)) 154 | for lang, image := range m { 155 | list = append(list, driverRef{Lang: lang, Ref: image}) 156 | } 157 | return c.installDrivers(list) 158 | } 159 | 160 | func (c *DriverInstallCommand) Validate() error { 161 | if !c.All && !c.Recommended && (c.Args.Language == "") { 162 | return fmt.Errorf("error `image` positional argument is mandatory") 163 | } 164 | 165 | if c.All && c.Recommended { 166 | return fmt.Errorf("error --all and --recommended are exclusive") 167 | } 168 | 169 | return nil 170 | } 171 | 172 | type driverRef struct { 173 | Lang string 174 | Ref string 175 | } 176 | 177 | func (c *DriverInstallCommand) installDrivers(refs []driverRef) error { 178 | if len(refs) == 0 { 179 | return nil 180 | } else if len(refs) == 1 { 181 | return c.installSingleDriver(refs[0]) 182 | } 183 | refNum := len(refs) 184 | ctx := context.Background() 185 | const workers = 3 186 | type resp struct { 187 | Ref string 188 | Err error 189 | } 190 | var ( 191 | wg sync.WaitGroup 192 | last error 193 | errs []error 194 | jobs = make(chan driverRef) 195 | out = make(chan resp, len(refs)) 196 | ) 197 | for i := 0; i < workers; i++ { 198 | wg.Add(1) 199 | go func() { 200 | defer wg.Done() 201 | 202 | for ref := range jobs { 203 | err := c.installDriver(ctx, ref) 204 | out <- resp{ 205 | Ref: ref.Ref, Err: err, 206 | } 207 | } 208 | }() 209 | } 210 | 211 | mref := make(map[string]driverRef, len(refs)) 212 | for _, ref := range refs { 213 | mref[ref.Ref] = ref 214 | } 215 | status := make(map[string]string, len(refs)) 216 | 217 | spin := spinner.New(spinner.CharSets[9], 100*time.Millisecond) 218 | clist := make([][3]string, 0, len(mref)) 219 | todo := len(refs) 220 | done := 0 221 | 222 | accept := func(r resp) { 223 | todo-- 224 | str := " + " 225 | if r.Err != nil { 226 | if daemon.ErrAlreadyInstalled.Is(r.Err) && c.Force { 227 | done++ 228 | } else { 229 | last = r.Err 230 | errs = append(errs, r.Err) 231 | str = "ERR" 232 | } 233 | } else { 234 | done++ 235 | } 236 | status[r.Ref] = str 237 | } 238 | 239 | draining := false 240 | first := true 241 | install: 242 | for todo >= 0 { 243 | clist = clist[:0] 244 | for _, ref := range mref { 245 | clist = append(clist, [3]string{ref.Lang, status[ref.Ref], ref.Ref}) 246 | } 247 | sort.Slice(clist, func(i, j int) bool { 248 | return clist[i][0] < clist[j][0] 249 | }) 250 | 251 | if first { 252 | first = false 253 | } else { 254 | fmt.Print(fmt.Sprintf("\033[%dA", refNum+3)) //delete previous N lines in terminal 255 | fmt.Print("\033[J") 256 | } 257 | 258 | if todo == 0 { 259 | fmt.Printf("\nInstalled %d/%d drivers:\n", done, len(clist)) 260 | } else { 261 | fmt.Printf("\nInstalling %d/%d drivers:\n", todo, len(clist)) 262 | } 263 | for _, cols := range clist { 264 | fmt.Printf("%12s %3s %s\n", cols[0], cols[1], cols[2]) 265 | } 266 | if todo == 0 { 267 | break 268 | } 269 | fmt.Print("Please wait ") 270 | spin.Start() 271 | 272 | if !draining { 273 | if len(refs) != 0 { 274 | cur := refs[0] 275 | // send jobs and receive responses 276 | select { 277 | case jobs <- cur: 278 | refs = refs[1:] 279 | status[cur.Ref] = "..." 280 | case r := <-out: 281 | accept(r) 282 | } 283 | spin.Stop() 284 | fmt.Println() 285 | continue install 286 | } 287 | // no jobs left 288 | close(jobs) 289 | draining = true 290 | } 291 | 292 | // drain responses 293 | r := <-out 294 | spin.Stop() 295 | fmt.Println() 296 | accept(r) 297 | } 298 | fmt.Println() 299 | if len(errs) != 0 { 300 | for _, err := range errs { 301 | fmt.Fprintln(os.Stderr, "error:", err) 302 | } 303 | } else { 304 | fmt.Println("Done") 305 | } 306 | return last 307 | } 308 | 309 | func (c *DriverInstallCommand) installDriver(ctx context.Context, ref driverRef) error { 310 | ref.Ref = c.getImageReference(ref.Ref) 311 | r, err := c.srv.InstallDriver(ctx, &protocol.InstallDriverRequest{ 312 | Language: ref.Lang, 313 | ImageReference: ref.Ref, 314 | Update: c.Update, 315 | }) 316 | if st, ok := status.FromError(err); ok && st.Code() == codes.AlreadyExists { 317 | return daemon.ErrAlreadyInstalled.New(ref.Lang, ref.Ref) 318 | } else if ok && st.Code() == codes.Unauthenticated { 319 | return daemon.ErrUnauthorized.New(ref.Lang, ref.Ref) 320 | } else if err == nil && len(r.Errors) == 0 { 321 | return nil 322 | } 323 | if err == nil { 324 | err = fmt.Errorf("%v", r.Errors) 325 | } 326 | return err 327 | } 328 | 329 | func (c *DriverInstallCommand) installSingleDriver(ref driverRef) error { 330 | ltext := "" 331 | if ref.Lang != "" { 332 | ltext = fmt.Sprintf("%s language ", ref.Lang) 333 | } 334 | fmt.Printf("Installing %sdriver from %q... ", ltext, ref.Ref) 335 | s := spinner.New(spinner.CharSets[9], 100*time.Millisecond) // Build our new spinner 336 | s.Start() 337 | 338 | err := c.installDriver(context.Background(), ref) 339 | 340 | s.Stop() 341 | if err == nil { 342 | fmt.Println("Done") 343 | return nil 344 | } 345 | if daemon.ErrAlreadyInstalled.Is(err) && c.Force { 346 | fmt.Fprintf(os.Stderr, "Warning: %s\n", err) 347 | return nil 348 | } 349 | if daemon.ErrUnauthorized.Is(err) { 350 | if strings.Contains(ref.Ref, "://bblfsh") { 351 | return fmt.Errorf("driver does not exist") 352 | } else { 353 | return fmt.Errorf("driver does not exist or is private") 354 | } 355 | } 356 | return err 357 | } 358 | 359 | func (c *DriverInstallCommand) getImageReference(ref string) string { 360 | parts := strings.SplitN(ref, ":", 2) 361 | if _, ok := SupportedTransports[parts[0]]; !ok { 362 | return DefaultTransport + ref 363 | } 364 | 365 | return ref 366 | } 367 | -------------------------------------------------------------------------------- /bblfshd-seccomp.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultAction": "SCMP_ACT_ERRNO", 3 | "archMap": [ 4 | { 5 | "architecture": "SCMP_ARCH_X86_64", 6 | "subArchitectures": [ 7 | "SCMP_ARCH_X86", 8 | "SCMP_ARCH_X32" 9 | ] 10 | }, 11 | { 12 | "architecture": "SCMP_ARCH_AARCH64", 13 | "subArchitectures": [ 14 | "SCMP_ARCH_ARM" 15 | ] 16 | }, 17 | { 18 | "architecture": "SCMP_ARCH_MIPS64", 19 | "subArchitectures": [ 20 | "SCMP_ARCH_MIPS", 21 | "SCMP_ARCH_MIPS64N32" 22 | ] 23 | }, 24 | { 25 | "architecture": "SCMP_ARCH_MIPS64N32", 26 | "subArchitectures": [ 27 | "SCMP_ARCH_MIPS", 28 | "SCMP_ARCH_MIPS64" 29 | ] 30 | }, 31 | { 32 | "architecture": "SCMP_ARCH_MIPSEL64", 33 | "subArchitectures": [ 34 | "SCMP_ARCH_MIPSEL", 35 | "SCMP_ARCH_MIPSEL64N32" 36 | ] 37 | }, 38 | { 39 | "architecture": "SCMP_ARCH_MIPSEL64N32", 40 | "subArchitectures": [ 41 | "SCMP_ARCH_MIPSEL", 42 | "SCMP_ARCH_MIPSEL64" 43 | ] 44 | }, 45 | { 46 | "architecture": "SCMP_ARCH_S390X", 47 | "subArchitectures": [ 48 | "SCMP_ARCH_S390" 49 | ] 50 | } 51 | ], 52 | "syscalls": [ 53 | { 54 | "names": [ 55 | "accept", 56 | "accept4", 57 | "access", 58 | "adjtimex", 59 | "alarm", 60 | "bind", 61 | "brk", 62 | "capget", 63 | "capset", 64 | "chdir", 65 | "chmod", 66 | "chown", 67 | "chown32", 68 | "clock_getres", 69 | "clock_gettime", 70 | "clock_nanosleep", 71 | "close", 72 | "connect", 73 | "copy_file_range", 74 | "creat", 75 | "dup", 76 | "dup2", 77 | "dup3", 78 | "epoll_create", 79 | "epoll_create1", 80 | "epoll_ctl", 81 | "epoll_ctl_old", 82 | "epoll_pwait", 83 | "epoll_wait", 84 | "epoll_wait_old", 85 | "eventfd", 86 | "eventfd2", 87 | "execve", 88 | "execveat", 89 | "exit", 90 | "exit_group", 91 | "faccessat", 92 | "fadvise64", 93 | "fadvise64_64", 94 | "fallocate", 95 | "fanotify_mark", 96 | "fchdir", 97 | "fchmod", 98 | "fchmodat", 99 | "fchown", 100 | "fchown32", 101 | "fchownat", 102 | "fcntl", 103 | "fcntl64", 104 | "fdatasync", 105 | "fgetxattr", 106 | "flistxattr", 107 | "flock", 108 | "fork", 109 | "fremovexattr", 110 | "fsetxattr", 111 | "fstat", 112 | "fstat64", 113 | "fstatat64", 114 | "fstatfs", 115 | "fstatfs64", 116 | "fsync", 117 | "ftruncate", 118 | "ftruncate64", 119 | "futex", 120 | "futimesat", 121 | "getcpu", 122 | "getcwd", 123 | "getdents", 124 | "getdents64", 125 | "getegid", 126 | "getegid32", 127 | "geteuid", 128 | "geteuid32", 129 | "getgid", 130 | "getgid32", 131 | "getgroups", 132 | "getgroups32", 133 | "getitimer", 134 | "getpeername", 135 | "getpgid", 136 | "getpgrp", 137 | "getpid", 138 | "getppid", 139 | "getpriority", 140 | "getrandom", 141 | "getresgid", 142 | "getresgid32", 143 | "getresuid", 144 | "getresuid32", 145 | "getrlimit", 146 | "get_robust_list", 147 | "getrusage", 148 | "getsid", 149 | "getsockname", 150 | "getsockopt", 151 | "get_thread_area", 152 | "gettid", 153 | "gettimeofday", 154 | "getuid", 155 | "getuid32", 156 | "getxattr", 157 | "inotify_add_watch", 158 | "inotify_init", 159 | "inotify_init1", 160 | "inotify_rm_watch", 161 | "io_cancel", 162 | "ioctl", 163 | "io_destroy", 164 | "io_getevents", 165 | "io_pgetevents", 166 | "ioprio_get", 167 | "ioprio_set", 168 | "io_setup", 169 | "io_submit", 170 | "io_uring_enter", 171 | "io_uring_register", 172 | "io_uring_setup", 173 | "ipc", 174 | "kill", 175 | "lchown", 176 | "lchown32", 177 | "lgetxattr", 178 | "link", 179 | "linkat", 180 | "listen", 181 | "listxattr", 182 | "llistxattr", 183 | "_llseek", 184 | "lremovexattr", 185 | "lseek", 186 | "lsetxattr", 187 | "lstat", 188 | "lstat64", 189 | "madvise", 190 | "memfd_create", 191 | "mincore", 192 | "mkdir", 193 | "mkdirat", 194 | "mknod", 195 | "mknodat", 196 | "mlock", 197 | "mlock2", 198 | "mlockall", 199 | "mmap", 200 | "mmap2", 201 | "mprotect", 202 | "mq_getsetattr", 203 | "mq_notify", 204 | "mq_open", 205 | "mq_timedreceive", 206 | "mq_timedsend", 207 | "mq_unlink", 208 | "mremap", 209 | "msgctl", 210 | "msgget", 211 | "msgrcv", 212 | "msgsnd", 213 | "msync", 214 | "munlock", 215 | "munlockall", 216 | "munmap", 217 | "nanosleep", 218 | "newfstatat", 219 | "_newselect", 220 | "open", 221 | "openat", 222 | "pause", 223 | "pipe", 224 | "pipe2", 225 | "poll", 226 | "ppoll", 227 | "prctl", 228 | "pread64", 229 | "preadv", 230 | "preadv2", 231 | "prlimit64", 232 | "pselect6", 233 | "pwrite64", 234 | "pwritev", 235 | "pwritev2", 236 | "read", 237 | "readahead", 238 | "readlink", 239 | "readlinkat", 240 | "readv", 241 | "recv", 242 | "recvfrom", 243 | "recvmmsg", 244 | "recvmsg", 245 | "remap_file_pages", 246 | "removexattr", 247 | "rename", 248 | "renameat", 249 | "renameat2", 250 | "restart_syscall", 251 | "rmdir", 252 | "rt_sigaction", 253 | "rt_sigpending", 254 | "rt_sigprocmask", 255 | "rt_sigqueueinfo", 256 | "rt_sigreturn", 257 | "rt_sigsuspend", 258 | "rt_sigtimedwait", 259 | "rt_tgsigqueueinfo", 260 | "sched_getaffinity", 261 | "sched_getattr", 262 | "sched_getparam", 263 | "sched_get_priority_max", 264 | "sched_get_priority_min", 265 | "sched_getscheduler", 266 | "sched_rr_get_interval", 267 | "sched_setaffinity", 268 | "sched_setattr", 269 | "sched_setparam", 270 | "sched_setscheduler", 271 | "sched_yield", 272 | "seccomp", 273 | "select", 274 | "semctl", 275 | "semget", 276 | "semop", 277 | "semtimedop", 278 | "send", 279 | "sendfile", 280 | "sendfile64", 281 | "sendmmsg", 282 | "sendmsg", 283 | "sendto", 284 | "setfsgid", 285 | "setfsgid32", 286 | "setfsuid", 287 | "setfsuid32", 288 | "setgid", 289 | "setgid32", 290 | "setgroups", 291 | "setgroups32", 292 | "setitimer", 293 | "setpgid", 294 | "setpriority", 295 | "setregid", 296 | "setregid32", 297 | "setresgid", 298 | "setresgid32", 299 | "setresuid", 300 | "setresuid32", 301 | "setreuid", 302 | "setreuid32", 303 | "setrlimit", 304 | "set_robust_list", 305 | "setsid", 306 | "setsockopt", 307 | "set_thread_area", 308 | "set_tid_address", 309 | "setuid", 310 | "setuid32", 311 | "setxattr", 312 | "shmat", 313 | "shmctl", 314 | "shmdt", 315 | "shmget", 316 | "shutdown", 317 | "sigaltstack", 318 | "signalfd", 319 | "signalfd4", 320 | "sigprocmask", 321 | "sigreturn", 322 | "socket", 323 | "socketcall", 324 | "socketpair", 325 | "splice", 326 | "stat", 327 | "stat64", 328 | "statfs", 329 | "statfs64", 330 | "statx", 331 | "symlink", 332 | "symlinkat", 333 | "sync", 334 | "sync_file_range", 335 | "syncfs", 336 | "sysinfo", 337 | "tee", 338 | "tgkill", 339 | "time", 340 | "timer_create", 341 | "timer_delete", 342 | "timerfd_create", 343 | "timerfd_gettime", 344 | "timerfd_settime", 345 | "timer_getoverrun", 346 | "timer_gettime", 347 | "timer_settime", 348 | "times", 349 | "tkill", 350 | "truncate", 351 | "truncate64", 352 | "ugetrlimit", 353 | "umask", 354 | "uname", 355 | "unlink", 356 | "unlinkat", 357 | "utime", 358 | "utimensat", 359 | "utimes", 360 | "vfork", 361 | "vmsplice", 362 | "wait4", 363 | "waitid", 364 | "waitpid", 365 | "write", 366 | "writev", 367 | "mount", 368 | "unshare", 369 | "pivot_root", 370 | "keyctl", 371 | "umount2", 372 | "sethostname" 373 | ], 374 | "action": "SCMP_ACT_ALLOW", 375 | "args": [], 376 | "comment": "", 377 | "includes": {}, 378 | "excludes": {} 379 | }, 380 | { 381 | "names": [ 382 | "ptrace" 383 | ], 384 | "action": "SCMP_ACT_ALLOW", 385 | "args": null, 386 | "comment": "", 387 | "includes": { 388 | "minKernel": "4.8" 389 | }, 390 | "excludes": {} 391 | }, 392 | { 393 | "names": [ 394 | "personality" 395 | ], 396 | "action": "SCMP_ACT_ALLOW", 397 | "args": [ 398 | { 399 | "index": 0, 400 | "value": 0, 401 | "valueTwo": 0, 402 | "op": "SCMP_CMP_EQ" 403 | } 404 | ], 405 | "comment": "", 406 | "includes": {}, 407 | "excludes": {} 408 | }, 409 | { 410 | "names": [ 411 | "personality" 412 | ], 413 | "action": "SCMP_ACT_ALLOW", 414 | "args": [ 415 | { 416 | "index": 0, 417 | "value": 8, 418 | "valueTwo": 0, 419 | "op": "SCMP_CMP_EQ" 420 | } 421 | ], 422 | "comment": "", 423 | "includes": {}, 424 | "excludes": {} 425 | }, 426 | { 427 | "names": [ 428 | "personality" 429 | ], 430 | "action": "SCMP_ACT_ALLOW", 431 | "args": [ 432 | { 433 | "index": 0, 434 | "value": 131072, 435 | "valueTwo": 0, 436 | "op": "SCMP_CMP_EQ" 437 | } 438 | ], 439 | "comment": "", 440 | "includes": {}, 441 | "excludes": {} 442 | }, 443 | { 444 | "names": [ 445 | "personality" 446 | ], 447 | "action": "SCMP_ACT_ALLOW", 448 | "args": [ 449 | { 450 | "index": 0, 451 | "value": 131080, 452 | "valueTwo": 0, 453 | "op": "SCMP_CMP_EQ" 454 | } 455 | ], 456 | "comment": "", 457 | "includes": {}, 458 | "excludes": {} 459 | }, 460 | { 461 | "names": [ 462 | "personality" 463 | ], 464 | "action": "SCMP_ACT_ALLOW", 465 | "args": [ 466 | { 467 | "index": 0, 468 | "value": 4294967295, 469 | "valueTwo": 0, 470 | "op": "SCMP_CMP_EQ" 471 | } 472 | ], 473 | "comment": "", 474 | "includes": {}, 475 | "excludes": {} 476 | }, 477 | { 478 | "names": [ 479 | "sync_file_range2" 480 | ], 481 | "action": "SCMP_ACT_ALLOW", 482 | "args": [], 483 | "comment": "", 484 | "includes": { 485 | "arches": [ 486 | "ppc64le" 487 | ] 488 | }, 489 | "excludes": {} 490 | }, 491 | { 492 | "names": [ 493 | "arm_fadvise64_64", 494 | "arm_sync_file_range", 495 | "sync_file_range2", 496 | "breakpoint", 497 | "cacheflush", 498 | "set_tls" 499 | ], 500 | "action": "SCMP_ACT_ALLOW", 501 | "args": [], 502 | "comment": "", 503 | "includes": { 504 | "arches": [ 505 | "arm", 506 | "arm64" 507 | ] 508 | }, 509 | "excludes": {} 510 | }, 511 | { 512 | "names": [ 513 | "arch_prctl" 514 | ], 515 | "action": "SCMP_ACT_ALLOW", 516 | "args": [], 517 | "comment": "", 518 | "includes": { 519 | "arches": [ 520 | "amd64", 521 | "x32" 522 | ] 523 | }, 524 | "excludes": {} 525 | }, 526 | { 527 | "names": [ 528 | "modify_ldt" 529 | ], 530 | "action": "SCMP_ACT_ALLOW", 531 | "args": [], 532 | "comment": "", 533 | "includes": { 534 | "arches": [ 535 | "amd64", 536 | "x32", 537 | "x86" 538 | ] 539 | }, 540 | "excludes": {} 541 | }, 542 | { 543 | "names": [ 544 | "s390_pci_mmio_read", 545 | "s390_pci_mmio_write", 546 | "s390_runtime_instr" 547 | ], 548 | "action": "SCMP_ACT_ALLOW", 549 | "args": [], 550 | "comment": "", 551 | "includes": { 552 | "arches": [ 553 | "s390", 554 | "s390x" 555 | ] 556 | }, 557 | "excludes": {} 558 | }, 559 | { 560 | "names": [ 561 | "open_by_handle_at" 562 | ], 563 | "action": "SCMP_ACT_ALLOW", 564 | "args": [], 565 | "comment": "", 566 | "includes": { 567 | "caps": [ 568 | "CAP_DAC_READ_SEARCH" 569 | ] 570 | }, 571 | "excludes": {} 572 | }, 573 | { 574 | "names": [ 575 | "bpf", 576 | "clone", 577 | "fanotify_init", 578 | "lookup_dcookie", 579 | "mount", 580 | "name_to_handle_at", 581 | "perf_event_open", 582 | "quotactl", 583 | "setdomainname", 584 | "sethostname", 585 | "setns", 586 | "syslog", 587 | "umount", 588 | "umount2", 589 | "unshare" 590 | ], 591 | "action": "SCMP_ACT_ALLOW", 592 | "args": [], 593 | "comment": "", 594 | "includes": { 595 | "caps": [ 596 | "CAP_SYS_ADMIN" 597 | ] 598 | }, 599 | "excludes": {} 600 | }, 601 | { 602 | "names": [ 603 | "clone" 604 | ], 605 | "action": "SCMP_ACT_ALLOW", 606 | "args": [ 607 | { 608 | "index": 0, 609 | "value": 2114060288, 610 | "valueTwo": 0, 611 | "op": "SCMP_CMP_MASKED_EQ" 612 | } 613 | ], 614 | "comment": "", 615 | "includes": {}, 616 | "excludes": { 617 | "caps": [ 618 | "CAP_SYS_ADMIN" 619 | ], 620 | "arches": [ 621 | "s390", 622 | "s390x" 623 | ] 624 | } 625 | }, 626 | { 627 | "names": [ 628 | "clone" 629 | ], 630 | "action": "SCMP_ACT_ALLOW", 631 | "args": [ 632 | { 633 | "index": 1, 634 | "value": 2114060288, 635 | "valueTwo": 0, 636 | "op": "SCMP_CMP_MASKED_EQ" 637 | } 638 | ], 639 | "comment": "s390 parameter ordering for clone is different", 640 | "includes": { 641 | "arches": [ 642 | "s390", 643 | "s390x" 644 | ] 645 | }, 646 | "excludes": { 647 | "caps": [ 648 | "CAP_SYS_ADMIN" 649 | ] 650 | } 651 | }, 652 | { 653 | "names": [ 654 | "reboot" 655 | ], 656 | "action": "SCMP_ACT_ALLOW", 657 | "args": [], 658 | "comment": "", 659 | "includes": { 660 | "caps": [ 661 | "CAP_SYS_BOOT" 662 | ] 663 | }, 664 | "excludes": {} 665 | }, 666 | { 667 | "names": [ 668 | "chroot" 669 | ], 670 | "action": "SCMP_ACT_ALLOW", 671 | "args": [], 672 | "comment": "", 673 | "includes": { 674 | "caps": [ 675 | "CAP_SYS_CHROOT" 676 | ] 677 | }, 678 | "excludes": {} 679 | }, 680 | { 681 | "names": [ 682 | "delete_module", 683 | "init_module", 684 | "finit_module", 685 | "query_module" 686 | ], 687 | "action": "SCMP_ACT_ALLOW", 688 | "args": [], 689 | "comment": "", 690 | "includes": { 691 | "caps": [ 692 | "CAP_SYS_MODULE" 693 | ] 694 | }, 695 | "excludes": {} 696 | }, 697 | { 698 | "names": [ 699 | "acct" 700 | ], 701 | "action": "SCMP_ACT_ALLOW", 702 | "args": [], 703 | "comment": "", 704 | "includes": { 705 | "caps": [ 706 | "CAP_SYS_PACCT" 707 | ] 708 | }, 709 | "excludes": {} 710 | }, 711 | { 712 | "names": [ 713 | "kcmp", 714 | "process_vm_readv", 715 | "process_vm_writev", 716 | "ptrace" 717 | ], 718 | "action": "SCMP_ACT_ALLOW", 719 | "args": [], 720 | "comment": "", 721 | "includes": { 722 | "caps": [ 723 | "CAP_SYS_PTRACE" 724 | ] 725 | }, 726 | "excludes": {} 727 | }, 728 | { 729 | "names": [ 730 | "iopl", 731 | "ioperm" 732 | ], 733 | "action": "SCMP_ACT_ALLOW", 734 | "args": [], 735 | "comment": "", 736 | "includes": { 737 | "caps": [ 738 | "CAP_SYS_RAWIO" 739 | ] 740 | }, 741 | "excludes": {} 742 | }, 743 | { 744 | "names": [ 745 | "settimeofday", 746 | "stime", 747 | "clock_settime" 748 | ], 749 | "action": "SCMP_ACT_ALLOW", 750 | "args": [], 751 | "comment": "", 752 | "includes": { 753 | "caps": [ 754 | "CAP_SYS_TIME" 755 | ] 756 | }, 757 | "excludes": {} 758 | }, 759 | { 760 | "names": [ 761 | "vhangup" 762 | ], 763 | "action": "SCMP_ACT_ALLOW", 764 | "args": [], 765 | "comment": "", 766 | "includes": { 767 | "caps": [ 768 | "CAP_SYS_TTY_CONFIG" 769 | ] 770 | }, 771 | "excludes": {} 772 | }, 773 | { 774 | "names": [ 775 | "get_mempolicy", 776 | "mbind", 777 | "set_mempolicy" 778 | ], 779 | "action": "SCMP_ACT_ALLOW", 780 | "args": [], 781 | "comment": "", 782 | "includes": { 783 | "caps": [ 784 | "CAP_SYS_NICE" 785 | ] 786 | }, 787 | "excludes": {} 788 | }, 789 | { 790 | "names": [ 791 | "syslog" 792 | ], 793 | "action": "SCMP_ACT_ALLOW", 794 | "args": [], 795 | "comment": "", 796 | "includes": { 797 | "caps": [ 798 | "CAP_SYSLOG" 799 | ] 800 | }, 801 | "excludes": {} 802 | } 803 | ] 804 | } 805 | -------------------------------------------------------------------------------- /daemon/service.go: -------------------------------------------------------------------------------- 1 | // +build linux,cgo 2 | 3 | package daemon 4 | 5 | import ( 6 | "context" 7 | "crypto/sha1" 8 | "encoding/hex" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "strconv" 13 | "time" 14 | "unicode/utf8" 15 | 16 | "gopkg.in/src-d/go-log.v1" 17 | 18 | "github.com/opentracing/opentracing-go" 19 | "github.com/prometheus/client_golang/prometheus" 20 | 21 | "github.com/bblfsh/bblfshd/daemon/protocol" 22 | "github.com/bblfsh/sdk/v3/driver/manifest" 23 | protocol2 "github.com/bblfsh/sdk/v3/protocol" 24 | xcontext "golang.org/x/net/context" 25 | manifest1 "gopkg.in/bblfsh/sdk.v1/manifest" 26 | protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" 27 | ) 28 | 29 | var ( 30 | _ protocol2.DriverServer = (*ServiceV2)(nil) 31 | _ protocol2.DriverHostServer = (*ServiceV2)(nil) 32 | 33 | parseKillDelay = time.Second 34 | ) 35 | 36 | func hashSHA1(content string) string { 37 | h := sha1.New() 38 | io.WriteString(h, content) 39 | return hex.EncodeToString(h.Sum(nil)) 40 | } 41 | 42 | func hashGit(content string) string { 43 | h := sha1.New() 44 | io.WriteString(h, "blob ") 45 | io.WriteString(h, strconv.Itoa(len(content))) 46 | io.WriteString(h, "\x00") 47 | io.WriteString(h, content) 48 | return hex.EncodeToString(h.Sum(nil)) 49 | } 50 | 51 | type ServiceV2 struct { 52 | daemon *Daemon 53 | } 54 | 55 | func NewServiceV2(d *Daemon) *ServiceV2 { 56 | return &ServiceV2{daemon: d} 57 | } 58 | 59 | // Parse implements protocol2.DriverServer. 60 | func (s *ServiceV2) Parse(rctx xcontext.Context, req *protocol2.ParseRequest) (resp *protocol2.ParseResponse, gerr error) { 61 | parseCallsV2.Add(1) 62 | defer prometheus.NewTimer(parseLatencyV2).ObserveDuration() 63 | parseContentSizeV2.Observe(float64(len(req.Content))) 64 | 65 | sp, ctx := opentracing.StartSpanFromContext(rctx, "bblfshd.v2.Parse") 66 | defer sp.Finish() 67 | 68 | resp = &protocol2.ParseResponse{} 69 | start := time.Now() 70 | defer func() { 71 | s.logResponse(gerr, req.Filename, req.Language, req.Content, time.Since(start)) 72 | }() 73 | 74 | if req.Content == "" { 75 | log.Debugf("empty request received, returning empty UAST") 76 | return resp, nil 77 | } 78 | 79 | if !utf8.ValidString(req.Content) { 80 | parseErrorsV2.Add(1) 81 | err := ErrUnknownEncoding.New() 82 | log.Debugf("parse v2 (%s): %s", req.Filename, err) 83 | return nil, err 84 | } 85 | 86 | language, dp, err := s.selectPool(ctx, req.Language, req.Content, req.Filename) 87 | if err != nil { 88 | parseErrorsV2.Add(1) 89 | log.Errorf(err, "error selecting pool") 90 | return nil, err 91 | } 92 | 93 | req.Language = language 94 | 95 | err = dp.ExecuteCtx(ctx, func(ctx context.Context, driver Driver) error { 96 | resp, err = parseV2(ctx, dp, driver, req) 97 | return err 98 | }) 99 | if err != nil { 100 | parseErrorsV2.Add(1) 101 | } 102 | if resp != nil { 103 | resp.Language = language 104 | } 105 | return resp, err 106 | } 107 | 108 | func parseV2(ctx context.Context, pool *DriverPool, drv Driver, req *protocol2.ParseRequest) (*protocol2.ParseResponse, error) { 109 | var ( 110 | resp *protocol2.ParseResponse 111 | err error 112 | ) 113 | done := make(chan struct{}) 114 | go func() { 115 | resp, err = drv.ServiceV2().Parse(ctx, req) 116 | close(done) 117 | }() 118 | 119 | var ( 120 | ctxKill context.Context 121 | cancel context.CancelFunc 122 | ) 123 | if deadline, ok := ctx.Deadline(); ok { 124 | ctxKill, cancel = context.WithDeadline(context.Background(), deadline.Add(parseKillDelay)) 125 | defer cancel() 126 | } else { 127 | ctxKill = ctx 128 | } 129 | 130 | select { 131 | case <-done: 132 | return resp, err 133 | 134 | case <-ctxKill.Done(): 135 | pool.killDriver(drv, "parseV2", ctxKill.Err()) 136 | return nil, ctxKill.Err() 137 | } 138 | } 139 | 140 | // ServerVersion implements protocol2.DriverHostServer. 141 | func (s *ServiceV2) ServerVersion(rctx xcontext.Context, _ *protocol2.VersionRequest) (*protocol2.VersionResponse, error) { 142 | versionCallsV2.Add(1) 143 | 144 | sp, _ := opentracing.StartSpanFromContext(rctx, "bblfshd.v2.ServerVersion") 145 | defer sp.Finish() 146 | 147 | resp := &protocol2.Version{Version: s.daemon.version} 148 | if !s.daemon.build.IsZero() { 149 | resp.Build = s.daemon.build 150 | } 151 | return &protocol2.VersionResponse{Version: resp}, nil 152 | } 153 | 154 | // SupportedLanguages implements protocol2.DriverHostServer. 155 | func (s *ServiceV2) SupportedLanguages(rctx xcontext.Context, _ *protocol2.SupportedLanguagesRequest) (_ *protocol2.SupportedLanguagesResponse, gerr error) { 156 | languagesCallsV2.Add(1) 157 | 158 | sp, _ := opentracing.StartSpanFromContext(rctx, "bblfshd.v2.SupportedLanguages") 159 | defer sp.Finish() 160 | 161 | start := time.Now() 162 | defer func() { 163 | s.logResponse(gerr, "", "", "", time.Since(start)) 164 | }() 165 | 166 | drivers, err := s.daemon.runtime.ListDrivers() 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | out := make([]*protocol2.Manifest, 0, len(drivers)) 172 | for _, d := range drivers { 173 | if d.Manifest == nil { 174 | return nil, errors.New("empty manifest returned by driver") 175 | } 176 | out = append(out, protocol2.NewManifest(d.Manifest)) 177 | } 178 | return &protocol2.SupportedLanguagesResponse{Languages: out}, nil 179 | } 180 | 181 | func (s *ServiceV2) logResponse(err error, filename, language, content string, elapsed time.Duration) { 182 | fields := log.Fields{"elapsed": elapsed} 183 | if filename != "" { 184 | fields["filename"] = filename 185 | } 186 | 187 | if language != "" { 188 | fields["language"] = language 189 | } 190 | 191 | if content != "" { 192 | fields["sha1"] = hashSHA1(content) 193 | fields["githash"] = hashGit(content) 194 | } 195 | 196 | l := log.With(fields) 197 | text := fmt.Sprintf("request processed content %d bytes", len(content)) 198 | 199 | if err != nil { 200 | l.Errorf(err, "%s", text) 201 | } else { 202 | l.Infof("%s", text) 203 | } 204 | } 205 | 206 | func (s *ServiceV2) detectLanguage(rctx context.Context, content, filename string) (string, error) { 207 | sp, _ := opentracing.StartSpanFromContext(rctx, "bblfshd.detectLanguage") 208 | defer sp.Finish() 209 | 210 | language := GetLanguage(filename, []byte(content)) 211 | if language == "" { 212 | return "", ErrLanguageDetection.New() 213 | } 214 | log.Debugf("detected language %q, filename %q", language, filename) 215 | return language, nil 216 | } 217 | 218 | func (s *ServiceV2) selectPool(rctx context.Context, language, content, filename string) (string, *DriverPool, error) { 219 | sp, ctx := opentracing.StartSpanFromContext(rctx, "bblfshd.pool.select") 220 | defer sp.Finish() 221 | 222 | if language == "" { 223 | lang, err := s.detectLanguage(ctx, content, filename) 224 | if err != nil { 225 | return "", nil, err 226 | } 227 | language = lang 228 | } else { // always re-map enry->bblfsh language names 229 | language = normalize(language) 230 | } 231 | 232 | dp, err := s.daemon.DriverPool(ctx, language) 233 | if err != nil { 234 | return language, nil, ErrUnexpected.Wrap(err) 235 | } 236 | 237 | return language, dp, nil 238 | } 239 | 240 | var _ protocol1.Service = (*Service)(nil) 241 | 242 | type Service struct { 243 | daemon *Daemon 244 | } 245 | 246 | func NewService(d *Daemon) *Service { 247 | return &Service{daemon: d} 248 | } 249 | 250 | // Parse implements protocol1.Service. 251 | func (d *Service) Parse(req *protocol1.ParseRequest) *protocol1.ParseResponse { 252 | parseCallsV1.Add(1) 253 | parseContentSizeV1.Observe(float64(len(req.Content))) 254 | 255 | resp := &protocol1.ParseResponse{} 256 | start := time.Now() 257 | defer func() { 258 | if resp == nil || resp.Status != protocol1.Ok || len(resp.Errors) != 0 { 259 | parseErrorsV1.Add(1) 260 | } 261 | dt := time.Since(start) 262 | parseLatencyV1.Observe(dt.Seconds()) 263 | resp.Elapsed = dt 264 | d.logResponse(resp.Status, req.Filename, req.Language, req.Content, resp.Elapsed) 265 | }() 266 | 267 | if req.Content == "" { 268 | log.Debugf("empty request received, returning empty UAST") 269 | return resp 270 | } 271 | if !utf8.ValidString(req.Content) { 272 | err := ErrUnknownEncoding.New() 273 | log.Debugf("parse v1 (%s): %s", req.Filename, err) 274 | resp.Response = newResponseFromError(err) 275 | return resp 276 | } 277 | 278 | language, dp, err := d.selectPool(context.TODO(), req.Language, req.Content, req.Filename) 279 | if err != nil { 280 | log.Errorf(err, "error selecting pool") 281 | resp.Response = newResponseFromError(err) 282 | resp.Language = language 283 | return resp 284 | } 285 | 286 | req.Language = language 287 | 288 | err = dp.Execute(func(ctx context.Context, driver Driver) error { 289 | resp, err = parseV1(ctx, dp, driver, req) 290 | return err 291 | }, req.Timeout) 292 | 293 | if err != nil { 294 | resp = &protocol1.ParseResponse{} 295 | resp.Response = newResponseFromError(err) 296 | } 297 | 298 | resp.Language = language 299 | return resp 300 | } 301 | 302 | func parseV1(ctx context.Context, pool *DriverPool, drv Driver, req *protocol1.ParseRequest) (*protocol1.ParseResponse, error) { 303 | var ( 304 | resp *protocol1.ParseResponse 305 | err error 306 | ) 307 | done := make(chan struct{}) 308 | go func() { 309 | resp, err = drv.Service().Parse(ctx, req) 310 | close(done) 311 | }() 312 | 313 | var ( 314 | ctxKill context.Context 315 | cancel context.CancelFunc 316 | ) 317 | if deadline, ok := ctx.Deadline(); ok { 318 | ctxKill, cancel = context.WithDeadline(context.Background(), deadline.Add(parseKillDelay)) 319 | defer cancel() 320 | } else { 321 | ctxKill = ctx 322 | } 323 | 324 | select { 325 | case <-done: 326 | return resp, err 327 | 328 | case <-ctxKill.Done(): 329 | pool.killDriver(drv, "parseV1", ctxKill.Err()) 330 | return nil, ctxKill.Err() 331 | } 332 | } 333 | 334 | func (d *Service) logResponse(s protocol1.Status, filename, language, content string, elapsed time.Duration) { 335 | fields := log.Fields{"elapsed": elapsed} 336 | if filename != "" { 337 | fields["filename"] = filename 338 | } 339 | 340 | if language != "" { 341 | fields["language"] = language 342 | } 343 | 344 | if content != "" { 345 | fields["sha1"] = hashSHA1(content) 346 | fields["githash"] = hashGit(content) 347 | } 348 | 349 | l := log.With(fields) 350 | text := fmt.Sprintf("request processed content %d bytes, status %s", len(content), s) 351 | 352 | switch s { 353 | case protocol1.Ok: 354 | l.Infof("%s", text) 355 | case protocol1.Error: 356 | l.Warningf("%s", text) 357 | case protocol1.Fatal: 358 | l.Errorf(errors.New("protocol1 fatal error"), "%s", text) 359 | } 360 | } 361 | 362 | // NativeParse implements protocol1.Service. 363 | func (d *Service) NativeParse(req *protocol1.NativeParseRequest) *protocol1.NativeParseResponse { 364 | parseCallsV1.Add(1) 365 | parseContentSizeV1.Observe(float64(len(req.Content))) 366 | 367 | resp := &protocol1.NativeParseResponse{} 368 | start := time.Now() 369 | defer func() { 370 | if resp == nil || resp.Status != protocol1.Ok || len(resp.Errors) != 0 { 371 | parseErrorsV1.Add(1) 372 | } 373 | dt := time.Since(start) 374 | parseLatencyV1.Observe(dt.Seconds()) 375 | resp.Elapsed = dt 376 | d.logResponse(resp.Status, req.Language, req.Language, req.Content, resp.Elapsed) 377 | }() 378 | 379 | if req.Content == "" { 380 | log.Debugf("empty request received, returning empty AST") 381 | return resp 382 | } 383 | 384 | if !utf8.ValidString(req.Content) { 385 | err := ErrUnknownEncoding.New() 386 | log.Debugf("native parse v1 (%s): %s", req.Filename, err) 387 | resp.Response = newResponseFromError(err) 388 | return resp 389 | } 390 | 391 | language, dp, err := d.selectPool(context.TODO(), req.Language, req.Content, req.Filename) 392 | if err != nil { 393 | log.Errorf(err, "error selecting pool") 394 | resp.Response = newResponseFromError(err) 395 | return resp 396 | } 397 | 398 | req.Language = language 399 | 400 | err = dp.Execute(func(ctx context.Context, driver Driver) error { 401 | resp, err = driver.Service().NativeParse(ctx, req) 402 | return err 403 | }, req.Timeout) 404 | 405 | if err != nil { 406 | resp = &protocol1.NativeParseResponse{} 407 | resp.Response = newResponseFromError(err) 408 | } 409 | 410 | resp.Language = language 411 | return resp 412 | } 413 | 414 | func (s *Service) selectPool(ctx context.Context, language, content, filename string) (string, *DriverPool, error) { 415 | if language == "" { 416 | language = GetLanguage(filename, []byte(content)) 417 | if language == "" { 418 | return language, nil, ErrLanguageDetection.New() 419 | } 420 | log.Debugf("detected language %q, filename %q", language, filename) 421 | } else { // always re-map enry->bblfsh language names 422 | language = normalize(language) 423 | } 424 | 425 | dp, err := s.daemon.DriverPool(ctx, language) 426 | if err != nil { 427 | return language, nil, ErrUnexpected.Wrap(err) 428 | } 429 | 430 | return language, dp, nil 431 | } 432 | 433 | // Version implements protocol1.Service. 434 | func (d *Service) Version(req *protocol1.VersionRequest) *protocol1.VersionResponse { 435 | versionCallsV1.Add(1) 436 | 437 | resp := &protocol1.VersionResponse{Version: d.daemon.version, Build: d.daemon.build} 438 | start := time.Now() 439 | defer func() { 440 | resp.Elapsed = time.Since(start) 441 | d.logResponse(resp.Status, "", "", "", resp.Elapsed) 442 | }() 443 | return resp 444 | } 445 | 446 | // manifestV2toV1 converts driver manifest from v2 API format to v1. 447 | func manifestV2toV1(m *manifest.Manifest) *manifest1.Manifest { 448 | dm := &manifest1.Manifest{ 449 | Name: m.Name, 450 | Language: m.Language, 451 | Version: m.Version, 452 | Status: manifest1.DevelopmentStatus(m.Status), 453 | Features: make([]manifest1.Feature, 0, len(m.Features)), 454 | } 455 | dm.Runtime.GoVersion = m.Runtime.GoVersion 456 | dm.Runtime.NativeVersion = []string{m.Runtime.NativeVersion} 457 | if m.Documentation != nil { 458 | dm.Documentation.Description = m.Documentation.Description 459 | dm.Documentation.Caveats = m.Documentation.Caveats 460 | } 461 | if !m.Build.IsZero() { 462 | dm.Build = &m.Build 463 | } 464 | for _, f := range m.Features { 465 | dm.Features = append(dm.Features, manifest1.Feature(f)) 466 | } 467 | return dm 468 | } 469 | 470 | // SupportedLanguages implements protocol1.Service. 471 | func (d *Service) SupportedLanguages(req *protocol1.SupportedLanguagesRequest) *protocol1.SupportedLanguagesResponse { 472 | languagesCallsV1.Add(1) 473 | 474 | resp := &protocol1.SupportedLanguagesResponse{} 475 | start := time.Now() 476 | defer func() { 477 | resp.Elapsed = time.Since(start) 478 | d.logResponse(resp.Status, "", "", "", resp.Elapsed) 479 | }() 480 | 481 | drivers, err := d.daemon.runtime.ListDrivers() 482 | if err != nil { 483 | resp.Response = newResponseFromError(err) 484 | return resp 485 | } 486 | 487 | driverRes := make([]protocol1.DriverManifest, len(drivers)) 488 | for i, driver := range drivers { 489 | m := manifestV2toV1(driver.Manifest) 490 | driverRes[i] = protocol1.NewDriverManifest(m) 491 | } 492 | 493 | resp.Languages = driverRes 494 | return resp 495 | } 496 | 497 | type ControlService struct { 498 | *Daemon 499 | } 500 | 501 | func NewControlService(d *Daemon) *ControlService { 502 | return &ControlService{Daemon: d} 503 | } 504 | 505 | func (s *ControlService) DriverPoolStates() map[string]*protocol.DriverPoolState { 506 | out := make(map[string]*protocol.DriverPoolState, 0) 507 | for language, pool := range s.Daemon.Current() { 508 | out[language] = pool.State() 509 | } 510 | 511 | return out 512 | } 513 | 514 | func (s *ControlService) DriverInstanceStates() ([]*protocol.DriverInstanceState, error) { 515 | var out []*protocol.DriverInstanceState 516 | for _, pool := range s.Daemon.Current() { 517 | for _, driver := range pool.Current() { 518 | status, err := driver.State() 519 | if err != nil { 520 | return nil, err 521 | } 522 | 523 | out = append(out, status) 524 | } 525 | } 526 | 527 | return out, nil 528 | } 529 | 530 | func (s *ControlService) DriverStates() ([]*protocol.DriverImageState, error) { 531 | list, err := s.Daemon.runtime.ListDrivers() 532 | if err != nil { 533 | return nil, err 534 | } 535 | 536 | var out []*protocol.DriverImageState 537 | for _, d := range list { 538 | out = append(out, &protocol.DriverImageState{ 539 | Reference: d.Reference, 540 | Language: d.Manifest.Language, 541 | Version: d.Manifest.Version, 542 | Build: d.Manifest.Build, 543 | Status: string(d.Manifest.Status), 544 | GoVersion: string(d.Manifest.Runtime.GoVersion), 545 | NativeVersion: []string{d.Manifest.Runtime.NativeVersion}, 546 | }) 547 | } 548 | 549 | return out, nil 550 | } 551 | --------------------------------------------------------------------------------