├── .gitignore ├── .travis.yml ├── CODEOWNERS ├── DCO ├── Jenkinsfile ├── LICENSE ├── MAINTAINERS ├── Makefile ├── README.md ├── assets ├── build │ └── bindata.go └── skeleton │ └── bindata.go ├── build ├── build.go ├── build_test.go ├── ci.go ├── dev.go ├── exec.go ├── git.go ├── golang.go ├── init.go ├── init_test.go ├── push.go ├── sdk_update.go └── test.go ├── cmd ├── bblfsh-drivers-release │ ├── cmd │ │ ├── apply.go │ │ └── collect.go │ └── main.go ├── bblfsh-drivers-updater │ ├── update.go │ └── utils │ │ └── utils.go ├── bblfsh-sdk │ ├── cmd │ │ ├── ast2gv.go │ │ ├── build.go │ │ ├── info.go │ │ ├── init.go │ │ ├── request.go │ │ └── update.go │ └── main.go ├── common.go ├── util.go └── util_test.go ├── common.go ├── doc.go ├── driver ├── driver.go ├── errors │ └── errors.go ├── fixtures │ └── fixtures.go ├── impl.go ├── integration │ ├── common.go │ ├── consts │ │ └── consts.go │ ├── suite.go │ └── suite_test.go ├── manifest │ ├── discovery │ │ ├── discovery.go │ │ └── discovery_test.go │ ├── manifest.go │ └── manifest_test.go ├── native │ ├── internal │ │ ├── crash │ │ │ ├── main.go │ │ │ ├── manifest.toml │ │ │ └── mock │ │ ├── simple │ │ │ ├── main.go │ │ │ ├── manifest.toml │ │ │ └── mock │ │ └── slow │ │ │ ├── main.go │ │ │ ├── manifest.toml │ │ │ └── mock │ ├── jsonlines │ │ ├── decoder.go │ │ ├── decoder_test.go │ │ ├── doc.go │ │ ├── encoder.go │ │ └── encoder_test.go │ ├── main.go │ ├── native.go │ └── native_test.go ├── server │ ├── common.go │ ├── grpc.go │ ├── server.go │ └── server_test.go └── transforms.go ├── etc ├── build │ └── Dockerfile.tpl └── skeleton │ ├── .gitignore │ ├── .travis.yml │ ├── Jenkinsfile.tpl │ ├── LICENSE │ ├── README.md.tpl │ ├── build.go │ ├── build.yml.tpl │ ├── driver │ ├── fixtures │ │ └── fixtures_test.go.tpl │ ├── impl │ │ └── impl.go │ ├── main.go.tpl │ ├── normalizer │ │ ├── annotation.go │ │ ├── normalizer.go │ │ └── transforms.go.tpl │ └── sdk_test.go │ ├── git │ └── hooks │ │ └── pre-commit │ ├── go.mod.tpl │ ├── manifest.toml.tpl │ ├── native │ ├── README.md.tpl │ └── native.sh │ ├── test.go │ └── update.go ├── go.mod ├── go.sum ├── internal ├── buildmanifest │ └── manifest.go └── docker │ └── docker.go ├── protocol ├── driver.go ├── driver.pb.go ├── driver.proto ├── driver_test.go └── v1 │ └── legacy.go ├── tools.go └── uast ├── helpers.go ├── helpers_test.go ├── iter.go ├── iter_test.go ├── nodes ├── external.go ├── hash.go ├── iter.go ├── iter_test.go ├── node.go ├── node_test.go └── nodesproto │ ├── nodes.pb.go │ ├── nodes.proto │ ├── nodesproto.go │ ├── nodesproto_test.go │ └── pio │ ├── io.go │ ├── io_test.go │ └── varint.go ├── query ├── iter.go ├── query.go ├── testdata │ └── large.go.sem.uast └── xpath │ ├── query.go │ ├── query_test.go │ └── xpath.go ├── role ├── generated.pb.go ├── generated.proto ├── role.go ├── role_string.go └── role_test.go ├── transformer ├── arrays.go ├── ast.go ├── conv.go ├── errors.go ├── legacy.go ├── ops.go ├── ops_test.go ├── positioner │ ├── positions.go │ ├── positions_test.go │ ├── tokens.go │ └── tokens_test.go ├── semantic.go ├── semantic_test.go ├── transformer.go └── transformer_test.go ├── types.go ├── types_test.go ├── uast.go ├── uast_test.go ├── uastyaml ├── uastyaml.go └── uastyaml_test.go ├── viewer └── viewer.go └── yaml └── uastyml.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | vendor/ 10 | .idea 11 | 12 | # Architecture specific extensions/prefixes 13 | *.[568vq] 14 | [568vq].out 15 | 16 | *.cgo1.go 17 | *.cgo2.c 18 | _cgo_defun.c 19 | _cgo_gotypes.go 20 | _cgo_export.* 21 | 22 | _testmain.go 23 | 24 | *.exe 25 | *.test 26 | *.prof 27 | bblfsh-sdk 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - '1.12.x' 5 | - '1.13.x' 6 | - tip 7 | 8 | services: 9 | - docker 10 | 11 | go_import_path: github.com/bblfsh/sdk 12 | 13 | matrix: 14 | fast_finish: true 15 | allow_failures: 16 | - go: tip 17 | 18 | env: 19 | - GO111MODULE=on 20 | 21 | before_install: 22 | - cd $GOPATH/src/github.com/bblfsh/sdk 23 | - make validate-commit 24 | - go mod download 25 | 26 | script: 27 | - make test-coverage 28 | 29 | after_success: 30 | - bash <(curl -s https://codecov.io/bash) 31 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @bblfsh/maintainers 2 | -------------------------------------------------------------------------------- /DCO: -------------------------------------------------------------------------------- 1 | Developer's Certificate of Origin 1.1 2 | 3 | By making a contribution to this project, I certify that: 4 | 5 | (a) The contribution was created in whole or in part by me and I 6 | have the right to submit it under the open source license 7 | indicated in the file; or 8 | 9 | (b) The contribution is based upon previous work that, to the best 10 | of my knowledge, is covered under an appropriate open source 11 | license and I have the right under that license to submit that 12 | work with modifications, whether created in whole or in part 13 | by me, under the same open source license (unless I am 14 | permitted to submit under a different license), as indicated 15 | in the file; or 16 | 17 | (c) The contribution was provided directly to me by some other 18 | person who certified (a), (b) or (c) and I have not modified 19 | it. 20 | 21 | (d) I understand and agree that this project and the contribution 22 | are public and that a record of the contribution (including all 23 | personal information I submit with it, including my sign-off) is 24 | maintained indefinitely and may be redistributed consistent with 25 | this project or the open source license(s) involved. 26 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { 3 | kubernetes { 4 | label 'sdk-drivers-updater' 5 | defaultContainer 'sdk-drivers-updater' 6 | yaml """ 7 | spec: 8 | nodeSelector: 9 | srcd.host/type: jenkins-worker 10 | affinity: 11 | podAntiAffinity: 12 | requiredDuringSchedulingIgnoredDuringExecution: 13 | - labelSelector: 14 | matchExpressions: 15 | - key: jenkins 16 | operator: In 17 | values: 18 | - slave 19 | topologyKey: kubernetes.io/hostname 20 | containers: 21 | - name: sdk-drivers-updater 22 | image: bblfsh/performance:latest 23 | imagePullPolicy: Always 24 | securityContext: 25 | privileged: true 26 | command: 27 | - dockerd 28 | tty: true 29 | """ 30 | } 31 | } 32 | triggers { 33 | GenericTrigger( 34 | genericVariables: [ 35 | [key: 'target', value: '$.target'], 36 | [key: 'title', value: '$.title'], 37 | [key: 'text', value: '$.text'], 38 | [key: 'sdk_version', value: '$.sdk_version'], 39 | [key: 'branch', value: '$.branch'], 40 | [key: 'commit_msg', value: '$.commit_msg'], 41 | [key: 'script', value: '$.script'] 42 | ], 43 | token: 'update', 44 | causeString: 'Triggered on $target', 45 | 46 | printContributedVariables: true, 47 | printPostContent: true, 48 | 49 | regexpFilterText: '$target', 50 | regexpFilterExpression: 'master' 51 | ) 52 | } 53 | stages { 54 | stage('Run updater') { 55 | when { branch 'master' } 56 | steps { 57 | withCredentials([usernamePassword(credentialsId: '87b3cad8-8b12-4e91-8f47-33f3d7d45620', passwordVariable: 'token', usernameVariable: 'user')]) { 58 | sh 'echo ${script} > /etc/script.sh ; chmod +x /etc/script.sh' 59 | sh 'GITHUB_TOKEN=${token} go run cmd/bblfsh-drivers-updater/update.go --script="/etc/script.sh" --sdk-version="${sdk_version}" --branch="${branch}" --title="${title}" --text="${text}" --commit-msg="${commit_msg}" --dockerfile=true' 60 | } 61 | } 62 | } 63 | } 64 | post { 65 | success { 66 | slackSend (color: '#2eb886', message: "SUCCESS: `${env.JOB_NAME}` <${env.BUILD_URL}|build #${env.BUILD_NUMBER}>") 67 | } 68 | failure { 69 | slackSend (color: '#b82e60', message: "FAILED: `${env.JOB_NAME}` <${env.BUILD_URL}|build #${env.BUILD_NUMBER}>") 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Denys Smirnov (@dennwc) 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Package configuration 2 | PROJECT := bblfsh-sdk 3 | 4 | # Environment 5 | BASE_PATH := $(shell pwd) 6 | 7 | # Assets configuration 8 | ASSETS_PATH := $(BASE_PATH)/etc 9 | ASSETS := $(shell ls $(ASSETS_PATH)) 10 | ASSETS_PACKAGE := assets 11 | BINDATA_FILE := bindata.go 12 | BINDATA_CMD := go run github.com/kevinburke/go-bindata/go-bindata/ 13 | 14 | # Go parameters 15 | GO_CMD = go 16 | GO_TEST = $(GO_CMD) test -v 17 | GO_GET = $(GO_CMD) get -v 18 | 19 | # Coverage 20 | COVERAGE_REPORT = coverage.txt 21 | COVERAGE_PROFILE = profile.out 22 | COVERAGE_MODE = atomic 23 | 24 | all: bindata 25 | 26 | bindata: $(ASSETS) 27 | 28 | $(ASSETS): 29 | chmod -R go=r $(ASSETS_PATH)/$@; \ 30 | $(BINDATA_CMD) \ 31 | -pkg $@ \ 32 | -modtime 1 \ 33 | -nocompress \ 34 | -prefix $(ASSETS_PATH)/$@ \ 35 | -o $(ASSETS_PACKAGE)/$@/$(BINDATA_FILE) \ 36 | $(ASSETS_PATH)/$@/... 37 | $(GO_CMD) fmt ./$(ASSETS_PACKAGE)/... 38 | 39 | test: 40 | $(GO_TEST) ./... 41 | 42 | test-coverage: 43 | echo "" > $(COVERAGE_REPORT); \ 44 | for dir in `$(GO_CMD) list ./... | egrep -v '/(vendor|etc)/'`; do \ 45 | $(GO_TEST) $$dir -coverprofile=$(COVERAGE_PROFILE) -covermode=$(COVERAGE_MODE); \ 46 | if [ $$? != 0 ]; then \ 47 | exit 2; \ 48 | fi; \ 49 | if [ -f $(COVERAGE_PROFILE) ]; then \ 50 | cat $(COVERAGE_PROFILE) >> $(COVERAGE_REPORT); \ 51 | rm $(COVERAGE_PROFILE); \ 52 | fi; \ 53 | done 54 | 55 | 56 | validate-commit: bindata 57 | git status --untracked-files=no --porcelain | grep -qe '..*'; \ 58 | if [ $$? -eq 0 ] ; then \ 59 | git diff|cat; \ 60 | echo >&2 "generated bindata is out of sync"; \ 61 | echo >&2 "Re-run 'make bindata' and commit the updates."; \ 62 | exit 2; \ 63 | fi 64 | 65 | .PHONY: bindata test test-coverage validate-commit driver-tpl $(ASSETS) 66 | -------------------------------------------------------------------------------- /build/build_test.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/require" 13 | "google.golang.org/grpc" 14 | 15 | "github.com/bblfsh/sdk/v3/driver" 16 | "github.com/bblfsh/sdk/v3/driver/manifest" 17 | "github.com/bblfsh/sdk/v3/internal/docker" 18 | "github.com/bblfsh/sdk/v3/protocol" 19 | "github.com/bblfsh/sdk/v3/uast/nodes" 20 | ) 21 | 22 | func TestIsApkOrApt(t *testing.T) { 23 | for _, c := range []struct { 24 | cmd string 25 | exp bool 26 | }{ 27 | {`echo`, false}, 28 | {`apk add xxx`, true}, 29 | {`apt install xxx`, true}, 30 | {`apt-get install xxx`, true}, 31 | {`apt update && apt install xxx`, true}, 32 | {`apt update && echo`, false}, 33 | {`apt update || echo`, false}, 34 | {`apt update ; echo`, false}, 35 | {`apt install x || apt install y`, true}, 36 | {`apt install x; apt install y`, true}, 37 | } { 38 | t.Run(c.cmd, func(t *testing.T) { 39 | if c.exp != isApkOrApt(c.cmd) { 40 | t.Fatal("should be", c.exp) 41 | } 42 | }) 43 | } 44 | } 45 | 46 | func TestDriverBuildAndRun(t *testing.T) { 47 | // If changing any skeleton files, make sure to 'make bindata' before running this. 48 | const lang = "test" 49 | 50 | dir, err := ioutil.TempDir("", "driver_") 51 | require.NoError(t, err) 52 | defer os.RemoveAll(dir) 53 | 54 | err = InitDriver(dir, "test", nil) 55 | require.NoError(t, err) 56 | 57 | dir = filepath.Join(dir, "test-driver") 58 | 59 | sdkRoot, err := filepath.Abs("../") 60 | require.NoError(t, err) 61 | 62 | // replace the SDK with the dev version, so we can actually test our changes 63 | out, err := exec.Command( 64 | "go", "mod", "edit", 65 | "-replace=github.com/bblfsh/sdk/v3="+sdkRoot, 66 | filepath.Join(dir, "go.mod"), 67 | ).CombinedOutput() 68 | if err != nil { 69 | require.Fail(t, "mod edit failed", "%s", string(out)) 70 | } 71 | 72 | b, err := NewDriver(dir) 73 | require.NoError(t, err) 74 | 75 | id, err := b.Build("") 76 | require.NoError(t, err) 77 | 78 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) 79 | defer cancel() 80 | 81 | d := &dockerDriver{ctx: ctx, img: id} 82 | err = d.Start() 83 | require.NoError(t, err) 84 | defer d.Close() 85 | 86 | const src = "foo" 87 | ast, err := d.Parse(ctx, src, &driver.ParseOptions{ 88 | Mode: driver.ModeNative, 89 | Language: lang, 90 | }) 91 | require.NoError(t, err) 92 | require.Equal(t, nodes.Object{ 93 | "Encoding": nodes.String("utf8"), 94 | "content": nodes.String(src), 95 | }, ast) 96 | } 97 | 98 | var ( 99 | _ driver.Driver = (*dockerDriver)(nil) 100 | _ driver.Module = (*dockerDriver)(nil) 101 | ) 102 | 103 | type dockerDriver struct { 104 | ctx context.Context 105 | img string 106 | 107 | cli *docker.Client 108 | cont *docker.Container 109 | cc *grpc.ClientConn 110 | d driver.Driver 111 | } 112 | 113 | func (d *dockerDriver) Start() error { 114 | cli, err := docker.Dial() 115 | if err != nil { 116 | return err 117 | } 118 | cont, err := docker.Run(cli, docker.CreateContainerOptions{ 119 | Context: d.ctx, 120 | Config: &docker.Config{ 121 | Image: d.img, 122 | }, 123 | HostConfig: &docker.HostConfig{ 124 | AutoRemove: true, 125 | }, 126 | }) 127 | if err != nil { 128 | return err 129 | } 130 | d.cli = cli 131 | d.cont = cont 132 | 133 | cc, err := grpc.DialContext(d.ctx, cont.NetworkSettings.IPAddress+":9432", 134 | grpc.WithInsecure(), grpc.WithBlock(), 135 | ) 136 | if err != nil { 137 | _ = d.Close() 138 | return err 139 | } 140 | d.cc = cc 141 | d.d = protocol.AsDriver(cc) 142 | return nil 143 | } 144 | 145 | func (d *dockerDriver) Close() error { 146 | if d.cc != nil { 147 | _ = d.cc.Close() 148 | } 149 | return d.cli.RemoveContainer(docker.RemoveContainerOptions{ 150 | ID: d.cont.ID, 151 | RemoveVolumes: true, 152 | Force: true, 153 | }) 154 | } 155 | 156 | func (d *dockerDriver) Parse(ctx context.Context, src string, opts *driver.ParseOptions) (nodes.Node, error) { 157 | return d.d.Parse(ctx, src, opts) 158 | } 159 | 160 | func (d *dockerDriver) Version(ctx context.Context) (driver.Version, error) { 161 | return d.d.Version(ctx) 162 | } 163 | 164 | func (d *dockerDriver) Languages(ctx context.Context) ([]manifest.Manifest, error) { 165 | return d.d.Languages(ctx) 166 | } 167 | -------------------------------------------------------------------------------- /build/ci.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import "os" 4 | 5 | func isCI() bool { 6 | return os.Getenv("CI") != "" || 7 | os.Getenv("CONTINUOUS_INTEGRATION") != "" || 8 | os.Getenv("TRAVIS") != "" 9 | } 10 | 11 | func ciBranch() string { 12 | return os.Getenv("TRAVIS_BRANCH") 13 | } 14 | 15 | func ciTag() string { 16 | return os.Getenv("TRAVIS_TAG") 17 | } 18 | 19 | func ciIsPR() bool { 20 | pr := os.Getenv("TRAVIS_PULL_REQUEST") 21 | return pr != "false" && pr != "" 22 | } 23 | -------------------------------------------------------------------------------- /build/dev.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "io" 8 | "strings" 9 | "time" 10 | 11 | "github.com/bblfsh/sdk/v3/driver" 12 | "github.com/bblfsh/sdk/v3/internal/docker" 13 | "github.com/bblfsh/sdk/v3/protocol" 14 | "google.golang.org/grpc" 15 | protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" 16 | ) 17 | 18 | const ( 19 | cliPort = "9432" 20 | dockerSchema = "docker-daemon:" 21 | ) 22 | 23 | type ServerInstance struct { 24 | cli *docker.Client 25 | user *grpc.ClientConn 26 | bblfshd *docker.Container 27 | } 28 | 29 | func (d *ServerInstance) installFromDocker(ctx context.Context, lang, id string) error { 30 | if !strings.Contains(id, ":") { 31 | id += ":latest" 32 | } 33 | cmd := []string{"bblfshctl", "driver", "install", lang, dockerSchema + id} 34 | printCommand("docker", append([]string{"exec", id}, cmd...)...) 35 | e, err := d.cli.CreateExec(docker.CreateExecOptions{ 36 | Context: ctx, 37 | Container: d.bblfshd.ID, 38 | AttachStdout: true, AttachStderr: true, 39 | Cmd: cmd, 40 | }) 41 | if err != nil { 42 | return err 43 | } 44 | buf := bytes.NewBuffer(nil) 45 | err = d.cli.StartExec(e.ID, docker.StartExecOptions{ 46 | Context: ctx, 47 | OutputStream: buf, ErrorStream: buf, 48 | }) 49 | if err != nil { 50 | return err 51 | } else if str := buf.String(); strings.Contains(strings.ToLower(str), "error") { 52 | return errors.New(strings.TrimSpace(str)) 53 | } 54 | return nil 55 | } 56 | func (d *ServerInstance) ClientV1(ctx context.Context) (protocol1.ProtocolServiceClient, error) { 57 | if d.user == nil { 58 | addr := d.bblfshd.NetworkSettings.IPAddress 59 | conn, err := grpc.DialContext(ctx, addr+":"+cliPort, grpc.WithInsecure(), grpc.WithBlock()) 60 | if err != nil { 61 | return nil, err 62 | } 63 | d.user = conn 64 | } 65 | return protocol1.NewProtocolServiceClient(d.user), nil 66 | } 67 | func (d *ServerInstance) ClientV2(ctx context.Context) (driver.Driver, error) { 68 | if d.user == nil { 69 | addr := d.bblfshd.NetworkSettings.IPAddress 70 | conn, err := grpc.DialContext(ctx, addr+":"+cliPort, grpc.WithInsecure(), grpc.WithBlock()) 71 | if err != nil { 72 | return nil, err 73 | } 74 | d.user = conn 75 | } 76 | return protocol.AsDriver(d.user), nil 77 | } 78 | func (s *ServerInstance) DumpLogs(w io.Writer) error { 79 | return getLogs(s.cli, s.bblfshd.ID, w) 80 | } 81 | func (d *ServerInstance) Close() error { 82 | if d.user != nil { 83 | _ = d.user.Close() 84 | } 85 | return d.cli.RemoveContainer(docker.RemoveContainerOptions{ 86 | ID: d.bblfshd.ID, Force: true, 87 | }) 88 | } 89 | 90 | // RunWithDriver starts a bblfshd server and installs a specified driver to it. 91 | func RunWithDriver(bblfshdVers, lang, id string) (*ServerInstance, error) { 92 | cli, err := docker.Dial() 93 | if err != nil { 94 | return nil, err 95 | } 96 | const ( 97 | bblfshd = "bblfsh/bblfshd" 98 | // needed to install driver from Docker instance 99 | sock = docker.Socket + ":" + docker.Socket 100 | ) 101 | image := bblfshd 102 | if bblfshdVers != "" { 103 | image += ":" + bblfshdVers 104 | } 105 | 106 | printCommand("docker", "run", "--rm", "--privileged", "-v", sock, image) 107 | c, err := docker.Run(cli, docker.CreateContainerOptions{ 108 | Config: &docker.Config{ 109 | Image: image, 110 | }, 111 | HostConfig: &docker.HostConfig{ 112 | AutoRemove: true, 113 | Privileged: true, 114 | Binds: []string{sock}, 115 | }, 116 | }) 117 | if err != nil { 118 | return nil, err 119 | } 120 | s := &ServerInstance{cli: cli, bblfshd: c} 121 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute*3) 122 | defer cancel() 123 | if err := s.installFromDocker(ctx, lang, id); err != nil { 124 | s.Close() 125 | return nil, err 126 | } 127 | return s, nil 128 | } 129 | 130 | func getLogs(cli *docker.Client, id string, w io.Writer) error { 131 | return cli.AttachToContainer(docker.AttachToContainerOptions{ 132 | Container: id, OutputStream: w, ErrorStream: w, 133 | Logs: true, Stdout: true, Stderr: true, 134 | }) 135 | } 136 | -------------------------------------------------------------------------------- /build/exec.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "os/exec" 8 | "strings" 9 | ) 10 | 11 | func create(path string) (io.WriteCloser, error) { 12 | return os.Create(path) 13 | } 14 | 15 | func printCommand(cmd string, args ...string) { 16 | fmt.Fprintln(os.Stderr, "+", cmd, strings.Join(args, " ")) 17 | } 18 | 19 | func execIn(wd string, out io.Writer, cmd string, args ...string) error { 20 | printCommand(cmd, args...) 21 | c := exec.Command(cmd, args...) 22 | c.Dir = wd 23 | c.Stderr = os.Stderr 24 | c.Stdout = out 25 | return c.Run() 26 | } 27 | 28 | func cmdIn(wd string, cmd string, args ...string) Cmd { 29 | printCommand(cmd, args...) 30 | c := exec.Command(cmd, args...) 31 | c.Dir = wd 32 | return Cmd{c} 33 | } 34 | 35 | type Cmd struct { 36 | *exec.Cmd 37 | } 38 | 39 | func (c Cmd) Run() error { 40 | err := c.Cmd.Run() 41 | if e, ok := err.(*exec.ExitError); ok { 42 | err = fmt.Errorf("%s\n%s", e.ProcessState, string(e.Stderr)) 43 | } 44 | return err 45 | } 46 | -------------------------------------------------------------------------------- /build/git.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os/exec" 7 | ) 8 | 9 | func gitRev(path string) (string, error) { 10 | cmd := exec.Command("git", "rev-parse", "HEAD") 11 | cmd.Dir = path 12 | data, err := cmd.Output() 13 | if err != nil { 14 | // maybe we don't have any commits yet 15 | cmd = exec.Command("git", "rev-list", "-n", "1", "--all") 16 | cmd.Dir = path 17 | data2, err2 := cmd.Output() 18 | data2 = bytes.TrimSpace(data2) 19 | if err2 != nil || len(data2) != 0 { 20 | // return original errors 21 | return "", fmt.Errorf("cannot determine git hash: %v", err) 22 | } 23 | // no commits yet 24 | return "", nil 25 | } 26 | return string(bytes.TrimSpace(data)), nil 27 | } 28 | 29 | func gitHasChanges(path string) (bool, error) { 30 | cmd := exec.Command("git", "status", "--porcelain", "-uno") 31 | cmd.Dir = path 32 | data, err := cmd.Output() 33 | if err != nil { 34 | return false, err 35 | } 36 | data = bytes.TrimSpace(data) 37 | return len(data) != 0, nil 38 | } 39 | 40 | func gitInit(path string) error { 41 | if out, err := exec.Command("git", "init", path).CombinedOutput(); err != nil { 42 | return fmt.Errorf("cannot init a repository: %v\n%s\n", err, string(out)) 43 | } 44 | return nil 45 | } 46 | 47 | func gitExec(root, cmd string, files ...string) error { 48 | git := cmdIn(root, "git", append([]string{cmd}, files...)...) 49 | if err := git.Run(); err != nil { 50 | return fmt.Errorf("cannot %s a file to git: %v\n", cmd, err) 51 | } 52 | return nil 53 | } 54 | 55 | func gitAdd(root string, files ...string) error { 56 | return gitExec(root, "add", files...) 57 | } 58 | 59 | func gitRm(root string, files ...string) error { 60 | return gitExec(root, "rm", files...) 61 | } 62 | -------------------------------------------------------------------------------- /build/golang.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | func (d *Driver) modPrepare() error { 9 | if _, err := os.Stat(d.path("go.sum")); os.IsNotExist(err) { 10 | err = modTidy(d.root) 11 | if err != nil { 12 | return err 13 | } 14 | } else if err != nil { 15 | return err 16 | } 17 | if _, err := os.Stat(d.path("vendor")); os.IsNotExist(err) { 18 | err = modVendor(d.root) 19 | if err != nil { 20 | return err 21 | } 22 | } else if err != nil { 23 | return err 24 | } 25 | return nil 26 | } 27 | 28 | func goCmd(dir string, stdout io.Writer, args ...string) Cmd { 29 | cmd := cmdIn(dir, "go", args...) 30 | cmd.Stdout = stdout 31 | cmd.Env = append(os.Environ(), "GO111MODULE=on") 32 | return cmd 33 | } 34 | 35 | func modExec(dir string, args ...string) error { 36 | return goCmd(dir, nil, append([]string{"mod"}, args...)...).Run() 37 | } 38 | 39 | func modTidy(dir string) error { 40 | return modExec(dir, "tidy") 41 | } 42 | 43 | func modInit(dir, pkg string) error { 44 | return modExec(dir, "init", pkg) 45 | } 46 | 47 | func modVendor(dir string) error { 48 | return modExec(dir, "vendor") 49 | } 50 | -------------------------------------------------------------------------------- /build/init.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "text/template" 9 | 10 | "github.com/bblfsh/sdk/v3/assets/skeleton" 11 | ) 12 | 13 | // InitOptions is a set of options available for the driver init. 14 | type InitOptions struct { 15 | Debug PrintfFunc 16 | Notice PrintfFunc 17 | Warning PrintfFunc 18 | } 19 | 20 | func (opt *InitOptions) toUpdateOpt() *UpdateOptions { 21 | return &UpdateOptions{ 22 | Debug: opt.Debug, 23 | Notice: opt.Notice, 24 | Warning: opt.Warning, 25 | } 26 | } 27 | 28 | // InitDriver initializes a new driver in the specified root driver directory. 29 | func InitDriver(root, language string, opt *InitOptions) error { 30 | if language == "" { 31 | return errors.New("'language' argument is mandatory") 32 | } 33 | if opt == nil { 34 | opt = &InitOptions{} 35 | } 36 | 37 | opt.Notice.printf("initializing driver %q, creating new manifest\n", language) 38 | 39 | if _, err := os.Stat(root); err == nil { 40 | root = filepath.Join(root, strings.ToLower(language)+"-driver") 41 | } else if !os.IsNotExist(err) { 42 | return err 43 | } 44 | 45 | opt.Notice.printf("initializing new repo %q\n", root) 46 | if err := gitInit(root); err != nil { 47 | return err 48 | } 49 | 50 | // generate manifests first, other files will use data from them 51 | for _, name := range []string{ 52 | manifestTpl, 53 | buildManifestTpl, 54 | } { 55 | tpl := string(skeleton.MustAsset(name)) 56 | 57 | t, err := template.New(name).Funcs(funcs).Parse(tpl) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | file := filepath.Join(root, strings.TrimSuffix(name, tplExt)) 63 | info := mustAssetInfo(name) 64 | 65 | f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_EXCL, info.Mode()) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | if err := t.Execute(f, map[string]string{ 71 | "Language": language, 72 | }); err != nil { 73 | _ = f.Close() 74 | return err 75 | } 76 | if err := f.Close(); err != nil { 77 | return err 78 | } 79 | } 80 | 81 | if err := UpdateSDK(root, opt.toUpdateOpt()); err != nil { 82 | return err 83 | } 84 | 85 | if err := gitAdd(root, 86 | "Dockerfile", 87 | "go.mod", 88 | "go.sum", 89 | ); err != nil { 90 | opt.Warning.printf("%v\n", err) 91 | } 92 | return nil 93 | } 94 | -------------------------------------------------------------------------------- /build/init_test.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestDriverInit(t *testing.T) { 13 | dir, err := ioutil.TempDir("", "driver_") 14 | require.NoError(t, err) 15 | defer os.RemoveAll(dir) 16 | 17 | const lang = "test" 18 | 19 | err = InitDriver(dir, lang, &InitOptions{ 20 | Warning: func(format string, args ...interface{}) (int, error) { 21 | t.Logf(format, args...) 22 | return 0, nil 23 | }, 24 | Notice: func(format string, args ...interface{}) (int, error) { 25 | t.Logf(format, args...) 26 | return 0, nil 27 | }, 28 | }) 29 | require.NoError(t, err) 30 | 31 | dir = filepath.Join(dir, lang+"-driver") 32 | 33 | err = UpdateSDK(dir, &UpdateOptions{ 34 | DryRun: true, 35 | Warning: func(format string, args ...interface{}) (int, error) { 36 | t.Logf(format, args...) 37 | return 0, nil 38 | }, 39 | Notice: func(format string, args ...interface{}) (int, error) { 40 | t.Logf(format, args...) 41 | return 0, nil 42 | }, 43 | }) 44 | require.NoError(t, err) 45 | } 46 | -------------------------------------------------------------------------------- /build/push.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "os/exec" 8 | "regexp" 9 | ) 10 | 11 | var reVers = regexp.MustCompile(`^v\d+`) 12 | 13 | func pushEnabled() bool { 14 | return !ciIsPR() && (ciBranch() == "master" || ciTag() != "") 15 | } 16 | 17 | func pushTagEnabled() bool { 18 | return pushEnabled() && (reVers.MatchString(ciBranch()) || reVers.MatchString(ciTag())) 19 | } 20 | 21 | func pushLatestEnabled() bool { 22 | return pushEnabled() && ciTag() != "" 23 | } 24 | 25 | func (d *Driver) Push(image string) error { 26 | if isCI() { 27 | log.Printf("CI variables: branch=%q, tag=%q, pr=%v", ciBranch(), ciTag(), ciIsPR()) 28 | } 29 | if !pushTagEnabled() && !pushLatestEnabled() { 30 | return fmt.Errorf("push disabled") 31 | } else if image == "" { 32 | return fmt.Errorf("image should be specified") 33 | } 34 | tag, err := d.VersionTag() 35 | if err != nil { 36 | return err 37 | } 38 | m, err := d.readManifest() 39 | if err != nil { 40 | return err 41 | } 42 | if user := os.Getenv("DOCKER_USERNAME"); user != "" { 43 | cmd := exec.Command("docker", "login", "-u="+user, "-p="+os.Getenv("DOCKER_PASSWORD")) 44 | cmd.Stdout = os.Stdout 45 | cmd.Stderr = os.Stderr 46 | if err := cmd.Run(); err != nil { 47 | return err 48 | } 49 | } 50 | push := func(id, name string) error { 51 | err = execIn("", nil, "docker", "tag", id, name) 52 | if err != nil { 53 | return err 54 | } 55 | return execIn("", nil, "docker", "push", name) 56 | } 57 | imageName := "bblfsh/" + m.Language + "-driver" 58 | if pushTagEnabled() { 59 | if err := push(image, imageName+":"+tag); err != nil { 60 | return err 61 | } 62 | } 63 | if pushLatestEnabled() { 64 | if err := push(image, imageName+":latest"); err != nil { 65 | return err 66 | } 67 | } 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /cmd/bblfsh-drivers-release/cmd/apply.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | 7 | "github.com/bblfsh/sdk/v3/cmd" 8 | "github.com/bblfsh/sdk/v3/driver/manifest/discovery" 9 | 10 | "github.com/google/go-github/github" 11 | "gopkg.in/src-d/go-errors.v1" 12 | "gopkg.in/src-d/go-log.v1" 13 | "gopkg.in/yaml.v2" 14 | ) 15 | 16 | const ApplyCommandDescription = "takes YAML file, generated with collect command and edited with new releases info and performs new releases if it's required" 17 | 18 | var ( 19 | errFailedToCreateRelease = errors.NewKind("%v: failed to create release: %v") 20 | errFailedToUpdateAllDrivers = errors.NewKind("release failed for one or several drivers, last error: %v") 21 | ) 22 | 23 | type ApplyCommand struct { 24 | cmd.Command 25 | 26 | DryRun bool `long:"dry-run" description:"performs extra debug info instead of the real action"` 27 | File string `long:"file" short:"f" env:"FILE" default:"drivers-releases.yml" description:"path to file with configuration"` 28 | ReleaseBranch string `long:"release-branch" env:"RELEASE_BRANCH" default:"master" description:"branch to release"` 29 | } 30 | 31 | func (c *ApplyCommand) Execute(args []string) error { 32 | ctx := context.Background() 33 | 34 | data, err := ioutil.ReadFile(c.File) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | desiredReleases := make(DriversReleases) 40 | if err := yaml.Unmarshal(data, desiredReleases); err != nil { 41 | return err 42 | } 43 | actualReleases, err := getDriversReleases(ctx) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | var lastErr error 49 | for language, release := range desiredReleases { 50 | if release.Tag == actualReleases[language].Tag { 51 | log.Warningf("%v-language: already tagged with %v release, please consider another iteration", language, release.Tag) 52 | continue 53 | } 54 | if err := c.releaseDriver(ctx, language+"-driver", release); err != nil { 55 | lastErr = err 56 | log.Warningf(err.Error()) 57 | } 58 | } 59 | if lastErr != nil { 60 | return errFailedToUpdateAllDrivers.New(lastErr) 61 | } 62 | 63 | return nil 64 | } 65 | 66 | // releaseDriver uses github API to create the release from previously parsed config file 67 | func (c *ApplyCommand) releaseDriver(ctx context.Context, repo string, release Release) error { 68 | client := discovery.GithubClient() 69 | 70 | releaseConfig := &github.RepositoryRelease{ 71 | TagName: github.String(release.Tag), 72 | TargetCommitish: github.String(c.ReleaseBranch), 73 | Name: github.String(release.Tag), 74 | Body: github.String(release.Description), 75 | Draft: github.Bool(false), 76 | Prerelease: github.Bool(false), 77 | } 78 | 79 | if c.DryRun { 80 | data, err := yaml.Marshal(releaseConfig) 81 | if err != nil { 82 | return err 83 | } 84 | log.Infof("%s-driver: release config:\n%v\n", repo, string(data)) 85 | return nil 86 | } 87 | 88 | releaseResp, _, err := client.Repositories.CreateRelease(ctx, owner, repo, releaseConfig) 89 | if err != nil { 90 | return errFailedToCreateRelease.New(repo, err) 91 | } 92 | log.Infof("%v: release %v has been successfully created", repo, *releaseResp.Name) 93 | return nil 94 | } 95 | -------------------------------------------------------------------------------- /cmd/bblfsh-drivers-release/cmd/collect.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "io/ioutil" 7 | "strings" 8 | 9 | "github.com/bblfsh/sdk/v3/cmd" 10 | "github.com/bblfsh/sdk/v3/driver/manifest/discovery" 11 | 12 | "github.com/google/go-github/github" 13 | "gopkg.in/src-d/go-errors.v1" 14 | "gopkg.in/src-d/go-log.v1" 15 | "gopkg.in/yaml.v2" 16 | ) 17 | 18 | const ( 19 | owner = discovery.GithubOrg 20 | 21 | CollectCommandDescription = "pulls info about all latest releases of drivers and all commits since those releases and dumps this info to a YAML file" 22 | ) 23 | 24 | var ( 25 | errFailedToGetCommitsDescription = errors.NewKind("%v: failed to get commits description: %v") 26 | errFailedToGetDriversReleases = errors.NewKind("failed to get drivers releases") 27 | ) 28 | 29 | // DriversReleases represents map with 30 | // - key: language 31 | // - value: Release object 32 | type DriversReleases map[string]Release 33 | 34 | // Release represents an object with the latest tag + concatenated descriptions of commits, performed after release 35 | type Release struct { 36 | // Tag corresponds to release tag in format v0.0.1 37 | Tag string `yaml:"tag"` 38 | // Description represents concatenated descriptions of commits 39 | Description string `yaml:"description"` 40 | } 41 | 42 | type CollectCommand struct { 43 | cmd.Command 44 | 45 | DryRun bool `long:"dry-run" description:"performs extra debug info instead of the real action"` 46 | File string `long:"file" short:"f" env:"FILE" default:"drivers-releases.yml" description:"path to file with configuration"` 47 | } 48 | 49 | func (c *CollectCommand) Execute(args []string) error { 50 | drivers, err := getDriversReleases(context.Background()) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | data, err := yaml.Marshal(drivers) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | if c.DryRun { 61 | log.Infof("\n%v\n", string(data)) 62 | return nil 63 | } 64 | if err := ioutil.WriteFile(c.File, data, 0644); err != nil { 65 | return err 66 | } 67 | 68 | log.Infof("file %v has been successfully written", c.File) 69 | return nil 70 | } 71 | 72 | // getDriversReleases retrieves DriversReleases using discovery and github API 73 | func getDriversReleases(ctx context.Context) (DriversReleases, error) { 74 | log.Infof("getting the list of supported drivers") 75 | drivers, err := discovery.OfficialDrivers(ctx, &discovery.Options{ 76 | NoBuildInfo: true, 77 | NoMaintainers: true, 78 | NoSDKVersion: true, 79 | NoStatic: true, 80 | }) 81 | if err != nil { 82 | return nil, errFailedToGetDriversReleases.Wrap(err) 83 | } 84 | 85 | results := make(DriversReleases) 86 | for _, d := range drivers { 87 | version, err := d.LatestVersion(ctx) 88 | if err != nil { 89 | return nil, errFailedToGetDriversReleases.Wrap(err) 90 | } 91 | 92 | var ( 93 | versionString = version.String() 94 | tag = "v" + versionString 95 | repo = d.Language + "-driver" 96 | ) 97 | 98 | log.Infof("driver: %v, version: %v", d.Language, version.String()) 99 | if versionString == "0.0.0" { 100 | log.Infof("skipping empty version for %v", repo) 101 | continue 102 | } 103 | 104 | description, err := getCommitsDescription(ctx, repo, tag) 105 | if err != nil { 106 | return nil, errFailedToGetDriversReleases.Wrap(err) 107 | } 108 | if description == "" { 109 | log.Infof("%v: no additional commits detected", repo) 110 | continue 111 | } 112 | 113 | log.Infof("post-commits: %v", description) 114 | results[d.Language] = Release{ 115 | Tag: tag, 116 | Description: description, 117 | } 118 | } 119 | 120 | return results, nil 121 | } 122 | 123 | // TODO(lwsanty): we probably need to adjust commits filtering by branch, but it's not crucial while we work on fork PRs basis 124 | // getCommitsDescription is used to obtain concatenated descriptions of commits, performed after release 125 | func getCommitsDescription(ctx context.Context, repo, tag string) (string, error) { 126 | client := discovery.GithubClient() 127 | release, _, err := client.Repositories.GetReleaseByTag(ctx, owner, repo, tag) 128 | if err != nil { 129 | return "", errFailedToGetCommitsDescription.New(repo, err) 130 | } 131 | 132 | var results []string 133 | for page := 1; ; page++ { 134 | rCommits, _, err := client.Repositories.ListCommits(ctx, owner, repo, &github.CommitsListOptions{ 135 | Since: release.PublishedAt.UTC(), 136 | ListOptions: github.ListOptions{ 137 | Page: page, PerPage: 100, 138 | }, 139 | }) 140 | if err != nil { 141 | return "", errFailedToGetCommitsDescription.New(repo, err) 142 | } 143 | 144 | log.Debugf("commits: %v", len(rCommits)) 145 | if len(rCommits) == 0 { 146 | break 147 | } 148 | for _, c := range rCommits { 149 | log.Debugf("appending: %v", *c.Commit.Message) 150 | results = append(results, processCommitMessage(*c.Commit.Message)) 151 | } 152 | } 153 | 154 | return strings.Join(results, "\n"), nil 155 | } 156 | 157 | // processCommitMessage performs required formatting to represent concatenated commits as a bullet list 158 | func processCommitMessage(msg string) string { 159 | var lines []string 160 | scanner := bufio.NewScanner(strings.NewReader(msg)) 161 | for scanner.Scan() { 162 | line := scanner.Text() 163 | 164 | if strings.Contains(line, "Signed-off") || line == "" { 165 | continue 166 | } 167 | lines = append(lines, line) 168 | } 169 | 170 | return "* " + strings.Join(lines, "\n") 171 | } 172 | -------------------------------------------------------------------------------- /cmd/bblfsh-drivers-release/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/bblfsh/sdk/v3/cmd/bblfsh-drivers-release/cmd" 8 | 9 | "github.com/jessevdk/go-flags" 10 | ) 11 | 12 | func main() { 13 | parser := flags.NewNamedParser("bblfsh-drivers-release", flags.Default) 14 | parser.AddCommand("collect", cmd.CollectCommandDescription, "", &cmd.CollectCommand{}) 15 | parser.AddCommand("apply", cmd.ApplyCommandDescription, "", &cmd.ApplyCommand{}) 16 | 17 | if _, err := parser.Parse(); err != nil { 18 | if _, ok := err.(*flags.Error); ok { 19 | parser.WriteHelp(os.Stdout) 20 | } 21 | fmt.Println() 22 | os.Exit(1) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /cmd/bblfsh-drivers-updater/update.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "io/ioutil" 7 | "os" 8 | "strings" 9 | 10 | "github.com/bblfsh/sdk/v3/cmd/bblfsh-drivers-updater/utils" 11 | "github.com/bblfsh/sdk/v3/driver/manifest/discovery" 12 | 13 | log "gopkg.in/src-d/go-log.v1" 14 | ) 15 | 16 | const ( 17 | commitMsg = "autogenerated changes" 18 | ) 19 | 20 | func main() { 21 | branchPtr := flag.String("branch", "patch-1", "branch to be created") 22 | titlePtr := flag.String("title", "", "PR title") 23 | textPtr := flag.String("text", "", "PR body text") 24 | SDKVersionPtr := flag.String("sdk-version", "", "sdk version to update to") 25 | scriptPathPtr := flag.String("script", "", "path to the script that will be executed") 26 | commitMsgPtr := flag.String("commit-msg", commitMsg, "commit message of the update") 27 | dockerfilePtr := flag.Bool("dockerfile", false, "use dockerfile to create a branch") 28 | explicitCredsPtr := flag.Bool("explicit-creds", false, "use explicit credentials inside dockerfile") 29 | dryRunPtr := flag.Bool("dry-run", false, "dry run") 30 | flag.Parse() 31 | 32 | handleErr := func(err error) { 33 | if err != nil { 34 | log.Errorf(err, "error") 35 | os.Exit(1) 36 | } 37 | } 38 | 39 | var scriptText string 40 | scriptData, err := ioutil.ReadFile(*scriptPathPtr) 41 | if err != nil { 42 | log.Errorf(err, "error") 43 | } 44 | scriptText = string(scriptData) 45 | if strings.TrimSpace(scriptText) == "" && strings.TrimSpace(*SDKVersionPtr) == "" { 46 | log.Infof("script and SDK version are empty, nothing to do here") 47 | os.Exit(0) 48 | } 49 | 50 | log.Infof("getting the list of supported drivers") 51 | drivers, err := discovery.OfficialDrivers(context.Background(), &discovery.Options{ 52 | NoStatic: true, 53 | NoMaintainers: true, 54 | NoBuildInfo: true, 55 | }) 56 | handleErr(err) 57 | 58 | log.Infof("%v drivers found", len(drivers)) 59 | for _, d := range drivers { 60 | log.Infof("Processing driver language: %+v, URL: %+v, SDK version: %+v", d.Language, d.RepositoryURL(), d.SDKVersion) 61 | tmpSDKVersion := *SDKVersionPtr 62 | switch { 63 | case d.InDevelopment(): 64 | log.Infof("skipping driver %v: not supported or still in development", d.Language) 65 | continue 66 | case tmpSDKVersion == d.SDKVersion: 67 | log.Infof("driver %v: sdk %v is already installed", d.Language, tmpSDKVersion) 68 | tmpSDKVersion = "" 69 | if strings.TrimSpace(scriptText) == "" { 70 | log.Infof("skipping driver %v: script is empty and version update is not required", d.Language) 71 | continue 72 | } 73 | fallthrough 74 | default: 75 | githubToken := os.Getenv("GITHUB_TOKEN") 76 | err := utils.PrepareBranch(d, githubToken, &utils.UpdateOptions{ 77 | Branch: *branchPtr, 78 | SDKVersion: tmpSDKVersion, 79 | Script: scriptText, 80 | CommitMsg: *commitMsgPtr, 81 | Dockerfile: *dockerfilePtr, 82 | ExplicitCredentials: *explicitCredsPtr, 83 | DryRun: *dryRunPtr, 84 | }) 85 | if utils.ErrNothingToCommit.Is(err) { 86 | log.Warningf("skipping driver %s: nothing to change", d.Language) 87 | continue 88 | } 89 | 90 | handleErr(err) 91 | pr := utils.PRInfo{ 92 | Branch: *branchPtr, 93 | Title: *titlePtr, 94 | Text: *textPtr, 95 | } 96 | if pr.Text == "" { 97 | pr.Text = *commitMsgPtr 98 | } 99 | handleErr(utils.PreparePR(d, githubToken, pr, *dryRunPtr)) 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /cmd/bblfsh-sdk/cmd/ast2gv.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "os/exec" 11 | "strconv" 12 | 13 | "github.com/bblfsh/sdk/v3/uast/nodes" 14 | "github.com/bblfsh/sdk/v3/uast/uastyaml" 15 | "github.com/ghodss/yaml" 16 | ) 17 | 18 | const Ast2GraphvizCommandDescription = "" + 19 | "Read '.native' files and generate graphviz diagrams" 20 | 21 | type Ast2GraphvizCommand struct { 22 | Args struct { 23 | SourceFiles []string `positional-arg-name:"sourcefile(s)" required:"true" description:"File(s) with the native AST"` 24 | } `positional-args:"yes"` 25 | Output string `long:"out" short:"o" default:"dot" description:"Output format (dot, svg, png)"` 26 | TypePred string `long:"type" short:"t" default:"@type" description:"Node type field in native AST"` 27 | Colors string `long:"colors" short:"c" default:"colors.yml" description:"File with node color definitions"` 28 | NoPos bool `long:"no-pos" description:"Omit position info"` 29 | NoNils bool `long:"no-nils" description:"Omit nil fields"` 30 | 31 | nodeColors map[string]string 32 | } 33 | 34 | func (c *Ast2GraphvizCommand) Execute(args []string) error { 35 | if err := c.readColors(c.Colors); err != nil { 36 | return err 37 | } 38 | var last error 39 | for _, name := range c.Args.SourceFiles { 40 | if err := c.processFile(name); err != nil { 41 | log.Printf("error processing %v: %v", name, err) 42 | last = err 43 | } 44 | } 45 | return last 46 | } 47 | 48 | func (c *Ast2GraphvizCommand) readColors(name string) error { 49 | f, err := os.Open(name) 50 | if os.IsNotExist(err) { 51 | return nil 52 | } else if err != nil { 53 | return err 54 | } 55 | defer f.Close() 56 | 57 | data, err := ioutil.ReadAll(f) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | var conf struct { 63 | Colors map[string]string `yaml:"colors"` 64 | } 65 | if err = yaml.Unmarshal(data, &conf); err != nil { 66 | return err 67 | } 68 | c.nodeColors = conf.Colors 69 | return nil 70 | } 71 | func (c *Ast2GraphvizCommand) processFile(name string) error { 72 | f, err := os.Open(name) 73 | if err != nil { 74 | return err 75 | } 76 | data, err := ioutil.ReadAll(f) 77 | f.Close() 78 | if err != nil { 79 | return err 80 | } 81 | ast, err := uastyaml.Unmarshal(data) 82 | if err != nil { 83 | return fmt.Errorf("cannot unmarshal uast: %v", err) 84 | } 85 | ext := c.Output 86 | if ext == "" { 87 | ext = "dot" 88 | } 89 | outName := name + "." + ext 90 | out, err := os.Create(outName) 91 | if err != nil { 92 | return err 93 | } 94 | defer out.Close() 95 | 96 | if ext == "dot" || ext == "gv" { 97 | return c.writeGraphviz(out, ast) 98 | } 99 | 100 | buf := bytes.NewBuffer(nil) 101 | if err := c.writeGraphviz(buf, ast); err != nil { 102 | return err 103 | } 104 | 105 | cmd := exec.Command("dot", "-T"+ext) 106 | cmd.Stdin = buf 107 | cmd.Stdout = out 108 | return cmd.Run() 109 | } 110 | 111 | func (c *Ast2GraphvizCommand) writeGraphviz(w io.Writer, n nodes.Node) error { 112 | fmt.Fprintln(w, "digraph AST {") 113 | defer fmt.Fprintln(w, "}") 114 | 115 | var last int 116 | nextID := func() string { 117 | last++ 118 | id := last 119 | return "n" + strconv.Itoa(id) 120 | } 121 | 122 | const ( 123 | circle = "ellipse" 124 | box = "box" 125 | diamond = "diamond" 126 | ) 127 | 128 | writeNode := func(id, label, shape, color string, small bool) { 129 | if shape == "" { 130 | shape = circle 131 | } 132 | opt := "" 133 | if small { 134 | const h = 0.4 135 | w := 0.8 136 | if label == "" { 137 | w = h 138 | } 139 | opt += fmt.Sprintf(" fontsize=10 margin=0 width=%.2f height=%.2f", w, h) 140 | } 141 | if color != "" { 142 | opt += fmt.Sprintf(" color=%q style=filled", color) 143 | } 144 | fmt.Fprintf(w, "\t%s [label=%q shape=%s%s]\n", id, label, shape, opt) 145 | } 146 | writePred := func(from, via, to string) { 147 | fmt.Fprintf(w, "\t%s -> %s [label=%q fontsize=10]\n", from, to, via) 148 | } 149 | writeLink := func(from, to string) { 150 | fmt.Fprintf(w, "\t%s -> %s\n", from, to) 151 | } 152 | _, _, _ = writeNode, writePred, writeLink 153 | 154 | var proc func(nodes.Node) string 155 | proc = func(n nodes.Node) string { 156 | id := nextID() 157 | switch n := n.(type) { 158 | case nodes.Array: 159 | writeNode(id, "", diamond, "", true) 160 | for _, s := range n { 161 | sid := proc(s) 162 | writeLink(id, sid) 163 | } 164 | case nodes.Object: 165 | n = n.CloneObject() 166 | tp, _ := n[c.TypePred].(nodes.String) 167 | delete(n, c.TypePred) 168 | if c.NoPos { 169 | delete(n, "@pos") 170 | } 171 | writeNode(id, string(tp), circle, c.nodeColors[string(tp)], tp == "") 172 | keys := n.Keys() 173 | for _, k := range keys { 174 | v := n[k] 175 | if c.NoNils && v == nil { 176 | continue 177 | } 178 | sid := proc(v) 179 | writePred(id, k, sid) 180 | } 181 | default: 182 | writeNode(id, fmt.Sprint(n), box, "", true) 183 | } 184 | return id 185 | } 186 | proc(n) 187 | return nil 188 | } 189 | -------------------------------------------------------------------------------- /cmd/bblfsh-sdk/cmd/build.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/bblfsh/sdk/v3/build" 7 | "github.com/bblfsh/sdk/v3/cmd" 8 | ) 9 | 10 | const BuildCommandDescription = "builds the driver" 11 | 12 | type BuildCommand struct { 13 | cmd.Command 14 | } 15 | 16 | func (c *BuildCommand) Execute(args []string) error { 17 | name := "" 18 | if len(args) != 0 { 19 | name = args[0] 20 | } 21 | d, err := build.NewDriver(c.Root) 22 | if err != nil { 23 | return err 24 | } 25 | id, err := d.Build(name) 26 | if err != nil { 27 | return err 28 | } 29 | fmt.Println(id) 30 | return nil 31 | } 32 | 33 | const TestCommandDescription = "tests the driver using fixtures" 34 | 35 | // TestCommand compares "gold" annotations from driver's ./fixtures directory 36 | // to the ones produced by bblfsh now. 37 | type TestCommand struct { 38 | cmd.Command 39 | Bblfshd string `long:"bblfshd" description:"bblfshd version to test with"` 40 | Bench bool `short:"b" long:"bench" description:"benchmark the driver"` 41 | } 42 | 43 | func (c *TestCommand) Execute(args []string) error { 44 | d, err := build.NewDriver(c.Root) 45 | if err != nil { 46 | return err 47 | } 48 | image := "" 49 | if len(args) != 0 { 50 | image = args[0] 51 | } 52 | return d.Test(c.Bblfshd, image, c.Bench) 53 | } 54 | 55 | const TagCommandDescription = "returns a version tag for the driver" 56 | 57 | type TagCommand struct { 58 | cmd.Command 59 | } 60 | 61 | func (c *TagCommand) Execute(args []string) error { 62 | d, err := build.NewDriver(c.Root) 63 | if err != nil { 64 | return err 65 | } 66 | tag, err := d.VersionTag() 67 | if err != nil { 68 | return err 69 | } 70 | fmt.Println(tag) 71 | return nil 72 | } 73 | 74 | const ReleaseCommandDescription = "prepare driver for the release" 75 | 76 | type ReleaseCommand struct { 77 | cmd.Command 78 | } 79 | 80 | func (c *ReleaseCommand) Execute(args []string) error { 81 | d, err := build.NewDriver(c.Root) 82 | if err != nil { 83 | return err 84 | } 85 | return d.FillManifest("") 86 | } 87 | 88 | const PushCommandDescription = "push driver image to docker registry (CI only)" 89 | 90 | type PushCommand struct { 91 | cmd.Command 92 | } 93 | 94 | func (c *PushCommand) Execute(args []string) error { 95 | if len(args) == 0 { 96 | return fmt.Errorf("image name should be specified") 97 | } 98 | d, err := build.NewDriver(c.Root) 99 | if err != nil { 100 | return err 101 | } 102 | return d.Push(args[0]) 103 | } 104 | -------------------------------------------------------------------------------- /cmd/bblfsh-sdk/cmd/info.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/bblfsh/sdk/v3/cmd" 9 | "github.com/bblfsh/sdk/v3/driver/manifest" 10 | ) 11 | 12 | const InfoCommandDescription = "prints info about the driver" 13 | 14 | type InfoCommand struct { 15 | cmd.Command 16 | } 17 | 18 | func (c *InfoCommand) Execute(args []string) error { 19 | m, err := manifest.Load(filepath.Join(c.Root, manifest.Filename)) 20 | if err != nil { 21 | return err 22 | } 23 | enc := json.NewEncoder(os.Stdout) 24 | enc.SetIndent("", " ") 25 | return enc.Encode(m) 26 | } 27 | -------------------------------------------------------------------------------- /cmd/bblfsh-sdk/cmd/init.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/bblfsh/sdk/v3/build" 5 | "github.com/bblfsh/sdk/v3/cmd" 6 | ) 7 | 8 | const InitCommandDescription = "initializes a driver for a given language and OS" 9 | 10 | type InitCommand struct { 11 | Args struct { 12 | Language string `positional-arg-name:"language" description:"target language of the driver"` 13 | } `positional-args:"yes"` 14 | 15 | cmd.Command 16 | } 17 | 18 | func (c *InitCommand) Execute(args []string) error { 19 | opt := &build.InitOptions{ 20 | Notice: cmd.Notice.Printf, 21 | Warning: cmd.Warning.Printf, 22 | } 23 | if c.Verbose { 24 | opt.Debug = cmd.Debug.Printf 25 | } 26 | return build.InitDriver(c.Root, c.Args.Language, opt) 27 | } 28 | -------------------------------------------------------------------------------- /cmd/bblfsh-sdk/cmd/request.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | 10 | "github.com/bblfsh/sdk/v3/cmd" 11 | ) 12 | 13 | const RequestCommandDescription = "returns parse request payload" 14 | 15 | type RequestCommand struct { 16 | Input string `short:"f" long:"file" description:"input source file"` 17 | Output string `short:"o" long:"output" description:"output json payload"` 18 | cmd.Command 19 | } 20 | 21 | type ParseRequest struct { 22 | Content string `json:"content"` 23 | Encoding string `json:"Encoding"` 24 | } 25 | 26 | func (r *RequestCommand) Execute(args []string) error { 27 | if len(r.Input) == 0 { 28 | return fmt.Errorf("no input source file") 29 | } 30 | 31 | src, err := ioutil.ReadFile(r.Input) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | var w io.Writer 37 | if len(r.Output) == 0 { 38 | w = os.Stdout 39 | } else { 40 | w, err = os.Create(r.Output) 41 | if err != nil { 42 | return err 43 | } 44 | } 45 | 46 | err = json.NewEncoder(w).Encode(ParseRequest{ 47 | Content: string(src), 48 | Encoding: "UTF-8", 49 | }) 50 | 51 | if c, ok := w.(io.Closer); ok { 52 | e := c.Close() 53 | if err == nil { 54 | err = e 55 | } 56 | } 57 | 58 | return err 59 | } 60 | -------------------------------------------------------------------------------- /cmd/bblfsh-sdk/cmd/update.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/bblfsh/sdk/v3/build" 5 | "github.com/bblfsh/sdk/v3/cmd" 6 | ) 7 | 8 | const UpdateCommandDescription = "updates an already initialized driver" 9 | 10 | type UpdateCommand struct { 11 | DryRun bool `long:"dry-run" description:"don't writes nothing just checks if something should be written"` 12 | 13 | cmd.Command 14 | } 15 | 16 | func (c *UpdateCommand) Options() *build.UpdateOptions { 17 | opt := &build.UpdateOptions{ 18 | DryRun: c.DryRun, 19 | Notice: cmd.Notice.Printf, 20 | Warning: cmd.Warning.Printf, 21 | } 22 | if c.Verbose { 23 | opt.Debug = cmd.Debug.Printf 24 | } 25 | return opt 26 | } 27 | 28 | func (c *UpdateCommand) Execute(args []string) error { 29 | return build.UpdateSDK(c.Root, c.Options()) 30 | } 31 | -------------------------------------------------------------------------------- /cmd/bblfsh-sdk/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/bblfsh/sdk/v3/cmd/bblfsh-sdk/cmd" 8 | 9 | "github.com/jessevdk/go-flags" 10 | ) 11 | 12 | var version string 13 | var build string 14 | 15 | func main() { 16 | parser := flags.NewNamedParser("bblfsh-sdk", flags.Default) 17 | parser.AddCommand("info", cmd.InfoCommandDescription, "", &cmd.InfoCommand{}) 18 | parser.AddCommand("update", cmd.UpdateCommandDescription, "", &cmd.UpdateCommand{}) 19 | parser.AddCommand("init", cmd.InitCommandDescription, "", &cmd.InitCommand{}) 20 | parser.AddCommand("build", cmd.BuildCommandDescription, "", &cmd.BuildCommand{}) 21 | parser.AddCommand("test", cmd.TestCommandDescription, "", &cmd.TestCommand{}) 22 | parser.AddCommand("tag", cmd.TagCommandDescription, "", &cmd.TagCommand{}) 23 | parser.AddCommand("release", cmd.ReleaseCommandDescription, "", &cmd.ReleaseCommand{}) 24 | parser.AddCommand("push", cmd.PushCommandDescription, "", &cmd.PushCommand{}) 25 | parser.AddCommand("ast2gv", cmd.Ast2GraphvizCommandDescription, "", &cmd.Ast2GraphvizCommand{}) 26 | parser.AddCommand("request", cmd.RequestCommandDescription, "", &cmd.RequestCommand{}) 27 | 28 | if _, err := parser.Parse(); err != nil { 29 | if _, ok := err.(*flags.Error); ok { 30 | parser.WriteHelp(os.Stdout) 31 | fmt.Printf("\nBuild information\n commit: %s\n date:%s\n", version, build) 32 | } 33 | 34 | os.Exit(1) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /cmd/common.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import "github.com/fatih/color" 4 | 5 | var ( 6 | Warning = color.New(color.FgRed) 7 | Notice = color.New(color.FgGreen) 8 | Debug = color.New(color.FgBlue) 9 | ) 10 | 11 | type Command struct { 12 | Verbose bool `long:"verbose" description:"show verbose debug information"` 13 | Root string `long:"root" default:"." description:"root of the driver"` 14 | } 15 | -------------------------------------------------------------------------------- /cmd/util.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | // Contains CLI helpers, shared between bblfshd and drivers. 4 | 5 | import ( 6 | "flag" 7 | "fmt" 8 | 9 | "google.golang.org/grpc" 10 | ) 11 | 12 | const ( 13 | // MaxMsgSizeCLIName is name of the CLI flag to set max msg size. 14 | maxMsgSizeCLIName = "grpc-max-message-size" 15 | // MaxMsgSizeCLIDesc is description for the CLI flag to set max msg size. 16 | maxMsgSizeCLIDesc = "max. message size to send/receive to/from clients (in MB)" 17 | 18 | // DefaulGRPCMaxSendRecvMsgSizeMB is maximum msg size for gRPC in MB. 19 | defaulGRPCMaxSendRecvMsgSizeMB = 100 20 | maxMsgSizeCapMB = 2048 21 | ) 22 | 23 | // GRPCSizeOptions returns a slice of gRPC server options with the max 24 | // message size the server can send/receive set. 25 | // Error is returned if requested size is bigger than 2GB. 26 | // It is intended to be shared by gRPC in bblfshd Server and Drivers. 27 | func GRPCSizeOptions(sizeMB int) ([]grpc.ServerOption, error) { 28 | if sizeMB >= maxMsgSizeCapMB || sizeMB <= 0 { 29 | return nil, fmt.Errorf("%s=%d value should be in between 1 and %dMB", 30 | maxMsgSizeCLIName, sizeMB, maxMsgSizeCapMB-1) 31 | } 32 | 33 | sizeBytes := sizeMB * 1024 * 1024 34 | return []grpc.ServerOption{ 35 | grpc.MaxRecvMsgSize(sizeBytes), 36 | grpc.MaxSendMsgSize(sizeBytes), 37 | }, nil 38 | } 39 | 40 | // FlagMaxGRPCMsgSizeMB sets the CLI configuation flag for max 41 | // gRPC send/recive msg size. 42 | func FlagMaxGRPCMsgSizeMB(fs *flag.FlagSet) *int { 43 | return fs.Int(maxMsgSizeCLIName, defaulGRPCMaxSendRecvMsgSizeMB, maxMsgSizeCLIDesc) 44 | } 45 | -------------------------------------------------------------------------------- /cmd/util_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestGRPCOptions_InvalidInput(t *testing.T) { 10 | // too small 11 | opts, err := GRPCSizeOptions(-1) 12 | require.Error(t, err, "a small value should not be applied") 13 | require.Nil(t, opts) 14 | 15 | // too big 16 | opts, err = GRPCSizeOptions(maxMsgSizeCapMB + 1) 17 | require.Error(t, err, "a big value should not be applied") 18 | require.Nil(t, opts) 19 | } 20 | 21 | func TestGRPCOptions_ValidInput(t *testing.T) { 22 | opts, err := GRPCSizeOptions(32) 23 | 24 | require.Nil(t, err) 25 | require.NotNil(t, opts) 26 | require.Len(t, opts, 2) 27 | // does not work as expected for Func values like grpc.ServerOption 28 | //require.Contains(t, opts, grpc.MaxRecvMsgSize(DefaulGRPCMaxSendRecvMsgSizeMB)) 29 | //require.Contains(t, opts, grpc.MaxSendMsgSize(DefaulGRPCMaxSendRecvMsgSizeMB)) 30 | } 31 | -------------------------------------------------------------------------------- /common.go: -------------------------------------------------------------------------------- 1 | package sdk 2 | 3 | const NativeBin = "/opt/driver/bin/native" 4 | const NativeBinTest = "/opt/driver/src/build/native" 5 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // SDK for Babelfish project. 2 | // 3 | // Babelfish SDK contains the tools and libraries required to create 4 | // a Babelfish driver for a programming language. 5 | package sdk 6 | -------------------------------------------------------------------------------- /driver/driver.go: -------------------------------------------------------------------------------- 1 | // Package driver contains all the logic to build a driver. 2 | package driver 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | "gopkg.in/src-d/go-errors.v1" 10 | 11 | derrors "github.com/bblfsh/sdk/v3/driver/errors" 12 | "github.com/bblfsh/sdk/v3/driver/manifest" 13 | "github.com/bblfsh/sdk/v3/uast/nodes" 14 | ) 15 | 16 | var ( 17 | // ErrDriverFailure is returned when the driver is malfunctioning. 18 | ErrDriverFailure = derrors.ErrDriverFailure 19 | 20 | // ErrSyntax is returned when driver cannot parse the source file. 21 | // Can be omitted for native driver implementations. 22 | ErrSyntax = derrors.ErrSyntax 23 | 24 | // ErrTransformFailure is returned if one of the UAST transformations fails. 25 | ErrTransformFailure = errors.NewKind("transform failed") 26 | 27 | // ErrModeNotSupported is returned if a UAST transformation mode is not supported by the driver. 28 | ErrModeNotSupported = errors.NewKind("transform mode not supported") 29 | 30 | // ErrLanguageDetection indicates that language was not detected by Enry. 31 | ErrLanguageDetection = errors.NewKind("could not autodetect language") 32 | 33 | // ErrUnknownEncoding is returned for parse requests with a file content in a non-UTF8 encoding. 34 | ErrUnknownEncoding = errors.NewKind("unknown source file encoding (expected UTF-8)") 35 | ) 36 | 37 | // ErrMissingDriver indicates that a driver image for the given language 38 | // cannot be found. 39 | type ErrMissingDriver struct { 40 | Language string 41 | } 42 | 43 | func (e *ErrMissingDriver) Error() string { 44 | return fmt.Sprintf("missing driver for language %q", e.Language) 45 | } 46 | 47 | // IsMissingDriver checks if an error is ErrMissingDriver. 48 | func IsMissingDriver(err error) bool { 49 | _, ok := err.(*ErrMissingDriver) 50 | return ok 51 | } 52 | 53 | const ( 54 | // ManifestLocation is the path of the manifest file in the driver image. 55 | ManifestLocation = "/opt/driver/etc/" + manifest.Filename 56 | ) 57 | 58 | // ErrMulti joins multiple errors. 59 | type ErrMulti = derrors.ErrMulti 60 | 61 | // Join multiple errors into a single error value. 62 | func JoinErrors(errs []error) error { 63 | return derrors.Join(errs) 64 | } 65 | 66 | type Mode int 67 | 68 | const ( 69 | ModeNative = Mode(1 << iota) 70 | ModePreprocessed 71 | ModeAnnotated 72 | ModeSemantic 73 | ) 74 | 75 | const ModeDefault = ModeSemantic 76 | 77 | // Parse mode parses a UAST mode string to an enum value. 78 | func ParseMode(mode string) (Mode, error) { 79 | switch mode { 80 | case "native": 81 | return ModeNative, nil 82 | case "annotated": 83 | return ModeAnnotated, nil 84 | case "semantic": 85 | return ModeSemantic, nil 86 | } 87 | return 0, fmt.Errorf("unsupported mode: %q", mode) 88 | } 89 | 90 | // Module is an interface for a generic module instance. 91 | type Module interface { 92 | Start() error 93 | Close() error 94 | } 95 | 96 | type ParseOptions struct { 97 | Mode Mode 98 | Language string 99 | Filename string 100 | } 101 | 102 | // Driver is an interface for a language driver that returns UAST. 103 | type Driver interface { 104 | // Parse reads the input string and constructs an AST representation of it. 105 | // 106 | // Language can be specified by providing ParseOptions. If the language is not set, 107 | // it will be set during the Parse call if implementation supports language detection. 108 | // 109 | // Depending on the mode, AST may be transformed to different UAST variants. 110 | // ErrModeNotSupported is returned for unsupported transformation modes. 111 | // 112 | // Syntax errors are indicated by returning ErrSyntax. 113 | // In this case a non-empty UAST may be returned, if driver supports partial parsing. 114 | // 115 | // Native driver failures are indicated by ErrDriverFailure and UAST transformation are indicated by ErrTransformFailure. 116 | // All other errors indicate a protocol or server failure. 117 | Parse(ctx context.Context, src string, opts *ParseOptions) (nodes.Node, error) 118 | 119 | // Version returns a version of the driver or the server, depending where is this interface is implemented. 120 | Version(ctx context.Context) (Version, error) 121 | 122 | // Languages returns a list of manifests for languages supported by this driver or the server. 123 | Languages(ctx context.Context) ([]manifest.Manifest, error) 124 | } 125 | 126 | // Version information for driver or the server. 127 | type Version struct { 128 | Version string // the version label for the driver, e.g., 'v1.2.3-tag' or 'undefined' 129 | Build time.Time // the timestamp when the driver was built 130 | } 131 | 132 | // DriverModule is an interface for a driver instance. 133 | type DriverModule interface { 134 | Module 135 | Driver 136 | } 137 | 138 | // Native is a base interface of a language driver that returns a native AST. 139 | type Native interface { 140 | Module 141 | // Parse reads the input string and constructs an AST representation of it. 142 | // All errors are considered ErrSyntax, unless they are wrapped into ErrDriverFailure. 143 | Parse(ctx context.Context, src string) (nodes.Node, error) 144 | } 145 | -------------------------------------------------------------------------------- /driver/errors/errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "strings" 5 | 6 | "gopkg.in/src-d/go-errors.v1" 7 | ) 8 | 9 | var ( 10 | // ErrDriverFailure is returned when the driver is malfunctioning. 11 | ErrDriverFailure = errors.NewKind("driver failure") 12 | 13 | // ErrSyntax is returned when driver cannot parse the source file. 14 | // Can be omitted for native driver implementations. 15 | ErrSyntax = errors.NewKind("syntax error") 16 | ) 17 | 18 | // Join multiple errors into a single error value. 19 | // If there are only one error, it will be returned directly. 20 | // Zero or more than one error will be wrapped into ErrMulti. 21 | func Join(errs []error) error { 22 | if len(errs) == 1 { 23 | return errs[0] 24 | } 25 | return &ErrMulti{Errors: errs} 26 | } 27 | 28 | // ErrMulti joins multiple errors. 29 | type ErrMulti struct { 30 | Errors []error 31 | } 32 | 33 | func (e *ErrMulti) Error() string { 34 | var buf strings.Builder 35 | for _, err := range e.Errors { 36 | buf.WriteString(err.Error()) 37 | buf.WriteString("\n") 38 | } 39 | return buf.String() 40 | } 41 | -------------------------------------------------------------------------------- /driver/impl.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/opentracing/opentracing-go" 8 | 9 | "github.com/bblfsh/sdk/v3/driver/manifest" 10 | "github.com/bblfsh/sdk/v3/uast/nodes" 11 | ) 12 | 13 | // NewDriver returns a new Driver instance based on the given ObjectToNode and list of transformers. 14 | func NewDriverFrom(d Native, m *manifest.Manifest, t Transforms) (DriverModule, error) { 15 | if d == nil { 16 | return nil, fmt.Errorf("no driver implementation") 17 | } else if m == nil { 18 | return nil, fmt.Errorf("no manifest") 19 | } 20 | return &driverImpl{d: d, m: m, t: t}, nil 21 | } 22 | 23 | // Driver implements a bblfsh driver, a driver is on charge of transforming a 24 | // source code into an AST and a UAST. To transform the AST into a UAST, a 25 | // `uast.ObjectToNode`` and a series of `tranformer.Transformer` are used. 26 | // 27 | // The `Parse` and `NativeParse` requests block the driver until the request is 28 | // done, since the communication with the native driver is a single-channel 29 | // synchronous communication over stdin/stdout. 30 | type driverImpl struct { 31 | d Native 32 | 33 | m *manifest.Manifest 34 | t Transforms 35 | } 36 | 37 | func (d *driverImpl) Start() error { 38 | return d.d.Start() 39 | } 40 | 41 | func (d *driverImpl) Close() error { 42 | return d.d.Close() 43 | } 44 | 45 | // Parse process a protocol.ParseRequest, calling to the native driver. It a 46 | // parser request is done to the internal native driver and the the returned 47 | // native AST is transform to UAST. 48 | func (d *driverImpl) Parse(rctx context.Context, src string, opts *ParseOptions) (nodes.Node, error) { 49 | sp, ctx := opentracing.StartSpanFromContext(rctx, "bblfsh.driver.Parse") 50 | defer sp.Finish() 51 | 52 | if opts == nil { 53 | opts = &ParseOptions{} 54 | } 55 | ast, err := d.d.Parse(ctx, src) 56 | if err != nil { 57 | if !ErrDriverFailure.Is(err) { 58 | // all other errors are considered syntax errors 59 | err = ErrSyntax.Wrap(err) 60 | } else { 61 | ast = nil 62 | } 63 | return ast, err 64 | } 65 | if opts.Language == "" { 66 | opts.Language = d.m.Language 67 | } 68 | 69 | ast, err = d.t.Do(ctx, opts.Mode, src, ast) 70 | if err != nil { 71 | err = ErrTransformFailure.Wrap(err) 72 | } 73 | return ast, err 74 | } 75 | 76 | // Version returns driver version. 77 | func (d *driverImpl) Version(ctx context.Context) (Version, error) { 78 | return Version{ 79 | Version: d.m.Version, 80 | Build: d.m.Build, 81 | }, nil 82 | } 83 | 84 | // Languages returns a single driver manifest for the language supported by the driver. 85 | func (d *driverImpl) Languages(ctx context.Context) ([]manifest.Manifest, error) { 86 | return []manifest.Manifest{*d.m}, nil // TODO: clone 87 | } 88 | -------------------------------------------------------------------------------- /driver/integration/common.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "path" 5 | "strings" 6 | ) 7 | 8 | // RemoveExtension removes the last extension from a file 9 | func RemoveExtension(filename string) string { 10 | i := strings.LastIndex(filename, path.Ext(filename)) 11 | return filename[:i] 12 | } 13 | -------------------------------------------------------------------------------- /driver/integration/consts/consts.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | const ( 4 | Endpoint = "SERVER_ENDPOINT" 5 | Language = "LANGUAGE" 6 | DriverPath = "DRIVER_PATH" 7 | ) 8 | -------------------------------------------------------------------------------- /driver/integration/suite.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | "time" 11 | 12 | integration "github.com/bblfsh/sdk/v3/driver/integration/consts" 13 | "github.com/bblfsh/sdk/v3/driver/manifest" 14 | protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" 15 | 16 | "github.com/pmezard/go-difflib/difflib" 17 | "github.com/stretchr/testify/require" 18 | "google.golang.org/grpc" 19 | ) 20 | 21 | const ( 22 | Endpoint = integration.Endpoint 23 | Language = integration.Language 24 | DriverPath = integration.DriverPath 25 | ) 26 | const ( 27 | DefaultFixtureLocation = "fixtures" 28 | ) 29 | 30 | var Suite *suite 31 | 32 | func Setup(endpoint, language, driverPath string) { 33 | if endpoint == "" { 34 | endpoint = "localhost:9432" 35 | } 36 | if language == "" { 37 | m, err := manifest.Load(manifest.Filename) 38 | if err != nil { 39 | panic(err) 40 | } 41 | language = m.Language 42 | } 43 | if driverPath == "" { 44 | driverPath = "./" 45 | } 46 | setup(endpoint, language, driverPath) 47 | } 48 | 49 | func setup(endpoint, language, driverPath string) { 50 | Suite = &suite{ 51 | Endpoint: endpoint, 52 | Language: language, 53 | Fixtures: filepath.Join(driverPath, DefaultFixtureLocation), 54 | } 55 | } 56 | 57 | func init() { 58 | setup(os.Getenv(Endpoint), os.Getenv(Language), os.Getenv(DriverPath)) 59 | } 60 | 61 | type suite struct { 62 | // Language of the driver being test. 63 | Language string 64 | // Endpoint of the grpc server to test. 65 | Endpoint string 66 | // Fixture to use against the driver 67 | Fixtures string 68 | 69 | c protocol1.ProtocolServiceClient 70 | } 71 | 72 | func (s *suite) SetUpTest(t *testing.T) { 73 | if s.Endpoint == "" || s.Language == "" { 74 | t.SkipNow() 75 | } 76 | t.Logf("dialing %v", s.Endpoint) 77 | 78 | r := require.New(t) 79 | // TODO: use client-go as soon NativeParse request is availabe on it. 80 | conn, err := grpc.Dial(s.Endpoint, grpc.WithTimeout(time.Second*2), grpc.WithInsecure(), grpc.WithBlock()) 81 | r.Nil(err) 82 | 83 | s.c = protocol1.NewProtocolServiceClient(conn) 84 | } 85 | 86 | func (s *suite) TestParse(t *testing.T) { 87 | files, err := filepath.Glob(fmt.Sprintf("%s/*", s.Fixtures)) 88 | if err != nil { 89 | panic(err) 90 | } 91 | 92 | for _, f := range files { 93 | if !isSource(f) { 94 | continue 95 | } 96 | 97 | t.Run(filepath.Base(f), func(t *testing.T) { 98 | s.doTestParse(t, f) 99 | }) 100 | } 101 | } 102 | 103 | func (s *suite) TestNativeParse(t *testing.T) { 104 | files, err := filepath.Glob(fmt.Sprintf("%s/*", s.Fixtures)) 105 | if err != nil { 106 | panic(err) 107 | } 108 | 109 | for _, f := range files { 110 | if !isSource(f) { 111 | continue 112 | } 113 | 114 | t.Run(filepath.Base(f), func(t *testing.T) { 115 | s.doTestNativeParse(t, f) 116 | }) 117 | } 118 | } 119 | 120 | func (s *suite) doTestParse(t *testing.T, filename string) { 121 | r := require.New(t) 122 | 123 | source := getSourceCode(r, filename) 124 | req := &protocol1.ParseRequest{ 125 | Language: s.Language, 126 | Content: source, 127 | } 128 | 129 | res, err := s.c.Parse(context.Background(), req) 130 | r.Nil(err) 131 | 132 | expected := getUAST(r, filename) 133 | EqualText(r, expected, res.String()) 134 | } 135 | 136 | func (s *suite) doTestNativeParse(t *testing.T, filename string) { 137 | r := require.New(t) 138 | 139 | source := getSourceCode(r, filename) 140 | req := &protocol1.NativeParseRequest{ 141 | Language: s.Language, 142 | Content: source, 143 | } 144 | 145 | res, err := s.c.NativeParse(context.Background(), req) 146 | r.Nil(err) 147 | 148 | expected := getAST(r, filename) 149 | EqualText(r, expected, res.String()) 150 | } 151 | 152 | func EqualText(r *require.Assertions, expected, actual string) { 153 | if expected == actual { 154 | return 155 | } 156 | 157 | diff := difflib.ContextDiff{ 158 | A: difflib.SplitLines(expected), 159 | B: difflib.SplitLines(actual), 160 | FromFile: "expected", 161 | ToFile: "actual", 162 | Context: 3, 163 | Eol: "\n", 164 | } 165 | 166 | patch, err := difflib.GetContextDiffString(diff) 167 | r.Nil(err) 168 | 169 | if patch != "" { 170 | r.Fail("response doesn't match", patch) 171 | } 172 | } 173 | 174 | func getSourceCode(r *require.Assertions, filename string) string { 175 | return getFileContent(r, filename, "") 176 | } 177 | 178 | func getUAST(r *require.Assertions, filename string) string { 179 | return getFileContent(r, filename, "uast") 180 | } 181 | 182 | func getAST(r *require.Assertions, filename string) string { 183 | return getFileContent(r, filename, "native") 184 | } 185 | 186 | func getFileContent(r *require.Assertions, filename, extension string) string { 187 | if len(extension) > 0 { 188 | filename = fmt.Sprintf("%s.%s", filename, extension) 189 | } 190 | content, err := ioutil.ReadFile(filename) 191 | r.Nil(err) 192 | 193 | return string(content) 194 | } 195 | 196 | func isSource(f string) bool { 197 | ext := filepath.Ext(f) 198 | return ext != ".native" && ext != ".uast" 199 | } 200 | -------------------------------------------------------------------------------- /driver/integration/suite_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestParse(t *testing.T) { 8 | Suite.SetUpTest(t) 9 | Suite.TestParse(t) 10 | } 11 | 12 | func TestNativeParse(t *testing.T) { 13 | Suite.SetUpTest(t) 14 | Suite.TestNativeParse(t) 15 | } 16 | -------------------------------------------------------------------------------- /driver/manifest/discovery/discovery_test.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/bblfsh/sdk/v3/driver/manifest" 8 | "github.com/blang/semver" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestOfficialDrivers(t *testing.T) { 13 | if testing.Short() { 14 | t.SkipNow() 15 | } 16 | ctx := context.Background() 17 | drivers, err := OfficialDrivers(ctx, nil) 18 | if isRateLimit(err) { 19 | t.Skip(err) 20 | } 21 | require.NoError(t, err) 22 | require.True(t, len(drivers) >= 15, "drivers: %d", len(drivers)) 23 | 24 | // make sure that IDs are distinct 25 | m := make(map[string]Driver) 26 | for _, d := range drivers { 27 | m[d.Language] = d 28 | } 29 | 30 | for _, exp := range []Driver{ 31 | {Manifest: manifest.Manifest{Language: "go", Name: "Go"}}, 32 | {Manifest: manifest.Manifest{Language: "javascript", Name: "JavaScript"}}, 33 | } { 34 | got := m[exp.Language] 35 | require.Equal(t, exp.Language, got.Language) 36 | require.Equal(t, exp.Name, got.Name) 37 | require.NotEmpty(t, got.Maintainers) 38 | require.NotEmpty(t, got.Features) 39 | 40 | if exp.Language == "go" { 41 | const latest = "2.7.1" 42 | vers, err := got.Versions(ctx) 43 | require.NoError(t, err) 44 | require.NotEmpty(t, vers) 45 | require.True(t, len(vers) >= 18, "versions: %d", len(vers)) 46 | require.True(t, semver.MustParse(latest).LTE(vers[0])) 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /driver/manifest/manifest_test.go: -------------------------------------------------------------------------------- 1 | package manifest 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | var fixture = ` 13 | name = "Foo" 14 | language = "foo" 15 | build = 0001-01-01T00:00:00Z 16 | status = "" 17 | features = ["ast", "uast", "roles"] 18 | 19 | [documentation] 20 | description = "foo" 21 | `[1:] 22 | 23 | func TestEncode(t *testing.T) { 24 | m := &Manifest{} 25 | m.Name = "Foo" 26 | m.Language = "foo" 27 | m.Features = []Feature{AST, UAST, Roles} 28 | m.Documentation = &Documentation{ 29 | Description: "foo", 30 | } 31 | 32 | buf := bytes.NewBuffer(nil) 33 | err := m.Encode(buf) 34 | assert.Nil(t, err) 35 | 36 | assert.Equal(t, fixture, buf.String()) 37 | } 38 | 39 | func TestDecode(t *testing.T) { 40 | m := &Manifest{} 41 | 42 | buf := bytes.NewBufferString(fixture) 43 | err := m.Decode(buf) 44 | assert.Nil(t, err) 45 | 46 | assert.Equal(t, "foo", m.Language) 47 | } 48 | 49 | func TestCurrentSDKVersion(t *testing.T) { 50 | require.Equal(t, 3, CurrentSDKMajor()) 51 | } 52 | 53 | func TestParseMaintainers(t *testing.T) { 54 | m := parseMaintainers(strings.NewReader(` 55 | John Doe (@john_at_github) 56 | Bob 57 | `)) 58 | require.Equal(t, []Maintainer{ 59 | {Name: "John Doe", Email: "john@domain.com", Github: "john_at_github"}, 60 | {Name: "Bob", Email: "bob@domain.com"}, 61 | }, m) 62 | } 63 | 64 | var casesVersion = []struct { 65 | name string 66 | files map[string]string 67 | expect string 68 | }{ 69 | { 70 | name: "no files", 71 | expect: "1", 72 | }, 73 | { 74 | name: "dep lock v1", 75 | files: map[string]string{ 76 | "Gopkg.lock": ` 77 | [[projects]] 78 | name = "gopkg.in/bblfsh/sdk.v1" 79 | version = "v1.16.1" 80 | `, 81 | }, 82 | expect: "1.16.1", 83 | }, 84 | { 85 | name: "dep lock both", 86 | files: map[string]string{ 87 | "Gopkg.lock": ` 88 | [[projects]] 89 | name = "gopkg.in/bblfsh/sdk.v1" 90 | version = "v1.16.1" 91 | 92 | [[projects]] 93 | name = "gopkg.in/bblfsh/sdk.v2" 94 | version = "v2.2.1" 95 | `, 96 | }, 97 | expect: "2.2.1", 98 | }, 99 | { 100 | name: "dep lock no vers", 101 | files: map[string]string{ 102 | "Gopkg.lock": ` 103 | [[projects]] 104 | name = "gopkg.in/bblfsh/sdk.v1" 105 | 106 | [[projects]] 107 | name = "gopkg.in/bblfsh/sdk.v2" 108 | `, 109 | }, 110 | expect: "2", 111 | }, 112 | { 113 | name: "dep toml x", 114 | files: map[string]string{ 115 | "Gopkg.toml": ` 116 | [[constraint]] 117 | name = "gopkg.in/bblfsh/sdk.v1" 118 | version = "1.16.x" 119 | `, 120 | }, 121 | expect: "1.16", 122 | }, 123 | { 124 | name: "go mod", 125 | files: map[string]string{ 126 | "go.mod": `module github.com/bblfsh/some-driver 127 | 128 | require ( 129 | gopkg.in/bblfsh/sdk.v1 v1.16.1 130 | github.com/bblfsh/sdk/v3 v3.1.2 131 | ) 132 | `, 133 | }, 134 | expect: "3.1.2", 135 | }, 136 | { 137 | name: "go mod legacy", 138 | files: map[string]string{ 139 | "go.mod": `module github.com/bblfsh/some-driver 140 | 141 | require ( 142 | gopkg.in/bblfsh/sdk.v1 v1.16.1 143 | ) 144 | `, 145 | }, 146 | expect: "1.16.1", 147 | }, 148 | { 149 | name: "go mod incompatible", 150 | files: map[string]string{ 151 | "go.mod": `module github.com/bblfsh/some-driver 152 | 153 | require ( 154 | gopkg.in/bblfsh/sdk.v1 v1.16.1 155 | github.com/bblfsh/sdk/v3 v3.1.2+incompatible 156 | ) 157 | `, 158 | }, 159 | expect: "3.1.2", 160 | }, 161 | { 162 | name: "go mod no tag", 163 | files: map[string]string{ 164 | "go.mod": `module github.com/bblfsh/some-driver 165 | 166 | require ( 167 | gopkg.in/bblfsh/sdk.v1 v1.16.1 168 | github.com/bblfsh/sdk/v3 v3.0.0-20190326155454-bbb149502c30 169 | ) 170 | `, 171 | }, 172 | expect: "3.0.0", 173 | }, 174 | } 175 | 176 | func TestSDKVersion(t *testing.T) { 177 | for _, c := range casesVersion { 178 | c := c 179 | t.Run(c.name, func(t *testing.T) { 180 | vers, err := SDKVersion(func(path string) ([]byte, error) { 181 | data, ok := c.files[path] 182 | if !ok { 183 | return nil, nil 184 | } 185 | return []byte(data), nil 186 | }) 187 | require.NoError(t, err) 188 | require.Equal(t, c.expect, vers) 189 | }) 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /driver/native/internal/crash/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/bblfsh/sdk/v3/driver/native" 7 | "github.com/bblfsh/sdk/v3/uast/nodes" 8 | ) 9 | 10 | type mockDriver struct{} 11 | 12 | func (mockDriver) Start() error { 13 | panic("died") 14 | return nil 15 | } 16 | 17 | func (mockDriver) Parse(ctx context.Context, src string) (nodes.Node, error) { 18 | panic("unreachable") 19 | } 20 | 21 | func (mockDriver) Close() error { 22 | return nil 23 | } 24 | 25 | func main() { 26 | native.Main(mockDriver{}) 27 | } 28 | -------------------------------------------------------------------------------- /driver/native/internal/crash/manifest.toml: -------------------------------------------------------------------------------- 1 | language = "fixture" 2 | status = "beta" 3 | version = "42" 4 | build = 2015-10-21T04:29:00Z 5 | commit = "424242+" 6 | 7 | [runtime] 8 | os = "alpine" 9 | -------------------------------------------------------------------------------- /driver/native/internal/crash/mock: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | dir=$(dirname $0) 3 | cd $dir 4 | exec go run ./main.go $@ 5 | -------------------------------------------------------------------------------- /driver/native/internal/simple/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/bblfsh/sdk/v3/driver/native" 9 | "github.com/bblfsh/sdk/v3/uast/nodes" 10 | ) 11 | 12 | type mockDriver struct{} 13 | 14 | func (mockDriver) Start() error { 15 | return nil 16 | } 17 | 18 | func (mockDriver) Parse(ctx context.Context, src string) (nodes.Node, error) { 19 | switch src { 20 | case "die": 21 | // just die, prints stack trace on stderr 22 | panic("died") 23 | case "print-and-die": 24 | // protocol runs on stdout, break it and then exit 25 | fmt.Println("crash command received") 26 | os.Exit(0) 27 | } 28 | return nodes.Object{ 29 | "root": nodes.Object{ 30 | "key": nodes.String(src), 31 | }, 32 | }, nil 33 | } 34 | 35 | func (mockDriver) Close() error { 36 | return nil 37 | } 38 | 39 | func main() { 40 | native.Main(mockDriver{}) 41 | } 42 | -------------------------------------------------------------------------------- /driver/native/internal/simple/manifest.toml: -------------------------------------------------------------------------------- 1 | language = "fixture" 2 | status = "beta" 3 | version = "42" 4 | build = 2015-10-21T04:29:00Z 5 | commit = "424242+" 6 | 7 | [runtime] 8 | os = "alpine" 9 | -------------------------------------------------------------------------------- /driver/native/internal/simple/mock: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | dir=$(dirname $0) 3 | cd $dir 4 | exec go run ./main.go $@ 5 | -------------------------------------------------------------------------------- /driver/native/internal/slow/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/bblfsh/sdk/v3/driver/native" 8 | "github.com/bblfsh/sdk/v3/uast/nodes" 9 | ) 10 | 11 | type mockDriver struct{} 12 | 13 | func (mockDriver) Start() error { 14 | time.Sleep(time.Second * 3) 15 | return nil 16 | } 17 | 18 | func (mockDriver) Parse(ctx context.Context, src string) (nodes.Node, error) { 19 | return nodes.Object{ 20 | "root": nodes.Object{ 21 | "key": nodes.String(src), 22 | }, 23 | }, nil 24 | } 25 | 26 | func (mockDriver) Close() error { 27 | return nil 28 | } 29 | 30 | func main() { 31 | native.Main(mockDriver{}) 32 | } 33 | -------------------------------------------------------------------------------- /driver/native/internal/slow/manifest.toml: -------------------------------------------------------------------------------- 1 | language = "fixture" 2 | status = "beta" 3 | version = "42" 4 | build = 2015-10-21T04:29:00Z 5 | commit = "424242+" 6 | 7 | [runtime] 8 | os = "alpine" 9 | -------------------------------------------------------------------------------- /driver/native/internal/slow/mock: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | dir=$(dirname $0) 3 | cd $dir 4 | exec go run ./main.go $@ 5 | -------------------------------------------------------------------------------- /driver/native/jsonlines/decoder.go: -------------------------------------------------------------------------------- 1 | package jsonlines 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "io" 7 | ) 8 | 9 | const ( 10 | // DefaultBufferSize is the default buffer size for decoding. It will 11 | // be used whenever the given reader is not buffered. 12 | DefaultBufferSize = 1024 * 1024 * 4 13 | ) 14 | 15 | type lineReader interface { 16 | ReadBytes(delim byte) ([]byte, error) 17 | } 18 | 19 | // Decoder decodes JSON lines. 20 | type Decoder interface { 21 | // Decode decodes the next JSON line into the given value. 22 | Decode(interface{}) error 23 | } 24 | 25 | type decoder struct { 26 | r lineReader 27 | } 28 | 29 | // NewDecoder creates a new decoder with the given reader. If the given reader 30 | // is not buffered, it will be wrapped with a *bufio.Reader. 31 | func NewDecoder(r io.Reader) Decoder { 32 | var lr lineReader 33 | if v, ok := r.(lineReader); ok { 34 | lr = v 35 | } else { 36 | lr = bufio.NewReaderSize(r, DefaultBufferSize) 37 | } 38 | 39 | return &decoder{r: lr} 40 | } 41 | 42 | // Decode decodes the next line in the reader. 43 | // It does not check JSON for well-formedness before decoding, so in case of 44 | // error, the structure might be half-filled. 45 | func (d *decoder) Decode(v interface{}) error { 46 | line, err := d.r.ReadBytes('\n') 47 | if err != nil { 48 | return err 49 | } 50 | 51 | switch o := v.(type) { 52 | case json.Unmarshaler: 53 | return o.UnmarshalJSON(line) 54 | default: 55 | return json.Unmarshal(line, v) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /driver/native/jsonlines/decoder_test.go: -------------------------------------------------------------------------------- 1 | package jsonlines 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestDecoder(t *testing.T) { 13 | require := require.New(t) 14 | 15 | input := `{"example":1} 16 | {"example":2} 17 | {"example":3} 18 | ` 19 | d := NewDecoder(strings.NewReader(input)) 20 | out := map[string]int{} 21 | 22 | err := d.Decode(&out) 23 | require.NoError(err) 24 | require.Equal(1, out["example"]) 25 | 26 | err = d.Decode(&out) 27 | require.NoError(err) 28 | require.Equal(2, out["example"]) 29 | 30 | err = d.Decode(&out) 31 | require.NoError(err) 32 | require.Equal(3, out["example"]) 33 | 34 | err = d.Decode(&out) 35 | require.Equal(io.EOF, err) 36 | } 37 | 38 | func TestDecoderWithBufferedReader(t *testing.T) { 39 | require := require.New(t) 40 | 41 | input := `{"example":1} 42 | {"example":2} 43 | {"example":3} 44 | ` 45 | d := NewDecoder(bufio.NewReader(strings.NewReader(input))) 46 | out := map[string]int{} 47 | 48 | err := d.Decode(&out) 49 | require.NoError(err) 50 | require.Equal(1, out["example"]) 51 | 52 | err = d.Decode(&out) 53 | require.NoError(err) 54 | require.Equal(2, out["example"]) 55 | 56 | err = d.Decode(&out) 57 | require.NoError(err) 58 | require.Equal(3, out["example"]) 59 | 60 | err = d.Decode(&out) 61 | require.Equal(io.EOF, err) 62 | } 63 | -------------------------------------------------------------------------------- /driver/native/jsonlines/doc.go: -------------------------------------------------------------------------------- 1 | // Package json lines mimicks standard library json Encoder and Decoder, 2 | // but to encode and decode one JSON per line. 3 | package jsonlines 4 | -------------------------------------------------------------------------------- /driver/native/jsonlines/encoder.go: -------------------------------------------------------------------------------- 1 | package jsonlines 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | ) 7 | 8 | // Encoder encodes JSON lines. 9 | type Encoder interface { 10 | // Encode encodes the next value into a JSON line. 11 | Encode(interface{}) error 12 | } 13 | 14 | type encoder struct { 15 | w io.Writer 16 | } 17 | 18 | // NewEncoder creates a new encoder using the given writer. 19 | func NewEncoder(w io.Writer) Encoder { 20 | return &encoder{w} 21 | } 22 | 23 | func (e *encoder) Encode(v interface{}) error { 24 | b, err := json.Marshal(v) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | _, err = e.w.Write(append(b, '\n')) 30 | return err 31 | } 32 | -------------------------------------------------------------------------------- /driver/native/jsonlines/encoder_test.go: -------------------------------------------------------------------------------- 1 | package jsonlines 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestEncoder(t *testing.T) { 11 | require := require.New(t) 12 | 13 | buf := bytes.NewBuffer(nil) 14 | d := NewEncoder(buf) 15 | 16 | err := d.Encode(map[string]int{"example": 1}) 17 | require.NoError(err) 18 | require.Equal("{\"example\":1}\n", buf.String()) 19 | buf.Truncate(0) 20 | 21 | err = d.Encode(map[string]int{"example": 2}) 22 | require.NoError(err) 23 | require.Equal("{\"example\":2}\n", buf.String()) 24 | buf.Truncate(0) 25 | } 26 | -------------------------------------------------------------------------------- /driver/native/main.go: -------------------------------------------------------------------------------- 1 | package native 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | 9 | "github.com/bblfsh/sdk/v3/driver" 10 | "github.com/bblfsh/sdk/v3/driver/native/jsonlines" 11 | ) 12 | 13 | // Main is a main function for running a native Go driver as an Exec-based module that uses internal json protocol. 14 | func Main(d driver.Native) { 15 | if err := d.Start(); err != nil { 16 | panic(err) 17 | } 18 | defer d.Close() 19 | srv := &nativeServer{d: d} 20 | c := struct { 21 | io.Reader 22 | io.Writer 23 | }{ 24 | os.Stdin, 25 | os.Stdout, 26 | } 27 | if err := srv.Serve(c); err != nil { 28 | panic(err) 29 | } 30 | } 31 | 32 | func errToStrings(err error) []string { 33 | if e, ok := err.(*driver.ErrMulti); ok { 34 | str := make([]string, 0, len(e.Errors)) 35 | for _, e := range e.Errors { 36 | str = append(str, e.Error()) 37 | } 38 | return str 39 | } 40 | return []string{err.Error()} 41 | } 42 | 43 | type nativeServer struct { 44 | d driver.Native 45 | } 46 | 47 | func (s *nativeServer) parse(ctx context.Context, req *parseRequest) *parseResponse { 48 | src, err := req.Encoding.Decode(req.Content) 49 | if err != nil { 50 | return &parseResponse{ 51 | Status: statusFatal, 52 | Errors: errToStrings(err), 53 | } 54 | } 55 | ast, err := s.d.Parse(ctx, src) 56 | if driver.ErrDriverFailure.Is(err) { 57 | return &parseResponse{ 58 | Status: statusFatal, 59 | Errors: errToStrings(err), 60 | } 61 | } 62 | if err != nil { 63 | return &parseResponse{ 64 | Status: statusError, 65 | AST: ast, Errors: errToStrings(err), 66 | } 67 | } 68 | return &parseResponse{Status: statusOK, AST: ast} 69 | } 70 | 71 | func (s *nativeServer) Serve(c io.ReadWriter) error { 72 | ctx := context.Background() 73 | enc := jsonlines.NewEncoder(c) 74 | dec := jsonlines.NewDecoder(c) 75 | for { 76 | var req parseRequest 77 | err := dec.Decode(&req) 78 | if err == io.EOF { 79 | return nil 80 | } else if err != nil { 81 | resp := &parseResponse{ 82 | Status: statusFatal, 83 | Errors: []string{fmt.Sprintf("failed to decode request: %v", err)}, 84 | } 85 | if err = enc.Encode(resp); err != nil { 86 | return err 87 | } 88 | continue 89 | } 90 | resp := s.parse(ctx, &req) 91 | if err = enc.Encode(resp); err != nil { 92 | return err 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /driver/server/common.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/bblfsh/sdk/v3/driver" 5 | "github.com/bblfsh/sdk/v3/driver/manifest" 6 | "github.com/bblfsh/sdk/v3/driver/native" 7 | ) 8 | 9 | var DefaultDriver driver.Native = native.NewDriver("") 10 | 11 | var ( 12 | // ManifestLocation location of the manifest file. Should not override 13 | // this variable unless you know what are you doing. 14 | ManifestLocation = driver.ManifestLocation 15 | ) 16 | 17 | // Run is a common main function used as an entry point for drivers. 18 | // It panics in case of an error. 19 | func Run(t driver.Transforms) { 20 | RunNative(DefaultDriver, t) 21 | } 22 | 23 | // RunNative is like Run but allows to provide a custom driver native driver implementation. 24 | func RunNative(d driver.Native, t driver.Transforms) { 25 | m, err := manifest.Load(ManifestLocation) 26 | if err != nil { 27 | panic(err) 28 | } 29 | dr, err := driver.NewDriverFrom(d, m, t) 30 | if err != nil { 31 | panic(err) 32 | } 33 | s := NewServer(dr) 34 | if err := s.Start(); err != nil { 35 | panic(err) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /driver/server/grpc.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/bblfsh/sdk/v3/driver" 9 | "github.com/bblfsh/sdk/v3/driver/manifest" 10 | protocol2 "github.com/bblfsh/sdk/v3/protocol" 11 | uast1 "github.com/bblfsh/sdk/v3/protocol/v1" 12 | "github.com/bblfsh/sdk/v3/uast/nodes" 13 | "google.golang.org/grpc" 14 | protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" 15 | ) 16 | 17 | // NewGRPCServer creates a gRPC server instance that dispatches requests to a provided driver. 18 | // 19 | // It will automatically include default server options for bblfsh protocol. 20 | func NewGRPCServer(drv driver.DriverModule, opts ...grpc.ServerOption) *grpc.Server { 21 | opts = append(opts, protocol2.ServerOptions()...) 22 | return NewGRPCServerCustom(drv, opts...) 23 | } 24 | 25 | // NewGRPCServerCustom is the same as NewGRPCServer, but it won't include any options except the ones that were passed. 26 | func NewGRPCServerCustom(drv driver.DriverModule, opts ...grpc.ServerOption) *grpc.Server { 27 | srv := grpc.NewServer(opts...) 28 | 29 | protocol1.DefaultService = service{drv} 30 | protocol1.RegisterProtocolServiceServer( 31 | srv, 32 | protocol1.NewProtocolServiceServer(), 33 | ) 34 | protocol2.RegisterDriver(srv, drv) 35 | 36 | return srv 37 | } 38 | 39 | type service struct { 40 | d driver.DriverModule 41 | } 42 | 43 | func errResp(err error) protocol1.Response { 44 | return protocol1.Response{Status: protocol1.Fatal, Errors: []string{err.Error()}} 45 | } 46 | 47 | func newDriverManifest(manifest *manifest.Manifest) protocol1.DriverManifest { 48 | features := make([]string, len(manifest.Features)) 49 | for i, feature := range manifest.Features { 50 | features[i] = string(feature) 51 | } 52 | return protocol1.DriverManifest{ 53 | Name: manifest.Name, 54 | Language: manifest.Language, 55 | Version: manifest.Version, 56 | Status: string(manifest.Status), 57 | Features: features, 58 | } 59 | } 60 | 61 | func containsLang(lang string, list []manifest.Manifest) bool { 62 | for _, m := range list { 63 | if m.Language == lang { 64 | return true 65 | } 66 | for _, l := range m.Aliases { 67 | if l == lang { 68 | return true 69 | } 70 | } 71 | } 72 | return false 73 | } 74 | 75 | // SupportedLanguages implements protocol1.Service. 76 | func (s service) SupportedLanguages(_ *protocol1.SupportedLanguagesRequest) *protocol1.SupportedLanguagesResponse { 77 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 78 | defer cancel() 79 | list, err := s.d.Languages(ctx) 80 | if err != nil { 81 | return &protocol1.SupportedLanguagesResponse{Response: errResp(err)} 82 | } 83 | resp := &protocol1.SupportedLanguagesResponse{ 84 | Response: protocol1.Response{Status: protocol1.Ok}, 85 | } 86 | for _, m := range list { 87 | resp.Languages = append(resp.Languages, newDriverManifest(&m)) 88 | } 89 | return resp 90 | } 91 | 92 | func (s service) parse(mode driver.Mode, req *protocol1.ParseRequest) (nodes.Node, protocol1.Response) { 93 | start := time.Now() 94 | ctx := context.Background() 95 | if req.Timeout > 0 { 96 | var cancel func() 97 | ctx, cancel = context.WithTimeout(ctx, req.Timeout) 98 | defer cancel() 99 | } 100 | list, err := s.d.Languages(ctx) 101 | if err != nil { 102 | r := errResp(err) 103 | r.Elapsed = time.Since(start) 104 | return nil, r 105 | } 106 | if !containsLang(req.Language, list) { 107 | r := errResp(ErrUnsupportedLanguage.New(req.Language)) 108 | r.Elapsed = time.Since(start) 109 | return nil, r 110 | } 111 | ast, err := s.d.Parse(ctx, req.Content, &driver.ParseOptions{ 112 | Mode: mode, 113 | Language: req.Language, 114 | Filename: req.Filename, 115 | }) 116 | dt := time.Since(start) 117 | var r protocol1.Response 118 | if err != nil { 119 | r = errResp(err) 120 | } else { 121 | r = protocol1.Response{Status: protocol1.Ok} 122 | } 123 | r.Elapsed = dt 124 | return ast, r 125 | } 126 | 127 | // Parse implements protocol1.Service. 128 | func (s service) Parse(req *protocol1.ParseRequest) *protocol1.ParseResponse { 129 | ast, resp := s.parse(driver.ModeAnnotated, req) 130 | if resp.Status != protocol1.Ok { 131 | return &protocol1.ParseResponse{Response: resp} 132 | } 133 | nd, err := uast1.ToNode(ast) 134 | if err != nil { 135 | r := errResp(err) 136 | r.Elapsed = resp.Elapsed 137 | return &protocol1.ParseResponse{Response: r} 138 | } 139 | return &protocol1.ParseResponse{ 140 | Response: resp, 141 | Language: req.Language, 142 | Filename: req.Filename, 143 | UAST: nd, 144 | } 145 | } 146 | 147 | // NativeParse implements protocol1.Service. 148 | func (s service) NativeParse(req *protocol1.NativeParseRequest) *protocol1.NativeParseResponse { 149 | ast, resp := s.parse(driver.ModeNative, (*protocol1.ParseRequest)(req)) 150 | if resp.Status != protocol1.Ok { 151 | return &protocol1.NativeParseResponse{Response: resp} 152 | } 153 | data, err := json.Marshal(ast) 154 | if err != nil { 155 | r := errResp(err) 156 | r.Elapsed = resp.Elapsed 157 | return &protocol1.NativeParseResponse{Response: r} 158 | } 159 | return &protocol1.NativeParseResponse{ 160 | Response: resp, 161 | Language: req.Language, 162 | AST: string(data), 163 | } 164 | } 165 | 166 | // Version implements protocol1.Service. 167 | func (s service) Version(req *protocol1.VersionRequest) *protocol1.VersionResponse { 168 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 169 | defer cancel() 170 | 171 | m, err := s.d.Version(ctx) 172 | if err != nil { 173 | return &protocol1.VersionResponse{Response: errResp(err)} 174 | } 175 | return &protocol1.VersionResponse{ 176 | Version: m.Version, 177 | Build: m.Build, 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /driver/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "net" 9 | "os" 10 | 11 | jaegercfg "github.com/uber/jaeger-client-go/config" 12 | "google.golang.org/grpc" 13 | 14 | cmdutil "github.com/bblfsh/sdk/v3/cmd" 15 | "github.com/bblfsh/sdk/v3/driver" 16 | "gopkg.in/src-d/go-errors.v1" 17 | "gopkg.in/src-d/go-log.v1" 18 | ) 19 | 20 | var ( 21 | // ErrInvalidLogger is returned by the driver server when the logger configuration is wrong. 22 | ErrInvalidLogger = errors.NewKind("invalid logger configuration") 23 | // ErrInvalidTracer is returned by the driver server when the tracing configuration is wrong. 24 | ErrInvalidTracer = errors.NewKind("invalid tracer configuration") 25 | // ErrUnsupportedLanguage is returned by the language server if the language in the request 26 | // is not supported by the driver. 27 | ErrUnsupportedLanguage = errors.NewKind("unsupported language: %q") 28 | ) 29 | 30 | var ( 31 | network *string 32 | address *string 33 | verbose *string 34 | maxMessageSize *int 35 | logs struct { 36 | level *string 37 | format *string 38 | fields *string 39 | } 40 | ) 41 | 42 | // Server is a grpc server for the communication with the driver. 43 | type Server struct { 44 | grpc *grpc.Server 45 | // Logger a logger to be used by the server. 46 | Logger log.Logger 47 | 48 | d driver.DriverModule 49 | 50 | // closers is a list of things to be closed 51 | // TODO: proper driver shutdown logic; it's unused right now 52 | closers []io.Closer 53 | } 54 | 55 | // NewServer returns a new server for a given Driver. 56 | func NewServer(d driver.DriverModule) *Server { 57 | return &Server{d: d} 58 | } 59 | 60 | // Start executes the binary driver and start to listen in the network and 61 | // address defined by the args. 62 | func (s *Server) Start() error { 63 | if err := s.initialize(); err != nil { 64 | return err 65 | } 66 | 67 | s.Logger.Debugf("executing native binary ...") 68 | if err := s.d.Start(); err != nil { 69 | return err 70 | } 71 | 72 | l, err := net.Listen(*network, *address) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | s.Logger.Infof("server listening in %s (%s)", *address, *network) 78 | 79 | return s.grpc.Serve(l) 80 | } 81 | 82 | func (s *Server) initialize() error { 83 | s.initializeFlags() 84 | if err := s.initializeLogger(); err != nil { 85 | return err 86 | } 87 | list, err := s.d.Languages(context.Background()) 88 | if err != nil { 89 | return err 90 | } else if len(list) != 1 { 91 | return fmt.Errorf("expected exactly one manifest, got %d", len(list)) 92 | } 93 | m := list[0] 94 | if err := s.initializeTracing(m.Language + "-driver"); err != nil { 95 | return err 96 | } 97 | 98 | grpcOpts, err := cmdutil.GRPCSizeOptions(*maxMessageSize) 99 | if err != nil { 100 | s.Logger.Errorf(err, "cannot initialize grpc server with maxMessageSize %d", *maxMessageSize) 101 | os.Exit(1) 102 | } 103 | 104 | build := "unknown" 105 | if !m.Build.IsZero() { 106 | build = m.Build.Format("2006-01-02T15:04:05Z") 107 | } 108 | s.Logger.Infof("%s-driver version: %s (build: %s)", 109 | m.Language, 110 | m.Version, 111 | build, 112 | ) 113 | s.grpc = NewGRPCServer(s.d, grpcOpts...) 114 | return nil 115 | } 116 | 117 | func (s *Server) initializeFlags() { 118 | const ( 119 | defaultNetwork = "tcp" 120 | defaultAddress = "0.0.0.0:9432" 121 | defaultVerbose = "info" 122 | defaultFormat = "text" 123 | ) 124 | 125 | cmd := flag.NewFlagSet("server", flag.ExitOnError) 126 | network = cmd.String("network", defaultNetwork, "network type: tcp, tcp4, tcp6, unix or unixpacket.") 127 | address = cmd.String("address", defaultAddress, "address to listen.") 128 | maxMessageSize = cmdutil.FlagMaxGRPCMsgSizeMB(cmd) 129 | 130 | logs.level = cmd.String("log-level", defaultVerbose, "log level: panic, fatal, error, warning, info, debug.") 131 | logs.format = cmd.String("log-format", defaultFormat, "format of the logs: text or json.") 132 | logs.fields = cmd.String("log-fields", "", "extra fields to add to every log line in json format.") 133 | 134 | cmd.Parse(os.Args[1:]) 135 | } 136 | 137 | func (s *Server) initializeLogger() error { 138 | // TODO(lwsanty): fix in go-log 139 | log.DefaultFactory = &log.LoggerFactory{ 140 | Level: *logs.level, 141 | Format: *logs.format, 142 | Fields: *logs.fields, 143 | } 144 | 145 | var err error 146 | s.Logger, err = log.DefaultFactory.New(nil) 147 | if err != nil { 148 | return ErrInvalidLogger.Wrap(err) 149 | } 150 | 151 | return nil 152 | } 153 | 154 | func (s *Server) initializeTracing(serviceName string) error { 155 | c, err := jaegercfg.FromEnv() 156 | if err != nil { 157 | return ErrInvalidTracer.Wrap(err) 158 | } 159 | closer, err := c.InitGlobalTracer(serviceName) 160 | if err != nil { 161 | return ErrInvalidTracer.Wrap(err) 162 | } 163 | s.closers = append(s.closers, closer) 164 | return nil 165 | } 166 | -------------------------------------------------------------------------------- /driver/server/server_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "testing" 5 | 6 | protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" 7 | 8 | "github.com/bblfsh/sdk/v3/driver" 9 | "github.com/bblfsh/sdk/v3/driver/manifest" 10 | "github.com/bblfsh/sdk/v3/driver/native" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func init() { 15 | ManifestLocation = "../native/internal/simple/manifest.toml" 16 | } 17 | 18 | func newDriver(path string) (*service, error) { 19 | if path == "" { 20 | path = "../native/internal/simple/mock" 21 | } 22 | m, err := manifest.Load(ManifestLocation) 23 | if err != nil { 24 | return nil, err 25 | } 26 | d, err := driver.NewDriverFrom(native.NewDriverAt(path, native.UTF8), m, driver.Transforms{}) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return &service{d: d}, nil 31 | } 32 | 33 | func TestDriverParserParse(t *testing.T) { 34 | require := require.New(t) 35 | 36 | d, err := newDriver("") 37 | require.NoError(err) 38 | require.NotNil(d) 39 | 40 | err = d.d.Start() 41 | require.NoError(err) 42 | 43 | r := d.Parse(&protocol1.ParseRequest{ 44 | Language: "fixture", 45 | Filename: "foo.f", 46 | Content: "foo", 47 | }) 48 | 49 | require.NotNil(r) 50 | require.Empty(r.Errors, "%v", r.Errors) 51 | require.Equal(protocol1.Ok, r.Status) 52 | require.Equal("fixture", r.Language) 53 | require.Equal("foo.f", r.Filename) 54 | require.True(r.Elapsed.Nanoseconds() > 0) 55 | require.Equal(` { 56 | . Children: { 57 | . . 0: { 58 | . . . Properties: { 59 | . . . . internalRole: root 60 | . . . . key: foo 61 | . . . } 62 | . . } 63 | . } 64 | } 65 | `, r.UAST.String()) 66 | 67 | err = d.d.Close() 68 | require.NoError(err) 69 | } 70 | 71 | func TestDriverParserParse_MissingLanguage(t *testing.T) { 72 | require := require.New(t) 73 | 74 | d, err := newDriver("") 75 | require.NoError(err) 76 | require.NotNil(d) 77 | 78 | err = d.d.Start() 79 | require.NoError(err) 80 | 81 | r := d.Parse(&protocol1.ParseRequest{ 82 | Content: "foo", 83 | }) 84 | 85 | require.NotNil(r) 86 | require.Equal(len(r.Errors), 1) 87 | require.Equal(r.Status, protocol1.Fatal) 88 | require.Equal(r.Elapsed.Nanoseconds() > 0, true) 89 | require.Nil(r.UAST) 90 | 91 | err = d.d.Close() 92 | require.NoError(err) 93 | } 94 | func TestDriverParserParse_Malfunctioning(t *testing.T) { 95 | require := require.New(t) 96 | 97 | d, err := newDriver("echo") 98 | require.NoError(err) 99 | require.NotNil(d) 100 | 101 | err = d.d.Start() 102 | require.NoError(err) 103 | 104 | r := d.Parse(&protocol1.ParseRequest{ 105 | Language: "fixture", 106 | Content: "foo", 107 | }) 108 | 109 | require.NotNil(r) 110 | 111 | require.Equal(r.Status, protocol1.Fatal) 112 | require.Equal(r.Elapsed.Nanoseconds() > 0, true) 113 | require.Equal(len(r.Errors), 1) 114 | require.Nil(r.UAST) 115 | 116 | err = d.d.Close() 117 | require.NoError(err) 118 | } 119 | 120 | func TestDriverParserNativeParse(t *testing.T) { 121 | require := require.New(t) 122 | 123 | d, err := newDriver("") 124 | require.NoError(err) 125 | require.NotNil(d) 126 | 127 | err = d.d.Start() 128 | require.NoError(err) 129 | 130 | r := d.NativeParse(&protocol1.NativeParseRequest{ 131 | Language: "fixture", 132 | Content: "foo", 133 | }) 134 | 135 | require.NotNil(r) 136 | require.Equal(len(r.Errors), 0) 137 | require.Equal(r.Status, protocol1.Ok) 138 | require.Equal(r.Language, "fixture") 139 | require.Equal(r.Elapsed.Nanoseconds() > 0, true) 140 | require.Equal(r.AST, "{\"root\":{\"key\":\"foo\"}}") 141 | 142 | err = d.d.Close() 143 | require.NoError(err) 144 | } 145 | 146 | func TestDriverParserVersion(t *testing.T) { 147 | require := require.New(t) 148 | 149 | d, err := newDriver("") 150 | require.NoError(err) 151 | require.NotNil(d) 152 | 153 | v := d.Version(nil) 154 | require.Equal(v.Version, "42") 155 | require.Equal(v.Build.String(), "2015-10-21 04:29:00 +0000 UTC") 156 | } 157 | -------------------------------------------------------------------------------- /driver/transforms.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/opentracing/opentracing-go" 7 | 8 | "github.com/bblfsh/sdk/v3/uast/nodes" 9 | "github.com/bblfsh/sdk/v3/uast/transformer" 10 | ) 11 | 12 | // Transforms describes a set of AST transformations the driver requires. 13 | // 14 | // The pipeline can be illustrated as: 15 | // ( AST )--------------> ( ModeNative ) 16 | // V 17 | // [ Preprocess ] 18 | // [ PreprocessCode ]--------> ( ModePreprocessed ) 19 | // | 20 | // |----------------\ 21 | // | [ Normalize ] 22 | // [ Annotations ] <-------/ 23 | // | 24 | // V 25 | // [ Code ]-------------\ 26 | // | [ Namespace ] 27 | // | | 28 | // V V 29 | // ( ModeAnnotated ) ( ModeSemantic ) 30 | type Transforms struct { 31 | // Namespace for native AST nodes of this language. Only enabled in Semantic mode. 32 | // 33 | // Namespace will be set at the end of the pipeline, thus all transforms can 34 | // use type names without the driver namespace. 35 | Namespace string 36 | 37 | // Preprocess stage normalizes native AST for both Annotated and Semantic stages. 38 | // 39 | // It usually: 40 | // * changes type key to uast.KeyType 41 | // * restructures positional information 42 | // * fixes any issues with native AST structure 43 | Preprocess []transformer.Transformer 44 | 45 | // PreprocessCode stage runs code-assisted transformations after the Preprocess stage. 46 | // It can be used to fix node tokens or positional information based on the source. 47 | PreprocessCode []transformer.CodeTransformer 48 | 49 | // Normalize stage converts a known native AST structures to a canonicalized 50 | // high-level AST representation (UAST). It is executed after PreprocessCode 51 | // and before the Annotations stage. 52 | Normalize []transformer.Transformer 53 | 54 | // Annotations stage applies UAST role annotations and is executed after 55 | // Semantic stage, or after PreprocessCode if Semantic is disabled. 56 | // 57 | // It also changes token key to uast.KeyToken. It should not be done in the 58 | // Preprocess stage, because Semantic annotations are easier on clean native AST. 59 | Annotations []transformer.Transformer 60 | } 61 | 62 | // Do applies AST transformation pipeline for specified AST subtree. 63 | // 64 | // Mode can be specified to stop the pipeline at a specific abstraction level. 65 | func (t Transforms) Do(rctx context.Context, mode Mode, code string, nd nodes.Node) (nodes.Node, error) { 66 | sp, ctx := opentracing.StartSpanFromContext(rctx, "uast.Transform") 67 | defer sp.Finish() 68 | 69 | if mode > ModeSemantic { 70 | return nil, ErrModeNotSupported.New() 71 | } 72 | if mode == 0 { 73 | mode = ModeDefault 74 | } 75 | if mode == ModeNative { 76 | return nd, nil 77 | } 78 | 79 | return t.do(ctx, mode, code, nd) 80 | } 81 | 82 | func (t Transforms) do(ctx context.Context, mode Mode, code string, nd nodes.Node) (nodes.Node, error) { 83 | var err error 84 | runAll := func(name string, list []transformer.Transformer) error { 85 | sp, _ := opentracing.StartSpanFromContext(ctx, "uast.Transform."+name) 86 | defer sp.Finish() 87 | 88 | for _, t := range list { 89 | nd, err = t.Do(nd) 90 | if err != nil { 91 | return err 92 | } 93 | } 94 | return nil 95 | } 96 | runAllCode := func(name string, list []transformer.CodeTransformer) error { 97 | sp, _ := opentracing.StartSpanFromContext(ctx, "uast.Transform."+name) 98 | defer sp.Finish() 99 | 100 | for _, ct := range list { 101 | t := ct.OnCode(code) 102 | nd, err = t.Do(nd) 103 | if err != nil { 104 | return err 105 | } 106 | } 107 | return nil 108 | } 109 | 110 | // Preprocess AST and optionally use the second pre-processing stage 111 | // that can access the source code (to fix tokens, for example). 112 | if err := runAll("preprocess", t.Preprocess); err != nil { 113 | return nd, err 114 | } 115 | if err := runAllCode("preprocess-code", t.PreprocessCode); err != nil { 116 | return nd, err 117 | } 118 | 119 | // First run Semantic mode (UAST canonicalization). 120 | // It's considered a more high-level representation, but it needs 121 | // a clean AST to run, so we execute it before Annotated mode. 122 | if mode >= ModeSemantic { 123 | if err := runAll("semantic", t.Normalize); err != nil { 124 | return nd, err 125 | } 126 | } 127 | 128 | // Next run Annotated mode. It won't see nodes converted by Semantic nodes, 129 | // because it expects a clean native AST. 130 | // This is intentional — Semantic nodes are already defined with specific 131 | // roles in mind, thus they shouldn't be annotated further on this stage. 132 | if mode >= ModeAnnotated { 133 | if err := runAll("annotated", t.Annotations); err != nil { 134 | return nd, err 135 | } 136 | } 137 | 138 | // All native nodes should have a namespace in Semantic mode. 139 | // Set if it was specified in the transform configuration. 140 | if mode >= ModeSemantic && t.Namespace != "" { 141 | tr := transformer.DefaultNamespace(t.Namespace) 142 | if err := runAll("namespace", []transformer.Transformer{tr}); err != nil { 143 | return nd, err 144 | } 145 | } 146 | 147 | return nd, nil 148 | } 149 | -------------------------------------------------------------------------------- /etc/build/Dockerfile.tpl: -------------------------------------------------------------------------------- 1 | # This file can be used directly with Docker. 2 | # 3 | # Prerequisites: 4 | # go mod vendor 5 | # bblfsh-sdk release 6 | # 7 | # However, the preferred way is: 8 | # go run ./build.go driver:tag 9 | # 10 | # This will regenerate all necessary files before building the driver. 11 | 12 | #============================== 13 | # Stage 1: Native Driver Build 14 | #============================== 15 | {{if and (ne .Native.Build.Gopath "") (eq .Native.Build.Image "") -}} 16 | FROM golang:{{ .Runtime.Version }} as native 17 | {{- else -}} 18 | FROM {{ .Native.Build.Image }} as native 19 | {{- end}} 20 | 21 | {{- if ne (len .Native.Build.BuildAssets) 0}} 22 | 23 | # add dependency files 24 | {{range .Native.Build.BuildAssets -}} 25 | ADD {{ .Path }} {{ .Dest }} 26 | {{end}} 27 | {{- end}} 28 | 29 | {{- if ne (len .Native.Build.Deps) 0}} 30 | 31 | # install build dependencies 32 | {{range .Native.Build.Deps -}} 33 | RUN {{ . }} 34 | {{end}} 35 | {{- end}} 36 | 37 | {{if ne .Native.Build.Gopath "" -}} 38 | ENV DRIVER_REPO=github.com/bblfsh/{{ .Language }}-driver 39 | ENV DRIVER_REPO_PATH={{ .Native.Build.Gopath }}/src/$DRIVER_REPO 40 | 41 | ADD go.* $DRIVER_REPO_PATH/ 42 | ADD vendor $DRIVER_REPO_PATH/vendor 43 | ADD driver $DRIVER_REPO_PATH/driver 44 | ADD native $DRIVER_REPO_PATH/native 45 | WORKDIR $DRIVER_REPO_PATH/native 46 | ENV GO111MODULE=on GOFLAGS=-mod=vendor 47 | {{- else -}} 48 | ADD native /native 49 | WORKDIR /native 50 | {{- end}} 51 | 52 | # build native driver 53 | {{range .Native.Build.Build -}} 54 | RUN {{ . }} 55 | {{end}} 56 | 57 | #================================ 58 | # Stage 1.1: Native Driver Tests 59 | #================================ 60 | FROM native as native_test 61 | {{if ne .Native.Build.Gopath ""}} 62 | # workaround for https://github.com/golang/go/issues/28065 63 | ENV CGO_ENABLED=0 64 | {{end -}} 65 | 66 | {{- if ne (len .Native.Test.Deps) 0}} 67 | # install test dependencies 68 | {{range .Native.Test.Deps -}} 69 | RUN {{ . }} 70 | {{end}} 71 | {{- end}} 72 | # run native driver tests 73 | {{range .Native.Test.Test -}} 74 | RUN {{ . }} 75 | {{end}} 76 | 77 | #================================= 78 | # Stage 2: Go Driver Server Build 79 | #================================= 80 | {{if ne .Native.Build.Gopath "" -}} 81 | FROM native as driver 82 | {{- else -}} 83 | FROM golang:{{ .Runtime.Version }} as driver 84 | 85 | ENV DRIVER_REPO=github.com/bblfsh/{{ .Language }}-driver 86 | ENV DRIVER_REPO_PATH=/go/src/$DRIVER_REPO 87 | 88 | ADD go.* $DRIVER_REPO_PATH/ 89 | ADD vendor $DRIVER_REPO_PATH/vendor 90 | ADD driver $DRIVER_REPO_PATH/driver 91 | {{- end}} 92 | 93 | WORKDIR $DRIVER_REPO_PATH/ 94 | 95 | ENV GO111MODULE=on GOFLAGS=-mod=vendor 96 | 97 | # workaround for https://github.com/golang/go/issues/28065 98 | ENV CGO_ENABLED=0 99 | 100 | # build server binary 101 | RUN go build -o /tmp/driver ./driver/main.go 102 | # build tests 103 | RUN go test -c -o /tmp/fixtures.test ./driver/fixtures/ 104 | 105 | #======================= 106 | # Stage 3: Driver Build 107 | #======================= 108 | FROM {{ .Native.Image }} 109 | 110 | {{range .Native.Deps -}} 111 | RUN {{ . }} 112 | {{end}} 113 | 114 | LABEL maintainer="source{d}" \ 115 | bblfsh.language="{{ .Language }}" 116 | 117 | WORKDIR /opt/driver 118 | 119 | {{- if ne (len .Native.ImageAssets) 0}} 120 | 121 | # copy static files from driver source directory 122 | {{range .Native.ImageAssets -}} 123 | ADD ./native/{{ .Path }} ./bin/{{ .Dest }} 124 | {{end}} 125 | {{- end}} 126 | 127 | # copy build artifacts for native driver 128 | {{range .Native.Build.Artifacts -}} 129 | COPY --from=native {{ .Path }} ./bin/{{ .Dest }} 130 | {{end}} 131 | 132 | # copy driver server binary 133 | COPY --from=driver /tmp/driver ./bin/ 134 | 135 | # copy tests binary 136 | COPY --from=driver /tmp/fixtures.test ./bin/ 137 | # move stuff to make tests work 138 | RUN ln -s /opt/driver ../build 139 | VOLUME /opt/fixtures 140 | 141 | # copy driver manifest and static files 142 | ADD .manifest.release.toml ./etc/manifest.toml 143 | 144 | ENTRYPOINT ["/opt/driver/bin/driver"] -------------------------------------------------------------------------------- /etc/skeleton/.gitignore: -------------------------------------------------------------------------------- 1 | .sdk 2 | build 3 | vendor/ -------------------------------------------------------------------------------- /etc/skeleton/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - '1.13.x' 5 | 6 | services: 7 | - docker 8 | 9 | env: 10 | global: 11 | - GO111MODULE=on 12 | - BBLFSHD_VERSION=v2.12.1 13 | 14 | install: 15 | - go mod download 16 | - docker pull bblfsh/bblfshd:$BBLFSHD_VERSION 17 | 18 | script: 19 | - go test ./driver/... 20 | - go run build.go ci-build 21 | - go run test.go --bblfshd $BBLFSHD_VERSION ci-build 22 | 23 | after_success: 24 | - go run github.com/bblfsh/sdk/v3/cmd/bblfsh-sdk push ci-build 25 | -------------------------------------------------------------------------------- /etc/skeleton/Jenkinsfile.tpl: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { 3 | kubernetes { 4 | label '{{.Manifest.Language}}-driver-bblfsh-performance' 5 | defaultContainer '{{.Manifest.Language}}-driver-bblfsh-performance' 6 | yaml """ 7 | spec: 8 | nodeSelector: 9 | srcd.host/type: jenkins-worker 10 | affinity: 11 | podAntiAffinity: 12 | requiredDuringSchedulingIgnoredDuringExecution: 13 | - labelSelector: 14 | matchExpressions: 15 | - key: jenkins 16 | operator: In 17 | values: 18 | - slave 19 | topologyKey: kubernetes.io/hostname 20 | containers: 21 | - name: {{.Manifest.Language}}-driver-bblfsh-performance 22 | image: bblfsh/performance:latest 23 | imagePullPolicy: Always 24 | securityContext: 25 | privileged: true 26 | command: 27 | - dockerd 28 | tty: true 29 | """ 30 | } 31 | } 32 | environment { 33 | DRIVER_LANGUAGE = "{{.Manifest.Language}}" 34 | DRIVER_REPO = "https://github.com/bblfsh/{{.Manifest.Language}}-driver.git" 35 | DRIVER_SRC_FIXTURES = "${env.WORKSPACE}/fixtures" 36 | BENCHMARK_FILE = "${env.WORKSPACE}/bench.log" 37 | LOG_LEVEL = "debug" 38 | PROM_ADDRESS = "http://prom-pushgateway-prometheus-pushgateway.monitoring.svc.cluster.local:9091" 39 | PROM_JOB = "bblfsh_perfomance" 40 | } 41 | // TODO(lwsanty): https://github.com/src-d/infrastructure/issues/992 42 | // this is polling for every 2 minutes 43 | // however it's better to use trigger curl http://yourserver/jenkins/git/notifyCommit?url= 44 | // https://kohsuke.org/2011/12/01/polling-must-die-triggering-jenkins-builds-from-a-git-hook/ 45 | // the problem is that it requires Jenkins to be accessible from the hook side 46 | // probably Travis CI could trigger Jenkins after all unit tests have passed... 47 | triggers { pollSCM('H/2 * * * *') } 48 | stages { 49 | stage('Run transformations benchmark') { 50 | when { branch 'master' } 51 | steps { 52 | sh "set -o pipefail ; go test -run=NONE -bench=/transform ./driver/... | tee ${env.BENCHMARK_FILE}" 53 | } 54 | } 55 | stage('Store transformations benchmark to prometheus') { 56 | when { branch 'master' } 57 | steps { 58 | sh "/root/bblfsh-performance parse-and-store --language=${env.DRIVER_LANGUAGE} --commit=${env.GIT_COMMIT} --storage=prom ${env.BENCHMARK_FILE}" 59 | } 60 | } 61 | stage('Run driver-native benchmark') { 62 | when { branch 'master' } 63 | steps { 64 | sh "/root/bblfsh-performance driver-native --language=${env.DRIVER_LANGUAGE} --commit=${env.GIT_COMMIT} --native=/root/utils/native-driver-performance --storage=prom ${env.DRIVER_SRC_FIXTURES}" 65 | } 66 | } 67 | stage('Run driver benchmark') { 68 | when { branch 'master' } 69 | steps { 70 | sh "/root/bblfsh-performance driver --language=${env.DRIVER_LANGUAGE} --commit=${env.GIT_COMMIT} --storage=prom ${env.DRIVER_SRC_FIXTURES}" 71 | } 72 | } 73 | stage('Run end-to-end benchmark') { 74 | when { branch 'master' } 75 | steps { 76 | sh "/root/bblfsh-performance end-to-end --language=${env.DRIVER_LANGUAGE} --commit=${env.GIT_COMMIT} --docker-tag=latest --custom-driver=true --storage=prom ${env.DRIVER_SRC_FIXTURES}" 77 | } 78 | } 79 | } 80 | post { 81 | success { 82 | slackSend (color: '#2eb886', message: "SUCCESS: `${env.JOB_NAME}` <${env.BUILD_URL}|build #${env.BUILD_NUMBER}>") 83 | } 84 | failure { 85 | slackSend (color: '#b82e60', message: "FAILED: `${env.JOB_NAME}` <${env.BUILD_URL}|build #${env.BUILD_NUMBER}>") 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /etc/skeleton/README.md.tpl: -------------------------------------------------------------------------------- 1 | # {{.Manifest.Name}} driver for [Babelfish](https://github.com/bblfsh/bblfshd) ![Driver Status](https://img.shields.io/badge/status-{{.Manifest.Status | escape_shield}}-{{template "color-status" .}}.svg) [![Build Status](https://travis-ci.org/bblfsh/{{.Manifest.Language}}-driver.svg?branch=master)](https://travis-ci.org/bblfsh/{{.Manifest.Language | escape_shield }}-driver) ![Native Version](https://img.shields.io/badge/{{.Manifest.Language}}%20version-{{.Manifest.Runtime.NativeVersion | escape_shield}}-aa93ea.svg) ![Go Version](https://img.shields.io/badge/go%20version-{{.Manifest.Runtime.GoVersion | escape_shield}}-63afbf.svg) 2 | 3 | {{if .Manifest.Documentation}}{{.Manifest.Documentation.Description}} 4 | 5 | {{if .Manifest.Documentation.Caveats -}} 6 | Caveats 7 | ------- 8 | 9 | {{.Manifest.Documentation.Caveats}} 10 | {{end -}}{{end -}} 11 | 12 | 13 | Development Environment 14 | ----------------------- 15 | 16 | Requirements: 17 | - `docker` 18 | - Go {{.Manifest.Runtime.GoVersion}}+ 19 | 20 | To initialize the build system execute: `go test ./driver`, at the root of the project. This will generate the `Dockerfile` for this driver. 21 | 22 | To run the tests just execute `go run test.go`, this will start the test over the native and the Go components of the driver using Docker. 23 | 24 | The build is done executing `go run build.go`. To evaluate the result using a docker container, execute: 25 | `go run build.go test-driver && docker run -it test-driver`. 26 | 27 | If the project is located under `$GOPATH`, run all the above with `GO111MODULE=on` environment variable, 28 | or move the project to any other directory outside of `$GOPATH`. 29 | 30 | License 31 | ------- 32 | 33 | GPLv3, see [LICENSE](LICENSE) 34 | 35 | 36 | {{define "color-status" -}} 37 | {{if eq .Manifest.Status "planning" -}} 38 | e08dd1 39 | {{- else if eq .Manifest.Status "pre-alpha" -}} 40 | d6ae86 41 | {{- else if eq .Manifest.Status "alpha" -}} 42 | db975c 43 | {{- else if eq .Manifest.Status "beta" -}} 44 | dbd25c 45 | {{- else if eq .Manifest.Status "stable" -}} 46 | 9ddb5c 47 | {{- else if eq .Manifest.Status "mature" -}} 48 | 60db5c 49 | {{- else -}} 50 | d1d1d1 51 | {{- end}} 52 | {{- end}} 53 | -------------------------------------------------------------------------------- /etc/skeleton/build.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/bblfsh/sdk/v3/build" 9 | ) 10 | 11 | func main() { 12 | flag.Parse() 13 | if err := runBuild("."); err != nil { 14 | fmt.Fprintln(os.Stderr, err) 15 | os.Exit(1) 16 | } 17 | } 18 | 19 | func runBuild(root string) error { 20 | args := flag.Args() 21 | name := "" 22 | if len(args) != 0 { 23 | name = args[0] 24 | } 25 | d, err := build.NewDriver(root) 26 | if err != nil { 27 | return err 28 | } 29 | id, err := d.Build(name) 30 | if err != nil { 31 | return err 32 | } 33 | fmt.Println(id) 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /etc/skeleton/build.yml.tpl: -------------------------------------------------------------------------------- 1 | sdk: '2' 2 | go-runtime: 3 | version: '1.13' 4 | native: 5 | # TODO: an image used as a driver runtime 6 | image: 'debian:latest' 7 | # TODO: files copied from the source to the driver image 8 | static: 9 | - path: 'native.sh' 10 | dest: 'native' 11 | build: 12 | # TODO: an image to build a driver 13 | image: 'debian:latest' 14 | # TODO: build system dependencies, can not use the source 15 | deps: 16 | - 'echo dependencies' 17 | # TODO: build steps 18 | run: 19 | - 'echo build' 20 | # TODO: files copied from the builder to the driver image 21 | # artifacts: 22 | # - path: '/native/native-binary' 23 | # dest: 'native-binary' 24 | test: 25 | # TODO: native driver tests 26 | run: 27 | - 'echo tests' -------------------------------------------------------------------------------- /etc/skeleton/driver/fixtures/fixtures_test.go.tpl: -------------------------------------------------------------------------------- 1 | package fixtures 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/bblfsh/{{.Manifest.Language}}-driver/driver/normalizer" 8 | "github.com/bblfsh/sdk/v3/driver" 9 | "github.com/bblfsh/sdk/v3/driver/fixtures" 10 | "github.com/bblfsh/sdk/v3/driver/native" 11 | "github.com/bblfsh/sdk/v3/uast/transformer/positioner" 12 | ) 13 | 14 | const projectRoot = "../../" 15 | 16 | var Suite = &fixtures.Suite{ 17 | Lang: "{{.Manifest.Language}}", 18 | Ext: ".ext", // TODO: specify correct file extension for source files in ./fixtures 19 | Path: filepath.Join(projectRoot, fixtures.Dir), 20 | NewDriver: func() driver.Native { 21 | return native.NewDriverAt(filepath.Join(projectRoot, "build/bin/native"), native.UTF8) 22 | }, 23 | Transforms: normalizer.Transforms, 24 | //BenchName: "fixture-name", // TODO: specify a largest file 25 | Semantic: fixtures.SemanticConfig{ 26 | BlacklistTypes: []string{ 27 | // TODO: list native types that should be converted to semantic UAST 28 | }, 29 | }, 30 | VerifyTokens: []positioner.VerifyToken{ 31 | // TODO: list nodes that needs to be checked for token correctness 32 | }, 33 | } 34 | 35 | func Test{{expName .Manifest.Language}}Driver(t *testing.T) { 36 | Suite.RunTests(t) 37 | } 38 | 39 | func Benchmark{{expName .Manifest.Language}}Driver(b *testing.B) { 40 | Suite.RunBenchmarks(b) 41 | } 42 | -------------------------------------------------------------------------------- /etc/skeleton/driver/impl/impl.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "github.com/bblfsh/sdk/v3/driver/native" 5 | "github.com/bblfsh/sdk/v3/driver/server" 6 | ) 7 | 8 | func init() { 9 | // Can be overridden to link a native driver into a Go driver server. 10 | server.DefaultDriver = native.NewDriver(native.UTF8) 11 | } 12 | -------------------------------------------------------------------------------- /etc/skeleton/driver/main.go.tpl: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "github.com/bblfsh/{{.Manifest.Language}}-driver/driver/impl" 5 | "github.com/bblfsh/{{.Manifest.Language}}-driver/driver/normalizer" 6 | 7 | "github.com/bblfsh/sdk/v3/driver/server" 8 | ) 9 | 10 | func main() { 11 | server.Run(normalizer.Transforms) 12 | } 13 | -------------------------------------------------------------------------------- /etc/skeleton/driver/normalizer/annotation.go: -------------------------------------------------------------------------------- 1 | package normalizer 2 | 3 | import ( 4 | "github.com/bblfsh/sdk/v3/uast/role" 5 | . "github.com/bblfsh/sdk/v3/uast/transformer" 6 | ) 7 | 8 | // Native is the of list `transformer.Transformer` to apply to a native AST. 9 | // To learn more about the Transformers and the available ones take a look to: 10 | // https://godoc.org/github.com/bblfsh/sdk/v3/uast/transformer 11 | var Native = Transformers([][]Transformer{ 12 | // The main block of transformation rules. 13 | {Mappings(Annotations...)}, 14 | { 15 | // RolesDedup is used to remove duplicate roles assigned by multiple 16 | // transformation rules. 17 | RolesDedup(), 18 | }, 19 | }...) 20 | 21 | // Annotations is a list of individual transformations to annotate a native AST with roles. 22 | var Annotations = []Mapping{ 23 | AnnotateType("internal-type", nil, role.Incomplete), 24 | } 25 | -------------------------------------------------------------------------------- /etc/skeleton/driver/normalizer/normalizer.go: -------------------------------------------------------------------------------- 1 | package normalizer 2 | 3 | import ( 4 | . "github.com/bblfsh/sdk/v3/uast/transformer" 5 | "github.com/bblfsh/sdk/v3/uast/transformer/positioner" 6 | ) 7 | 8 | var Preprocess = Transformers([][]Transformer{ 9 | { 10 | // ResponseMetadata is a transform that trims response metadata from AST. 11 | // 12 | // https://godoc.org/github.com/bblfsh/sdk/uast/transformer#ResponseMetadata 13 | ResponseMetadata{ 14 | TopLevelIsRootNode: false, 15 | }, 16 | }, 17 | {Mappings(Preprocessors...)}, 18 | }...) 19 | 20 | var Normalize = Transformers([][]Transformer{ 21 | 22 | {Mappings(Normalizers...)}, 23 | }...) 24 | 25 | // Preprocessors is a block of AST preprocessing rules rules. 26 | var Preprocessors = []Mapping{ 27 | // ObjectToNode defines how to normalize common fields of native AST 28 | // (like node type, token, positional information). 29 | // 30 | // https://godoc.org/github.com/bblfsh/sdk/uast/transformer#ObjectToNode 31 | ObjectToNode{ 32 | InternalTypeKey: "...", // native AST type key name 33 | }.Mapping(), 34 | } 35 | 36 | // PreprocessCode is a preprocessor stage that can use the source code to 37 | // fix tokens and positional information. 38 | var PreprocessCode = []CodeTransformer{ 39 | positioner.FromOffset(), 40 | } 41 | 42 | // Normalizers is the main block of normalization rules to convert native AST to semantic UAST. 43 | var Normalizers = []Mapping{} 44 | -------------------------------------------------------------------------------- /etc/skeleton/driver/normalizer/transforms.go.tpl: -------------------------------------------------------------------------------- 1 | package normalizer 2 | 3 | import "github.com/bblfsh/sdk/v3/driver" 4 | 5 | var Transforms = driver.Transforms{ 6 | Namespace: "{{.Manifest.Language}}", 7 | Preprocess: Preprocess, 8 | PreprocessCode: PreprocessCode, 9 | Normalize: Normalize, 10 | Annotations: Native, 11 | } 12 | -------------------------------------------------------------------------------- /etc/skeleton/driver/sdk_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/bblfsh/sdk/v3/build" 7 | ) 8 | 9 | func TestSDKUpToDate(t *testing.T) { 10 | printf := func(format string, args ...interface{}) (int, error) { 11 | t.Logf(format, args...) 12 | return 0, nil 13 | } 14 | err := build.UpdateSDK("../", &build.UpdateOptions{ 15 | DryRun: true, 16 | Debug: printf, 17 | Notice: printf, 18 | Warning: printf, 19 | }) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /etc/skeleton/git/hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | last_stash=$(git rev-parse -q --verify refs/stash) 3 | git stash save -q --keep-index "automatic stash on pre-commit at $(git branch --points-at HEAD)"; 4 | new_stash=$(git rev-parse -q --verify refs/stash) 5 | 6 | go test `git rev-parse --show-toplevel`/driver 7 | status=$? 8 | 9 | if [ "$last_stash" != "$new_stash" ]; then 10 | git reset --hard -q && git stash apply --index -q && git stash drop -q 11 | fi 12 | 13 | exit $status 14 | -------------------------------------------------------------------------------- /etc/skeleton/go.mod.tpl: -------------------------------------------------------------------------------- 1 | module github.com/bblfsh/{{.Manifest.Language}}-driver 2 | 3 | go 1.11 4 | -------------------------------------------------------------------------------- /etc/skeleton/manifest.toml.tpl: -------------------------------------------------------------------------------- 1 | # Manifest contains metadata about the driver. To learn about the different 2 | # values in this manifest refer to: 3 | # https://github.com/bblfsh/sdk/blob/master/driver/manifest/manifest.go 4 | 5 | # human-readable language name 6 | name = "{{.Language}}" 7 | # language identifier 8 | language = "{{.Language}}" 9 | 10 | # status describes the current development status of the driver, the valid 11 | # values for status are: `planning`, `pre-alpha`, `alpha`, `beta`, `stable`, 12 | # `mature` or `inactive`. 13 | status = "planning" 14 | features = ["ast"] 15 | 16 | # documentation block is use to render the README.md file. 17 | [documentation] 18 | description = """ 19 | """ 20 | -------------------------------------------------------------------------------- /etc/skeleton/native/README.md.tpl: -------------------------------------------------------------------------------- 1 | The native driver (the one producing the native AST) should be located here. 2 | 3 | See: 4 | 5 | https://doc.bblf.sh/driver/sdk.html 6 | -------------------------------------------------------------------------------- /etc/skeleton/native/native.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This implements a trivial native driver that returns its input unchanged. 3 | # TODO: Implement a parser for the target language. 4 | while read -r req 5 | do 6 | resp='{"status":"ok", "ast": ' 7 | resp+=$req 8 | resp+='}' 9 | echo $resp 10 | done 11 | exit 1 12 | -------------------------------------------------------------------------------- /etc/skeleton/test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/bblfsh/sdk/v3/build" 9 | ) 10 | 11 | var ( 12 | fBblfshd = flag.String("bblfshd", "", "bblfshd version to test with") 13 | fBench = flag.Bool("bench", false, "benchmark the driver") 14 | ) 15 | 16 | func main() { 17 | flag.Parse() 18 | if err := runTest("."); err != nil { 19 | fmt.Fprintln(os.Stderr, err) 20 | os.Exit(1) 21 | } 22 | } 23 | 24 | func runTest(root string) error { 25 | args := flag.Args() 26 | image := "" 27 | if len(args) != 0 { 28 | image = args[0] 29 | } 30 | d, err := build.NewDriver(root) 31 | if err != nil { 32 | return err 33 | } 34 | return d.Test(*fBblfshd, image, *fBench) 35 | } 36 | -------------------------------------------------------------------------------- /etc/skeleton/update.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/bblfsh/sdk/v3/build" 9 | ) 10 | 11 | func main() { 12 | flag.Parse() 13 | if err := runUpdate("."); err != nil { 14 | fmt.Fprintln(os.Stderr, err) 15 | os.Exit(1) 16 | } 17 | } 18 | 19 | func runUpdate(root string) error { 20 | return build.UpdateSDK(root, &build.UpdateOptions{ 21 | Notice: fmt.Printf, 22 | Warning: fmt.Printf, 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/bblfsh/sdk/v3 2 | 3 | go 1.11 4 | 5 | require ( 6 | bitbucket.org/creachadair/shell v0.0.6 7 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect 8 | github.com/BurntSushi/toml v0.3.1 9 | github.com/Microsoft/go-winio v0.4.13 // indirect 10 | github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect 11 | github.com/antchfx/xpath v0.0.0-20190319080838-ce1d48779e67 12 | github.com/blang/semver v3.5.1+incompatible 13 | github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect 14 | github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect 15 | github.com/docker/go-connections v0.4.0 // indirect 16 | github.com/docker/go-units v0.4.0 // indirect 17 | github.com/fatih/color v1.7.0 18 | github.com/ghodss/yaml v1.0.0 19 | github.com/gogo/protobuf v1.2.1 20 | github.com/golang/protobuf v1.3.1 21 | github.com/google/go-cmp v0.2.0 // indirect 22 | github.com/google/go-github v15.0.0+incompatible 23 | github.com/google/go-github/v27 v27.0.4 24 | github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect 25 | github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 26 | github.com/jessevdk/go-flags v1.4.0 27 | github.com/kevinburke/go-bindata v3.13.0+incompatible 28 | github.com/mattn/go-colorable v0.1.2 // indirect 29 | github.com/mcuadros/go-lookup v0.0.0-20171110082742-5650f26be767 // indirect 30 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect 31 | github.com/onsi/ginkgo v1.8.0 // indirect 32 | github.com/onsi/gomega v1.5.0 // indirect 33 | github.com/opencontainers/go-digest v1.0.0-rc1 // indirect 34 | github.com/opencontainers/image-spec v1.0.1 // indirect 35 | github.com/opencontainers/runc v1.0.0-rc5 // indirect 36 | github.com/opentracing/opentracing-go v1.1.0 37 | github.com/ory/dockertest v3.3.4+incompatible 38 | github.com/pmezard/go-difflib v1.0.0 39 | github.com/rogpeppe/go-internal v1.3.0 40 | github.com/sirupsen/logrus v1.4.2 // indirect 41 | github.com/src-d/envconfig v1.0.0 // indirect 42 | github.com/stretchr/testify v1.3.0 43 | github.com/uber-go/atomic v1.4.0 // indirect 44 | github.com/uber/jaeger-client-go v2.15.0+incompatible 45 | github.com/uber/jaeger-lib v1.5.0 // indirect 46 | github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect 47 | go.uber.org/atomic v1.4.0 // indirect 48 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect 49 | golang.org/x/net v0.0.0-20190628185345-da137c7871d7 50 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 51 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect 52 | golang.org/x/text v0.3.2 // indirect 53 | google.golang.org/genproto v0.0.0-20190516172635-bb713bdc0e52 // indirect 54 | google.golang.org/grpc v1.20.1 55 | gopkg.in/bblfsh/sdk.v1 v1.17.0 56 | gopkg.in/src-d/go-errors.v1 v1.0.0 57 | gopkg.in/src-d/go-log.v1 v1.0.2 58 | gopkg.in/yaml.v2 v2.2.2 59 | gotest.tools v2.2.0+incompatible // indirect 60 | ) 61 | -------------------------------------------------------------------------------- /internal/buildmanifest/manifest.go: -------------------------------------------------------------------------------- 1 | // Package buildmanifest provides shared types for driver build manifests. 2 | package buildmanifest 3 | 4 | import "gopkg.in/yaml.v2" 5 | 6 | const ( 7 | // Filename of a build manifest relative to the package directory. 8 | Filename = "build.yml" 9 | // CurrentFormat is the SDK version corresponding to the current manifest format. 10 | // 11 | // See Manifest.Format. 12 | CurrentFormat = "2" 13 | ) 14 | 15 | // Artifact is a file copied from one path to another during the build. 16 | type Artifact struct { 17 | Path string `yaml:"path"` 18 | Dest string `yaml:"dest"` 19 | } 20 | 21 | // Manifest used for the declarative build system for drivers. 22 | type Manifest struct { 23 | // SDK version corresponding to manifest file format. Used to track future changes to this file. 24 | // The current version is given by CurrentFormat. 25 | Format string `yaml:"sdk"` 26 | 27 | // Native is a build manifest for steps related to a native driver build and runtime. 28 | Native struct { 29 | // Image is a Docker image used as the native driver runtime in the final image. 30 | // The format is image:version. 31 | Image string `yaml:"image"` 32 | 33 | // ImageAssets is a list of files that will be copied from native driver's source directory 34 | // to the final driver image. Note that those files won't be modified by the build. 35 | ImageAssets []Artifact `yaml:"static"` 36 | 37 | // Deps is a list of apt/apk commands executed in the final driver image. 38 | // This directive should be avoided since a different Docker image may be used instead of it. 39 | Deps []string `yaml:"deps"` 40 | 41 | Build struct { 42 | // Gopath directive sets GOPATH for the native driver build. Only used by Go drivers. 43 | // Should be a single absolute path inside the build container. 44 | Gopath string `yaml:"gopath"` 45 | 46 | // Image is a Docker image used to build the native driver. The format is image:version. 47 | Image string `yaml:"image"` 48 | 49 | // Deps is a list of shell commands to pull native driver dependencies. 50 | // Note that those commands are executed before copying the driver files to the 51 | // container, so they can be cached. See Build also. 52 | Deps []string `yaml:"deps"` 53 | 54 | // BuildAssets is a list of files that are copied from the native driver source to the 55 | // build container for the native driver. It is similar to Docker ADD command. 56 | BuildAssets []Artifact `yaml:"add"` 57 | 58 | // Build is a list of shell commands to build the native driver. Those commands 59 | // can access files copied by BuildAssets directives. Files produced by the build should 60 | // be mentioned in the Artifacts directive to be copied to the final image. 61 | Build []string `yaml:"run"` 62 | 63 | // Artifacts is a list of files copied from the native build container to the 64 | // final driver image. 65 | Artifacts []Artifact `yaml:"artifacts"` 66 | } `yaml:"build"` 67 | 68 | Test struct { 69 | // Deps is a list of shell commands to install native driver test dependencies. 70 | Deps []string `yaml:"deps"` 71 | 72 | // Test is a list of shell commands to test the native driver. Those commands 73 | // are run over the build container for a native driver. 74 | Test []string `yaml:"run"` 75 | } `yaml:"test"` 76 | } `yaml:"native"` 77 | 78 | // Runtime is a manifest for the driver server runtime (Go). Used to build the Go server binary. 79 | Runtime struct { 80 | // Version of Go used to build and run the driver server. 81 | Version string `yaml:"version"` 82 | } `yaml:"go-runtime"` 83 | } 84 | 85 | // Decode updates m by decoding a YAML manifest from data. 86 | func (m *Manifest) Decode(data []byte) error { 87 | return yaml.Unmarshal(data, m) 88 | } 89 | -------------------------------------------------------------------------------- /internal/docker/docker.go: -------------------------------------------------------------------------------- 1 | package docker 2 | 3 | import ( 4 | "io" 5 | "os" 6 | 7 | "github.com/ory/dockertest/docker" 8 | ) 9 | 10 | const ( 11 | FileName = "Dockerfile" 12 | Socket = "/var/run/docker.sock" 13 | SocketURL = "unix://" + Socket 14 | ) 15 | 16 | func Installed() bool { 17 | _, err := os.Stat(Socket) 18 | return err == nil 19 | } 20 | 21 | func Dial() (*Client, error) { 22 | return docker.NewClient(SocketURL) 23 | } 24 | 25 | type Client = docker.Client 26 | 27 | type Config = docker.Config 28 | 29 | type HostConfig = docker.HostConfig 30 | type Container = docker.Container 31 | 32 | type BuildImageOptions = docker.BuildImageOptions 33 | 34 | type CreateContainerOptions = docker.CreateContainerOptions 35 | type RemoveContainerOptions = docker.RemoveContainerOptions 36 | type AttachToContainerOptions = docker.AttachToContainerOptions 37 | 38 | type CreateExecOptions = docker.CreateExecOptions 39 | type StartExecOptions = docker.StartExecOptions 40 | 41 | type CloseWaiter = docker.CloseWaiter 42 | 43 | func Run(cli *Client, opt CreateContainerOptions) (*Container, error) { 44 | c, err := cli.CreateContainer(opt) 45 | if err != nil { 46 | return nil, err 47 | } 48 | err = cli.StartContainer(c.ID, nil) 49 | if err != nil { 50 | cli.RemoveContainer(docker.RemoveContainerOptions{ 51 | ID: c.ID, Force: true, 52 | }) 53 | return nil, err 54 | } 55 | return cli.InspectContainer(c.ID) 56 | } 57 | 58 | func RunAndWait(cli *Client, stdout, stderr io.Writer, opt CreateContainerOptions) error { 59 | c, err := cli.CreateContainer(opt) 60 | if err != nil { 61 | return err 62 | } 63 | defer cli.RemoveContainer(docker.RemoveContainerOptions{ 64 | ID: c.ID, Force: true, 65 | }) 66 | 67 | cw, err := cli.AttachToContainerNonBlocking(docker.AttachToContainerOptions{ 68 | Container: c.ID, 69 | OutputStream: stdout, ErrorStream: stderr, 70 | Stdout: stdout != nil, Stderr: stderr != nil, 71 | Logs: true, Stream: true, 72 | }) 73 | if err != nil { 74 | return err 75 | } 76 | defer cw.Close() 77 | 78 | err = cli.StartContainer(c.ID, nil) 79 | if err != nil { 80 | return err 81 | } 82 | return cw.Wait() 83 | } 84 | -------------------------------------------------------------------------------- /protocol/driver.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package gopkg.in.bblfsh.sdk.v2.protocol; 3 | 4 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 5 | import "google/protobuf/timestamp.proto"; 6 | 7 | option (gogoproto.protosizer_all) = true; 8 | option (gogoproto.sizer_all) = false; 9 | option (gogoproto.marshaler_all) = true; 10 | option (gogoproto.unmarshaler_all) = true; 11 | option (gogoproto.goproto_getters_all) = false; 12 | 13 | // support for Any message used in google.rpc.Status. 14 | option (gogoproto.goproto_registration) = true; 15 | 16 | option go_package = "protocol"; 17 | 18 | // ParseRequest is a request to parse a file and get its UAST. 19 | message ParseRequest { 20 | // Content stores the content of a source file. Required. 21 | string content = 1; 22 | // Language can be set optionally to disable automatic language detection. 23 | string language = 2; 24 | // Filename can be set optionally to assist automatic language detection. 25 | string filename = 3; 26 | // Mode sets a transformation pipeline used for UAST. 27 | Mode mode = 4; 28 | } 29 | 30 | enum Mode { 31 | // DefaultMode selects the transformation mode that is considered to produce UAST of the best quality. 32 | DEFAULT_MODE = 0x0 [(gogoproto.enumvalue_customname) = "DefaultMode"]; 33 | // Native disables any UAST transformations and emits a native language AST as returned by the parser. 34 | NATIVE = 0x1 [(gogoproto.enumvalue_customname) = "Native"]; 35 | // Preprocessed runs only basic transformation over native AST (normalize positional info, type fields). 36 | PREPROCESSED = 0x2 [(gogoproto.enumvalue_customname) = "Preprocessed"]; 37 | // Annotated UAST is based on native AST, but provides role annotations for nodes. 38 | ANNOTATED = 0x4 [(gogoproto.enumvalue_customname) = "Annotated"]; 39 | // Semantic UAST normalizes native AST nodes to a unified structure where possible. 40 | SEMANTIC = 0x8 [(gogoproto.enumvalue_customname) = "Semantic"]; 41 | } 42 | 43 | // ParseResponse is the reply to ParseRequest. 44 | message ParseResponse { 45 | // UAST is a binary encoding of the resulting UAST. 46 | bytes uast = 1; 47 | // Language that was automatically detected. 48 | string language = 2; 49 | // Errors is a list of parsing errors. 50 | // Only set if parser was able to return a response. Otherwise gRPC error codes are used. 51 | repeated ParseError errors = 3; 52 | } 53 | 54 | message ParseError { 55 | // Text is an error message. 56 | string text = 1; 57 | } 58 | 59 | service Driver { 60 | // Parse returns an UAST for a given source file. 61 | rpc Parse (ParseRequest) returns (ParseResponse); 62 | } 63 | 64 | message Version { 65 | string version = 1; 66 | google.protobuf.Timestamp build = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; 67 | } 68 | 69 | enum DevelopmentStatus { 70 | DEV_INACTIVE = 0 [(gogoproto.enumvalue_customname) = "Inactive"]; 71 | DEV_PLANNING = 1 [(gogoproto.enumvalue_customname) = "Planning"]; 72 | DEV_PREALPHA = 2 [(gogoproto.enumvalue_customname) = "PreAlpha"]; 73 | DEV_ALPHA = 3 [(gogoproto.enumvalue_customname) = "Alpha"]; 74 | DEV_BETA = 4 [(gogoproto.enumvalue_customname) = "Beta"]; 75 | DEV_STABLE = 5 [(gogoproto.enumvalue_customname) = "Stable"]; 76 | DEV_MATURE = 6 [(gogoproto.enumvalue_customname) = "Mature"]; 77 | } 78 | 79 | message Manifest { 80 | // Name is a human-readable language or driver name. 81 | string name = 1; 82 | // Language is a Babelfish language identifier. 83 | string language = 2; 84 | // Aliases is a list of alternative language identifiers from Enry/Linguist. 85 | repeated string aliases = 3; 86 | // Version of the language driver. 87 | Version version = 4; 88 | // Status of the driver development. 89 | DevelopmentStatus status = 5; 90 | // Features this driver supports. 91 | repeated string features = 6; 92 | } 93 | 94 | message VersionRequest {} 95 | 96 | message VersionResponse { 97 | Version version = 1; 98 | } 99 | 100 | message SupportedLanguagesRequest {} 101 | 102 | message SupportedLanguagesResponse { 103 | // Languages is a list of driver manifests for each language supported by the server. 104 | repeated Manifest languages = 1; 105 | } 106 | 107 | service DriverHost { 108 | // ServerVersion returns version information of this server. 109 | rpc ServerVersion(VersionRequest) returns (VersionResponse); 110 | // SupportedLanguages returns a list of languages supported by the server. 111 | rpc SupportedLanguages (SupportedLanguagesRequest) returns (SupportedLanguagesResponse); 112 | } 113 | 114 | // ErrorDetails adds bblfsh-specific information to gRPC errors (google.rpc.Status). 115 | message ErrorDetails { 116 | // support for Any message used in google.rpc.Status. 117 | option (gogoproto.messagename) = true; 118 | oneof reason { 119 | bool invalid_file_encoding = 1; 120 | string unsupported_language = 2; 121 | bool cannot_detect_language = 3; 122 | bool unsupported_transform_mode = 4; 123 | bool transform_failure = 5; 124 | bool driver_failure = 6; 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /protocol/driver_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net" 7 | "testing" 8 | "unsafe" 9 | 10 | "github.com/stretchr/testify/require" 11 | "google.golang.org/grpc" 12 | serrors "gopkg.in/src-d/go-errors.v1" 13 | 14 | "github.com/bblfsh/sdk/v3/driver" 15 | "github.com/bblfsh/sdk/v3/driver/manifest" 16 | "github.com/bblfsh/sdk/v3/uast/nodes" 17 | ) 18 | 19 | var _ driver.Driver = (*driverMock)(nil) 20 | 21 | type driverMock struct { 22 | name string 23 | uast nodes.Node 24 | vers driver.Version 25 | list []manifest.Manifest 26 | err error 27 | } 28 | 29 | func (d *driverMock) Parse(ctx context.Context, src string, opts *driver.ParseOptions) (nodes.Node, error) { 30 | return d.uast, d.err 31 | } 32 | 33 | func (d *driverMock) Version(ctx context.Context) (driver.Version, error) { 34 | return d.vers, d.err 35 | } 36 | 37 | func (d *driverMock) Languages(ctx context.Context) ([]manifest.Manifest, error) { 38 | return d.list, d.err 39 | } 40 | 41 | func defaultUAST() nodes.Node { 42 | return nodes.Object{"k": nodes.String("v")} 43 | } 44 | 45 | func kindOfError(e *serrors.Error) *serrors.Kind { 46 | // FIXME(dennwc): fix upstream 47 | return (*struct { 48 | kind *serrors.Kind 49 | })(unsafe.Pointer(e)).kind 50 | } 51 | 52 | func equalError(t testing.TB, exp, got error) { 53 | if exp, ok := exp.(*serrors.Error); ok { 54 | if got, ok := got.(*serrors.Error); ok { 55 | kexp, kgot := kindOfError(exp), kindOfError(got) 56 | if kexp != kgot { 57 | require.Fail(t, "unexpected error kind", "%v vs %v", kexp, kgot) 58 | } 59 | equalError(t, exp.Cause(), got.Cause()) 60 | return 61 | } 62 | } 63 | require.Equal(t, exp, got) 64 | } 65 | 66 | func TestDriverError(t *testing.T) { 67 | var cases = []driverMock{ 68 | {name: "success", uast: defaultUAST()}, 69 | {name: "partial parse", uast: defaultUAST(), err: driver.ErrSyntax.Wrap(errors.New("invalid source"))}, 70 | {name: "non-utf8", err: driver.ErrUnknownEncoding.New()}, 71 | {name: "language detection failure", err: driver.ErrLanguageDetection.New()}, 72 | {name: "unsupported mode", err: driver.ErrModeNotSupported.New()}, 73 | {name: "transform failure", err: driver.ErrTransformFailure.Wrap(errors.New("test failure"))}, 74 | {name: "driver failure", err: driver.ErrDriverFailure.Wrap(errors.New("test failure"))}, 75 | } 76 | for _, c := range cases { 77 | c := c 78 | t.Run(c.name, func(t *testing.T) { 79 | var d driver.Driver = &c 80 | srv := grpc.NewServer(ServerOptions()...) 81 | RegisterDriver(srv, d) 82 | errc := make(chan error, 1) 83 | lis, err := net.Listen("tcp", "localhost:0") 84 | require.NoError(t, err) 85 | defer lis.Close() 86 | go func() { 87 | if err := srv.Serve(lis); err != nil { 88 | errc <- err 89 | } 90 | }() 91 | 92 | opts := append([]grpc.DialOption{grpc.WithInsecure()}, DialOptions()...) 93 | cc, err := grpc.Dial(lis.Addr().String(), opts...) 94 | if err != nil { 95 | // maybe Serve failed to start the server? 96 | select { 97 | case serr := <-errc: 98 | err = serr 99 | default: 100 | } 101 | } 102 | require.NoError(t, err) 103 | defer cc.Close() 104 | 105 | cd := AsDriver(cc) 106 | 107 | nd, err := cd.Parse(context.Background(), "test", nil) 108 | exp, eerr := d.Parse(context.Background(), "test", nil) 109 | equalError(t, eerr, err) 110 | require.Equal(t, exp, nd) 111 | }) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /protocol/v1/legacy.go: -------------------------------------------------------------------------------- 1 | package uast1 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | 8 | "github.com/bblfsh/sdk/v3/uast" 9 | "github.com/bblfsh/sdk/v3/uast/nodes" 10 | "github.com/bblfsh/sdk/v3/uast/role" 11 | uast1 "gopkg.in/bblfsh/sdk.v1/uast" 12 | ) 13 | 14 | // ToNode converts a generic AST node to Node object used in the protocol. 15 | func ToNode(n nodes.Node) (*uast1.Node, error) { 16 | nd, err := asNode(n, "") 17 | if err != nil { 18 | return nil, err 19 | } 20 | switch len(nd) { 21 | case 0: 22 | return nil, nil 23 | case 1: 24 | return nd[0], nil 25 | default: 26 | return &uast1.Node{Children: nd}, nil 27 | } 28 | } 29 | 30 | func arrayAsNode(n nodes.Array, field string) ([]*uast1.Node, error) { 31 | arr := make([]*uast1.Node, 0, len(n)) 32 | for _, s := range n { 33 | nd, err := asNode(s, field) 34 | if err != nil { 35 | return arr, err 36 | } 37 | arr = append(arr, nd...) 38 | } 39 | return arr, nil 40 | } 41 | 42 | func pos(p *uast.Position) *uast1.Position { 43 | return (*uast1.Position)(p) 44 | } 45 | func roles(arr role.Roles) []uast1.Role { 46 | out := make([]uast1.Role, 0, len(arr)) 47 | for _, r := range arr { 48 | out = append(out, uast1.Role(r)) 49 | } 50 | return out 51 | } 52 | 53 | func objectAsNode(n nodes.Object, field string) ([]*uast1.Node, error) { 54 | ps := uast.PositionsOf(n) 55 | typ := uast.TypeOf(n) 56 | if i := strings.Index(typ, ":"); i >= 0 { 57 | typ = typ[i+1:] 58 | } 59 | nd := &uast1.Node{ 60 | InternalType: typ, 61 | Token: uast.TokenOf(n), 62 | Roles: roles(uast.RolesOf(n)), 63 | StartPosition: pos(ps.Start()), 64 | EndPosition: pos(ps.End()), 65 | Properties: make(map[string]string), 66 | } 67 | if field != "" { 68 | nd.Properties[uast1.InternalRoleKey] = field 69 | } 70 | for k, np := range ps { 71 | switch k { 72 | case uast.KeyStart, uast.KeyEnd: 73 | // already processed 74 | continue 75 | } 76 | sn, err := asNode(np.ToObject(), k) 77 | if err != nil { 78 | return nil, err 79 | } 80 | p := *pos(&np) 81 | sp := p 82 | ep := sp 83 | ep.Col++ 84 | ep.Offset++ 85 | sn[0].StartPosition = &sp 86 | sn[0].EndPosition = &ep 87 | nd.Children = append(nd.Children, sn...) 88 | } 89 | 90 | for k, v := range n { 91 | switch k { 92 | case uast.KeyType, uast.KeyToken, uast.KeyRoles, uast.KeyPos: 93 | // already processed 94 | continue 95 | } 96 | if nv, ok := v.(nodes.Value); ok { 97 | nd.Properties[k] = fmt.Sprint(nv.Native()) 98 | } else { 99 | sn, err := asNode(v, k) 100 | if err != nil { 101 | return nil, err 102 | } 103 | nd.Children = append(nd.Children, sn...) 104 | } 105 | } 106 | sort.Stable(byOffset(nd.Children)) 107 | return []*uast1.Node{nd}, nil 108 | } 109 | 110 | func valueAsNode(n nodes.Value, field string) ([]*uast1.Node, error) { 111 | nd := &uast1.Node{ 112 | Token: fmt.Sprint(n), 113 | Properties: make(map[string]string), 114 | } 115 | if field != "" { 116 | nd.Properties[uast1.InternalRoleKey] = field 117 | } 118 | return []*uast1.Node{nd}, nil 119 | } 120 | 121 | func asNode(n nodes.Node, field string) ([]*uast1.Node, error) { 122 | switch n := n.(type) { 123 | case nil: 124 | return nil, nil 125 | case nodes.Array: 126 | return arrayAsNode(n, field) 127 | case nodes.Object: 128 | return objectAsNode(n, field) 129 | case nodes.Value: 130 | return valueAsNode(n, field) 131 | default: 132 | return nil, fmt.Errorf("argument should be a node or a list, got: %T", n) 133 | } 134 | } 135 | 136 | type byOffset []*uast1.Node 137 | 138 | func (s byOffset) Len() int { return len(s) } 139 | func (s byOffset) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 140 | func (s byOffset) Less(i, j int) bool { 141 | a := s[i] 142 | b := s[j] 143 | apos := startPosition(a) 144 | bpos := startPosition(b) 145 | if apos != nil && bpos != nil { 146 | if apos.Offset != bpos.Offset { 147 | return apos.Offset < bpos.Offset 148 | } 149 | } else if (apos == nil && bpos != nil) || (apos != nil && bpos == nil) { 150 | return bpos != nil 151 | } 152 | field1, ok1 := a.Properties[uast1.InternalRoleKey] 153 | field2, ok2 := b.Properties[uast1.InternalRoleKey] 154 | if ok1 && ok2 { 155 | return field1 < field2 156 | } 157 | return false 158 | } 159 | 160 | func startPosition(n *uast1.Node) *uast1.Position { 161 | if n.StartPosition != nil { 162 | return n.StartPosition 163 | } 164 | 165 | var min *uast1.Position 166 | for _, c := range n.Children { 167 | other := startPosition(c) 168 | if other == nil { 169 | continue 170 | } 171 | 172 | if min == nil || other.Offset < min.Offset { 173 | min = other 174 | } 175 | } 176 | 177 | return min 178 | } 179 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | package sdk 4 | 5 | // this ensures that we depend on a specific version of the tool 6 | import _ "github.com/kevinburke/go-bindata/go-bindata" 7 | -------------------------------------------------------------------------------- /uast/helpers_test.go: -------------------------------------------------------------------------------- 1 | package uast 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/bblfsh/sdk/v3/uast/nodes" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestAllImportPaths(t *testing.T) { 11 | root := nodes.Array{ 12 | toNode(RuntimeImport{ 13 | Path: Identifier{Name: "a"}, 14 | }), 15 | toNode(InlineImport{ 16 | Path: String{Value: "a"}, 17 | }), 18 | toNode(RuntimeReImport{ 19 | Path: QualifiedIdentifier{Names: []Identifier{ 20 | {Name: "a"}, 21 | {Name: "b"}, 22 | }}, 23 | }), 24 | toNode(Import{ 25 | Path: Alias{ 26 | Node: Identifier{Name: "c"}, 27 | }, 28 | }), 29 | } 30 | paths := AllImportPaths(root) 31 | require.Equal(t, []string{"a", "a/b", "c"}, paths) 32 | } 33 | -------------------------------------------------------------------------------- /uast/iter.go: -------------------------------------------------------------------------------- 1 | package uast 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/bblfsh/sdk/v3/uast/nodes" 7 | ) 8 | 9 | // NewPositionalIterator creates a new iterator that enumerates all object nodes, sorting them by positions in the source file. 10 | // Nodes with no positions will be enumerated last. 11 | func NewPositionalIterator(root nodes.External) nodes.Iterator { 12 | return &positionIter{root: root} 13 | } 14 | 15 | type positionIter struct { 16 | root nodes.External 17 | nodes []nodes.External 18 | } 19 | 20 | type nodesByPos struct { 21 | nodes []nodes.External 22 | pos []Position 23 | } 24 | 25 | func (arr nodesByPos) Len() int { 26 | return len(arr.nodes) 27 | } 28 | 29 | func (arr nodesByPos) Less(i, j int) bool { 30 | return arr.pos[i].Less(arr.pos[j]) 31 | } 32 | 33 | func (arr nodesByPos) Swap(i, j int) { 34 | arr.nodes[i], arr.nodes[j] = arr.nodes[j], arr.nodes[i] 35 | arr.pos[i], arr.pos[j] = arr.pos[j], arr.pos[i] 36 | } 37 | 38 | func (it *positionIter) sort() { 39 | // in general, we cannot expect that parent node's positional info will include all the children 40 | // because of this we are forced to enumerate all nodes and sort them by positions on the first call to Next 41 | var plist []Position 42 | noPos := func() { 43 | plist = append(plist, Position{}) 44 | } 45 | posType := TypeOf(Positions{}) 46 | nodes.WalkPreOrderExt(it.root, func(n nodes.External) bool { 47 | if n == nil || n.Kind() != nodes.KindObject { 48 | return true 49 | } 50 | obj, ok := n.(nodes.ExternalObject) 51 | if !ok { 52 | return true 53 | } 54 | if TypeOf(n) == posType { 55 | // skip position nodes 56 | return false 57 | } 58 | it.nodes = append(it.nodes, n) 59 | m, _ := obj.ValueAt(KeyPos) 60 | if m == nil || m.Kind() != nodes.KindObject { 61 | noPos() 62 | return true 63 | } 64 | var ps Positions 65 | if err := NodeAs(m, &ps); err != nil { 66 | noPos() 67 | return true 68 | } 69 | if p := ps.Start(); p != nil { 70 | plist = append(plist, *p) 71 | } else { 72 | noPos() 73 | } 74 | return true 75 | }) 76 | sort.Sort(nodesByPos{nodes: it.nodes, pos: plist}) 77 | plist = nil 78 | } 79 | 80 | // Next implements nodes.Iterator. 81 | func (it *positionIter) Next() bool { 82 | if it.nodes == nil { 83 | it.sort() 84 | return len(it.nodes) != 0 85 | } 86 | if len(it.nodes) == 0 { 87 | return false 88 | } 89 | it.nodes = it.nodes[1:] 90 | return len(it.nodes) != 0 91 | } 92 | 93 | // Node implements nodes.Iterator. 94 | func (it *positionIter) Node() nodes.External { 95 | if len(it.nodes) == 0 { 96 | return nil 97 | } 98 | return it.nodes[0] 99 | } 100 | -------------------------------------------------------------------------------- /uast/iter_test.go: -------------------------------------------------------------------------------- 1 | package uast 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/bblfsh/sdk/v3/uast/nodes" 9 | ) 10 | 11 | func toNode(o interface{}) nodes.Node { 12 | n, err := ToNode(o) 13 | if err != nil { 14 | panic(err) 15 | } 16 | return n 17 | } 18 | 19 | func pos(off, line, col uint32) GenNode { 20 | return GenNode{ 21 | Positions: Positions{ 22 | KeyStart: Position{Offset: off, Line: line, Col: col}, 23 | }, 24 | } 25 | } 26 | 27 | func expect(t testing.TB, it nodes.Iterator, exp ...nodes.External) { 28 | var got []nodes.External 29 | for it.Next() { 30 | got = append(got, it.Node()) 31 | } 32 | require.Equal(t, len(exp), len(got), "%v", got) 33 | require.Equal(t, exp, got) 34 | } 35 | 36 | func TestPosIterator(t *testing.T) { 37 | root := nodes.Array{ 38 | toNode(Identifier{ 39 | GenNode: pos(3, 4, 1), 40 | Name: "A", 41 | }), 42 | toNode(Identifier{ 43 | GenNode: pos(0, 1, 1), 44 | Name: "B", 45 | }), 46 | toNode(Identifier{ 47 | GenNode: pos(0, 0, 0), 48 | Name: "C", 49 | }), 50 | } 51 | a, b, c := root[0], root[1], root[2] 52 | it := NewPositionalIterator(root) 53 | expect(t, it, b, a, c) 54 | } 55 | -------------------------------------------------------------------------------- /uast/nodes/external.go: -------------------------------------------------------------------------------- 1 | package nodes 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // External is a node interface that can be implemented by other packages. 8 | type External interface { 9 | // Kind returns a node kind. 10 | Kind() Kind 11 | // Value returns a primitive value of the node or nil if node is not a value. 12 | Value() Value 13 | // SameAs check if the node is exactly the same node as n2. This usually means checking node pointers. 14 | SameAs(n2 External) bool 15 | } 16 | 17 | // ExternalArray is an analog of Array type. 18 | type ExternalArray interface { 19 | External 20 | // Size returns the number of child nodes. 21 | Size() int 22 | // ValueAt returns a array value by an index. 23 | ValueAt(i int) External 24 | } 25 | 26 | // ExternalObject is an analog of Object type. 27 | type ExternalObject interface { 28 | External 29 | // Size returns the number of fields in an object. 30 | Size() int 31 | // Keys returns a sorted list of keys (object field names). 32 | Keys() []string 33 | // ValueAt returns an object field by key. It returns false if key does not exist. 34 | ValueAt(key string) (External, bool) 35 | } 36 | 37 | // toNodeExt converts the external node to a native node type. 38 | // The returned value is the copy of an original node. 39 | func toNodeExt(n External) (Node, error) { 40 | if n == nil { 41 | return nil, nil 42 | } 43 | switch kind := n.Kind(); kind { 44 | case KindNil: 45 | return nil, nil 46 | case KindObject: 47 | o, ok := n.(ExternalObject) 48 | if !ok { 49 | return nil, fmt.Errorf("node type %T returns a %v kind, but doesn't implement the interface", n, kind) 50 | } 51 | keys := o.Keys() 52 | m := make(Object, len(keys)) 53 | for _, k := range keys { 54 | nv, ok := o.ValueAt(k) 55 | if !ok { 56 | return nil, fmt.Errorf("node type %T: key %q is listed, but cannot be fetched", n, k) 57 | } 58 | v, err := toNodeExt(nv) 59 | if err != nil { 60 | return nil, err 61 | } 62 | m[k] = v 63 | } 64 | return m, nil 65 | case KindArray: 66 | a, ok := n.(ExternalArray) 67 | if !ok { 68 | return nil, fmt.Errorf("node type %T returns a %v kind, but doesn't implement the interface", n, kind) 69 | } 70 | sz := a.Size() 71 | m := make(Array, sz) 72 | for i := 0; i < sz; i++ { 73 | nv := a.ValueAt(i) 74 | v, err := toNodeExt(nv) 75 | if err != nil { 76 | return nil, err 77 | } 78 | m[i] = v 79 | } 80 | return m, nil 81 | default: 82 | return n.Value(), nil 83 | } 84 | } 85 | 86 | // equalExt compares two external nodes. 87 | func equalExt(n1, n2 External) bool { 88 | k1, k2 := n1.Kind(), n2.Kind() 89 | switch k1 { 90 | case KindObject: 91 | if k2 != KindObject { 92 | return false 93 | } 94 | o1, ok := n1.(ExternalObject) 95 | if !ok { 96 | return false 97 | } 98 | o2, ok := n2.(ExternalObject) 99 | if !ok { 100 | return false 101 | } 102 | if o1.Size() != o2.Size() { 103 | return false 104 | } 105 | 106 | for _, k := range o1.Keys() { 107 | v1, ok1 := o1.ValueAt(k) 108 | v2, ok2 := o2.ValueAt(k) 109 | if !ok1 || !ok2 || !Equal(v1, v2) { 110 | return false 111 | } 112 | } 113 | return true 114 | case KindArray: 115 | if k2 != KindArray { 116 | return false 117 | } 118 | a1, ok := n1.(ExternalArray) 119 | if !ok { 120 | return false 121 | } 122 | a2, ok := n2.(ExternalArray) 123 | if !ok { 124 | return false 125 | } 126 | sz := a1.Size() 127 | if sz != a2.Size() { 128 | return false 129 | } 130 | for i := 0; i < sz; i++ { 131 | v1 := a1.ValueAt(i) 132 | v2 := a2.ValueAt(i) 133 | if !Equal(v1, v2) { 134 | return false 135 | } 136 | } 137 | return true 138 | default: 139 | return Equal(n1.Value(), n2.Value()) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /uast/nodes/hash.go: -------------------------------------------------------------------------------- 1 | package nodes 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/binary" 6 | "fmt" 7 | "hash" 8 | "io" 9 | "math" 10 | "sort" 11 | ) 12 | 13 | // HashSize is the size of hash used for nodes. 14 | const HashSize = sha256.Size 15 | 16 | type Hash [HashSize]byte 17 | 18 | var DefaultHasher = NewHasher() 19 | 20 | // HashOf computes a hash of a node with all it's children. 21 | // Shorthand for DefaultHasher.HashOf with default settings. 22 | func HashOf(n External) Hash { 23 | return DefaultHasher.HashOf(n) 24 | } 25 | 26 | // NewHasher creates a new hashing config with default options. 27 | func NewHasher() *Hasher { 28 | return &Hasher{} 29 | } 30 | 31 | // Hasher allows to configure node hashing. 32 | type Hasher struct { 33 | // KeyFilter allows to skip field in objects by returning false from the function. 34 | // Hash will still reflect the presence or absence of these key, but it won't hash a value of that field. 35 | KeyFilter func(key string) bool 36 | } 37 | 38 | // HashOf computes a hash of a node with all it's children. 39 | // Caller should not rely on a specific hash value, since the hash size and the algorithm might change. 40 | func (h *Hasher) HashOf(n External) Hash { 41 | hash := sha256.New() 42 | err := h.HashTo(hash, n) 43 | if err != nil { 44 | panic(err) 45 | } 46 | var v Hash 47 | sz := len(hash.Sum(v[:0])) 48 | if sz != HashSize { 49 | panic("unexpected hash size") 50 | } 51 | return v 52 | } 53 | 54 | // HashTo hashes the node with a custom hash function. See HashOf for details. 55 | func (h *Hasher) HashTo(hash hash.Hash, n External) error { 56 | return h.hashTo(hash, n) 57 | } 58 | 59 | var hashEndianess = binary.LittleEndian 60 | 61 | func (h *Hasher) hashTo(w io.Writer, n External) error { 62 | kind := KindOf(n) 63 | 64 | // write kind first (uint32) 65 | var buf [4]byte 66 | hashEndianess.PutUint32(buf[:], uint32(kind)) 67 | if _, err := w.Write(buf[:]); err != nil { 68 | return err 69 | } 70 | switch kind { 71 | case KindNil: 72 | return nil 73 | case KindArray: 74 | arr, ok := n.(ExternalArray) 75 | if !ok { 76 | return fmt.Errorf("node is an array, but an interface implementation is missing: %T", n) 77 | } 78 | return h.hashArray(w, arr) 79 | case KindObject: 80 | obj, ok := n.(ExternalObject) 81 | if !ok { 82 | return fmt.Errorf("node is an object, but an interface implementation is missing: %T", n) 83 | } 84 | return h.hashObject(w, obj) 85 | } 86 | if kind.In(KindsValues) { 87 | v := n.Value() 88 | return h.hashValue(w, v) 89 | } 90 | return fmt.Errorf("unsupported type: %T (%s)", n, kind) 91 | } 92 | 93 | func (h *Hasher) hashArray(w io.Writer, arr ExternalArray) error { 94 | sz := arr.Size() 95 | var buf [4]byte 96 | hashEndianess.PutUint32(buf[:], uint32(sz)) 97 | _, err := w.Write(buf[:]) 98 | if err != nil { 99 | return err 100 | } 101 | for i := 0; i < sz; i++ { 102 | v := arr.ValueAt(i) 103 | if err = h.hashTo(w, v); err != nil { 104 | return err 105 | } 106 | } 107 | return nil 108 | } 109 | 110 | func (h *Hasher) hashObject(w io.Writer, obj ExternalObject) error { 111 | sz := obj.Size() 112 | var buf [4]byte 113 | hashEndianess.PutUint32(buf[:], uint32(sz)) 114 | _, err := w.Write(buf[:]) 115 | if err != nil { 116 | return err 117 | } 118 | keys := obj.Keys() 119 | if !sort.StringsAreSorted(keys) { 120 | return fmt.Errorf("object keys are not sorted: %T", obj) 121 | } 122 | for _, key := range keys { 123 | v, ok := obj.ValueAt(key) 124 | if !ok { 125 | return fmt.Errorf("key %q is listed, but the value is missing in %T", key, obj) 126 | } 127 | if err = h.hashValue(w, String(key)); err != nil { 128 | return err 129 | } 130 | if h.KeyFilter != nil && !h.KeyFilter(key) { 131 | continue 132 | } 133 | if err = h.hashTo(w, v); err != nil { 134 | return err 135 | } 136 | } 137 | return nil 138 | } 139 | 140 | func (h *Hasher) hashValue(w io.Writer, v Value) error { 141 | switch v := v.(type) { 142 | case nil: 143 | return nil 144 | case Bool: 145 | var err error 146 | if v { 147 | _, err = w.Write([]byte{1}) 148 | } else { 149 | _, err = w.Write([]byte{0}) 150 | } 151 | return err 152 | case Int: 153 | var buf [8]byte 154 | hashEndianess.PutUint64(buf[:], uint64(v)) 155 | _, err := w.Write(buf[:]) 156 | return err 157 | case Uint: 158 | var buf [8]byte 159 | hashEndianess.PutUint64(buf[:], uint64(v)) 160 | _, err := w.Write(buf[:]) 161 | return err 162 | case Float: 163 | var buf [8]byte 164 | hashEndianess.PutUint64(buf[:], math.Float64bits(float64(v))) 165 | _, err := w.Write(buf[:]) 166 | return err 167 | case String: 168 | var buf [4]byte 169 | hashEndianess.PutUint32(buf[:], uint32(len(v))) 170 | _, err := w.Write(buf[:]) 171 | if err != nil { 172 | return err 173 | } 174 | _, err = w.Write([]byte(v)) 175 | return err 176 | default: 177 | return fmt.Errorf("unsupported value type: %T (%s)", v, v.Kind()) 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /uast/nodes/iter_test.go: -------------------------------------------------------------------------------- 1 | package nodes_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/bblfsh/sdk/v3/uast/nodes" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestIterPreOrder(t *testing.T) { 11 | a := nodes.Object{ 12 | "k1": nodes.Int(1), 13 | "k2": nodes.Int(2), 14 | } 15 | b := nodes.String("v") 16 | root := nodes.Array{a, b} 17 | 18 | it := nodes.NewIterator(root, nodes.PreOrder) 19 | got := allNodes(it) 20 | exp := []nodes.Node{ 21 | root, 22 | a, nodes.Int(1), nodes.Int(2), 23 | b, 24 | } 25 | if !nodes.Equal(nodes.Array(exp), nodes.Array(got)) { 26 | require.Equal(t, nodes.Array(exp), nodes.Array(got)) 27 | } 28 | } 29 | 30 | func TestIterPostOrder(t *testing.T) { 31 | a := nodes.Object{ 32 | "k1": nodes.Int(1), 33 | "k2": nodes.Int(2), 34 | } 35 | b := nodes.String("v") 36 | root := nodes.Array{a, b} 37 | 38 | it := nodes.NewIterator(root, nodes.PostOrder) 39 | got := allNodes(it) 40 | exp := []nodes.Node{ 41 | nodes.Int(1), nodes.Int(2), a, 42 | b, 43 | root, 44 | } 45 | if !nodes.Equal(nodes.Array(exp), nodes.Array(got)) { 46 | require.Equal(t, nodes.Array(exp), nodes.Array(got)) 47 | } 48 | } 49 | 50 | func TestIterLevelOrder(t *testing.T) { 51 | a := nodes.Object{ 52 | "k1": nodes.Int(1), 53 | "k2": nodes.Int(2), 54 | } 55 | b := nodes.String("v") 56 | root := nodes.Array{a, b} 57 | 58 | it := nodes.NewIterator(root, nodes.LevelOrder) 59 | got := allNodes(it) 60 | exp := []nodes.Node{ 61 | root, 62 | a, b, 63 | nodes.Int(1), nodes.Int(2), 64 | } 65 | if !nodes.Equal(nodes.Array(exp), nodes.Array(got)) { 66 | require.Equal(t, nodes.Array(exp), nodes.Array(got)) 67 | } 68 | } 69 | 70 | func TestIterChildren(t *testing.T) { 71 | a := nodes.Object{ 72 | "k1": nodes.Int(1), 73 | "k2": nodes.Array{nodes.Int(2)}, 74 | } 75 | b := nodes.String("v") 76 | c := nodes.Object{ 77 | "k3": b, 78 | } 79 | d := nodes.Array{c} 80 | root := nodes.Object{ 81 | "a": a, 82 | "b": b, 83 | "c": c, 84 | "d": d, 85 | } 86 | 87 | it := nodes.NewIterator(root, nodes.ChildrenOrder) 88 | got := allNodes(it) 89 | exp := []nodes.Node{a,c,c} 90 | if !nodes.Equal(nodes.Array(exp), nodes.Array(got)) { 91 | require.Equal(t, nodes.Array(exp), nodes.Array(got)) 92 | } 93 | require.Equal(t, nodes.ChildrenCount(root), len(exp)) 94 | } 95 | 96 | func allNodes(it nodes.Iterator) []nodes.Node { 97 | var out []nodes.Node 98 | for it.Next() { 99 | n, _ := it.Node().(nodes.Node) 100 | out = append(out, n) 101 | } 102 | return out 103 | } 104 | -------------------------------------------------------------------------------- /uast/nodes/nodesproto/nodes.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package gopkg.in.bblfsh.sdk.v2.uast.nodes; 3 | 4 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 5 | 6 | option (gogoproto.protosizer_all) = true; 7 | option (gogoproto.sizer_all) = false; 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.unmarshaler_all) = true; 10 | option go_package = "nodesproto"; 11 | 12 | // GraphHeader is the first message written over the wire before transfering graph nodes. 13 | // 14 | // It should be preceded by the magic number "\x00bgr" (4 bytes) and the version number 0x1 15 | // written as a little-endian 4 byte integer. Next, the length of the header message should be 16 | // written in varint encoding, directly followed by the message itself. 17 | message GraphHeader { 18 | // LastID is a last node ID used by ID allocator for this graph. Implementation may reserve some 19 | // IDs space by setting LastID > max(nodes.ID). If not set, max(nodes.ID) is assumed. 20 | // Tools that manipulate the graph, but want to preserve IDs of the nodes should allocate IDs 21 | // starting from LastID+1. 22 | uint64 last_id = 1; 23 | // Root is an optional ID of a global root for this graph. This field is used to preserve 24 | // compatibility with tools that expect a tree-shaped data. 25 | // Implementation may also store multiple roots by referencing an Array node, or may store 26 | // multiple named roots by referencing an Object node. 27 | // If not set explicitly, implementations that expect a tree should search the graph for 28 | // unused nodes of type Object or Array and treat them as an array of roots. 29 | uint64 root = 2; 30 | // Metadata is an optional ID for a metadata node for this file. Should reference an Object. 31 | // If set, implementations that expect a tree should exclude it from the list of roots. 32 | uint64 metadata = 3; 33 | } 34 | 35 | // Node represents any node that can be stored in the graph. 36 | // 37 | // A list of Node messages follows directly after GraphHeader and each such message should be preceded by its length 38 | // written in varint encoding. Nodes in the list should always be sorted by ID (ascending). 39 | // ID of a node can be zero, in this case ID will be assigned automatically as prevNode.ID+1 (starting from 1). 40 | // 41 | // In general there is 3 kinds of nodes: Values, Arrays, Objects. 42 | // If Value oneof field is set, all other fields are ignored, and node is decoded as a value node (leaf). 43 | // If any of Keys or KeysFrom fields are set, the node is an Object (set of key-value pairs). 44 | // Only Keys/KeysFrom and Values fields are considered in this case. 45 | // In other cases a node is an Array, and only Values field is considered. 46 | message Node { 47 | // ID is a unique file-local ID of the node. 48 | // To implement global IDs, application should write additional data to the graph 49 | // or keep a mapping from file-local IDs to global ones. 50 | uint64 id = 1; 51 | // Value is a union for primitive value types. 52 | // These values are always leaf nodes, they should never store any references to other nodes, 53 | // or other graph-specific information. 54 | // It is assumed that encoded Value can be copied to a different Graph without any changes. 55 | oneof value { 56 | string string = 2; 57 | int64 int = 3; 58 | uint64 uint = 4; 59 | double float = 5; 60 | bool bool = 6; 61 | } 62 | // Keys is an ordered set of Object keys. Corresponding values are stored in Values field. 63 | repeated uint64 keys = 7; 64 | // KeysFrom can refer to a node ID previously seen on the wire. In this case, Keys from that node 65 | // are copied to Keys field of the current node. Thus, full list of Keys can be omitted. 66 | uint64 keys_from = 10; 67 | // Values stores an ordered list of node IDs. Zero ID represents a null node. 68 | // For Array node this field represent an array itself, and for Object nodes 69 | // this field is a set of values that corresponds to keys defined by Keys or KeysFrom. 70 | repeated uint64 values = 8; 71 | // IsObject is a helper field to distinguish between empty arrays and empty objects. 72 | bool is_object = 9; 73 | // ValuesOffs is an offset added to all value IDs. Used for compression. 74 | uint64 values_offs = 11; 75 | } -------------------------------------------------------------------------------- /uast/nodes/nodesproto/nodesproto_test.go: -------------------------------------------------------------------------------- 1 | package nodesproto 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "testing" 7 | 8 | "github.com/bblfsh/sdk/v3/uast/nodes" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | var treeCases = []struct { 13 | name string 14 | size int 15 | in nodes.Node 16 | out nodes.Node 17 | json string 18 | }{ 19 | { 20 | name: "nested array", 21 | size: 46, 22 | in: nodes.Array{ 23 | nodes.Object{ 24 | "k": nodes.Array{ 25 | nodes.String("A"), 26 | }, 27 | "k2": nodes.Int(42), 28 | }, 29 | }, 30 | }, 31 | { 32 | name: "nested object", 33 | size: 73, 34 | in: nodes.Object{ 35 | "@type": nodes.String("node"), 36 | "k": nodes.Array{ 37 | nodes.String("A"), 38 | }, 39 | "k2": nodes.Int(42), 40 | "k3": nodes.Object{ 41 | "@type": nodes.String("node"), 42 | }, 43 | }, 44 | }, 45 | { 46 | name: "same keys", 47 | size: 48, 48 | in: nodes.Object{ 49 | "@type": nodes.String("node"), 50 | "k": nodes.Object{ 51 | "@type": nodes.String("node"), 52 | "k": nil, 53 | }, 54 | }, 55 | json: `{ 56 | "root": 1, 57 | "last": 6, 58 | "nodes": { 59 | "1": { 60 | "id": 1, 61 | "kind": 2, 62 | "keys": [ 63 | 2, 64 | 3 65 | ], 66 | "values": [ 67 | 4, 68 | 5 69 | ] 70 | }, 71 | "2": { 72 | "id": 2, 73 | "kind": 8, 74 | "val": "@type" 75 | }, 76 | "3": { 77 | "id": 3, 78 | "kind": 8, 79 | "val": "k" 80 | }, 81 | "4": { 82 | "id": 4, 83 | "kind": 8, 84 | "val": "node" 85 | }, 86 | "5": { 87 | "id": 5, 88 | "kind": 2, 89 | "keys": [ 90 | 2, 91 | 3 92 | ], 93 | "values": [ 94 | 4, 95 | 0 96 | ] 97 | } 98 | } 99 | }`, 100 | }, 101 | { 102 | name: "dups", 103 | size: 61, 104 | in: nodes.Array{ 105 | nodes.Object{ 106 | "@type": nodes.String("node"), 107 | "k": nodes.Array{nodes.String("n1"), nodes.String("n2")}, 108 | }, 109 | nodes.Object{ 110 | "@type": nodes.String("node"), 111 | "k": nodes.Array{nodes.String("n1"), nodes.String("n2")}, 112 | }, 113 | }, 114 | }, 115 | { 116 | name: "empty object", 117 | size: 22, 118 | in: nodes.Array{ 119 | nodes.Array{}, 120 | nodes.Object{}, 121 | }, 122 | json: `{ 123 | "root": 1, 124 | "last": 4, 125 | "nodes": { 126 | "1": { 127 | "id": 1, 128 | "kind": 4, 129 | "values": [ 130 | 2, 131 | 3 132 | ] 133 | }, 134 | "2": { 135 | "id": 2, 136 | "kind": 4 137 | }, 138 | "3": { 139 | "id": 3, 140 | "kind": 2 141 | } 142 | } 143 | }`, 144 | }, 145 | } 146 | 147 | func TestTree(t *testing.T) { 148 | for _, c := range treeCases { 149 | t.Run(c.name, func(t *testing.T) { 150 | in := c.in 151 | exp := c.out 152 | if exp == nil { 153 | exp = c.in.Clone() 154 | } 155 | buf := bytes.NewBuffer(nil) 156 | err := WriteTo(buf, in) 157 | require.NoError(t, err) 158 | require.Equal(t, int(c.size), int(buf.Len())) 159 | 160 | out, err := ReadTree(bytes.NewReader(buf.Bytes())) 161 | require.NoError(t, err) 162 | require.True(t, nodes.Equal(exp, out)) 163 | 164 | if c.json != "" { 165 | raw, err := ReadRaw(bytes.NewReader(buf.Bytes())) 166 | require.NoError(t, err) 167 | got, err := json.MarshalIndent(raw, "", "\t") 168 | require.NoError(t, err) 169 | require.Equal(t, c.json, string(got)) 170 | } 171 | }) 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /uast/nodes/nodesproto/pio/io.go: -------------------------------------------------------------------------------- 1 | // Extensions for Protocol Buffers to create more go like structures. 2 | // 3 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 4 | // http://github.com/gogo/protobuf/gogoproto 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | package pio 30 | 31 | import ( 32 | "github.com/gogo/protobuf/proto" 33 | ) 34 | 35 | type Writer interface { 36 | WriteMsg(proto.Message) (int, error) 37 | } 38 | 39 | type Reader interface { 40 | ReadMsg(msg proto.Message) error 41 | SkipMsg() error 42 | } 43 | 44 | type marshaler interface { 45 | MarshalTo(data []byte) (n int, err error) 46 | } 47 | 48 | func getSize(v interface{}) (int, bool) { 49 | if sz, ok := v.(interface { 50 | Size() (n int) 51 | }); ok { 52 | return sz.Size(), true 53 | } else if sz, ok := v.(interface { 54 | ProtoSize() (n int) 55 | }); ok { 56 | return sz.ProtoSize(), true 57 | } else { 58 | return 0, false 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /uast/nodes/nodesproto/pio/io_test.go: -------------------------------------------------------------------------------- 1 | // Extensions for Protocol Buffers to create more go like structures. 2 | // 3 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 4 | // http://github.com/gogo/protobuf/gogoproto 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | package pio_test 30 | 31 | import ( 32 | "bytes" 33 | goio "io" 34 | "math/rand" 35 | "testing" 36 | "time" 37 | 38 | io "github.com/bblfsh/sdk/v3/uast/nodes/nodesproto/pio" 39 | "github.com/gogo/protobuf/test" 40 | ) 41 | 42 | func iotest(writer io.Writer, reader io.Reader) error { 43 | size := 1000 44 | msgs := make([]*test.NinOptNative, size) 45 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 46 | for i := range msgs { 47 | msgs[i] = test.NewPopulatedNinOptNative(r, true) 48 | //issue 31 49 | if i == 5 { 50 | msgs[i] = &test.NinOptNative{} 51 | } 52 | //issue 31 53 | if i == 999 { 54 | msgs[i] = &test.NinOptNative{} 55 | } 56 | _, err := writer.WriteMsg(msgs[i]) 57 | if err != nil { 58 | return err 59 | } 60 | } 61 | i := 0 62 | for { 63 | msg := &test.NinOptNative{} 64 | if err := reader.ReadMsg(msg); err != nil { 65 | if err == goio.EOF { 66 | break 67 | } 68 | return err 69 | } 70 | if err := msg.VerboseEqual(msgs[i]); err != nil { 71 | return err 72 | } 73 | i++ 74 | } 75 | if i != size { 76 | panic("not enough messages read") 77 | } 78 | return nil 79 | } 80 | 81 | func TestVarintNormal(t *testing.T) { 82 | buf := bytes.NewBuffer(nil) 83 | writer := io.NewWriter(buf) 84 | reader := io.NewReader(buf, 1024*1024) 85 | if err := iotest(writer, reader); err != nil { 86 | t.Error(err) 87 | } 88 | } 89 | 90 | func TestVarintNoClose(t *testing.T) { 91 | buf := bytes.NewBuffer(nil) 92 | writer := io.NewWriter(buf) 93 | reader := io.NewReader(buf, 1024*1024) 94 | if err := iotest(writer, reader); err != nil { 95 | t.Error(err) 96 | } 97 | } 98 | 99 | //issue 32 100 | func TestVarintMaxSize(t *testing.T) { 101 | buf := bytes.NewBuffer(nil) 102 | writer := io.NewWriter(buf) 103 | reader := io.NewReader(buf, 20) 104 | if err := iotest(writer, reader); err != goio.ErrShortBuffer { 105 | t.Error(err) 106 | } 107 | } 108 | 109 | func TestVarintError(t *testing.T) { 110 | buf := bytes.NewBuffer(nil) 111 | buf.Write([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}) 112 | reader := io.NewReader(buf, 1024*1024) 113 | msg := &test.NinOptNative{} 114 | err := reader.ReadMsg(msg) 115 | if err == nil { 116 | t.Fatalf("Expected error") 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /uast/nodes/nodesproto/pio/varint.go: -------------------------------------------------------------------------------- 1 | // Extensions for Protocol Buffers to create more go like structures. 2 | // 3 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 4 | // http://github.com/gogo/protobuf/gogoproto 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | package pio 30 | 31 | import ( 32 | "bufio" 33 | "encoding/binary" 34 | "errors" 35 | "io" 36 | 37 | "github.com/gogo/protobuf/proto" 38 | ) 39 | 40 | var ( 41 | errSmallBuffer = errors.New("Buffer Too Small") 42 | errLargeValue = errors.New("Value is Larger than 64 bits") 43 | ) 44 | 45 | func NewWriter(w io.Writer) Writer { 46 | return &varintWriter{w: w, lenBuf: make([]byte, binary.MaxVarintLen64)} 47 | } 48 | 49 | type varintWriter struct { 50 | w io.Writer 51 | lenBuf []byte 52 | buffer []byte 53 | } 54 | 55 | func (w *varintWriter) WriteMsg(msg proto.Message) (_ int, err error) { 56 | var data []byte 57 | if m, ok := msg.(marshaler); ok { 58 | n, ok := getSize(m) 59 | if !ok { 60 | data, err = proto.Marshal(msg) 61 | if err != nil { 62 | return 0, err 63 | } 64 | } 65 | if n >= len(w.buffer) { 66 | w.buffer = make([]byte, n) 67 | } 68 | _, err = m.MarshalTo(w.buffer) 69 | if err != nil { 70 | return 0, err 71 | } 72 | data = w.buffer[:n] 73 | } else { 74 | data, err = proto.Marshal(msg) 75 | if err != nil { 76 | return 0, err 77 | } 78 | } 79 | length := uint64(len(data)) 80 | n := binary.PutUvarint(w.lenBuf, length) 81 | n, err = w.w.Write(w.lenBuf[:n]) 82 | if err != nil { 83 | return n, err 84 | } 85 | nd, err := w.w.Write(data) 86 | return n + nd, err 87 | } 88 | 89 | func NewReader(r io.Reader, maxSize int) Reader { 90 | return &varintReader{r: bufio.NewReader(r), maxSize: maxSize} 91 | } 92 | 93 | type varintReader struct { 94 | r *bufio.Reader 95 | buf []byte 96 | maxSize int 97 | 98 | readLen bool 99 | len int 100 | } 101 | 102 | func (r *varintReader) readLength() error { 103 | if r.readLen { 104 | return nil 105 | } 106 | length64, err := binary.ReadUvarint(r.r) 107 | if err != nil { 108 | return err 109 | } 110 | length := int(length64) 111 | r.readLen, r.len = true, length 112 | return nil 113 | } 114 | 115 | func (r *varintReader) SkipMsg() error { 116 | if err := r.readLength(); err != nil { 117 | return err 118 | } 119 | if r.len < 0 { 120 | return io.ErrShortBuffer 121 | } 122 | r.readLen = false 123 | if _, err := r.r.Discard(r.len); err != nil { 124 | return err 125 | } 126 | return nil 127 | } 128 | 129 | func (r *varintReader) ReadMsg(msg proto.Message) error { 130 | if err := r.readLength(); err != nil { 131 | return err 132 | } 133 | if r.len < 0 || r.len > r.maxSize { 134 | return io.ErrShortBuffer 135 | } 136 | r.readLen = false 137 | if len(r.buf) < r.len { 138 | r.buf = make([]byte, r.len) 139 | } 140 | buf := r.buf[:r.len] 141 | if _, err := io.ReadFull(r.r, buf); err != nil { 142 | return err 143 | } 144 | return proto.Unmarshal(buf, msg) 145 | } 146 | -------------------------------------------------------------------------------- /uast/query/iter.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "github.com/bblfsh/sdk/v3/uast" 5 | "github.com/bblfsh/sdk/v3/uast/nodes" 6 | ) 7 | 8 | type Empty = nodes.Empty 9 | 10 | type IterOrder = nodes.IterOrder 11 | 12 | const ( 13 | IterAny = nodes.IterAny 14 | PreOrder = nodes.PreOrder 15 | PostOrder = nodes.PostOrder 16 | LevelOrder = nodes.LevelOrder 17 | ChildrenOrder = nodes.ChildrenOrder 18 | PositionOrder = ChildrenOrder + iota + 1 19 | ) 20 | 21 | // NewIterator creates a new iterator with a given order. 22 | // It's an extension of nodes.NewIterator that additionally supports PositionOrder. 23 | func NewIterator(root nodes.External, order IterOrder) Iterator { 24 | if root == nil { 25 | return Empty{} 26 | } 27 | switch order { 28 | case PositionOrder: 29 | return uast.NewPositionalIterator(root) 30 | } 31 | return nodes.NewIterator(root, order) 32 | } 33 | -------------------------------------------------------------------------------- /uast/query/query.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "github.com/bblfsh/sdk/v3/uast/nodes" 5 | ) 6 | 7 | type Interface interface { 8 | // Prepare parses the query and prepares it for repeated execution. 9 | Prepare(query string) (Query, error) 10 | // Execute prepares and runs a query for a given subtree. 11 | Execute(root nodes.External, query string) (Iterator, error) 12 | } 13 | 14 | type Query interface { 15 | // Execute runs a query for a given subtree. 16 | Execute(root nodes.External) (Iterator, error) 17 | } 18 | 19 | type Iterator = nodes.Iterator 20 | 21 | // AllNodes iterates over all nodes and returns them as a slice. 22 | func AllNodes(it Iterator) []nodes.External { 23 | var out []nodes.External 24 | for it.Next() { 25 | out = append(out, it.Node()) 26 | } 27 | return out 28 | } 29 | 30 | // Count counts the nodes in the iterator. Iterator will be exhausted as a result. 31 | func Count(it Iterator) int { 32 | var n int 33 | for it.Next() { 34 | n++ 35 | } 36 | return n 37 | } 38 | -------------------------------------------------------------------------------- /uast/query/xpath/xpath.go: -------------------------------------------------------------------------------- 1 | package xpath 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/antchfx/xpath" 7 | 8 | "github.com/bblfsh/sdk/v3/uast/nodes" 9 | "github.com/bblfsh/sdk/v3/uast/query" 10 | ) 11 | 12 | func New() query.Interface { 13 | return &index{} 14 | } 15 | 16 | type index struct{} 17 | 18 | func (t *index) newNavigator(n nodes.External) xpath.NodeNavigator { 19 | return newNavigator(n) 20 | } 21 | 22 | func (t *index) Prepare(query string) (query.Query, error) { 23 | exp, err := xpath.Compile(query) 24 | if err != nil { 25 | return nil, err 26 | } 27 | return &xQuery{idx: t, exp: exp}, nil 28 | } 29 | 30 | func (t *index) Execute(root nodes.External, query string) (query.Iterator, error) { 31 | q, err := t.Prepare(query) 32 | if err != nil { 33 | return nil, err 34 | } 35 | return q.Execute(root) 36 | } 37 | 38 | type xQuery struct { 39 | idx *index 40 | exp *xpath.Expr 41 | } 42 | 43 | func (q *xQuery) Execute(root nodes.External) (_ query.Iterator, gerr error) { 44 | // This workaround should be temporary. xpath library is not 45 | // managing panics correctly (it should output a nice error instead) 46 | // TODO(ncordon): fix the xpath library instead of recovering from the panic 47 | defer func() { 48 | if r := recover(); r != nil { 49 | gerr = fmt.Errorf("Error executing the xPath query, maybe wrong syntax? \nRecovered from %v", r) 50 | } 51 | }() 52 | 53 | nav := q.idx.newNavigator(root) 54 | val := q.exp.Evaluate(nav) 55 | 56 | if it, ok := val.(*xpath.NodeIterator); ok { 57 | return &iterator{it: it}, nil 58 | } 59 | var v nodes.Value 60 | 61 | switch val := val.(type) { 62 | case bool: 63 | v = nodes.Bool(val) 64 | case float64: 65 | if float64(int64(val)) == val { 66 | v = nodes.Int(val) 67 | } else { 68 | v = nodes.Float(val) 69 | } 70 | case int: 71 | v = nodes.Int(val) 72 | case uint: 73 | v = nodes.Uint(val) 74 | case string: 75 | v = nodes.String(val) 76 | default: 77 | return nil, fmt.Errorf("unsupported type: %T", val) 78 | } 79 | 80 | return &valIterator{val: v}, nil 81 | } 82 | 83 | type valIterator struct { 84 | state int 85 | val nodes.Value 86 | } 87 | 88 | func (it *valIterator) Next() bool { 89 | switch it.state { 90 | case 0: 91 | it.state++ 92 | return true 93 | case 1: 94 | it.state++ 95 | } 96 | return false 97 | } 98 | 99 | func (it *valIterator) Node() nodes.External { 100 | if it.state == 1 { 101 | return it.val 102 | } 103 | return nil 104 | } 105 | 106 | type iterator struct { 107 | it *xpath.NodeIterator 108 | } 109 | 110 | func (it *iterator) Next() bool { 111 | return it.it.MoveNext() 112 | } 113 | 114 | func (it *iterator) Node() nodes.External { 115 | c := it.it.Current() 116 | if c == nil { 117 | return nil 118 | } 119 | nav := c.(*nodeNavigator) 120 | if nav.cur == nil { 121 | return nil 122 | } 123 | return nav.cur.n 124 | } 125 | -------------------------------------------------------------------------------- /uast/role/role_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=Role"; DO NOT EDIT. 2 | 3 | package role 4 | 5 | import "strconv" 6 | 7 | const _Role_name = "InvalidIdentifierQualifiedOperatorBinaryUnaryLeftRightInfixPostfixBitwiseBooleanUnsignedLeftShiftRightShiftOrXorAndExpressionStatementEqualNotLessThanLessThanOrEqualGreaterThanGreaterThanOrEqualIdenticalContainsIncrementDecrementNegativePositiveDereferenceTakeAddressFileAddSubstractMultiplyDivideModuloPackageDeclarationImportPathnameAliasFunctionBodyNameReceiverArgumentValueArgsListBaseImplementsInstanceSubtypeSubpackageModuleFriendWorldIfConditionThenElseSwitchCaseDefaultForInitializationUpdateIteratorWhileDoWhileBreakContinueGotoBlockScopeReturnTryCatchFinallyThrowAssertCallCalleePositionalNoopLiteralByteByteStringCharacterListMapNullNumberRegexpSetStringTupleTypeEntryKeyPrimitiveAssignmentThisCommentDocumentationWhitespaceIncompleteUnannotatedVisibilityAnnotationAnonymousEnumerationArithmeticRelationalVariable" 8 | 9 | var _Role_index = [...]uint16{0, 7, 17, 26, 34, 40, 45, 49, 54, 59, 66, 73, 80, 88, 97, 107, 109, 112, 115, 125, 134, 139, 142, 150, 165, 176, 194, 203, 211, 220, 229, 237, 245, 256, 267, 271, 274, 283, 291, 297, 303, 310, 321, 327, 335, 340, 348, 352, 356, 364, 372, 377, 385, 389, 399, 407, 414, 424, 430, 436, 441, 443, 452, 456, 460, 466, 470, 477, 480, 494, 500, 508, 513, 520, 525, 533, 537, 542, 547, 553, 556, 561, 568, 573, 579, 583, 589, 599, 603, 610, 614, 624, 633, 637, 640, 644, 650, 656, 659, 665, 670, 674, 679, 682, 691, 701, 705, 712, 725, 735, 745, 756, 766, 776, 785, 796, 806, 816, 824} 10 | 11 | func (i Role) String() string { 12 | if i < 0 || i >= Role(len(_Role_index)-1) { 13 | return "Role(" + strconv.FormatInt(int64(i), 10) + ")" 14 | } 15 | return _Role_name[_Role_index[i]:_Role_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /uast/role/role_test.go: -------------------------------------------------------------------------------- 1 | package role 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestFromString(t *testing.T) { 10 | require.Equal(t, List, FromString(List.String())) 11 | } 12 | 13 | func TestRoleValid(t *testing.T) { 14 | require.True(t, Variable.Valid()) 15 | require.False(t, (Variable + 1).Valid()) 16 | require.False(t, (Invalid).Valid()) 17 | require.False(t, Role(-1).Valid()) 18 | } 19 | -------------------------------------------------------------------------------- /uast/transformer/conv.go: -------------------------------------------------------------------------------- 1 | package transformer 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | ) 7 | 8 | // Quote uses strconv.Quote/Unquote to wrap provided string value. 9 | func Quote(op Op) Op { 10 | return StringConv(op, func(s string) (string, error) { 11 | ns, err := strconv.Unquote(s) 12 | if err != nil { 13 | return "", fmt.Errorf("%v (%s)", err, s) 14 | } 15 | return ns, nil 16 | }, func(s string) (string, error) { 17 | return strconv.Quote(s), nil 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /uast/transformer/errors.go: -------------------------------------------------------------------------------- 1 | package transformer 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "gopkg.in/src-d/go-errors.v1" 8 | 9 | "github.com/bblfsh/sdk/v3/uast" 10 | "github.com/bblfsh/sdk/v3/uast/nodes" 11 | ) 12 | 13 | var ( 14 | // ErrVariableRedeclared is returned when a transformation defines the same variable twice. 15 | ErrVariableRedeclared = errors.NewKind("variable %q redeclared (%v vs %v)") 16 | // ErrVariableNotDefined is returned when an operation excepts a variable to be defined, but it was not set 17 | // in current transformation state. 18 | // 19 | // Receiving this error might mean that transformations have an incorrect order and tries to use a variable 20 | // in Lookup or If, for example, before another transformation step that sets this variable is executed. 21 | // If this is the case, see Fields vs Obj comparison in regards to execution order. 22 | ErrVariableNotDefined = errors.NewKind("variable %q is not defined") 23 | // ErrVariableUnused is returned when a first half of transformation defines a variable and the second part 24 | // never uses it in any operations. 25 | // 26 | // If you receive this error and still think that every variable is used - double check conditional branches 27 | // like Opt, If, Case, etc. 28 | ErrVariableUnused = errors.NewKind("variables %q unused in the second part of the transform") 29 | // ErrExpectedObject is returned when transformation expected an object in the tree or variable, but got other type. 30 | ErrExpectedObject = errors.NewKind("expected object, got %T") 31 | // ErrExpectedList is returned when transformation expected an array in the tree or variable, but got other type. 32 | ErrExpectedList = errors.NewKind("expected list, got %T") 33 | // ErrExpectedValue is returned when transformation expected a value in the tree or variable, but got other type. 34 | ErrExpectedValue = errors.NewKind("expected value, got %T") 35 | // ErrUnhandledValueIn is returned when Lookup fails to find a value in the map. It's recommended to define all 36 | // values in the lookup map. If it's not possible, use nil value as a key to set default case for Lookup. 37 | ErrUnhandledValueIn = errors.NewKind("unhandled value: %v in %v") 38 | // ErrUnexpectedNode is an internal error returned by SDK when a transformation that creates a value receives another 39 | // value as an argument. This should not happen. 40 | ErrUnexpectedNode = errors.NewKind("expected node to be nil, got: %v") 41 | // ErrUnexpectedValue is returned when the value from state does not match constraints enforced by Construct. 42 | ErrUnexpectedValue = errors.NewKind("unexpected value: %v") 43 | // ErrUnexpectedType is returned in both Check and Construct when unexpected type is received as an argument or variable. 44 | ErrUnexpectedType = errors.NewKind("unexpected type: exp %T vs got %T") 45 | // ErrAmbiguousValue is returned when Lookup map contains the same value mapped to different keys. 46 | ErrAmbiguousValue = errors.NewKind("map has ambiguous value %v") 47 | // ErrUnusedField is returned when a transformation is not defined as partial, but does not process a specific key 48 | // found in object. This usually means that an AST has a field that is not covered by transformation code and it 49 | // should be added to the mapping. 50 | // 51 | // Use NewErrUnusedField for constructing this error. 52 | ErrUnusedField = errors.NewKind("unused field(s) on node %v: %v") 53 | // ErrDuplicateField is returned when trying to create a Fields definition with two items with the same name. 54 | ErrDuplicateField = errors.NewKind("duplicate field: %v") 55 | // ErrUndefinedField is returned when trying to create an object with a field that is not defined in the type spec. 56 | ErrUndefinedField = errors.NewKind("undefined field: %v") 57 | 58 | errAnd = errors.NewKind("op %d (%T)") 59 | errKey = errors.NewKind("key %q") 60 | errElem = errors.NewKind("elem %d (%T)") 61 | errAppend = errors.NewKind("append") 62 | errMapping = errors.NewKind("mapping %q") 63 | 64 | errCheck = errors.NewKind("check") 65 | errConstruct = errors.NewKind("construct") 66 | ) 67 | 68 | var _ error = (*MultiError)(nil) 69 | 70 | // NewMultiError creates an error that contains multiple other errors. 71 | // If a slice is empty, it returns nil. If there is only one error, it is returned directly. 72 | func NewMultiError(errs ...error) error { 73 | if len(errs) == 0 { 74 | return nil 75 | } else if len(errs) == 1 { 76 | return errs[0] 77 | } 78 | return &MultiError{Errs: errs} 79 | } 80 | 81 | // MultiError is an error that groups multiple other errors. 82 | type MultiError struct { 83 | Errs []error 84 | } 85 | 86 | func (e *MultiError) Error() string { 87 | if len(e.Errs) == 1 { 88 | return e.Errs[0].Error() 89 | } 90 | buf := bytes.NewBuffer(nil) 91 | fmt.Fprintf(buf, "received %d errors:\n", len(e.Errs)) 92 | for _, err := range e.Errs { 93 | fmt.Fprintf(buf, "\t%v\n", err) 94 | } 95 | return buf.String() 96 | } 97 | 98 | // NewErrUnusedField is a helper for creating ErrUnusedField. 99 | // 100 | // It will include a short type information for the node to simplify debugging. 101 | func NewErrUnusedField(n nodes.Object, fields []string) error { 102 | var t interface{} 103 | if typ, _ := n[uast.KeyType].(nodes.String); typ != "" { 104 | t = typ 105 | } else { 106 | t = n.Keys() 107 | } 108 | var f interface{} = fields 109 | if len(fields) == 1 { 110 | f = fields[0] 111 | } 112 | return ErrUnusedField.New(t, f) 113 | } 114 | -------------------------------------------------------------------------------- /uast/transformer/legacy.go: -------------------------------------------------------------------------------- 1 | package transformer 2 | 3 | import ( 4 | "github.com/bblfsh/sdk/v3/uast" 5 | "github.com/bblfsh/sdk/v3/uast/nodes" 6 | "github.com/bblfsh/sdk/v3/uast/role" 7 | ) 8 | 9 | const dedupCloneObj = false 10 | 11 | var _ Transformer = ResponseMetadata{} 12 | 13 | // ResponseMetadata is a transformation that is applied to the root of AST tree to trim any metadata that might be there. 14 | type ResponseMetadata struct { 15 | // TopLevelIsRootNode tells ToNode where to find the root node of 16 | // the AST. If true, the root will be its input argument. If false, 17 | // the root will be the value of the only key present in its input 18 | // argument. 19 | TopLevelIsRootNode bool 20 | } 21 | 22 | // Do applies the transformation described by this object. 23 | func (n ResponseMetadata) Do(root nodes.Node) (nodes.Node, error) { 24 | if obj, ok := root.(nodes.Object); ok && !n.TopLevelIsRootNode && len(obj) == 1 { 25 | for _, v := range obj { 26 | root = v 27 | break 28 | } 29 | } 30 | return root, nil 31 | } 32 | 33 | // ObjectToNode transform trees that are represented as nested JSON objects. 34 | // That is, an interface{} containing maps, slices, strings and integers. It 35 | // then converts from that structure to *Node. 36 | type ObjectToNode struct { 37 | // InternalTypeKey is the name of the key that the native AST uses 38 | // to differentiate the type of the AST nodes. This internal key will then be 39 | // checkable in the AnnotationRules with the `HasInternalType` predicate. This 40 | // field is mandatory. 41 | InternalTypeKey string 42 | // OffsetKey is the key used in the native AST to indicate the absolute offset, 43 | // from the file start position, where the code mapped to the AST node starts. 44 | OffsetKey string 45 | // EndOffsetKey is the key used in the native AST to indicate the absolute offset, 46 | // from the file start position, where the code mapped to the AST node ends. 47 | EndOffsetKey string 48 | // LineKey is the key used in the native AST to indicate 49 | // the line number where the code mapped to the AST node starts. 50 | LineKey string 51 | // EndLineKey is the key used in the native AST to indicate 52 | // the line number where the code mapped to the AST node ends. 53 | EndLineKey string 54 | // ColumnKey is a key that indicates the column inside the line 55 | ColumnKey string 56 | // EndColumnKey is a key that indicates the column inside the line where the node ends. 57 | EndColumnKey string 58 | } 59 | 60 | // Mapping construct a transformation from ObjectToNode definition. 61 | func (n ObjectToNode) Mapping() Mapping { 62 | var ( 63 | ast Fields 64 | // -> 65 | norm = make(Obj) 66 | normPos Fields 67 | ) 68 | 69 | if n.InternalTypeKey != "" { 70 | const vr = "itype" 71 | ast = append(ast, Field{Name: n.InternalTypeKey, Op: Var(vr)}) 72 | norm[uast.KeyType] = Var(vr) 73 | } 74 | 75 | if n.OffsetKey != "" { 76 | const vr = "pos_off_start" 77 | ast = append(ast, Field{Name: n.OffsetKey, Op: Var(vr)}) 78 | normPos = append(normPos, Field{Name: uast.KeyStart, Op: SavePosOffset(vr)}) 79 | } 80 | if n.EndOffsetKey != "" { 81 | const vr = "pos_off_end" 82 | ast = append(ast, Field{Name: n.EndOffsetKey, Op: Var(vr)}) 83 | normPos = append(normPos, Field{Name: uast.KeyEnd, Op: SavePosOffset(vr)}) 84 | } 85 | if n.LineKey != "" && n.ColumnKey != "" { 86 | const ( 87 | vre = "pos_start_exists" 88 | vrl = "pos_line_start" 89 | vrc = "pos_col_start" 90 | ) 91 | ast = append(ast, 92 | Field{Name: n.LineKey, Op: Var(vrl), Optional: vre}, 93 | Field{Name: n.ColumnKey, Op: Var(vrc), Optional: vre}, 94 | ) 95 | normPos = append(normPos, Field{Name: uast.KeyStart, Op: SavePosLineCol(vrl, vrc), Optional: vre}) 96 | } else if (n.LineKey != "" && n.ColumnKey == "") || (n.LineKey == "" && n.ColumnKey != "") { 97 | panic("both LineKey and ColumnKey should either be set or not") 98 | } 99 | if n.EndLineKey != "" && n.EndColumnKey != "" { 100 | const ( 101 | vre = "pos_end_exists" 102 | vrl = "pos_line_end" 103 | vrc = "pos_col_end" 104 | ) 105 | ast = append(ast, 106 | Field{Name: n.EndLineKey, Op: Var(vrl), Optional: vre}, 107 | Field{Name: n.EndColumnKey, Op: Var(vrc), Optional: vre}, 108 | ) 109 | normPos = append(normPos, Field{Name: uast.KeyEnd, Op: SavePosLineCol(vrl, vrc), Optional: vre}) 110 | } else if (n.EndLineKey != "" && n.EndColumnKey == "") || (n.EndLineKey == "" && n.EndColumnKey != "") { 111 | panic("both EndLineKey and EndColumnKey should either be set or not") 112 | } 113 | if len(normPos) != 0 { 114 | norm[uast.KeyPos] = UASTType(uast.Positions{}, normPos) 115 | } 116 | return MapPart("other", MapObj(ast, norm)) 117 | } 118 | 119 | // RolesDedup is an irreversible transformation that removes duplicate roles from AST nodes. 120 | func RolesDedup() TransformFunc { 121 | return TransformFunc(func(n nodes.Node) (nodes.Node, bool, error) { 122 | obj, ok := n.(nodes.Object) 123 | if !ok { 124 | return n, false, nil 125 | } 126 | roles := uast.RolesOf(obj) 127 | if len(roles) == 0 { 128 | return n, false, nil 129 | } 130 | m := make(map[role.Role]struct{}, len(roles)) 131 | out := make([]role.Role, 0, len(roles)) 132 | for _, r := range roles { 133 | if _, ok := m[r]; ok { 134 | continue 135 | } 136 | m[r] = struct{}{} 137 | out = append(out, r) 138 | } 139 | if len(out) == len(roles) { 140 | return n, false, nil 141 | } 142 | if dedupCloneObj { 143 | obj = obj.CloneObject() 144 | } 145 | obj[uast.KeyRoles] = uast.RoleList(out...) 146 | return obj, dedupCloneObj, nil 147 | }) 148 | } 149 | -------------------------------------------------------------------------------- /uast/transformer/positioner/tokens.go: -------------------------------------------------------------------------------- 1 | package positioner 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/bblfsh/sdk/v3/uast" 7 | "github.com/bblfsh/sdk/v3/uast/nodes" 8 | "github.com/bblfsh/sdk/v3/uast/transformer" 9 | ) 10 | 11 | var _ transformer.CodeTransformer = TokenFromSource{} 12 | 13 | // TokenFromSource extract node's token from the source code by using positional 14 | // information. 15 | type TokenFromSource struct { 16 | // Key is the name of the token field to update. Uses uast.KeyToken, if not set. 17 | // Only nodes with this field will be considered. 18 | Key string 19 | // Types is the list of node types that will be updated. Empty means all nodes. 20 | Types []string 21 | } 22 | 23 | // OnCode implements transformer.CodeTransformer. 24 | func (t TokenFromSource) OnCode(code string) transformer.Transformer { 25 | return &tokenFromSource{ 26 | tokenFilter: newTokenFilter(code, t.Key, t.Types), 27 | } 28 | } 29 | 30 | func newTokenFilter(source string, key string, types []string) tokenFilter { 31 | f := tokenFilter{ 32 | source: source, tokenKey: key, 33 | } 34 | if f.tokenKey == "" { 35 | f.tokenKey = uast.KeyToken 36 | } 37 | if len(types) != 0 { 38 | f.types = make(map[string]struct{}) 39 | for _, tp := range types { 40 | f.types[tp] = struct{}{} 41 | } 42 | } 43 | return f 44 | } 45 | 46 | type tokenFilter struct { 47 | source string 48 | tokenKey string 49 | types map[string]struct{} 50 | } 51 | 52 | func (f *tokenFilter) filterObj(node nodes.Node) (nodes.Object, bool) { 53 | obj, ok := node.(nodes.Object) 54 | if !ok { 55 | return nil, false 56 | } 57 | // nodes should already have a token 58 | if _, ok = obj[f.tokenKey]; !ok { 59 | return nil, false 60 | } 61 | // apply types filter 62 | if f.types != nil { 63 | typ, ok := obj[uast.KeyType].(nodes.String) 64 | if !ok || typ == "" { 65 | return nil, false 66 | } 67 | _, ok = f.types[string(typ)] 68 | if !ok { 69 | return nil, false 70 | } 71 | } 72 | return obj, true 73 | } 74 | 75 | func (f *tokenFilter) tokenFromPos(obj nodes.Object) (string, bool, error) { 76 | pos := uast.PositionsOf(obj) 77 | if len(pos) == 0 { 78 | return "", false, nil 79 | } 80 | start := pos.Start() 81 | end := pos.End() 82 | if start == nil || end == nil || !start.HasOffset() || !end.HasOffset() { 83 | return "", false, nil 84 | } 85 | si, ei := start.Offset, end.Offset 86 | if si > ei { 87 | return "", false, fmt.Errorf("start offset is larger than an end offset: %v", pos) 88 | } else if ei > uint32(len(f.source)) { 89 | return "", false, fmt.Errorf("offset out of bounds: %v", pos) 90 | } 91 | // offset is given in bytes 92 | token := f.source[si:ei] 93 | return token, true, nil 94 | } 95 | 96 | type tokenFromSource struct { 97 | tokenFilter 98 | } 99 | 100 | // Do implements transformer.Transformer. See TokenFromSource. 101 | func (t *tokenFromSource) Do(root nodes.Node) (nodes.Node, error) { 102 | var last error 103 | nodes.WalkPreOrder(root, func(node nodes.Node) bool { 104 | if last != nil { 105 | return false 106 | } 107 | obj, ok := t.filterObj(node) 108 | if !ok { 109 | // skip node, but recurse to children 110 | return true 111 | } 112 | token, ok, err := t.tokenFromPos(obj) 113 | if err != nil { 114 | last = err 115 | return false 116 | } else if !ok { 117 | return true // recurse 118 | } 119 | // it won't be nil, since we require both token and pos fields to exist 120 | obj[t.tokenKey] = nodes.String(token) 121 | return true 122 | }) 123 | return root, last 124 | } 125 | 126 | // VerifyToken check that node's token matches its positional information. 127 | type VerifyToken struct { 128 | // Key is the name of the token field to check. Uses uast.KeyToken, if not set. 129 | Key string 130 | // Types is the list of node types that will be checked. Empty means all nodes. 131 | Types []string 132 | } 133 | 134 | func (t VerifyToken) Verify(code string, root nodes.Node) error { 135 | key := t.Key 136 | if key == "" { 137 | key = uast.KeyToken 138 | } 139 | f := newTokenFilter(code, key, t.Types) 140 | 141 | var last error 142 | nodes.WalkPreOrder(root, func(node nodes.Node) bool { 143 | if last != nil { 144 | return false 145 | } 146 | obj, ok := f.filterObj(node) 147 | if !ok { 148 | // skip node, but recurse to children 149 | return true 150 | } 151 | token1, ok := obj[key].(nodes.String) 152 | if !ok { 153 | return true 154 | } 155 | token2, ok, err := f.tokenFromPos(obj) 156 | if err != nil { 157 | last = err 158 | return false 159 | } else if !ok { 160 | return true 161 | } 162 | if string(token1) != token2 { 163 | last = fmt.Errorf("wrong token for node %q: %q vs %q", 164 | uast.TypeOf(obj), token1, token2) 165 | return false 166 | } 167 | return true 168 | }) 169 | return last 170 | } 171 | -------------------------------------------------------------------------------- /uast/transformer/positioner/tokens_test.go: -------------------------------------------------------------------------------- 1 | package positioner 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/bblfsh/sdk/v3/uast" 9 | "github.com/bblfsh/sdk/v3/uast/nodes" 10 | ) 11 | 12 | func newPos(start, end int) nodes.Node { 13 | return uast.Positions{ 14 | uast.KeyStart: { 15 | Offset: uint32(start), 16 | Line: 1, 17 | Col: uint32(1 + start), 18 | }, 19 | uast.KeyEnd: { 20 | Offset: uint32(end), 21 | Line: 1, 22 | Col: uint32(1 + end), 23 | }, 24 | }.ToObject() 25 | } 26 | 27 | func TestTokenFromSource(t *testing.T) { 28 | testAst := nodes.Array{ 29 | nodes.Object{ 30 | uast.KeyType: nodes.String("test:Var"), 31 | uast.KeyPos: newPos(1, 13), 32 | "Names": nodes.Array{ 33 | nodes.Object{ 34 | uast.KeyType: nodes.String("test:Ident"), 35 | uast.KeyToken: nodes.String(""), 36 | uast.KeyPos: newPos(5, 6), 37 | }, 38 | nodes.Object{ 39 | uast.KeyType: nodes.String("test:Ident"), 40 | uast.KeyToken: nodes.String(""), 41 | uast.KeyPos: newPos(8, 9), 42 | }, 43 | }, 44 | "Type": nodes.Object{ 45 | uast.KeyType: nodes.String("test:Type"), 46 | uast.KeyToken: nodes.String(""), 47 | uast.KeyPos: newPos(10, 13), 48 | "Notes": nodes.String(""), 49 | }, 50 | }, 51 | } 52 | 53 | var cases = []struct { 54 | name string 55 | source string 56 | ast, exp nodes.Node 57 | conf TokenFromSource 58 | }{ 59 | { 60 | name: "fix all", 61 | source: " var a, b int", 62 | conf: TokenFromSource{}, // defaults 63 | ast: testAst, 64 | exp: nodes.Array{ 65 | nodes.Object{ 66 | uast.KeyType: nodes.String("test:Var"), 67 | uast.KeyPos: newPos(1, 13), 68 | "Names": nodes.Array{ 69 | nodes.Object{ 70 | uast.KeyType: nodes.String("test:Ident"), 71 | uast.KeyToken: nodes.String("a"), 72 | uast.KeyPos: newPos(5, 6), 73 | }, 74 | nodes.Object{ 75 | uast.KeyType: nodes.String("test:Ident"), 76 | uast.KeyToken: nodes.String("b"), 77 | uast.KeyPos: newPos(8, 9), 78 | }, 79 | }, 80 | "Type": nodes.Object{ 81 | uast.KeyType: nodes.String("test:Type"), 82 | uast.KeyToken: nodes.String("int"), 83 | uast.KeyPos: newPos(10, 13), 84 | "Notes": nodes.String(""), 85 | }, 86 | }, 87 | }, 88 | }, 89 | { 90 | name: "only type", 91 | source: " var a, b int", 92 | conf: TokenFromSource{ 93 | Types: []string{"test:Type"}, 94 | }, 95 | ast: testAst, 96 | exp: nodes.Array{ 97 | nodes.Object{ 98 | uast.KeyType: nodes.String("test:Var"), 99 | uast.KeyPos: newPos(1, 13), 100 | "Names": nodes.Array{ 101 | nodes.Object{ 102 | uast.KeyType: nodes.String("test:Ident"), 103 | uast.KeyToken: nodes.String(""), 104 | uast.KeyPos: newPos(5, 6), 105 | }, 106 | nodes.Object{ 107 | uast.KeyType: nodes.String("test:Ident"), 108 | uast.KeyToken: nodes.String(""), 109 | uast.KeyPos: newPos(8, 9), 110 | }, 111 | }, 112 | "Type": nodes.Object{ 113 | uast.KeyType: nodes.String("test:Type"), 114 | uast.KeyToken: nodes.String("int"), 115 | uast.KeyPos: newPos(10, 13), 116 | "Notes": nodes.String(""), 117 | }, 118 | }, 119 | }, 120 | }, 121 | { 122 | name: "specific field", 123 | source: " var a, b int", 124 | conf: TokenFromSource{ 125 | Key: "Notes", 126 | }, 127 | ast: testAst, 128 | exp: nodes.Array{ 129 | nodes.Object{ 130 | uast.KeyType: nodes.String("test:Var"), 131 | uast.KeyPos: newPos(1, 13), 132 | "Names": nodes.Array{ 133 | nodes.Object{ 134 | uast.KeyType: nodes.String("test:Ident"), 135 | uast.KeyToken: nodes.String(""), 136 | uast.KeyPos: newPos(5, 6), 137 | }, 138 | nodes.Object{ 139 | uast.KeyType: nodes.String("test:Ident"), 140 | uast.KeyToken: nodes.String(""), 141 | uast.KeyPos: newPos(8, 9), 142 | }, 143 | }, 144 | "Type": nodes.Object{ 145 | uast.KeyType: nodes.String("test:Type"), 146 | uast.KeyToken: nodes.String(""), 147 | uast.KeyPos: newPos(10, 13), 148 | "Notes": nodes.String("int"), 149 | }, 150 | }, 151 | }, 152 | }, 153 | } 154 | for _, c := range cases { 155 | t.Run(c.name, func(t *testing.T) { 156 | root := c.ast 157 | tr := c.conf.OnCode(c.source) 158 | got, err := tr.Do(root.Clone()) 159 | require.NoError(t, err) 160 | require.Equal(t, c.exp, got) 161 | }) 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /uast/transformer/semantic_test.go: -------------------------------------------------------------------------------- 1 | package transformer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestComment(t *testing.T) { 10 | var cases = []struct { 11 | name string 12 | text string 13 | exp commentElems 14 | }{ 15 | { 16 | name: "one line", 17 | text: "// some text", 18 | exp: commentElems{ 19 | StartToken: "//", EndToken: "", 20 | Prefix: " ", 21 | Text: "some text", 22 | }, 23 | }, 24 | { 25 | name: "space", 26 | text: "// ", 27 | exp: commentElems{ 28 | StartToken: "//", EndToken: "", 29 | Prefix: " ", 30 | Text: "", 31 | }, 32 | }, 33 | { 34 | name: "utf8 chars", 35 | text: "// “utf8 ½Å commentÅ”", 36 | exp: commentElems{ 37 | StartToken: "//", EndToken: "", 38 | Prefix: " ", Suffix: "", 39 | Text: "“utf8 ½Å commentÅ”", 40 | }, 41 | }, 42 | { 43 | name: "utf8 singleton", 44 | text: "/*\t\t\u00a0 ½ */", 45 | exp: commentElems{ 46 | StartToken: "/*", EndToken: "", 47 | Prefix:"\t\t\u00a0 ", Suffix:" */", 48 | Text: "½", 49 | }, 50 | }, 51 | { 52 | name: "new line", 53 | text: "// some text\n", 54 | exp: commentElems{ 55 | StartToken: "//", EndToken: "", 56 | Prefix: " ", Suffix: "\n", 57 | Text: "some text", 58 | }, 59 | }, 60 | { 61 | name: "multi-line single", 62 | text: "/* some text */", 63 | exp: commentElems{ 64 | StartToken: "/*", EndToken: "*/", 65 | Prefix: " ", Suffix: " ", 66 | Text: "some text", 67 | }, 68 | }, 69 | { 70 | name: "multi-line new line", 71 | text: 72 | `/* 73 | some text 74 | */`, 75 | exp: commentElems{ 76 | StartToken: "/*", EndToken: "*/", 77 | Prefix: "\n\t", Suffix: "\n", 78 | Text: "some text", 79 | }, 80 | }, 81 | { 82 | name: "multi-line", 83 | text: 84 | `/* 85 | some text 86 | line two 87 | */`, 88 | exp: commentElems{ 89 | StartToken: "/*", EndToken: "*/", 90 | Prefix: "\n\t", Indent: "\t", Suffix: "\n", 91 | Text: "some text\nline two", 92 | }, 93 | }, 94 | { 95 | name: "stylistic", 96 | text: 97 | `/* 98 | * some text 99 | * line two 100 | * line three 101 | */`, 102 | exp: commentElems{ 103 | StartToken: "/*", EndToken: "*/", 104 | Prefix: "\n * ", Indent: " * ", Suffix: "\n", 105 | Text: "some text\nline two\nline three", 106 | }, 107 | }, 108 | { 109 | name: "multiple single line", 110 | text: 111 | `// some text 112 | // line two`, 113 | exp: commentElems{ 114 | StartToken: "//", EndToken: "", 115 | Prefix: " ", Indent: "// ", Suffix: "", 116 | Text: "some text\nline two", 117 | }, 118 | }, 119 | { 120 | name: "stylistic inconsistent", 121 | text: 122 | `/* 123 | * some text 124 | * line two 125 | * line three 126 | */`, 127 | exp: commentElems{ 128 | StartToken: "/*", EndToken: "*/", 129 | Prefix: "\n * ", Indent: " * ", Suffix: "\n", 130 | Text: "some text\n line two\nline three", 131 | }, 132 | }, 133 | { 134 | name: "inconsistent", 135 | text: 136 | `/* 137 | * some text 138 | line two 139 | * line three 140 | */`, 141 | exp: commentElems{ 142 | StartToken: "/*", EndToken: "*/", 143 | Prefix: "\n * ", Indent: " ", Suffix: "\n", 144 | Text: "some text\n line two\n* line three", 145 | }, 146 | }, 147 | } 148 | for _, c := range cases { 149 | t.Run(c.name, func(t *testing.T) { 150 | v := commentElems{ 151 | StartToken: c.exp.StartToken, 152 | EndToken: c.exp.EndToken, 153 | } 154 | if !v.Split(c.text) { 155 | t.Error("split failed") 156 | } 157 | require.Equal(t, c.exp, v) 158 | require.Equal(t, c.text, v.Join()) 159 | }) 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /uast/uastyaml/uastyaml_test.go: -------------------------------------------------------------------------------- 1 | package uastyaml 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/bblfsh/sdk/v3/uast" 7 | . "github.com/bblfsh/sdk/v3/uast/nodes" 8 | "github.com/bblfsh/sdk/v3/uast/role" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | var casesYML = []struct { 13 | name string 14 | n Node 15 | exp string 16 | expn Node 17 | }{ 18 | { 19 | name: "nil", 20 | n: nil, exp: "~", 21 | }, 22 | { 23 | name: "empty array", 24 | n: Array{}, exp: "[]", 25 | }, 26 | { 27 | name: "nil array", 28 | n: (Array)(nil), exp: "[]", expn: Array{}, 29 | }, 30 | { 31 | name: "empty object", 32 | n: Object{}, exp: "{}", 33 | }, 34 | { 35 | name: "nil object", 36 | n: (Object)(nil), exp: "{}", expn: Object{}, 37 | }, 38 | { 39 | name: "string", 40 | n: String("a"), exp: "a", 41 | }, 42 | { 43 | name: "type", 44 | n: String(uast.KeyType), exp: "'@type'", 45 | }, 46 | { 47 | name: "one value", 48 | n: Array{String("a")}, exp: "[a]", 49 | }, 50 | { 51 | name: "three values", 52 | n: Array{String("a"), Int(1), Bool(true)}, 53 | exp: "[a, 1, true]", 54 | }, 55 | { 56 | name: "one object", 57 | n: Array{Object{}}, 58 | exp: "[\n {},\n]", 59 | }, 60 | { 61 | name: "values and object", 62 | n: Array{String("a"), Int(1), Object{}}, 63 | exp: "[\n a,\n 1,\n {},\n]", 64 | }, 65 | { 66 | name: "system fields", 67 | n: Object{ 68 | uast.KeyType: String("ns:type"), 69 | uast.KeyToken: String(":="), 70 | uast.KeyRoles: uast.RoleList(role.Alias, role.Expression, role.Anonymous), 71 | uast.KeyPos: Object{ 72 | uast.KeyType: String(uast.TypePositions), 73 | uast.KeyStart: uast.Position{Offset: 5, Line: 1, Col: 1}.ToObject(), 74 | uast.KeyEnd: uast.Position{Offset: 6, Line: 1, Col: 2}.ToObject(), 75 | }, 76 | "key": String("val"), 77 | "raw": String(":="), 78 | }, 79 | exp: `{ '@type': "ns:type", 80 | '@token': ":=", 81 | '@role': [Alias, Anonymous, Expression], 82 | '@pos': { '@type': "uast:Positions", 83 | start: { '@type': "uast:Position", 84 | offset: 5, 85 | line: 1, 86 | col: 1, 87 | }, 88 | end: { '@type': "uast:Position", 89 | offset: 6, 90 | line: 1, 91 | col: 2, 92 | }, 93 | }, 94 | key: "val", 95 | raw: ":=", 96 | }`, 97 | expn: Object{ 98 | uast.KeyType: String("ns:type"), 99 | uast.KeyToken: String(":="), 100 | uast.KeyRoles: uast.RoleList(role.Alias, role.Anonymous, role.Expression), 101 | uast.KeyPos: Object{ 102 | uast.KeyType: String(uast.TypePositions), 103 | uast.KeyStart: uast.Position{Offset: 5, Line: 1, Col: 1}.ToObject(), 104 | uast.KeyEnd: uast.Position{Offset: 6, Line: 1, Col: 2}.ToObject(), 105 | }, 106 | "key": String("val"), 107 | "raw": String(":="), 108 | }, 109 | }, 110 | } 111 | 112 | func TestNodeYML(t *testing.T) { 113 | for _, c := range casesYML { 114 | t.Run(c.name, func(t *testing.T) { 115 | data, err := Marshal(c.n) 116 | require.NoError(t, err) 117 | require.Equal(t, c.exp, string(data)) 118 | 119 | nn, err := Unmarshal(data) 120 | require.NoError(t, err) 121 | 122 | en := c.expn 123 | if en == nil { 124 | en = c.n 125 | } 126 | require.True(t, Equal(en, nn)) 127 | }) 128 | } 129 | } 130 | 131 | var casesStringKind = []struct { 132 | name string 133 | val string 134 | exp stringFormat 135 | }{ 136 | { 137 | name: "sys", 138 | val: "@type", 139 | exp: stringQuoted, 140 | }, 141 | { 142 | name: "plain", 143 | val: "start", 144 | exp: stringPlain, 145 | }, 146 | { 147 | name: "off", 148 | val: "off", 149 | exp: stringQuoted, 150 | }, 151 | { 152 | name: "new line", 153 | val: "a\nb", 154 | exp: stringDoubleQuoted, 155 | }, 156 | { 157 | name: "null", 158 | val: "~", 159 | exp: stringQuoted, 160 | }, 161 | } 162 | 163 | func TestStringKind(t *testing.T) { 164 | for _, c := range casesStringKind { 165 | t.Run(c.name, func(t *testing.T) { 166 | require.Equal(t, c.exp, bestStringFormat(c.val)) 167 | }) 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /uast/viewer/viewer.go: -------------------------------------------------------------------------------- 1 | package viewer 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | uast1 "github.com/bblfsh/sdk/v3/protocol/v1" 7 | "github.com/bblfsh/sdk/v3/uast/nodes" 8 | "github.com/bblfsh/sdk/v3/uast/role" 9 | ) 10 | 11 | // MarshalUAST writes a UAST file compatible with uast-viewer. 12 | func MarshalUAST(lang, code string, ast nodes.Node) ([]byte, error) { 13 | nd, err := uast1.ToNode(ast) 14 | if err != nil { 15 | return nil, err 16 | } 17 | data, _ := json.Marshal(map[string]interface{}{ 18 | "code": code, "uast": nd, 19 | "lang": lang, 20 | }) 21 | var o interface{} 22 | err = json.Unmarshal(data, &o) 23 | var fix func(interface{}) 24 | fix = func(n interface{}) { 25 | switch n := n.(type) { 26 | case []interface{}: 27 | for _, s := range n { 28 | fix(s) 29 | } 30 | case map[string]interface{}: 31 | for k, v := range n { 32 | if k == "Roles" { 33 | var arr []string 34 | for _, r := range v.([]interface{}) { 35 | rl := role.Role(r.(float64)) 36 | arr = append(arr, rl.String()) 37 | } 38 | n[k] = arr 39 | continue 40 | } else if k == "StartPosition" || k == "EndPosition" { 41 | m := v.(map[string]interface{}) 42 | rename := func(from, to string) { 43 | m[to] = m[from] 44 | delete(m, from) 45 | } 46 | rename("offset", "Offset") 47 | rename("line", "Line") 48 | rename("col", "Col") 49 | continue 50 | } 51 | fix(v) 52 | } 53 | } 54 | } 55 | fix(o) 56 | return json.Marshal(o) 57 | } 58 | -------------------------------------------------------------------------------- /uast/yaml/uastyml.go: -------------------------------------------------------------------------------- 1 | package uastyml 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/bblfsh/sdk/v3/uast/nodes" 7 | "github.com/bblfsh/sdk/v3/uast/uastyaml" 8 | ) 9 | 10 | // Marshal encode the UAST to a human-readable YAML. 11 | // 12 | // Deprecated: use uastyaml.Marshal instead 13 | func Marshal(n nodes.Node) ([]byte, error) { 14 | return uastyaml.Marshal(n) 15 | } 16 | 17 | // NewEncoder creates a YAML encoder for UAST. 18 | // 19 | // Deprecated: use uastyaml.NewEncoder instead 20 | func NewEncoder(w io.Writer) *uastyaml.Encoder { 21 | return uastyaml.NewEncoder(w) 22 | } 23 | 24 | // Encoder is a YAML encoder for UAST. 25 | // 26 | // Deprecated: use uastyaml.Encoder instead 27 | type Encoder = uastyaml.Encoder 28 | 29 | // Unmarshal decodes YAML to a UAST. 30 | // 31 | // Deprecated: use uastyaml.Unmarshal instead 32 | func Unmarshal(data []byte) (nodes.Node, error) { 33 | return uastyaml.Unmarshal(data) 34 | } 35 | --------------------------------------------------------------------------------