├── Dockerfile ├── README.md ├── go.mod ├── cmd └── gobuild │ └── main.go ├── gobuild.go ├── go.sum └── loader └── loader.go /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:experimental 2 | 3 | FROM docker.io/library/golang:1.11-alpine AS builder 4 | WORKDIR /src 5 | RUN apk add --no-cache git 6 | RUN --mount=target=. \ 7 | --mount=target=/go/pkg/mod,type=cache \ 8 | CGO_ENABLED=0 go build -o /out/gobuild --ldflags '-s -w -extldflags "-static"' ./cmd/gobuild 9 | 10 | FROM scratch 11 | COPY --from=builder /out/gobuild /bin/gobuild 12 | ENV PATH=/bin 13 | ENTRYPOINT ["/gobuild"] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### llb-gobuild 2 | 3 | Generate BuildKit LLB definition for building a golang package. 4 | 5 | ``` 6 | src := llb.Local("src") 7 | 8 | gb := gobuild.New(nil) 9 | // gb := gobuild.New(&gobuild.Opt{DevMode: true}) 10 | 11 | buildctl, err := gb.BuildExe(gobuild.BuildOpt{ 12 | Source: src, 13 | MountPath: "/go/src/github.com/moby/buildkit", 14 | Pkg: "github.com/moby/buildkit/cmd/buildctl", 15 | BuildTags: []string{}, 16 | }) 17 | if err != nil { 18 | return err 19 | } 20 | ``` 21 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tonistiigi/llb-gobuild 2 | 3 | require ( 4 | github.com/docker/distribution v2.7.1+incompatible // indirect 5 | github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect 6 | github.com/moby/buildkit v0.3.3 7 | github.com/opencontainers/go-digest v1.0.0-rc1 8 | github.com/pkg/errors v0.8.1 9 | github.com/tonistiigi/fsutil v0.0.0-20190130224639-b4281fa67095 // indirect 10 | golang.org/x/sys v0.0.0-20190201152629-afcc84fd7533 // indirect 11 | google.golang.org/grpc v1.18.0 // indirect 12 | ) 13 | -------------------------------------------------------------------------------- /cmd/gobuild/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "flag" 7 | "io" 8 | "os" 9 | "runtime" 10 | 11 | "github.com/moby/buildkit/client/llb" 12 | "github.com/moby/buildkit/solver/pb" 13 | digest "github.com/opencontainers/go-digest" 14 | gobuild "github.com/tonistiigi/llb-gobuild" 15 | "github.com/tonistiigi/llb-gobuild/loader" 16 | ) 17 | 18 | type opt struct { 19 | target string 20 | } 21 | 22 | func main() { 23 | var o opt 24 | flag.StringVar(&o.target, "target", "/out/buildkit.llb.definition", "target file") 25 | flag.Parse() 26 | 27 | out := os.Stdout 28 | 29 | if o.target != "" { 30 | f, err := os.Create(o.target) 31 | if err != nil { 32 | panic(err) 33 | } 34 | defer f.Close() 35 | out = f 36 | } 37 | 38 | var goOpt gobuild.BuildOptJSON 39 | if err := json.Unmarshal([]byte(os.Getenv("GOOPT")), &goOpt); err != nil { 40 | panic(err) 41 | } 42 | 43 | if goOpt.Pkg == "" { 44 | panic("no target pkg specified") 45 | } 46 | 47 | if goOpt.GOPATH == "" { 48 | goOpt.GOPATH = "/go" 49 | } 50 | 51 | if goOpt.GOOS == "" { 52 | goOpt.GOOS = runtime.GOOS 53 | } 54 | 55 | if goOpt.GOARCH == "" { 56 | goOpt.GOARCH = runtime.GOARCH 57 | } 58 | 59 | if err := generateLLB(goOpt, out); err != nil { 60 | panic(err) 61 | } 62 | } 63 | 64 | func generateLLB(opt gobuild.BuildOptJSON, out io.Writer) error { 65 | def, err := llb.ReadFrom(bytes.NewBuffer([]byte(opt.SourceDef))) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | src := llb.NewState(&output{digest.Digest(opt.Source), opt.SourceIndex}) 71 | 72 | l := loader.New(loader.GoOpt{ 73 | Source: src, 74 | MountPath: opt.MountPath, 75 | CgoEnabled: opt.CgoEnabled, 76 | BuildTags: opt.BuildTags, 77 | GOARCH: opt.GOARCH, 78 | GOOS: opt.GOOS, 79 | GOPATH: opt.GOPATH, 80 | }) 81 | 82 | st, err := l.BuildExe(opt.Pkg) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | dt, err := st.Marshal() 88 | if err != nil { 89 | return err 90 | } 91 | 92 | def.Def = append(def.Def, dt.Def...) 93 | 94 | // loadLLB(def) 95 | 96 | return llb.WriteTo(def, out) 97 | } 98 | 99 | type llbOp struct { 100 | Op pb.Op 101 | Digest digest.Digest 102 | } 103 | 104 | func loadLLB(bs [][]byte) error { 105 | var ops []llbOp 106 | for _, dt := range bs { 107 | var op pb.Op 108 | if err := (&op).Unmarshal(dt); err != nil { 109 | return err 110 | } 111 | dgst := digest.FromBytes(dt) 112 | ops = append(ops, llbOp{Op: op, Digest: dgst}) 113 | } 114 | enc := json.NewEncoder(os.Stdout) 115 | for _, op := range ops { 116 | if err := enc.Encode(op); err != nil { 117 | return err 118 | } 119 | } 120 | return nil 121 | } 122 | 123 | type output struct { 124 | dgst digest.Digest 125 | index int 126 | } 127 | 128 | func (o *output) ToInput(*llb.Constraints) (*pb.Input, error) { 129 | return &pb.Input{Digest: digest.Digest(o.dgst), Index: pb.OutputIndex(o.index)}, nil 130 | } 131 | 132 | func (o *output) Vertex() llb.Vertex { 133 | return &emptyVertex{} 134 | } 135 | 136 | type emptyVertex struct{} 137 | 138 | func (v *emptyVertex) Validate() error { 139 | return nil 140 | } 141 | func (v *emptyVertex) Marshal(*llb.Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) { 142 | return "", nil, nil, nil 143 | } 144 | func (v *emptyVertex) Output() llb.Output { 145 | return nil 146 | } 147 | func (v *emptyVertex) Inputs() []llb.Output { 148 | return nil 149 | } 150 | -------------------------------------------------------------------------------- /gobuild.go: -------------------------------------------------------------------------------- 1 | package gobuild 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | 7 | "github.com/moby/buildkit/client/llb" 8 | "github.com/moby/buildkit/client/llb/llbbuild" 9 | "github.com/moby/buildkit/solver/pb" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | type Opt struct { 14 | DevMode bool 15 | } 16 | 17 | func New(opt *Opt) *GoBuilder { 18 | devMode := false 19 | if opt != nil && opt.DevMode { 20 | devMode = true 21 | } 22 | return &GoBuilder{DevMode: devMode} 23 | } 24 | 25 | type BuildOpt struct { 26 | Source llb.State 27 | MountPath string 28 | Pkg string 29 | CgoEnabled bool 30 | BuildTags []string 31 | GOARCH string 32 | GOOS string 33 | } 34 | 35 | type BuildOptJSON struct { 36 | Source string 37 | SourceIndex int 38 | SourceDef []byte 39 | MountPath string 40 | Pkg string 41 | CgoEnabled bool 42 | BuildTags []string 43 | GOARCH string 44 | GOOS string 45 | GOPATH string 46 | } 47 | 48 | type GoBuilder struct { 49 | DevMode bool 50 | } 51 | 52 | func (gb *GoBuilder) BuildExe(opt BuildOpt) (*llb.State, error) { 53 | def, err := opt.Source.Marshal() 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | buf := &bytes.Buffer{} 59 | if err := llb.WriteTo(def, buf); err != nil { 60 | return nil, err 61 | } 62 | var op pb.Op 63 | if err := (&op).Unmarshal(def.Def[len(def.Def)-1]); err != nil { 64 | return nil, errors.Wrap(err, "failed to parse llb proto op") 65 | } 66 | if len(op.Inputs) == 0 { 67 | return nil, errors.Errorf("invalid source state") 68 | } 69 | 70 | dt, err := json.Marshal(BuildOptJSON{ 71 | Source: op.Inputs[0].Digest.String(), 72 | SourceIndex: int(op.Inputs[0].Index), 73 | SourceDef: buf.Bytes(), 74 | MountPath: opt.MountPath, 75 | Pkg: opt.Pkg, 76 | CgoEnabled: opt.CgoEnabled, 77 | BuildTags: opt.BuildTags, 78 | GOARCH: opt.GOARCH, 79 | GOOS: opt.GOOS, 80 | }) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | goBuild := llb.Image("docker.io/tonistiigi/llb-gobuild@sha256:511744f1570cfc9c88e67fb88181986f19841c6880aafa6fe8d5a3f6e7144f61") 86 | if gb.DevMode { 87 | goBuild = gobuildDev() 88 | } 89 | 90 | run := goBuild.Run(llb.Shlexf("gobuild %s", opt.Pkg), llb.AddEnv("GOOPT", string(dt))) 91 | run.AddMount(opt.MountPath, opt.Source, llb.Readonly) 92 | out := run.AddMount("/out", llb.Scratch()).With(llbbuild.Build()) 93 | return &out, nil 94 | } 95 | 96 | func gobuildDev() llb.State { 97 | gobDev := llb.Local("gobuild-dev") 98 | build := goBuildBase(). 99 | Run(llb.Shlex("apk add --no-cache git")). 100 | Dir("/src"). 101 | Run(llb.Shlex("go build -o /out/gobuild ./cmd/gobuild")) 102 | 103 | build.AddMount("/src", gobDev, llb.Readonly) 104 | build.AddMount("/go/pkg/mod", llb.Scratch(), llb.AsPersistentCacheDir("gocache", llb.CacheMountShared)) 105 | 106 | out := build.AddMount("/out", llb.Scratch()) 107 | 108 | alpine := llb.Image("docker.io/library/alpine:latest") 109 | return copy(out, "/gobuild", alpine, "/bin") 110 | } 111 | 112 | func goBuildBase() llb.State { 113 | goAlpine := llb.Image("docker.io/library/golang:1.11-alpine@sha256:31389db6001c5222bef9817a04ae8c8401ae8bed6fb965aac17b0e742f4c3e5e") 114 | return goAlpine. 115 | AddEnv("CGO_ENABLED", "0"). 116 | AddEnv("GOPATH", "/go"). 117 | AddEnv("PATH", "/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") //. 118 | //AddEnv("GOPATH", "/go").Run(llb.Shlex("apk add --no-cache gcc libc-dev")). 119 | // Root() 120 | } 121 | 122 | // copy copies files between 2 states using cp until there is no copyOp 123 | func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.State { 124 | cpImage := llb.Image("docker.io/library/alpine:latest") 125 | cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath)) 126 | cp.AddMount("/src", src, llb.Readonly) 127 | return cp.AddMount("/dest", dest) 128 | } 129 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= 3 | github.com/Microsoft/hcsshim v0.8.5/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= 4 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 5 | github.com/containerd/containerd v1.2.2 h1:N3tAHxrX+byqfAsENdDWLSMtFD4thUxK7kFElUl+8z8= 6 | github.com/containerd/containerd v1.2.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= 7 | github.com/containerd/continuity v0.0.0-20181001140422-bd77b46c8352/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= 8 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 9 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= 11 | github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 12 | github.com/docker/docker v0.7.3-0.20180531152204-71cd53e4a197/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 13 | github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 14 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 15 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 16 | github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= 17 | github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 18 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 19 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 20 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 21 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 22 | github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= 23 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 24 | github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg= 25 | github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= 26 | github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= 27 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 28 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 29 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 30 | github.com/moby/buildkit v0.3.3 h1:7eh9tOdFSuE84Q5wvmUjXhEvqnO7nNiwja45Hr59+uc= 31 | github.com/moby/buildkit v0.3.3/go.mod h1:nnELdKPRkUAQR6pAB3mRU3+IlbqL3SSaAWqQL8k/K+4= 32 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 33 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 34 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 35 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 36 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 37 | github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= 38 | github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= 39 | github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= 40 | github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= 41 | github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= 42 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 43 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 44 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 45 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 46 | github.com/sirupsen/logrus v1.0.3 h1:B5C/igNWoiULof20pKfY4VntcIPqKuwEmoLZrabbUrc= 47 | github.com/sirupsen/logrus v1.0.3/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= 48 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 49 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 50 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 51 | github.com/tonistiigi/fsutil v0.0.0-20190130224639-b4281fa67095 h1:b9hxvP2uTwPNrzo4q77+g25UgefFB3+f7i5IpRvlzDs= 52 | github.com/tonistiigi/fsutil v0.0.0-20190130224639-b4281fa67095/go.mod h1:UYukpgbLYvlEDwyQ/SqtLJ9QlgYORCPEwDln/Noj7xc= 53 | golang.org/x/crypto v0.0.0-20190129210102-ccddf3741a0c h1:MWY7h75sb9ioBR+s5Zgq1JYXxhbZvrSP2okwLi3ItmI= 54 | golang.org/x/crypto v0.0.0-20190129210102-ccddf3741a0c/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 55 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 56 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 57 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= 58 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 59 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 60 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= 61 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 62 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 63 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 64 | golang.org/x/sys v0.0.0-20190201152629-afcc84fd7533 h1:bLfqnzrpeG4usq5OvMCrwTdmMJ6aTmlCuo1eKl0mhkI= 65 | golang.org/x/sys v0.0.0-20190201152629-afcc84fd7533/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 66 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 67 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 68 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 69 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 70 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= 71 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 72 | google.golang.org/grpc v1.18.0 h1:IZl7mfBGfbhYx2p2rKRtYgDFw6SBz+kclmxYrCksPPA= 73 | google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 74 | gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= 75 | gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= 76 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 77 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 78 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 79 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 80 | gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= 81 | gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= 82 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 83 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 84 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 85 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 86 | gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= 87 | gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= 88 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 89 | -------------------------------------------------------------------------------- /loader/loader.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | import ( 4 | "fmt" 5 | "go/build" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "runtime" 10 | "sort" 11 | "strings" 12 | 13 | "github.com/moby/buildkit/client/llb" 14 | ) 15 | 16 | type GoOpt struct { 17 | Source llb.State 18 | MountPath string 19 | CgoEnabled bool 20 | BuildTags []string 21 | GOARCH string 22 | GOOS string 23 | GOPATH string 24 | } 25 | 26 | func New(opt GoOpt) *Loader { 27 | bctx := build.Context{ 28 | GOARCH: opt.GOARCH, 29 | GOOS: opt.GOOS, 30 | // GOROOT: runtime.GOROOT(), 31 | GOPATH: opt.GOPATH, 32 | Compiler: "gc", 33 | CgoEnabled: opt.CgoEnabled, 34 | BuildTags: opt.BuildTags, 35 | ReleaseTags: []string{"go1.1", "go1.2", "go1.3", "go1.4", "go1.5", "go1.6", "go1.7", "go1.8", "go1.9"}, 36 | } 37 | 38 | return &Loader{ 39 | v: newVendorDirs(opt.GOPATH), 40 | opt: opt, 41 | cache: map[string]*pkg{}, 42 | bctx: bctx, 43 | local: opt.Source, 44 | base: goBuildBase(), 45 | cgoBase: cgoBuildBase(), 46 | wd: opt.MountPath, 47 | } 48 | } 49 | 50 | type Loader struct { 51 | opt GoOpt 52 | v *vendorDirs 53 | cache map[string]*pkg 54 | bctx build.Context 55 | local llb.State 56 | base llb.State 57 | cgoBase llb.State 58 | wd string 59 | } 60 | 61 | func (l *Loader) BuildExe(pkg string) (*llb.State, error) { 62 | p, err := l.loadDir(filepath.Join(l.opt.GOPATH, "src", pkg)) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | cmd := llb.Shlexf("/usr/local/go/pkg/tool/linux_amd64/link -o /work/%s/_obj/exe/%s -linkmode=internal -L /work -extld=gcc -extldflags \"-static\" -buildmode=exe /work/%s.a", p.p.ImportPath, path.Base(p.p.ImportPath), p.p.ImportPath) 68 | 69 | st3 := l.base.Run(cmd) 70 | st3.AddMount(fmt.Sprintf("/work/%s.a", p.p.ImportPath), p.state, llb.SourcePath(path.Base(p.p.ImportPath)+".a"), llb.Readonly) 71 | out := st3.AddMount(fmt.Sprintf("/work/%s/_obj/exe", p.p.ImportPath), llb.Scratch()) 72 | 73 | for s, d := range p.alldeps { 74 | st3.AddMount(fmt.Sprintf("/work/%s.a", s), d, llb.SourcePath(path.Base(s)+".a"), llb.Readonly) 75 | } 76 | 77 | return &out, nil 78 | } 79 | 80 | func (l *Loader) loadDir(dir string) (*pkg, error) { 81 | if p, ok := l.cache[dir]; ok { 82 | return p, nil 83 | } 84 | p, err := l.bctx.ImportDir(dir, 0) 85 | if err != nil { 86 | return nil, err 87 | } 88 | l.v.add(dir) 89 | 90 | vendorIndex := strings.Index(p.ImportPath, "/vendor/") 91 | if vendorIndex != -1 { 92 | p.ImportPath = p.ImportPath[vendorIndex+8:] 93 | } 94 | 95 | pkg := &pkg{p: p, alldeps: map[string]llb.State{}, cgo: len(p.CgoFiles) > 0} 96 | 97 | for _, imp := range p.Imports { 98 | for _, vd := range l.v.dirs { 99 | if strings.HasPrefix(dir, filepath.Dir(vd)) { 100 | d := filepath.Join(vd, imp) 101 | fi, err := os.Stat(d) 102 | if err == nil && fi.IsDir() { 103 | p, err := l.loadDir(d) 104 | if err != nil { 105 | return nil, err 106 | } 107 | if p.cgo { 108 | pkg.cgo = true 109 | } 110 | pkg.AddDep(imp, p) 111 | break 112 | } 113 | } 114 | } 115 | } 116 | 117 | name := p.ImportPath 118 | if p.Name == "main" { 119 | name = p.Name 120 | } 121 | 122 | cmd := llb.Shlexf("/usr/local/go/pkg/tool/linux_amd64/compile -trimpath /work -o /work/%s.a -p %s -complete -I /work -pack %s", p.ImportPath, name, strings.Join(p.GoFiles, " ")) 123 | 124 | var callo llb.State 125 | var cgoimport llb.State 126 | var cgop llb.State 127 | var gofiles []string 128 | if len(p.CgoFiles) > 0 { 129 | cmd = llb.Shlexf("/usr/local/go/pkg/tool/linux_amd64/cgo --objdir /work/%s/_obj/ -- -I /work/%s/_obj/ -g -O2 %s %s", p.ImportPath, p.ImportPath, strings.Join(p.CgoCFLAGS, " "), strings.Join(p.CgoFiles, " ")) 130 | st := l.cgoBase.Run(cmd, llb.Dir("/root")) 131 | for _, f := range p.CgoFiles { 132 | st.AddMount(path.Join("/root", f), l.local, llb.SourcePath(strings.TrimPrefix(path.Join(dir, f), l.wd)), llb.Readonly) 133 | } 134 | cgop = st.AddMount(path.Join("/work", p.ImportPath, "_obj"), llb.Scratch()) 135 | 136 | cflags := []string{"_cgo_export.c", "_cgo_main.c"} 137 | for _, f := range p.CgoFiles { 138 | cflags = append(cflags, strings.TrimSuffix(f, ".go")+".cgo2.c") 139 | } 140 | 141 | type ob struct { 142 | st llb.State 143 | f string 144 | } 145 | 146 | cobj := []ob{} 147 | 148 | for _, f := range cflags { 149 | cmd = llb.Shlexf("gcc -I . -fPIC -m64 -pthread -gno-record-gcc-switches -fmessage-length=0 -I /work/%s/_obj/ -g -O2 %s -o /out/%s -c /work/%s/_obj/%s", p.ImportPath, strings.Join(p.CgoCFLAGS, " "), strings.TrimSuffix(f, ".c")+".o", p.ImportPath, f) // TODO: non-amd64 150 | st := l.cgoBase.Run(cmd, llb.Dir("/root")) 151 | st.AddMount(path.Join("/work", p.ImportPath, "_obj"), cgop, llb.Readonly) 152 | st.AddMount("/root", l.local, llb.SourcePath(strings.TrimPrefix(dir, l.wd)), llb.Readonly) 153 | cobj = append(cobj, ob{st: st.AddMount("/out", llb.Scratch()), f: f}) 154 | } 155 | 156 | for _, f := range p.CFiles { 157 | cmd = llb.Shlexf("gcc -I . -fPIC -m64 -pthread -gno-record-gcc-switches -fmessage-length=0 -I /work/%s/_obj/ -g -O2 %s -o /out/%s -c /root/%s", p.ImportPath, strings.Join(p.CgoCFLAGS, " "), strings.TrimSuffix(f, ".c")+".o", f) 158 | st := l.cgoBase.Run(cmd, llb.Dir("/root")) 159 | st.AddMount("/root", l.local, llb.SourcePath(strings.TrimPrefix(dir, l.wd)), llb.Readonly) 160 | cobj = append(cobj, ob{st: st.AddMount("/out", llb.Scratch()), f: f}) 161 | } 162 | 163 | workFiles := make([]string, 0, len(cobj)) 164 | for _, f := range cobj { 165 | workFiles = append(workFiles, path.Join("/root", strings.TrimSuffix(f.f, ".c")+".o")) 166 | } 167 | cmd = llb.Shlexf("gcc -fPIC -m64 -pthread -fmessage-length=0 -gno-record-gcc-switches -o /out/_cgo_.o %s -g -O2", strings.Join(workFiles, " ")) 168 | st = l.cgoBase.Run(cmd) 169 | for _, f := range cobj { 170 | st.AddMount(path.Join("/root", ext(f.f, "c", "o")), f.st, llb.SourcePath(ext(f.f, "c", "o")), llb.Readonly) 171 | } 172 | out := st.AddMount("/out", llb.Scratch()) 173 | 174 | cmd = llb.Shlexf("/usr/local/go/pkg/tool/linux_amd64/cgo -dynpackage %s -dynimport /root/_cgo_.o -dynout /out/_cgo_import.go", p.Name) 175 | st = l.cgoBase.Run(cmd) 176 | cgoimport = st.AddMount("/out", llb.Scratch()) 177 | st.AddMount("/root/_cgo_.o", out, llb.SourcePath("_cgo_.o"), llb.Readonly) 178 | 179 | workFiles = make([]string, 0, len(cobj)) 180 | for _, f := range cobj { 181 | if f.f != "_cgo_main.c" { 182 | workFiles = append(workFiles, path.Join("/root", ext(f.f, "c", "o"))) 183 | } 184 | } 185 | 186 | cmd = llb.Shlexf("gcc -fPIC -m64 -pthread -fmessage-length=0 -gno-record-gcc-switches -o /out/_all.o %s -g -O2 -Wl,-r -nostdlib -no-pie -Wl,--build-id=none", strings.Join(workFiles, " ")) 187 | st = l.cgoBase.Run(cmd) 188 | callo = st.AddMount("/out", llb.Scratch()) 189 | for _, f := range cobj { 190 | if f.f != "_cgo_main.c" { 191 | st.AddMount(path.Join("/root", ext(f.f, "c", "o")), f.st, llb.SourcePath(ext(f.f, "c", "o")), llb.Readonly) 192 | } 193 | } 194 | 195 | gofiles = make([]string, 0) 196 | for _, f := range p.CgoFiles { 197 | gofiles = append(gofiles, strings.TrimSuffix(f, ".go")+".cgo1.go") 198 | } 199 | gofiles = append(gofiles, "_cgo_import.go", "_cgo_gotypes.go") 200 | 201 | cmd = llb.Shlexf("/usr/local/go/pkg/tool/linux_amd64/compile -o /work/%s.a -p %s -trimpath /work -I /work -pack %s %s", p.ImportPath, name, strings.Join(p.GoFiles, " "), strings.Join(gofiles, " ")) 202 | 203 | } 204 | 205 | if len(p.SFiles) > 0 { 206 | cmd = llb.Shlexf("/usr/local/go/pkg/tool/linux_amd64/compile -o /work/%s.a -trimpath /work -p %s -I /work -pack -asmhdr /work/%s/_obj/go_asm.h %s ", p.ImportPath, name, p.ImportPath, strings.Join(p.GoFiles, " ")) 207 | 208 | } 209 | 210 | st := l.base.Run(cmd, llb.Dir("/root")) 211 | 212 | for _, mount := range pkg.mounts { 213 | st.AddMount(path.Join("/work", mount.target, mount.selector+".a"), mount.state, llb.SourcePath(mount.selector+".a"), llb.Readonly) 214 | } 215 | 216 | for _, f := range p.GoFiles { 217 | st.AddMount(path.Join("/root", f), l.local, llb.SourcePath(strings.TrimPrefix(path.Join(dir, f), l.wd)), llb.Readonly) 218 | } 219 | 220 | if len(p.CgoFiles) > 0 { 221 | for _, f := range p.CgoFiles { 222 | f = strings.TrimSuffix(f, ".go") + ".cgo1.go" 223 | st.AddMount(path.Join("/root", f), cgop, llb.SourcePath(f), llb.Readonly) 224 | } 225 | st.AddMount(path.Join("/root/_cgo_import.go"), cgoimport, llb.SourcePath("_cgo_import.go"), llb.Readonly) 226 | st.AddMount(path.Join("/root/_cgo_gotypes.go"), cgop, llb.SourcePath("_cgo_gotypes.go"), llb.Readonly) 227 | } 228 | 229 | var asmheader llb.State 230 | if len(p.SFiles) > 0 { 231 | asmheader = st.AddMount(path.Join("/work", p.ImportPath, "_obj"), llb.Scratch()) 232 | } 233 | 234 | out := st.AddMount(path.Join("/work", path.Dir(p.ImportPath)), llb.Scratch()) 235 | 236 | if len(p.SFiles) > 0 { 237 | cmd := llb.Shlexf("/usr/local/go/pkg/tool/linux_amd64/asm -I /work/%s/_obj/ -I /usr/local/go/pkg/include -D GOOS_%s -D GOARCH_%s -o /work/%s/_obj/asm_%s_%s.o -trimpath /work %s", p.ImportPath, runtime.GOOS, runtime.GOARCH, p.ImportPath, runtime.GOOS, runtime.GOARCH, strings.Join(p.SFiles, " ")) 238 | st := l.base.Run(cmd, llb.Dir("/root")) 239 | for _, f := range p.SFiles { 240 | st.AddMount(path.Join("/root", f), l.local, llb.SourcePath(strings.TrimPrefix(path.Join(dir, f), l.wd)), llb.Readonly) 241 | } 242 | for _, f := range p.HFiles { 243 | st.AddMount(path.Join("/root", f), l.local, llb.SourcePath(strings.TrimPrefix(path.Join(dir, f), l.wd)), llb.Readonly) 244 | } 245 | st.AddMount(path.Join("/work", p.ImportPath, "_obj/go_asm.h"), asmheader, llb.SourcePath("go_asm.h"), llb.Readonly) 246 | asmp := st.AddMount(path.Join("/work", p.ImportPath, "_obj"), llb.Scratch()) 247 | 248 | cmd = llb.Shlexf("/usr/local/go/pkg/tool/linux_amd64/pack r /work/%s.a /work/%s/_obj/asm_%s_%s.o", p.ImportPath, p.ImportPath, runtime.GOOS, runtime.GOARCH) 249 | st2 := l.base.Run(cmd) 250 | 251 | out = st2.AddMount(path.Join("/work", p.ImportPath+".a"), out, llb.SourcePath(path.Base(p.ImportPath)+".a")) 252 | st2.AddMount(path.Join("/work", p.ImportPath, fmt.Sprintf("_obj/asm_%s_%s.o", runtime.GOOS, runtime.GOARCH)), asmp, llb.SourcePath(fmt.Sprintf("asm_%s_%s.o", runtime.GOOS, runtime.GOARCH)), llb.Readonly) 253 | 254 | } 255 | 256 | if len(p.CgoFiles) > 0 { 257 | cmd = llb.Shlexf("/usr/local/go/pkg/tool/linux_amd64/pack r /work/%s.a /work/_all.o", p.ImportPath) 258 | st2 := l.base.Run(cmd) 259 | 260 | out = st2.AddMount(path.Join("/work", p.ImportPath+".a"), out, llb.SourcePath(path.Base(p.ImportPath)+".a")) 261 | st2.AddMount("/work/_all.o", callo, llb.SourcePath("_all.o"), llb.Readonly) 262 | } 263 | 264 | pkg.state = out 265 | 266 | l.cache[dir] = pkg 267 | 268 | return pkg, err 269 | } 270 | 271 | func ext(fn, old, nw string) string { 272 | return strings.TrimSuffix(fn, "."+old) + "." + nw 273 | } 274 | 275 | type pkg struct { 276 | p *build.Package 277 | mounts []*mount 278 | state llb.State 279 | alldeps map[string]llb.State 280 | cgo bool 281 | } 282 | 283 | type mount struct { 284 | target string 285 | selector string 286 | state llb.State 287 | } 288 | 289 | func (p *pkg) AddDep(name string, dep *pkg) { 290 | p.mounts = append(p.mounts, &mount{ 291 | target: path.Dir(name), 292 | selector: path.Base(name), 293 | state: dep.state, 294 | }) 295 | for str, d := range dep.alldeps { 296 | p.alldeps[str] = d 297 | } 298 | p.alldeps[name] = dep.state 299 | } 300 | 301 | type vendorDirs struct { 302 | checkedDirs map[string]struct{} 303 | dirs []string 304 | gopath string 305 | } 306 | 307 | func newVendorDirs(gopath string) *vendorDirs { 308 | return &vendorDirs{ 309 | checkedDirs: map[string]struct{}{}, 310 | gopath: gopath, 311 | dirs: []string{filepath.Join(gopath, "src")}, 312 | } 313 | } 314 | 315 | func (v *vendorDirs) add(d string) { 316 | if v.gopath == d { 317 | return 318 | } 319 | if _, ok := v.checkedDirs[d]; ok { 320 | return 321 | } 322 | v.checkedDirs[d] = struct{}{} 323 | vd := filepath.Join(d, "vendor") 324 | fi, err := os.Stat(vd) 325 | if err == nil { 326 | if fi.IsDir() { 327 | v.dirs = append(v.dirs, vd) 328 | sort.Slice(v.dirs, func(a, b int) bool { 329 | return v.dirs[a] > v.dirs[b] 330 | }) 331 | } 332 | } 333 | v.add(filepath.Dir(d)) 334 | } 335 | 336 | func goBuildBase() llb.State { 337 | goAlpine := llb.Image("docker.io/library/golang:1.11-alpine") 338 | return goAlpine. 339 | AddEnv("CGO_ENABLED", "0"). 340 | AddEnv("PATH", "/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"). //. 341 | AddEnv("GOPATH", "/go") 342 | } 343 | 344 | func cgoBuildBase() llb.State { 345 | return goBuildBase().Run(llb.Shlex("apk add --no-cache linux-headers gcc libc-dev")).Root() 346 | } 347 | --------------------------------------------------------------------------------