├── .gitignore ├── docs ├── templates │ ├── .gitignore │ ├── package.json │ ├── src │ │ └── compile.js │ ├── reference │ │ └── reference.md │ └── package-lock.json ├── index.md ├── intro │ └── quickstart.md ├── specification.md └── tutorial │ ├── lets-begin.md │ └── arguments.md ├── version.go ├── codegen ├── debugger_test.go ├── gateway_exec_windows.go ├── builtin_pipeline.go ├── error_handler.go ├── gateway_exec_unix.go ├── resolver.go ├── debug │ └── linespec.go └── builtin_string.go ├── pkg ├── stargzutil │ ├── fixtures │ │ ├── alpine_stargz_desc.json │ │ ├── alpine_desc.json │ │ ├── alpine_index_desc.json │ │ ├── alpine.json │ │ ├── alpine_stargz.json │ │ └── alpine_index.json │ ├── stargzutil.go │ └── stargzutil_test.go ├── llbutil │ ├── sourcemap.go │ ├── file_info.go │ ├── secret.go │ ├── secret_test.go │ ├── id.go │ ├── session.go │ └── readonly_mountpoints.go ├── sockproxy │ └── run.go ├── steer │ ├── input_steerer_test.go │ └── input_steerer.go ├── gitscheme │ ├── parse.go │ └── parse_test.go └── filebuffer │ └── filebuffer.go ├── cmd ├── hlb │ ├── main.go │ └── command │ │ ├── version.go │ │ ├── context.go │ │ ├── langserver.go │ │ ├── format.go │ │ ├── app.go │ │ └── lint.go ├── builtingen │ └── main.go └── docgen │ └── main.go ├── .golangci.yml ├── diagnostic ├── context.go ├── levenshtein.go └── error.go ├── solver ├── context.go ├── client.go ├── progress_test.go ├── multiwriter.go ├── directory.go ├── request.go └── tree.go ├── language ├── hlb.tmbundle │ └── Info.plist ├── hlb-pygments.py ├── hlb-rouge.rb └── hlb-sublime3.yaml ├── parser ├── ast │ ├── kind_set.go │ ├── context.go │ ├── deprecated.go │ ├── scope.go │ ├── match_test.go │ ├── match.go │ └── walk_test.go ├── docstrings.go ├── directory.go ├── util.go ├── parse.go └── parse_test.go ├── examples ├── node.hlb └── forward.hlb ├── rpc ├── dapserver │ ├── common.go │ ├── handles.go │ └── server.go └── langserver │ └── scope.go ├── mkdocs.yml ├── checker └── builtin.go ├── builtin ├── builtin.go └── gen │ ├── builtins.go │ └── documentation.go ├── module ├── tree.go ├── vendor.go └── resolve_test.go ├── LANGSERVER.md ├── README.md ├── linter ├── linter.go └── linter_test.go ├── go.hlb ├── hlb.go ├── client.go ├── .github └── workflows │ └── test.yml ├── local └── environment.go ├── scripts └── mkBuildkitdDroplet.sh └── mkdocs.hlb /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | build/ 3 | .idea/ 4 | -------------------------------------------------------------------------------- /docs/templates/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package hlb 2 | 3 | var Version = "0.3+unknown" 4 | -------------------------------------------------------------------------------- /codegen/debugger_test.go: -------------------------------------------------------------------------------- 1 | package codegen 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestDebugger(t *testing.T) { 8 | SubtestDebuggerSuite(t, func() Debugger { 9 | return NewDebugger(nil) 10 | }) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/stargzutil/fixtures/alpine_stargz_desc.json: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 3 | "digest": "sha256:4382407e6f4fab29345722ba819c33f9d158b1bce240839e889d3ff715f0ad93", 4 | "size": 739 5 | } 6 | -------------------------------------------------------------------------------- /pkg/stargzutil/fixtures/alpine_desc.json: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 3 | "digest": "sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3", 4 | "size": 528 5 | } 6 | -------------------------------------------------------------------------------- /pkg/stargzutil/fixtures/alpine_index_desc.json: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", 3 | "digest": "sha256:21a3deaa0d32a8057914f36584b5288d2e5ecc984380bc0118285c70fa8c9300", 4 | "size": 1638 5 | } 6 | -------------------------------------------------------------------------------- /cmd/hlb/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/openllb/hlb/cmd/hlb/command" 8 | ) 9 | 10 | func main() { 11 | app := command.App() 12 | if err := app.Run(os.Args); err != nil { 13 | fmt.Fprintf(os.Stderr, "%s\n", err) 14 | os.Exit(1) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /codegen/gateway_exec_windows.go: -------------------------------------------------------------------------------- 1 | package codegen 2 | 3 | import ( 4 | "context" 5 | 6 | gateway "github.com/moby/buildkit/frontend/gateway/client" 7 | ) 8 | 9 | func addResizeHandler(ctx context.Context, proc gateway.ContainerProcess) func() { 10 | // not implemented on windows 11 | return func() {} 12 | } 13 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | enable-all: false 3 | disable-all: true 4 | enable: 5 | - gofmt 6 | - goimports 7 | - gosimple 8 | - ineffassign 9 | - misspell 10 | - exportloopref 11 | - typecheck 12 | - unconvert 13 | - unused 14 | 15 | issues: 16 | exclude-use-default: false 17 | -------------------------------------------------------------------------------- /docs/templates/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "templates", 3 | "version": "1.0.0", 4 | "description": "", 5 | "scripts": { 6 | "compile": "node src/compile.js" 7 | }, 8 | "keywords": [], 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "handlebars": "^4.7.2" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /pkg/llbutil/sourcemap.go: -------------------------------------------------------------------------------- 1 | package llbutil 2 | 3 | import ( 4 | "github.com/alecthomas/participle/v2/lexer" 5 | "github.com/moby/buildkit/solver/pb" 6 | ) 7 | 8 | func PositionFromLexer(pos lexer.Position) pb.Position { 9 | return pb.Position{ 10 | Line: int32(pos.Line), 11 | Character: int32(pos.Column), 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /cmd/hlb/command/version.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/openllb/hlb" 7 | cli "github.com/urfave/cli/v2" 8 | ) 9 | 10 | var versionCommand = &cli.Command{ 11 | Name: "version", 12 | Usage: "prints hlb tool version", 13 | Action: func(c *cli.Context) error { 14 | fmt.Println(hlb.Version) 15 | return nil 16 | }, 17 | } 18 | -------------------------------------------------------------------------------- /cmd/hlb/command/context.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/logrusorgru/aurora" 8 | isatty "github.com/mattn/go-isatty" 9 | "github.com/moby/buildkit/util/appcontext" 10 | "github.com/openllb/hlb/diagnostic" 11 | ) 12 | 13 | func Context() context.Context { 14 | ctx := appcontext.Context() 15 | if isatty.IsTerminal(os.Stderr.Fd()) { 16 | ctx = diagnostic.WithColor(ctx, aurora.NewAurora(true)) 17 | } 18 | return ctx 19 | } 20 | -------------------------------------------------------------------------------- /diagnostic/context.go: -------------------------------------------------------------------------------- 1 | package diagnostic 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/logrusorgru/aurora" 7 | ) 8 | 9 | type colorKey struct{} 10 | 11 | func WithColor(ctx context.Context, color aurora.Aurora) context.Context { 12 | return context.WithValue(ctx, colorKey{}, color) 13 | } 14 | 15 | func Color(ctx context.Context) aurora.Aurora { 16 | color, ok := ctx.Value(colorKey{}).(aurora.Aurora) 17 | if !ok { 18 | return aurora.NewAurora(false) 19 | } 20 | return color 21 | } 22 | -------------------------------------------------------------------------------- /solver/context.go: -------------------------------------------------------------------------------- 1 | package solver 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/sync/semaphore" 7 | ) 8 | 9 | type concurrencyLimiterKey struct{} 10 | 11 | func WithConcurrencyLimiter(ctx context.Context, limiter *semaphore.Weighted) context.Context { 12 | return context.WithValue(ctx, concurrencyLimiterKey{}, limiter) 13 | } 14 | 15 | func ConcurrencyLimiter(ctx context.Context) *semaphore.Weighted { 16 | limiter, _ := ctx.Value(concurrencyLimiterKey{}).(*semaphore.Weighted) 17 | return limiter 18 | } 19 | -------------------------------------------------------------------------------- /solver/client.go: -------------------------------------------------------------------------------- 1 | package solver 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/moby/buildkit/client" 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | // BuildkitClient returns a basic buildkit client. 11 | func BuildkitClient(ctx context.Context, addr string) (*client.Client, error) { 12 | opts := []client.ClientOpt{} 13 | cln, err := client.New(ctx, addr, opts...) 14 | if err != nil { 15 | return cln, err 16 | } 17 | _, err = cln.ListWorkers(ctx) 18 | return cln, errors.Wrap(err, "unable to connect to buildkitd") 19 | } 20 | -------------------------------------------------------------------------------- /pkg/sockproxy/run.go: -------------------------------------------------------------------------------- 1 | package sockproxy 2 | 3 | import ( 4 | "io" 5 | "net" 6 | ) 7 | 8 | func Run(l net.Listener, dialer func() (net.Conn, error)) error { 9 | for { 10 | proxy, err := l.Accept() 11 | if err != nil { 12 | return err 13 | } 14 | conn, err := dialer() 15 | if err != nil { 16 | return err 17 | } 18 | 19 | go func() { 20 | defer proxy.Close() 21 | _, _ = io.Copy(conn, proxy) 22 | }() 23 | 24 | go func() { 25 | defer conn.Close() 26 | _, _ = io.Copy(proxy, conn) 27 | }() 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /docs/templates/src/compile.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const Handlebars = require('handlebars'); 3 | 4 | Handlebars.registerHelper('eq', function(a, b) { 5 | return a == b; 6 | }); 7 | 8 | var source = fs.readFileSync('./reference/reference.md', 'utf8'); 9 | var template = Handlebars.compile(source); 10 | 11 | var context = JSON.parse(fs.readFileSync('./data/reference.json', 'utf8')); 12 | var md = template(context); 13 | 14 | 15 | if (!fs.existsSync('./dist')){ 16 | fs.mkdirSync('./dist'); 17 | } 18 | fs.writeFileSync('./dist/reference.md', md); 19 | -------------------------------------------------------------------------------- /language/hlb.tmbundle/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | contactName 6 | HLB authors 7 | description 8 | Language support for HLB files. 9 | name 10 | hlb 11 | uuid 12 | 88c38584-8b5f-45be-93a6-e2c9da5b6e3f 13 | 14 | 15 | -------------------------------------------------------------------------------- /codegen/builtin_pipeline.go: -------------------------------------------------------------------------------- 1 | package codegen 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/moby/buildkit/client" 7 | "github.com/openllb/hlb/solver" 8 | ) 9 | 10 | type Stage struct{} 11 | 12 | func (s Stage) Call(ctx context.Context, cln *client.Client, val Value, opts Option, requests ...solver.Request) (Value, error) { 13 | if len(requests) == 0 { 14 | return val, nil 15 | } 16 | 17 | current, err := val.Request() 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | next := solver.Parallel(requests...) 23 | return NewValue(ctx, solver.Sequential(current, next)) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/stargzutil/fixtures/alpine.json: -------------------------------------------------------------------------------- 1 | { 2 | "schemaVersion": 2, 3 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 4 | "config": { 5 | "mediaType": "application/vnd.docker.container.image.v1+json", 6 | "size": 1471, 7 | "digest": "sha256:c059bfaa849c4d8e4aecaeb3a10c2d9b3d85f5165c66ad3a4d937758128c4d18" 8 | }, 9 | "layers": [ 10 | { 11 | "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", 12 | "size": 2818413, 13 | "digest": "sha256:59bf1c3509f33515622619af21ed55bbe26d24913cedbca106468a5fb37a50c3" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /cmd/builtingen/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | 10 | "github.com/openllb/hlb/builtin/gen" 11 | ) 12 | 13 | func main() { 14 | if len(os.Args) != 3 { 15 | log.Fatal("builtingen: must have exactly 2 args") 16 | } 17 | 18 | err := run(os.Args[1], os.Args[2]) 19 | if err != nil { 20 | fmt.Fprintf(os.Stderr, "builtingen: %s\n", err) 21 | os.Exit(1) 22 | } 23 | } 24 | 25 | func run(src, dest string) error { 26 | f, err := os.Open(src) 27 | if err != nil { 28 | return err 29 | } 30 | defer f.Close() 31 | 32 | dt, err := gen.GenerateBuiltins(context.Background(), f) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | return ioutil.WriteFile(dest, dt, 0644) 38 | } 39 | -------------------------------------------------------------------------------- /parser/ast/kind_set.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "sort" 5 | ) 6 | 7 | type KindSet struct { 8 | set map[Kind]struct{} 9 | } 10 | 11 | func NewKindSet(kinds ...Kind) *KindSet { 12 | set := make(map[Kind]struct{}) 13 | for _, kind := range kinds { 14 | if kind.Primary() == Option { 15 | set[Option] = struct{}{} 16 | } 17 | set[kind] = struct{}{} 18 | } 19 | return &KindSet{set} 20 | } 21 | 22 | func (ks *KindSet) Has(kind Kind) bool { 23 | _, ok := ks.set[kind] 24 | return ok 25 | } 26 | 27 | func (ks *KindSet) Kinds() (kinds []Kind) { 28 | for kind := range ks.set { 29 | kinds = append(kinds, kind) 30 | } 31 | sort.SliceStable(kinds, func(i, j int) bool { 32 | return kinds[i] < kinds[j] 33 | }) 34 | return kinds 35 | } 36 | -------------------------------------------------------------------------------- /pkg/llbutil/file_info.go: -------------------------------------------------------------------------------- 1 | package llbutil 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "time" 7 | 8 | "github.com/tonistiigi/fsutil/types" 9 | ) 10 | 11 | type FileInfo struct { 12 | *types.Stat 13 | } 14 | 15 | func (fi *FileInfo) Name() string { 16 | return filepath.Base(fi.Stat.Path) 17 | } 18 | 19 | func (fi *FileInfo) Size() int64 { 20 | return fi.Stat.Size_ 21 | } 22 | 23 | func (fi *FileInfo) Mode() os.FileMode { 24 | return os.FileMode(fi.Stat.Mode) 25 | } 26 | 27 | func (fi *FileInfo) ModTime() time.Time { 28 | return time.Unix(fi.Stat.ModTime/1e9, fi.Stat.ModTime%1e9) 29 | } 30 | 31 | func (fi *FileInfo) IsDir() bool { 32 | return fi.Mode().IsDir() 33 | } 34 | 35 | func (fi *FileInfo) Sys() interface{} { 36 | return fi.Stat 37 | } 38 | -------------------------------------------------------------------------------- /examples/node.hlb: -------------------------------------------------------------------------------- 1 | # Run `npm test` for the node project `left-pad`. 2 | fs default() { 3 | npmRun leftPad fs { nodeModules leftPad; } "test" 4 | } 5 | 6 | fs leftPad() { 7 | git "https://github.com/left-pad/left-pad.git" "master" 8 | } 9 | 10 | fs npmInstall(fs src) { 11 | image "node:alpine" with option { 12 | resolve 13 | } 14 | run "npm install" with option { 15 | dir "/src" 16 | mount src "/src" 17 | # Name the mounted filesystem as nodeModules 18 | mount fs { scratch; } "/src/node_modules" as nodeModules 19 | } 20 | } 21 | 22 | fs npmRun(fs src, fs nodeModules, string script) { 23 | image "node:alpine" 24 | run string { format "npm run %s" script; } with option { 25 | dir "/src" 26 | mount src "/src" 27 | mount nodeModules "/src/node_modules" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pkg/stargzutil/fixtures/alpine_stargz.json: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 3 | "schemaVersion": 2, 4 | "config": { 5 | "mediaType": "application/vnd.oci.image.config.v1+json", 6 | "digest": "sha256:b200a018d67c97778fbfc47175bc86c30f8040f583eb1809bbfc7c05d34aa778", 7 | "size": 703 8 | }, 9 | "layers": [ 10 | { 11 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 12 | "digest": "sha256:8ba6cec101df53fb9ce1031ceaa2c701b7c5b250cefad8c74806ff02406cb14a", 13 | "size": 2833104, 14 | "annotations": { 15 | "containerd.io/snapshot/stargz/toc.digest": "sha256:77dd89754ec6fda86dacacca3b1b1d2beac3f626177819d09ece10efcd4dbe3a", 16 | "io.containers.estargz.uncompressed-size": "5966336" 17 | } 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /rpc/dapserver/common.go: -------------------------------------------------------------------------------- 1 | package dapserver 2 | 3 | import dap "github.com/google/go-dap" 4 | 5 | func newEvent(event string) dap.Event { 6 | return dap.Event{ 7 | ProtocolMessage: dap.ProtocolMessage{ 8 | Seq: 0, 9 | Type: "event", 10 | }, 11 | Event: event, 12 | } 13 | } 14 | 15 | func newResponse(msg dap.RequestMessage) dap.Response { 16 | req := msg.GetRequest() 17 | return dap.Response{ 18 | ProtocolMessage: dap.ProtocolMessage{ 19 | Seq: 0, 20 | Type: "response", 21 | }, 22 | Command: req.Command, 23 | RequestSeq: req.Seq, 24 | Success: true, 25 | } 26 | } 27 | 28 | func newErrorResponse(msg dap.RequestMessage, err error) *dap.ErrorResponse { 29 | resp := &dap.ErrorResponse{ 30 | Response: newResponse(msg), 31 | } 32 | resp.Success = false 33 | resp.Message = err.Error() 34 | return resp 35 | } 36 | -------------------------------------------------------------------------------- /cmd/docgen/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | 11 | "github.com/openllb/hlb/builtin/gen" 12 | ) 13 | 14 | func main() { 15 | if len(os.Args) != 3 { 16 | log.Fatal("docgen: must have exactly 2 args") 17 | } 18 | 19 | err := run(os.Args[1], os.Args[2]) 20 | if err != nil { 21 | fmt.Fprintf(os.Stderr, "docgen: %s\n", err) 22 | os.Exit(1) 23 | } 24 | } 25 | 26 | func run(src, dest string) error { 27 | f, err := os.Open(src) 28 | if err != nil { 29 | return err 30 | } 31 | defer f.Close() 32 | 33 | doc, err := gen.GenerateDocumentation(context.Background(), f) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | dt, err := json.MarshalIndent(doc, "", " ") 39 | if err != nil { 40 | return err 41 | } 42 | 43 | return ioutil.WriteFile(dest, dt, 0644) 44 | } 45 | -------------------------------------------------------------------------------- /codegen/error_handler.go: -------------------------------------------------------------------------------- 1 | package codegen 2 | 3 | import ( 4 | "context" 5 | 6 | gateway "github.com/moby/buildkit/frontend/gateway/client" 7 | ) 8 | 9 | type gatewayError struct { 10 | context.Context 11 | Client gateway.Client 12 | err error 13 | } 14 | 15 | func withGatewayError(ctx context.Context, c gateway.Client, err error) *gatewayError { 16 | return &gatewayError{Context: ctx, Client: c, err: err} 17 | } 18 | 19 | func (e *gatewayError) Unwrap() error { 20 | return e.err 21 | } 22 | 23 | func (e *gatewayError) Error() string { 24 | return e.err.Error() 25 | } 26 | 27 | func (cg *CodeGen) errorHandler(ctx context.Context, c gateway.Client, gerr error) error { 28 | if cg.dbgr == nil { 29 | return gerr 30 | } 31 | 32 | s := cg.dbgr.recording[cg.dbgr.recordingIndex-1] 33 | return cg.dbgr.yield(s.Ctx, s.Scope, s.Node, s.Value, s.Options, withGatewayError(ctx, c, gerr)) 34 | } 35 | -------------------------------------------------------------------------------- /rpc/langserver/scope.go: -------------------------------------------------------------------------------- 1 | package langserver 2 | 3 | type Scope uint16 4 | 5 | const ( 6 | String Scope = iota 7 | Constant 8 | Numeric 9 | Variable 10 | Parameter 11 | Keyword 12 | Modifier 13 | Type 14 | Function 15 | Module 16 | Comment 17 | ) 18 | 19 | func (s Scope) String() string { 20 | return scopeAsString[s] 21 | } 22 | 23 | var ( 24 | // Conventional textmate scopes: 25 | // https://macromates.com/manual/en/language_grammars 26 | scopeAsString = map[Scope]string{ 27 | String: "string.hlb", 28 | Constant: "constant.language.hlb", 29 | Numeric: "constant.numeric.hlb", 30 | Variable: "variable.hlb", 31 | Parameter: "variable.parameter.hlb", 32 | Keyword: "keyword.hlb", 33 | Modifier: "storage.modifier.hlb", 34 | Type: "storage.type.hlb", 35 | Function: "entity.name.function.hlb", 36 | Module: "entity.name.namespace.hlb", 37 | Comment: "comment.hlb", 38 | } 39 | ) 40 | -------------------------------------------------------------------------------- /parser/docstrings.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "github.com/openllb/hlb/parser/ast" 4 | 5 | // AssignDocStrings assigns the comment group immediately before a function 6 | // declaration as the function's doc string. 7 | func AssignDocStrings(mod *ast.Module) { 8 | var ( 9 | lastCG *ast.CommentGroup 10 | ) 11 | 12 | ast.Match(mod, ast.MatchOpts{}, 13 | func(decl *ast.Decl) { 14 | if decl.Comments != nil { 15 | lastCG = decl.Comments 16 | } 17 | }, 18 | func(fun *ast.FuncDecl) { 19 | if lastCG != nil && lastCG.End().Line == fun.Pos.Line-1 { 20 | fun.Doc = lastCG 21 | } 22 | 23 | if fun.Body != nil { 24 | ast.Match(fun.Body, ast.MatchOpts{}, 25 | func(cg *ast.CommentGroup) { 26 | lastCG = cg 27 | }, 28 | func(call *ast.CallStmt) { 29 | if lastCG != nil && lastCG.End().Line == call.Pos.Line-1 { 30 | call.Doc = lastCG 31 | } 32 | }, 33 | ) 34 | } 35 | }, 36 | ) 37 | } 38 | -------------------------------------------------------------------------------- /cmd/hlb/command/langserver.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/openllb/hlb" 8 | "github.com/openllb/hlb/rpc/langserver" 9 | cli "github.com/urfave/cli/v2" 10 | ) 11 | 12 | var langserverCommand = &cli.Command{ 13 | Name: "langserver", 14 | Usage: "run hlb language server over stdio", 15 | Flags: []cli.Flag{ 16 | &cli.StringFlag{ 17 | Name: "logfile", 18 | Usage: "file to log output", 19 | Value: "/tmp/hlb-langserver.log", 20 | }, 21 | }, 22 | Action: func(c *cli.Context) error { 23 | f, err := os.Create(c.String("logfile")) 24 | if err != nil { 25 | return err 26 | } 27 | defer f.Close() 28 | log.SetOutput(f) 29 | 30 | cln, ctx, err := hlb.Client(Context(), c.String("addr")) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | s, err := langserver.NewServer(ctx, cln) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | return s.Listen(os.Stdin, os.Stdout) 41 | }, 42 | } 43 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: HLB (High-level build) 2 | repo_url: https://openllb.github.io 3 | edit_uri: edit 4 | repo_name: openllb/hlb 5 | pages: 6 | - Introduction: 7 | - "What is HLB?": index.md 8 | - "Quickstart": intro/quickstart.md 9 | - Tutorial: 10 | - "Let's begin!": tutorial/lets-begin.md 11 | - "Improving our program": tutorial/improving.md 12 | - "Using arguments": tutorial/arguments.md 13 | - Reference: reference.md 14 | - Specification: specification.md 15 | theme: 16 | name: material 17 | 18 | # There are many available formatting extensions available, please read: 19 | # https://facelessuser.github.io/pymdown-extensions/ 20 | markdown_extensions: 21 | - toc: 22 | permalink: True 23 | - pymdownx.tilde 24 | - admonition 25 | - pymdownx.superfences 26 | - codehilite: 27 | guess_lang: false 28 | 29 | extra_css: 30 | - css/asciinema-player.css 31 | 32 | extra_javascript: 33 | - js/asciinema-player.js 34 | -------------------------------------------------------------------------------- /parser/ast/context.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | ) 7 | 8 | type modulesKey struct{} 9 | 10 | func WithModules(ctx context.Context, mods *ModuleLookup) context.Context { 11 | return context.WithValue(ctx, modulesKey{}, mods) 12 | } 13 | 14 | func Modules(ctx context.Context) *ModuleLookup { 15 | mods, ok := ctx.Value(modulesKey{}).(*ModuleLookup) 16 | if !ok { 17 | return NewModules() 18 | } 19 | return mods 20 | } 21 | 22 | type ModuleLookup struct { 23 | mods map[string]*Module 24 | mu sync.RWMutex 25 | } 26 | 27 | func NewModules() *ModuleLookup { 28 | return &ModuleLookup{ 29 | mods: make(map[string]*Module), 30 | } 31 | } 32 | 33 | func (ml *ModuleLookup) Get(filename string) *Module { 34 | ml.mu.RLock() 35 | defer ml.mu.RUnlock() 36 | return ml.mods[filename] 37 | } 38 | 39 | func (ml *ModuleLookup) Set(filename string, mod *Module) { 40 | ml.mu.Lock() 41 | defer ml.mu.Unlock() 42 | ml.mods[filename] = mod 43 | } 44 | -------------------------------------------------------------------------------- /pkg/llbutil/secret.go: -------------------------------------------------------------------------------- 1 | package llbutil 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/tonistiigi/fsutil" 10 | ) 11 | 12 | func FilterLocalFiles(localPath string, includePatterns, excludePatterns []string) (localPaths []string, err error) { 13 | var fi os.FileInfo 14 | fi, err = os.Stat(localPath) 15 | if err != nil { 16 | return 17 | } 18 | 19 | switch { 20 | case fi.Mode().IsRegular(): 21 | localPaths = append(localPaths, localPath) 22 | return 23 | case fi.Mode().IsDir(): 24 | opt := &fsutil.FilterOpt{ 25 | IncludePatterns: includePatterns, 26 | ExcludePatterns: excludePatterns, 27 | } 28 | err = fsutil.Walk(context.TODO(), localPath, opt, func(walkPath string, info os.FileInfo, err error) error { 29 | if err != nil { 30 | return err 31 | } 32 | if info.Mode().IsRegular() { 33 | localPaths = append(localPaths, filepath.Join(localPath, walkPath)) 34 | } 35 | return nil 36 | }) 37 | return 38 | default: 39 | return localPaths, fmt.Errorf("unexpected file type at %s", localPath) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /codegen/gateway_exec_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | package codegen 5 | 6 | import ( 7 | "context" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | 12 | gateway "github.com/moby/buildkit/frontend/gateway/client" 13 | "golang.org/x/sys/unix" 14 | ) 15 | 16 | func addResizeHandler(ctx context.Context, proc gateway.ContainerProcess) func() { 17 | ch := make(chan os.Signal, 1) 18 | ch <- syscall.SIGWINCH // Initial resize. 19 | 20 | go forwardResize(ctx, ch, proc, int(os.Stdin.Fd())) 21 | 22 | signal.Notify(ch, syscall.SIGWINCH) 23 | return func() { signal.Stop(ch) } 24 | } 25 | 26 | func forwardResize(ctx context.Context, ch chan os.Signal, proc gateway.ContainerProcess, fd int) { 27 | for { 28 | select { 29 | case <-ctx.Done(): 30 | close(ch) 31 | return 32 | case <-ch: 33 | ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ) 34 | if err != nil { 35 | return 36 | } 37 | 38 | err = proc.Resize(ctx, gateway.WinSize{ 39 | Cols: uint32(ws.Col), 40 | Rows: uint32(ws.Row), 41 | }) 42 | if err != nil { 43 | return 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /checker/builtin.go: -------------------------------------------------------------------------------- 1 | package checker 2 | 3 | import ( 4 | "github.com/openllb/hlb/builtin" 5 | "github.com/openllb/hlb/parser/ast" 6 | ) 7 | 8 | // GlobalScope is a scope containing references to all builtins. 9 | var GlobalScope = NewBuiltinScope(builtin.Lookup) 10 | 11 | const ( 12 | BuiltinFilename = "" 13 | ) 14 | 15 | // NewBuiltinScope returns a new scope containing synthetic FuncDecl Objects for 16 | // builtins. 17 | func NewBuiltinScope(builtins builtin.BuiltinLookup) *ast.Scope { 18 | scope := ast.NewScope(nil, ast.BuiltinScope, builtin.Module) 19 | ast.Match(builtin.Module, ast.MatchOpts{}, 20 | func(fd *ast.FuncDecl) { 21 | obj := scope.Lookup(fd.Sig.Name.Text) 22 | if obj == nil { 23 | obj = &ast.Object{ 24 | Ident: fd.Sig.Name, 25 | Node: &ast.BuiltinDecl{ 26 | Module: builtin.Module, 27 | Name: fd.Sig.Name.String(), 28 | FuncDeclByKind: make(map[ast.Kind]*ast.FuncDecl), 29 | }, 30 | } 31 | } 32 | 33 | decl := obj.Node.(*ast.BuiltinDecl) 34 | decl.Kinds = append(decl.Kinds, fd.Kind()) 35 | decl.FuncDeclByKind[fd.Kind()] = fd 36 | scope.Insert(obj) 37 | }, 38 | ) 39 | 40 | return scope 41 | } 42 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | `hlb` is a high-level build language for [BuildKit](https://github.com/moby/buildkit/). 2 | 3 | Describe your build in containerized units of work, and BuildKit will build your target as efficiently as possible. 4 | 5 | ## Key features 6 | 7 | - Efficient, expressive, and repeatable builds 8 | - Share build caches 9 | - Extendable frontends to run Dockerfiles, or your publish your own 10 | 11 | 12 | 13 | ## Getting started 14 | 15 | On the left side is a table of contents, organized into sections that can be expanded to show topics they cover. Both the sections and topics are ordered from basic to advanced concepts. 16 | 17 | The guides are intended to contain practical explanations of how to write `hlb` builds, focusing on the most widely used features of `hlb`. For comprehensive documentation of every available instruction and option, see the [Reference](reference.md) and [Specification](specification.md). 18 | 19 | The guides begin with a [quick start](intro/quickstart.md) to ensure your environment is working correctly, followed by a [tutorial](tutorial/lets-begin.md) on how to write your first `hlb` program. 20 | -------------------------------------------------------------------------------- /rpc/dapserver/handles.go: -------------------------------------------------------------------------------- 1 | package dapserver 2 | 3 | const startHandle = 1000 4 | 5 | // handlesMap maps arbitrary values to unique sequential ids. 6 | // This provides convenient abstraction of references, offering 7 | // opacity and allowing simplification of complex identifiers. 8 | // Based on 9 | // https://github.com/microsoft/vscode-debugadapter-node/blob/master/adapter/src/handles.ts 10 | type handlesMap struct { 11 | nextHandle int 12 | handleToVal map[int]interface{} 13 | aliasToHandle map[string]int 14 | } 15 | 16 | func newHandlesMap() *handlesMap { 17 | return &handlesMap{ 18 | nextHandle: startHandle, 19 | handleToVal: make(map[int]interface{}), 20 | aliasToHandle: make(map[string]int), 21 | } 22 | } 23 | 24 | func (hs *handlesMap) create(alias string, value interface{}) int { 25 | next := hs.nextHandle 26 | hs.nextHandle++ 27 | hs.handleToVal[next] = value 28 | hs.aliasToHandle[alias] = next 29 | return next 30 | } 31 | 32 | func (hs *handlesMap) get(handle int) (interface{}, bool) { 33 | v, ok := hs.handleToVal[handle] 34 | return v, ok 35 | } 36 | 37 | func (hs *handlesMap) lookupHandle(alias string) (int, bool) { 38 | handle, ok := hs.aliasToHandle[alias] 39 | return handle, ok 40 | } 41 | -------------------------------------------------------------------------------- /builtin/builtin.go: -------------------------------------------------------------------------------- 1 | package builtin 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "github.com/openllb/hlb/parser" 8 | "github.com/openllb/hlb/parser/ast" 9 | "github.com/openllb/hlb/pkg/filebuffer" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | var ( 14 | Module *ast.Module 15 | 16 | FileBuffer *filebuffer.FileBuffer 17 | ) 18 | 19 | func init() { 20 | err := initSources() 21 | if err != nil { 22 | panic(err) 23 | } 24 | } 25 | 26 | func initSources() (err error) { 27 | ctx := filebuffer.WithBuffers(context.Background(), filebuffer.NewBuffers()) 28 | ctx = ast.WithModules(ctx, ast.NewModules()) 29 | 30 | Module, err = parser.Parse(ctx, &parser.NamedReader{ 31 | Reader: strings.NewReader(Reference), 32 | Value: "", 33 | }, filebuffer.WithEphemeral()) 34 | if err != nil { 35 | return errors.Wrapf(err, "failed to initialize filebuffer for builtins") 36 | } 37 | FileBuffer = filebuffer.Buffers(ctx).Get(Module.Pos.Filename) 38 | return 39 | } 40 | 41 | func Buffers() *filebuffer.BufferLookup { 42 | buffers := filebuffer.NewBuffers() 43 | buffers.Set(FileBuffer.Filename(), FileBuffer) 44 | return buffers 45 | } 46 | 47 | func Modules() *ast.ModuleLookup { 48 | modules := ast.NewModules() 49 | modules.Set(Module.Pos.Filename, Module) 50 | return modules 51 | } 52 | -------------------------------------------------------------------------------- /parser/directory.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/moby/buildkit/client/llb" 9 | digest "github.com/opencontainers/go-digest" 10 | "github.com/openllb/hlb/parser/ast" 11 | ) 12 | 13 | type localDirectory struct { 14 | root string 15 | dgst digest.Digest 16 | } 17 | 18 | // NewLocalDirectory returns an ast.Directory representing a directory on the 19 | // local system. It is also used to abstract the difference between reading 20 | // remote modules that has been vendored. 21 | func NewLocalDirectory(root string, dgst digest.Digest) ast.Directory { 22 | return &localDirectory{root, dgst} 23 | } 24 | 25 | func (r *localDirectory) Path() string { 26 | return r.root 27 | } 28 | 29 | func (r *localDirectory) Digest() digest.Digest { 30 | return r.dgst 31 | } 32 | 33 | func (r *localDirectory) Definition() *llb.Definition { 34 | return nil 35 | } 36 | 37 | func (r *localDirectory) Open(filename string) (io.ReadCloser, error) { 38 | if filepath.IsAbs(filename) { 39 | return os.Open(filename) 40 | } 41 | return os.Open(filepath.Join(r.root, filename)) 42 | } 43 | 44 | func (r *localDirectory) Stat(filename string) (os.FileInfo, error) { 45 | if filepath.IsAbs(filename) { 46 | return os.Stat(filename) 47 | } 48 | return os.Stat(filepath.Join(r.root, filename)) 49 | } 50 | -------------------------------------------------------------------------------- /parser/ast/deprecated.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "github.com/alecthomas/participle/v2/lexer" 5 | ) 6 | 7 | // DeprecatedImportDecl represents an import declaration. 8 | type DeprecatedImportDecl struct { 9 | Pos lexer.Position 10 | Import *Import `parser:"@@"` 11 | Ident *Ident `parser:"@@"` 12 | ImportFunc *ImportFunc `parser:"( @@"` 13 | ImportPath *ImportPath `parser:"| @@ )"` 14 | } 15 | 16 | func (d *DeprecatedImportDecl) Position() lexer.Position { return d.Pos } 17 | func (d *DeprecatedImportDecl) End() lexer.Position { 18 | switch { 19 | case d.ImportFunc != nil: 20 | return d.ImportFunc.End() 21 | case d.ImportPath != nil: 22 | return d.ImportPath.End() 23 | } 24 | return lexer.Position{} 25 | } 26 | 27 | // Import represents the function for a remote import. 28 | type ImportFunc struct { 29 | Pos lexer.Position 30 | From *From `parser:"@@"` 31 | Func *FuncLit `parser:"@@"` 32 | } 33 | 34 | func (i *ImportFunc) Position() lexer.Position { return i.Pos } 35 | func (i *ImportFunc) End() lexer.Position { return i.Func.End() } 36 | 37 | // ImportPath represents the relative path to a local import. 38 | type ImportPath struct { 39 | Pos lexer.Position 40 | Path *StringLit `parser:"@@"` 41 | } 42 | 43 | func (i *ImportPath) Position() lexer.Position { return i.Pos } 44 | func (i *ImportPath) End() lexer.Position { return i.Path.End() } 45 | -------------------------------------------------------------------------------- /parser/util.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/alecthomas/participle/v2/lexer" 10 | ) 11 | 12 | // ResolvePath resolves the path relative to root, and expands `~` to the 13 | // user's home directory. 14 | func ResolvePath(root, path string) (string, error) { 15 | path, err := ExpandHomeDir(path) 16 | if err != nil { 17 | return path, err 18 | } 19 | if filepath.IsAbs(path) { 20 | return path, nil 21 | } 22 | return filepath.Join(root, path), nil 23 | } 24 | 25 | // ExpandHomeDir expands the path to include the home directory if the path is 26 | // prefixed with `~`. If it isn't prefixed with `~`, the path is returned as-is. 27 | func ExpandHomeDir(path string) (string, error) { 28 | if len(path) == 0 { 29 | return path, nil 30 | } 31 | 32 | if path[0] != '~' { 33 | return path, nil 34 | } 35 | 36 | if len(path) > 1 && path[1] != '/' && path[1] != '\\' { 37 | return "", errors.New("cannot expand user-specific home dir") 38 | } 39 | 40 | // Works without cgo, available since go1.12 41 | dir, err := os.UserHomeDir() 42 | if err != nil { 43 | return "", err 44 | } 45 | 46 | return filepath.Join(dir, path[1:]), nil 47 | } 48 | 49 | // FormatPos returns a lexer.Position formatted as a string. 50 | func FormatPos(pos lexer.Position) string { 51 | return fmt.Sprintf("%s:%d:%d:", pos.Filename, pos.Line, pos.Column) 52 | } 53 | -------------------------------------------------------------------------------- /module/tree.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path/filepath" 7 | "sync" 8 | 9 | "github.com/moby/buildkit/client" 10 | "github.com/openllb/hlb/parser/ast" 11 | "github.com/xlab/treeprint" 12 | ) 13 | 14 | // NewTree resolves the import graph and returns a treeprint.Tree that can be 15 | // printed to display a visualization of the imports. Imports that transitively 16 | // import the same module will be duplicated in the tree. 17 | func NewTree(ctx context.Context, cln *client.Client, mod *ast.Module, long bool) (treeprint.Tree, error) { 18 | resolver, err := NewResolver(cln) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | var ( 24 | tree = treeprint.New() 25 | nodeByModule = make(map[*ast.Module]treeprint.Tree) 26 | mu sync.Mutex 27 | ) 28 | 29 | tree.SetValue(mod.Pos.Filename) 30 | nodeByModule[mod] = tree 31 | 32 | err = ResolveGraph(ctx, cln, resolver, mod, func(info VisitInfo) error { 33 | filename := info.Filename 34 | if info.Digest != "" { 35 | encoded := info.Digest.Encoded() 36 | if len(encoded) > 7 { 37 | encoded = encoded[:7] 38 | } 39 | prefix := fmt.Sprintf("%s:%s", info.Digest.Algorithm(), encoded) 40 | filename = filepath.Join(prefix, filename) 41 | } 42 | 43 | mu.Lock() 44 | node := nodeByModule[info.Parent] 45 | inode := node.AddMetaBranch(info.ImportDecl.Name.Text, filename) 46 | nodeByModule[info.Import] = inode 47 | mu.Unlock() 48 | return nil 49 | }) 50 | return tree, err 51 | } 52 | -------------------------------------------------------------------------------- /diagnostic/levenshtein.go: -------------------------------------------------------------------------------- 1 | package diagnostic 2 | 3 | func Suggestion(value string, candidates []string) string { 4 | if len(candidates) == 0 { 5 | return "" 6 | } 7 | min := -1 8 | index := -1 9 | for i, candidate := range candidates { 10 | dist := Levenshtein([]rune(value), []rune(candidate)) 11 | if min == -1 || dist < min { 12 | min = dist 13 | index = i 14 | } 15 | } 16 | failLimit := 1 17 | if len(value) > 3 { 18 | failLimit = 2 19 | } 20 | if min > failLimit { 21 | return "" 22 | } 23 | return candidates[index] 24 | } 25 | 26 | // Levenshtein returns the levenshtein distance between two rune arrays. 27 | // 28 | // This implementation translated from the optimized C code at 29 | // https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#C 30 | func Levenshtein(s1, s2 []rune) int { 31 | s1len := len(s1) 32 | s2len := len(s2) 33 | column := make([]int, len(s1)+1) 34 | 35 | for y := 1; y <= s1len; y++ { 36 | column[y] = y 37 | } 38 | for x := 1; x <= s2len; x++ { 39 | column[0] = x 40 | lastdiag := x - 1 41 | for y := 1; y <= s1len; y++ { 42 | olddiag := column[y] 43 | var incr int 44 | if s1[y-1] != s2[x-1] { 45 | incr = 1 46 | } 47 | 48 | column[y] = min3(column[y]+1, column[y-1]+1, lastdiag+incr) 49 | lastdiag = olddiag 50 | } 51 | } 52 | return column[s1len] 53 | } 54 | 55 | func min3(a, b, c int) int { 56 | if a < b { 57 | if a < c { 58 | return a 59 | } 60 | } else { 61 | if b < c { 62 | return b 63 | } 64 | } 65 | return c 66 | } 67 | -------------------------------------------------------------------------------- /LANGSERVER.md: -------------------------------------------------------------------------------- 1 | hlb langserver 2 | ============== 3 | 4 | Language server for [hlb](https://github.com/openllb/hlb) speaking [LSP](https://github.com/Microsoft/language-server-protocol). 5 | 6 | Capabilities 7 | ------------ 8 | 9 | | Capability | Support | 10 | |-----------------------|---------| 11 | | Hover | ✔ | 12 | | Jump to definition | ✔ | 13 | | Find references | | 14 | | Completion | | 15 | | Workspace symbols | | 16 | | Semantic highlighting | ✔ | 17 | 18 | Installation 19 | ------------ 20 | 21 | To build and install the `hlb langserver` run: 22 | 23 | ```sh 24 | go get -u github.com/openllb/hlb/cmd/hlb 25 | ``` 26 | 27 | Usage 28 | ----- 29 | 30 | Kakoune ([kak-lsp](https://github.com/ul/kak-lsp/)) 31 | ```toml 32 | [language.hlb] 33 | filetypes = ["hlb"] 34 | roots = [".git", ".hg"] 35 | command = "hlb-langserver" 36 | offset_encoding = "utf-8" 37 | 38 | [semantic_scopes] 39 | # Map textmate scopes to kakoune faces for semantic highlighting 40 | # the underscores are translated to dots, and indicate nesting. 41 | # That is, if variable_other_field is omitted, it will try the face for 42 | # variable_other and then variable 43 | # 44 | # To see a list of available scopes in the debug buffer, run lsp-semantic-available-scopes 45 | string="string" 46 | constant="value" 47 | variable="variable" 48 | keyword="keyword" 49 | storage_modifier="type" 50 | storage_type="type" 51 | entity_name_function="function" 52 | entity_name_namespace="module" 53 | comment="comment" 54 | ``` 55 | -------------------------------------------------------------------------------- /examples/forward.hlb: -------------------------------------------------------------------------------- 1 | fs default() { 2 | scratch 3 | run "/usr/local/bin/docker" "version" with option { 4 | ignoreCache 5 | mountDocker 6 | } 7 | } 8 | 9 | fs buildDockerCli() { 10 | image "golang:alpine" with option { resolve; } 11 | run "apk add -U git bash coreutils gcc musl-dev" 12 | env "CGO_ENABLED" "0" 13 | env "DISABLE_WARN_OUTSIDE_CONTAINER" "1" 14 | run "./scripts/build/binary" with option { 15 | dir "/go/src/github.com/docker/cli" 16 | mount fs { git "https://github.com/docker/cli.git" "v19.03.8"; } "/go/src/github.com/docker/cli" 17 | mount scratch "/go/src/github.com/docker/cli/build" as dockerCli 18 | } 19 | } 20 | 21 | option::run mountDocker() { 22 | mount dockerCli "/usr/local/bin" 23 | forward "unix:///run/docker.sock" "/var/run/docker.sock" 24 | } 25 | 26 | fs testSSH() { 27 | image "alpine" 28 | run "apk add -U openssh-client" 29 | mkdir "/root/.ssh" 0o700 30 | run "ssh-keyscan github.com >> /root/.ssh/known_hosts" 31 | run "ssh -q -T git@github.com || true" with option { 32 | ignoreCache 33 | ssh 34 | } 35 | } 36 | 37 | fs nginx() { 38 | image "busybox" 39 | run "docker rm -f hlb-nginx || true" with option { 40 | ignoreCache 41 | mountDocker 42 | } 43 | run "docker run --name hlb-nginx -d -p 8080:80 nginx" with option { 44 | ignoreCache 45 | mountDocker 46 | } 47 | } 48 | 49 | fs tcp() { 50 | image "alpine" with option { resolve; } 51 | run "apk add -U curl" 52 | run "curl --unix-socket /nginx.sock http://localhost" with option { 53 | ignoreCache 54 | forward "tcp://localhost:8080" "/nginx.sock" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /pkg/steer/input_steerer_test.go: -------------------------------------------------------------------------------- 1 | package steer 2 | 3 | import ( 4 | "io" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | type fixedReader struct { 12 | io.Reader 13 | byteClock chan struct{} 14 | } 15 | 16 | func (fr *fixedReader) Read(p []byte) (n int, err error) { 17 | b := make([]byte, 1) 18 | n, err = fr.Reader.Read(b) 19 | copy(p, b) 20 | <-fr.byteClock 21 | return 22 | } 23 | 24 | func TestInputSteerer(t *testing.T) { 25 | r := &fixedReader{ 26 | Reader: strings.NewReader("abc"), 27 | byteClock: make(chan (struct{}), 1), 28 | } 29 | 30 | pr, pw := io.Pipe() 31 | is := NewInputSteerer(r, pw) 32 | 33 | p := make([]byte, 1) 34 | r.byteClock <- struct{}{} 35 | n, err := pr.Read(p) 36 | require.NoError(t, err) 37 | require.Equal(t, 1, n) 38 | require.Equal(t, "a", string(p[:n])) 39 | 40 | pr2, pw2 := io.Pipe() 41 | is.Push(pw2) 42 | 43 | // A new pipe writer was pushed, so reading from the previous pipe reader 44 | // should block until it is popped off. 45 | done := make(chan struct{}) 46 | go func() { 47 | defer close(done) 48 | p = make([]byte, 1) 49 | r.byteClock <- struct{}{} 50 | n, err = pr.Read(p) 51 | require.NoError(t, err) 52 | require.Equal(t, 1, n) 53 | }() 54 | 55 | p2 := make([]byte, 1) 56 | r.byteClock <- struct{}{} 57 | n, err = pr2.Read(p2) 58 | require.NoError(t, err) 59 | require.Equal(t, 1, n) 60 | require.Equal(t, "b", string(p2[:n])) 61 | 62 | // After popping, the value read should be after what the popped off pipe 63 | // reader read. 64 | is.Pop() 65 | <-done 66 | require.Equal(t, "c", string(p[:n])) 67 | } 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hlb 2 | 3 | [![GoDoc](https://img.shields.io/static/v1?label=godoc&message=reference&color=blue)](https://pkg.go.dev/github.com/openllb/hlb) 4 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 5 | [![Test](https://github.com/openllb/hlb/workflows/Test/badge.svg)](https://github.com/openllb/hlb/actions?query=workflow%3ATest) 6 | 7 | `hlb` is a high-level build language for [BuildKit](https://github.com/moby/buildkit/). 8 | 9 | Describe your build in containerized units of work, and BuildKit will build your target as efficiently as possible. 10 | 11 | ## Getting started with HLB 12 | 13 | If you're on a MacOS or Linux (`linux-amd64`), head on over to [Releases](https://github.com/openllb/hlb/releases) to grab a static binary. 14 | 15 | Otherwise, you can compile HLB yourself using [go](https://golang.org/dl/): 16 | ```sh 17 | git clone https://github.com/openllb/hlb.git 18 | cd hlb 19 | go install ./cmd/hlb 20 | ``` 21 | 22 | Then you can run one of the examples in `./examples`: 23 | ```sh 24 | hlb run ./examples/node.hlb 25 | ``` 26 | 27 | ## Bring your own BuildKit 28 | 29 | By default, HLB uses the BuildKit embedded in a docker engine. HLB supports `BUILDKIT_HOST` the same way `buildctl` does, so you can run BuildKit in a container and connect to it: 30 | 31 | ```sh 32 | docker run -d --name buildkitd --privileged moby/buildkit:master 33 | export BUILDKIT_HOST=docker-container://buildkitd 34 | hlb run ./examples/node.hlb 35 | ``` 36 | 37 | ## Language server 38 | 39 | If your editor has a decent LSP plugin, HLB does support LSP over stdio via the `hlb langserver` subcommand. 40 | -------------------------------------------------------------------------------- /linter/linter.go: -------------------------------------------------------------------------------- 1 | package linter 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/openllb/hlb/diagnostic" 7 | "github.com/openllb/hlb/errdefs" 8 | "github.com/openllb/hlb/parser/ast" 9 | ) 10 | 11 | type Linter struct { 12 | errs []error 13 | } 14 | 15 | type LintOption func(*Linter) 16 | 17 | func Lint(ctx context.Context, mod *ast.Module, opts ...LintOption) error { 18 | l := Linter{} 19 | for _, opt := range opts { 20 | opt(&l) 21 | } 22 | l.Lint(ctx, mod) 23 | if len(l.errs) > 0 { 24 | return &diagnostic.Error{Diagnostics: l.errs} 25 | } 26 | return nil 27 | } 28 | 29 | func (l *Linter) Lint(ctx context.Context, mod *ast.Module) { 30 | ast.Match(mod, ast.MatchOpts{}, 31 | func(id *ast.ImportDecl) { 32 | if id.DeprecatedPath != nil { 33 | l.errs = append(l.errs, errdefs.WithDeprecated( 34 | mod, id.DeprecatedPath, 35 | `import path without keyword "from" is deprecated`, 36 | )) 37 | id.From = &ast.From{Text: "from"} 38 | id.Expr = &ast.Expr{ 39 | BasicLit: &ast.BasicLit{ 40 | Str: id.DeprecatedPath, 41 | }, 42 | } 43 | } 44 | }, 45 | func(t *ast.Type) { 46 | if string(t.Kind) == "group" { 47 | l.errs = append(l.errs, errdefs.WithDeprecated( 48 | mod, t, 49 | "type `group` is deprecated, use `pipeline` instead", 50 | )) 51 | t.Kind = ast.Pipeline 52 | } 53 | }, 54 | func(call *ast.CallStmt) { 55 | if call.Name != nil && call.Name.Ident.Text == "parallel" { 56 | l.errs = append(l.errs, errdefs.WithDeprecated( 57 | mod, call.Name, 58 | "function `parallel` is deprecated, use `stage` instead", 59 | )) 60 | call.Name.Ident.Text = "stage" 61 | } 62 | }, 63 | ) 64 | } 65 | -------------------------------------------------------------------------------- /cmd/hlb/command/format.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | 10 | "github.com/alecthomas/participle/v2/lexer" 11 | "github.com/openllb/hlb/parser" 12 | cli "github.com/urfave/cli/v2" 13 | ) 14 | 15 | var formatCommand = &cli.Command{ 16 | Name: "format", 17 | Aliases: []string{"fmt"}, 18 | Usage: "formats hlb programs", 19 | ArgsUsage: "[ <*.hlb> ... ]", 20 | Flags: []cli.Flag{ 21 | &cli.BoolFlag{ 22 | Name: "write", 23 | Aliases: []string{"w"}, 24 | Usage: "write result to (source) file instead of stdout", 25 | }, 26 | }, 27 | Action: func(c *cli.Context) error { 28 | rs, cleanup, err := collectReaders(c) 29 | if err != nil { 30 | return err 31 | } 32 | defer func() { 33 | err := cleanup() 34 | if err != nil { 35 | fmt.Fprint(os.Stderr, err.Error()) 36 | } 37 | }() 38 | 39 | return Format(Context(), rs, FormatInfo{ 40 | Write: c.Bool("write"), 41 | }) 42 | }, 43 | } 44 | 45 | type FormatInfo struct { 46 | Write bool 47 | } 48 | 49 | func Format(ctx context.Context, rs []io.Reader, info FormatInfo) error { 50 | mods, err := parser.ParseMultiple(ctx, rs) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | if info.Write { 56 | for i, mod := range mods { 57 | filename := lexer.NameOfReader(rs[i]) 58 | if filename == "" { 59 | return fmt.Errorf("Unable to write, file name unavailable") 60 | } 61 | info, err := os.Stat(filename) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | err = ioutil.WriteFile(filename, []byte(mod.String()), info.Mode()) 67 | if err != nil { 68 | return err 69 | } 70 | } 71 | } else { 72 | for _, mod := range mods { 73 | fmt.Printf("%s", mod) 74 | } 75 | } 76 | 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /go.hlb: -------------------------------------------------------------------------------- 1 | import go from image("openllb/go.hlb") 2 | 3 | export binary 4 | 5 | export crossBinaries 6 | 7 | export lint 8 | 9 | string versionCmd() { 10 | "git describe --match 'v[0-9]*' --tags --dirty='.dirty' --always | sed 's/^v//'" 11 | } 12 | 13 | fs golang() { 14 | image "golang:1.21-alpine" 15 | } 16 | 17 | fs build(fs src, string package, string verPackage) { 18 | golang 19 | run "apk add -U git gcc libc-dev" 20 | env "GO111MODULE" "on" 21 | dir "/go/src/hlb" 22 | run "v=$(${versionCmd}) && /usr/local/go/bin/go build -o /out/binary -ldflags \"-linkmode external -extldflags -static -X ${package}.Version=$v\" -a ${verPackage}" with option { 23 | cacheMounts src 24 | mount scratch "/out" as binary 25 | } 26 | } 27 | 28 | pipeline crossBinaries(fs src, string package, string verPackage) { 29 | go.buildCommonWithOptions src package option::template { 30 | stringField "base" "docker.elastic.co/beats-dev/golang-crossbuild" 31 | stringField "goVersion" "1.21.3" 32 | stringField "goBuildFlags" "-ldflags \"-X ${verPackage}.Version=$(${versionCmd})\"" 33 | } option::run { 34 | env "CGO_ENABLED" "0" 35 | } 36 | } 37 | 38 | fs lint(fs src) { 39 | golang 40 | run "apk add -U git gcc libc-dev" 41 | run "sh /golangci/install.sh -b /usr/bin v1.55.0" with option { 42 | mount http("https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh") "/golangci" 43 | } 44 | env "GO111MODULE" "on" 45 | env "PATH" "/usr/bin:/bin:/usr/local/go/bin" 46 | dir "/go/src/hlb" 47 | run "go get" with cacheMounts(src) 48 | run "/usr/bin/golangci-lint run --timeout 10m" with cacheMounts(src) 49 | } 50 | 51 | option::run cacheMounts(fs src) { 52 | mount src "/go/src/hlb" with readonly 53 | mount scratch "/root/.cache/go-build" with cache("hlb/go-build", "private") 54 | mount scratch "/go/pkg/mod" with cache("hlb/go-mod", "private") 55 | } 56 | -------------------------------------------------------------------------------- /hlb.go: -------------------------------------------------------------------------------- 1 | package hlb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/moby/buildkit/client" 9 | "github.com/openllb/hlb/builtin" 10 | "github.com/openllb/hlb/checker" 11 | "github.com/openllb/hlb/codegen" 12 | "github.com/openllb/hlb/diagnostic" 13 | "github.com/openllb/hlb/linter" 14 | "github.com/openllb/hlb/module" 15 | "github.com/openllb/hlb/parser/ast" 16 | "github.com/openllb/hlb/pkg/filebuffer" 17 | "github.com/openllb/hlb/solver" 18 | "golang.org/x/sync/semaphore" 19 | ) 20 | 21 | const defaultMaxConcurrency = 20 22 | 23 | // WithDefaultContext adds common context values to the context. 24 | func WithDefaultContext(ctx context.Context, cln *client.Client) context.Context { 25 | ctx = filebuffer.WithBuffers(ctx, builtin.Buffers()) 26 | ctx = ast.WithModules(ctx, builtin.Modules()) 27 | if cln != nil { 28 | ctx = codegen.WithImageResolver(ctx, codegen.NewCachedImageResolver(cln)) 29 | } 30 | return ctx 31 | } 32 | 33 | // Compile compiles targets in a module and returns a solver.Request. 34 | func Compile(ctx context.Context, cln *client.Client, w io.Writer, mod *ast.Module, targets []codegen.Target) (solver.Request, error) { 35 | err := checker.SemanticPass(mod) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | err = linter.Lint(ctx, mod) 41 | if err != nil { 42 | for _, span := range diagnostic.Spans(err) { 43 | fmt.Fprintln(w, span.Pretty(ctx)) 44 | } 45 | } 46 | 47 | err = checker.Check(mod) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | resolver, err := module.NewResolver(cln) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | cg := codegen.New(cln, resolver) 58 | if solver.ConcurrencyLimiter(ctx) == nil { 59 | ctx = solver.WithConcurrencyLimiter(ctx, semaphore.NewWeighted(defaultMaxConcurrency)) 60 | } 61 | return cg.Generate(ctx, mod, targets) 62 | } 63 | -------------------------------------------------------------------------------- /docs/intro/quickstart.md: -------------------------------------------------------------------------------- 1 | This guide will teach you how to setup `hlb` and run a build to output `hello world`. 2 | 3 | ## Installation 4 | 5 | If you're on a MacOS or Linux (`linux-amd64`), head on over to [Releases](https://github.com/openllb/hlb/releases) to grab a static binary. 6 | 7 | Otherwise, you can compile HLB yourself using [go](https://golang.org/dl/): 8 | ```sh 9 | git clone https://github.com/openllb/hlb.git 10 | cd hlb 11 | go install ./cmd/hlb 12 | ``` 13 | 14 | You'll also need to run `buildkitd` somewhere you can connect to. The easiest way if you have [Docker](https://www.docker.com/get-started), is to run a local buildkit container: 15 | ```sh 16 | # We're still waiting on some upstream PRs to be merged, but soon you'll be able to use standard moby/buildkit 17 | docker run -d --name buildkitd --privileged openllb/buildkit:experimental 18 | ``` 19 | 20 | ## Run your first build 21 | 22 | Now that you have installed `hlb`, we can run our first build. Typically, we will write our program in a file with a `.hlb` extension, but for our first build we can just pipe the program in from stdin. Try it yourself! 23 | 24 | ```sh 25 | export BUILDKIT_HOST=docker-container://buildkitd 26 | echo 'fs default() { scratch; mkfile "/output" 0o644 "hello world"; }' | hlb run --target default,download=. 27 | ``` 28 | 29 | Once the build has finished, you should end up with a file `output` in your working directory. 30 | 31 | ```sh 32 | $ cat output 33 | hello world 34 | ``` 35 | 36 | Congratulations! You've now ran your first `hlb` build and downloaded the output back to your system. 37 | 38 | !!! tip 39 | By default, once the build has finished, nothing is exported anywhere. You'll need to specify where the results go, e.g. to your host as a tarball, or pushed to a Docker registry. 40 | 41 | Now that we've verified `hlb` is functioning, it's time to start the [tutorial](../tutorial/lets-begin.md). 42 | -------------------------------------------------------------------------------- /pkg/steer/input_steerer.go: -------------------------------------------------------------------------------- 1 | package steer 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | ) 7 | 8 | // InputSteerer is a mechanism for directing input to one of a set of 9 | // Readers. This is used when the debugger runs an exec: we can't 10 | // interrupt a Read from the exec context, so if we naively passed the 11 | // primary reader into the exec, it would swallow the next debugger 12 | // command after the exec session ends. To work around this, have a 13 | // goroutine which continuously reads from the input, and steers data 14 | // into the appropriate writer depending whether we have an exec session 15 | // active. 16 | type InputSteerer struct { 17 | mu sync.Mutex 18 | ws []io.WriteCloser 19 | } 20 | 21 | func NewInputSteerer(inputReader io.Reader, ws ...io.WriteCloser) *InputSteerer { 22 | is := &InputSteerer{ws: ws} 23 | 24 | go func() { 25 | var p [4096]byte 26 | for { 27 | n, err := inputReader.Read(p[:]) 28 | var w io.WriteCloser 29 | is.mu.Lock() 30 | if len(is.ws) != 0 { 31 | w = is.ws[len(is.ws)-1] 32 | } 33 | is.mu.Unlock() 34 | if n != 0 && w != nil { 35 | w.Write(p[:n]) 36 | } 37 | if err != nil { 38 | is.mu.Lock() 39 | defer is.mu.Unlock() 40 | for _, w := range is.ws { 41 | if pw, ok := w.(*io.PipeWriter); ok { 42 | pw.CloseWithError(err) 43 | } else { 44 | w.Close() 45 | } 46 | } 47 | return 48 | } 49 | } 50 | }() 51 | return is 52 | } 53 | 54 | // Push pushes a new writer to steer input to, until Pop is called to steer it 55 | // back to the previous writer. 56 | func (is *InputSteerer) Push(w io.WriteCloser) { 57 | is.mu.Lock() 58 | defer is.mu.Unlock() 59 | is.ws = append(is.ws, w) 60 | } 61 | 62 | // Pop causes future input to be directed to the writer where it was going before 63 | // the last call to Push. 64 | func (is *InputSteerer) Pop() { 65 | is.mu.Lock() 66 | defer is.mu.Unlock() 67 | is.ws = is.ws[:len(is.ws)-1] 68 | } 69 | -------------------------------------------------------------------------------- /docs/templates/reference/reference.md: -------------------------------------------------------------------------------- 1 | {{#each Builtins}} 2 | ## {{Funcs.0.Type}} functions 3 | {{#each Funcs}} 4 | ### {{Type}} {{Name}}({{#each Params}}{{#if @first}}{{else}}, {{/if}}{{Type}} {{Name}}{{/each}}) 5 | 6 | {{#if Params}} 7 | {{#each Params}} 8 | !!! info "{{Type}} {{Name}}" 9 | {{ Doc }} 10 | {{/each}} 11 | {{/if}} 12 | 13 | {{Doc}} 14 | 15 | #!hlb 16 | {{#if (eq Type "fs")}} 17 | fs default() { 18 | {{else}} 19 | string myString() { 20 | {{/if}} 21 | {{Name}}{{#if Params}}{{#each Params}} {{#if (eq Type "string")}}"{{Name}}"{{else if (eq Type "int")}}0{{else if (eq Type "octal")}}0o644{{else if (eq Type "bool")}}false{{else if (eq Type "fs")}}scratch{{else}}{{/if}}{{/each}}{{/if}}{{#if Options}} with option { 22 | {{#each Options}} 23 | {{Name}}{{#if Params}}{{#each Params}} {{#if (eq Type "string")}}"{{Name}}"{{else if (eq Type "int")}}0{{else if (eq Type "octal")}}0o644{{else if (eq Type "bool")}}false{{else if (eq Type "fs")}}scratch{{else}}{{/if}}{{/each}}{{/if}} 24 | {{/each}} 25 | }{{/if}} 26 | } 27 | 28 | 29 | {{#if Options}} 30 | {{#each Options}} 31 | #### {{Type}} {{Name}}({{#each Params}}{{#if @first}}{{else}}, {{/if}}{{Type}} {{Name}}{{/each}}) 32 | 33 | {{#if Params}} 34 | {{#each Params}} 35 | !!! info "{{Type}} {{Name}}" 36 | {{ Doc }} 37 | {{/each}} 38 | {{/if}} 39 | 40 | {{Doc}} 41 | 42 | {{/each}} 43 | {{/if}} 44 | 45 | {{/each}} 46 | {{/each}} 47 | 48 | 61 | -------------------------------------------------------------------------------- /pkg/gitscheme/parse.go: -------------------------------------------------------------------------------- 1 | package gitscheme 2 | 3 | import ( 4 | "net/url" 5 | "strings" 6 | ) 7 | 8 | type URI struct { 9 | Scheme string 10 | User string 11 | Host string 12 | Path string 13 | Branch string 14 | Filename string 15 | } 16 | 17 | // Parse parses a Git URI scheme that supports referencing a file in a git 18 | // repository in a specific branch or commit. 19 | // 20 | // The branch and filepath are optional. 21 | // If branch is omitted, based on the host, the default branch for the 22 | // repository is retrieved. 23 | func Parse(uri string) (*URI, error) { 24 | // Example uri: git://github.com/openllb/hlb@main:build.hlb 25 | u, err := url.Parse(uri) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | // The ":" character was chosen to split git repository from the reference 31 | // to a file in the repository. Since Windows doesn't support ":" in directory 32 | // names, it seems to be a safe choice for git repositories. This is also 33 | // consistent with the SSH URI scheme. 34 | var gitPath, filename string 35 | parts := strings.Split(u.Path, ":") 36 | if len(parts) > 1 { 37 | gitPath, filename = strings.Join(parts[:len(parts)-1], ":"), parts[len(parts)-1] 38 | } else { 39 | gitPath = parts[0] 40 | } 41 | 42 | // The "@" character was chosen to specify the branch or commit of a git 43 | // repository. Since "@" is not a valid character for GitHub repository names, 44 | // it seems to be a safe choice. This is also consistent with `go get` 45 | // starting with Go 1.11 when using Go modules. 46 | var branch string 47 | parts = strings.SplitN(gitPath, "@", 2) 48 | if len(parts) > 1 { 49 | // "@" is a valid character in git branches, so we must join the rest. 50 | gitPath, branch = parts[0], parts[1] 51 | } 52 | 53 | return &URI{ 54 | Scheme: u.Scheme, 55 | User: u.User.String(), 56 | Host: u.Host, 57 | Path: gitPath, 58 | Branch: branch, 59 | Filename: filename, 60 | }, nil 61 | } 62 | -------------------------------------------------------------------------------- /solver/progress_test.go: -------------------------------------------------------------------------------- 1 | package solver 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "io/ioutil" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/creack/pty" 11 | "github.com/docker/buildx/util/progress" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestProgress(t *testing.T) { 16 | t.Parallel() 17 | 18 | type testCase struct { 19 | name string 20 | fn func(p Progress) error 21 | } 22 | 23 | ctx := context.Background() 24 | for _, tc := range []testCase{{ 25 | "empty opts", nil, 26 | }, { 27 | "output empty sync", 28 | func(p Progress) error { 29 | // Can sync from beginning. 30 | err := p.Sync() 31 | if err != nil { 32 | return err 33 | } 34 | 35 | // Can sync after sync. 36 | return p.Sync() 37 | }, 38 | }, { 39 | "output sync after write", 40 | func(p Progress) error { 41 | pw := p.MultiWriter().WithPrefix("", false) 42 | if err := progress.Wrap("test", pw.Write, func(l progress.SubLogger) error { 43 | return ProgressFromReader(l, io.NopCloser(strings.NewReader(""))) 44 | }); err != nil { 45 | return err 46 | } 47 | 48 | // Can sync after write. 49 | return p.Sync() 50 | }, 51 | }} { 52 | for _, mode := range []string{"tty", "plain"} { 53 | tc, mode := tc, mode 54 | t.Run(tc.name+" "+mode, func(t *testing.T) { 55 | ptm, pts, err := pty.Open() 56 | require.NoError(t, err) 57 | 58 | var opts []ProgressOption 59 | switch mode { 60 | case "tty": 61 | opts = append(opts, WithLogOutputTTY(pts)) 62 | case "plain": 63 | opts = append(opts, WithLogOutputPlain(pts)) 64 | } 65 | 66 | p, err := NewProgress(ctx, opts...) 67 | require.NoError(t, err) 68 | 69 | if tc.fn != nil { 70 | err = tc.fn(p) 71 | require.NoError(t, err) 72 | } 73 | 74 | err = p.Wait() 75 | require.NoError(t, err) 76 | 77 | err = pts.Close() 78 | require.NoError(t, err) 79 | 80 | data, _ := ioutil.ReadAll(ptm) 81 | t.Log("\n" + string(data)) 82 | 83 | err = ptm.Close() 84 | require.NoError(t, err) 85 | }) 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package hlb 2 | 3 | import ( 4 | "context" 5 | "net" 6 | 7 | "github.com/docker/buildx/store/storeutil" 8 | "github.com/docker/buildx/util/imagetools" 9 | dockercommand "github.com/docker/cli/cli/command" 10 | "github.com/docker/cli/cli/flags" 11 | "github.com/moby/buildkit/client" 12 | "github.com/openllb/hlb/codegen" 13 | "github.com/openllb/hlb/solver" 14 | ) 15 | 16 | // Client returns a BuildKit client specified by addr based on BuildKit's 17 | // connection helpers. 18 | // 19 | // If addr is empty, an attempt is made to connect to docker engine's embedded 20 | // BuildKit which supports a subset of the exporters and special `moby` 21 | // exporter. 22 | func Client(ctx context.Context, addr string) (*client.Client, context.Context, error) { 23 | // Attempt to connect to a healthy docker engine. 24 | dockerCli, auth, err := NewDockerCli(ctx) 25 | 26 | // If addr is empty, connect to BuildKit using connection helpers. 27 | if addr != "" { 28 | ctx = codegen.WithDockerAPI(ctx, dockerCli.Client(), auth, err, false) 29 | cln, err := solver.BuildkitClient(ctx, addr) 30 | return cln, ctx, err 31 | } 32 | 33 | // Otherwise, connect to docker engine's embedded BuildKit. 34 | ctx = codegen.WithDockerAPI(ctx, dockerCli.Client(), auth, err, true) 35 | cln, err := client.New(ctx, "", client.WithContextDialer(func(context.Context, string) (net.Conn, error) { 36 | return dockerCli.Client().DialHijack(ctx, "/grpc", "h2c", nil) 37 | }), client.WithSessionDialer(func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) { 38 | return dockerCli.Client().DialHijack(ctx, "/session", proto, meta) 39 | })) 40 | return cln, ctx, err 41 | } 42 | 43 | func NewDockerCli(ctx context.Context) (dockerCli *dockercommand.DockerCli, auth imagetools.Auth, err error) { 44 | dockerCli, err = dockercommand.NewDockerCli() 45 | if err != nil { 46 | return 47 | } 48 | 49 | err = dockerCli.Initialize(flags.NewClientOptions()) 50 | if err != nil { 51 | return 52 | } 53 | 54 | _, err = dockerCli.Client().ServerVersion(ctx) 55 | if err != nil { 56 | return 57 | } 58 | 59 | imageopt, err := storeutil.GetImageConfig(dockerCli, nil) 60 | if err != nil { 61 | return 62 | } 63 | 64 | auth = imageopt.Auth 65 | return 66 | } 67 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | name: Build 12 | runs-on: "ubuntu-latest" 13 | steps: 14 | - name: Set up Go 1.21 15 | uses: actions/setup-go@v5 16 | with: 17 | go-version: ^1.21 18 | id: go 19 | 20 | - name: Check out code into the Go module directory 21 | uses: actions/checkout@v4 22 | 23 | - name: Set up Docker Buildx 24 | uses: docker/setup-buildx-action@v3 25 | 26 | - name: Restore Cache 27 | uses: actions/cache@v4 28 | if: github.repository != 'openllb/hlb' 29 | id: cache 30 | with: 31 | path: ~/go/pkg/mod 32 | key: ${{ runner.os }}-${{ hashFiles('**/go.sum') }} 33 | 34 | - name: Get dependencies 35 | run: go get 36 | 37 | - name: Compile 38 | run: | 39 | go build -o build/hlb ./cmd/hlb 40 | go build -o build/docgen ./cmd/docgen 41 | go build -o build/builtingen ./cmd/builtingen 42 | 43 | - name: Go Tests 44 | run: go test -v ./... 45 | 46 | - name: Start BuildkitD 47 | if: github.repository != 'openllb/hlb' 48 | run: docker run --name buildkitd --rm -d --privileged openllb/buildkit:experimental 49 | 50 | - name: BuildkitD Wait 51 | if: github.repository != 'openllb/hlb' 52 | # waiting for buildkitd to report 1 worker (2 lines, 1 for column titles, one for the worker details) 53 | run: while true; do lineCount=$(docker exec buildkitd buildctl debug workers | wc -l); if [ $lineCount -gt 1 ]; then break; fi; sleep 1; done 54 | 55 | - name: GoLint 56 | run: ./build/hlb run -t lint 57 | 58 | - name: Ensure generated files 59 | run: | 60 | ./build/hlb run -t gen 61 | if ! git diff --quiet; then 62 | echo "ERROR: Generated files need updating:" >&2 63 | echo "" >&2 64 | git ls-files -m | sed 's/^/ /g' >&2 65 | echo "" >&2 66 | echo "To resolve run:" >&2 67 | echo " ./hlb run -t gen" >&2 68 | echo "and add changes to the git branch" >&2 69 | exit 1 70 | fi 71 | 72 | - name: Crosscompile 73 | run: ./build/hlb run 74 | -------------------------------------------------------------------------------- /local/environment.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "runtime" 7 | "strings" 8 | ) 9 | 10 | type contextKey string 11 | 12 | const ( 13 | environContextKey contextKey = "environ" 14 | cwdContextKey contextKey = "cwd" 15 | osContextKey contextKey = "os" 16 | archContextKey contextKey = "arch" 17 | ) 18 | 19 | func WithEnviron(ctx context.Context, environ []string) context.Context { 20 | if environ == nil { 21 | return ctx 22 | } 23 | return context.WithValue(ctx, environContextKey, environ) 24 | } 25 | 26 | func WithCwd(ctx context.Context, cwd string) (context.Context, error) { 27 | if cwd == "" { 28 | var err error 29 | cwd, err = os.Getwd() 30 | if err != nil { 31 | return ctx, err 32 | } 33 | } 34 | return context.WithValue(ctx, cwdContextKey, cwd), nil 35 | } 36 | 37 | func WithOs(ctx context.Context, os string) context.Context { 38 | if os == "" { 39 | os = runtime.GOOS 40 | } 41 | return context.WithValue(ctx, osContextKey, os) 42 | } 43 | 44 | func WithArch(ctx context.Context, arch string) context.Context { 45 | if arch == "" { 46 | arch = runtime.GOARCH 47 | } 48 | return context.WithValue(ctx, archContextKey, arch) 49 | } 50 | 51 | func Env(ctx context.Context, key string) string { 52 | if environ, ok := ctx.Value(environContextKey).([]string); ok { 53 | for _, env := range environ { 54 | envParts := strings.SplitN(env, "=", 2) 55 | if envParts[0] == key { 56 | if len(envParts) > 1 { 57 | return envParts[1] 58 | } 59 | return "" 60 | } 61 | } 62 | // did not find the key 63 | return "" 64 | } 65 | return os.Getenv(key) 66 | } 67 | 68 | func Environ(ctx context.Context) []string { 69 | if environ, ok := ctx.Value(environContextKey).([]string); ok { 70 | return environ 71 | } 72 | return os.Environ() 73 | } 74 | 75 | func Cwd(ctx context.Context) (string, error) { 76 | if workdir, ok := ctx.Value(cwdContextKey).(string); ok { 77 | return workdir, nil 78 | } 79 | return os.Getwd() 80 | } 81 | 82 | func Os(ctx context.Context) string { 83 | if os, ok := ctx.Value(osContextKey).(string); ok { 84 | return os 85 | } 86 | return runtime.GOOS 87 | } 88 | 89 | func Arch(ctx context.Context) string { 90 | if os, ok := ctx.Value(archContextKey).(string); ok { 91 | return os 92 | } 93 | return runtime.GOARCH 94 | } 95 | -------------------------------------------------------------------------------- /parser/parse.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | 8 | "github.com/alecthomas/participle/v2/lexer" 9 | "github.com/openllb/hlb/parser/ast" 10 | "github.com/openllb/hlb/pkg/filebuffer" 11 | "golang.org/x/sync/errgroup" 12 | ) 13 | 14 | func Parse(ctx context.Context, r io.Reader, opts ...filebuffer.Option) (*ast.Module, error) { 15 | mod := &ast.Module{} 16 | defer AssignDocStrings(mod) 17 | 18 | name := lexer.NameOfReader(r) 19 | if name == "" { 20 | name = "" 21 | } 22 | fb := filebuffer.New(name, opts...) 23 | 24 | r = &NewlinedReader{Reader: r} 25 | r = io.TeeReader(r, fb) 26 | defer func() { 27 | if mod.Pos.Filename != "" { 28 | filebuffer.Buffers(ctx).Set(mod.Pos.Filename, fb) 29 | } 30 | }() 31 | 32 | err := ast.Parser.Parse(name, r, mod) 33 | if err != nil { 34 | return nil, err 35 | } 36 | mod.Directory = NewLocalDirectory("", "") 37 | ast.Modules(ctx).Set(mod.Pos.Filename, mod) 38 | return mod, nil 39 | } 40 | 41 | func ParseMultiple(ctx context.Context, rs []io.Reader) ([]*ast.Module, error) { 42 | mods := make([]*ast.Module, len(rs)) 43 | 44 | var g errgroup.Group 45 | for i, r := range rs { 46 | i, r := i, r 47 | g.Go(func() error { 48 | mod, err := Parse(ctx, r) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | mods[i] = mod 54 | return nil 55 | }) 56 | } 57 | 58 | return mods, g.Wait() 59 | } 60 | 61 | type NamedReader struct { 62 | io.Reader 63 | Value string 64 | } 65 | 66 | func (nr *NamedReader) Name() string { 67 | return nr.Value 68 | } 69 | 70 | func (nr NamedReader) Close() error { 71 | return nil 72 | } 73 | 74 | // NewlinedReader appends one more newline after an EOF is reached, so that 75 | // parsing is made easier when inputs that don't end with a newline. 76 | type NewlinedReader struct { 77 | io.Reader 78 | newlined int 79 | } 80 | 81 | func (nr *NewlinedReader) Read(p []byte) (n int, err error) { 82 | if nr.newlined > 1 { 83 | return 0, io.EOF 84 | } else if nr.newlined == 1 { 85 | p[0] = byte('\n') 86 | nr.newlined++ 87 | return 1, nil 88 | } 89 | 90 | n, err = nr.Reader.Read(p) 91 | if err != nil { 92 | if errors.Is(err, io.EOF) { 93 | nr.newlined++ 94 | return n, nil 95 | } 96 | return n, err 97 | } 98 | return n, nil 99 | } 100 | -------------------------------------------------------------------------------- /pkg/stargzutil/fixtures/alpine_index.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifests": [ 3 | { 4 | "digest": "sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3", 5 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 6 | "platform": { 7 | "architecture": "amd64", 8 | "os": "linux" 9 | }, 10 | "size": 528 11 | }, 12 | { 13 | "digest": "sha256:e047bc2af17934d38c5a7fa9f46d443f1de3a7675546402592ef805cfa929f9d", 14 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 15 | "platform": { 16 | "architecture": "arm", 17 | "os": "linux", 18 | "variant": "v6" 19 | }, 20 | "size": 528 21 | }, 22 | { 23 | "digest": "sha256:8483ecd016885d8dba70426fda133c30466f661bb041490d525658f1aac73822", 24 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 25 | "platform": { 26 | "architecture": "arm", 27 | "os": "linux", 28 | "variant": "v7" 29 | }, 30 | "size": 528 31 | }, 32 | { 33 | "digest": "sha256:c74f1b1166784193ea6c8f9440263b9be6cae07dfe35e32a5df7a31358ac2060", 34 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 35 | "platform": { 36 | "architecture": "arm64", 37 | "os": "linux", 38 | "variant": "v8" 39 | }, 40 | "size": 528 41 | }, 42 | { 43 | "digest": "sha256:2689e157117d2da668ad4699549e55eba1ceb79cb7862368b30919f0488213f4", 44 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 45 | "platform": { 46 | "architecture": "386", 47 | "os": "linux" 48 | }, 49 | "size": 528 50 | }, 51 | { 52 | "digest": "sha256:2042a492bcdd847a01cd7f119cd48caa180da696ed2aedd085001a78664407d6", 53 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 54 | "platform": { 55 | "architecture": "ppc64le", 56 | "os": "linux" 57 | }, 58 | "size": 528 59 | }, 60 | { 61 | "digest": "sha256:49e322ab6690e73a4909f787bcbdb873631264ff4a108cddfd9f9c249ba1d58e", 62 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 63 | "platform": { 64 | "architecture": "s390x", 65 | "os": "linux" 66 | }, 67 | "size": 528 68 | } 69 | ], 70 | "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", 71 | "schemaVersion": 2 72 | } 73 | -------------------------------------------------------------------------------- /pkg/gitscheme/parse_test.go: -------------------------------------------------------------------------------- 1 | package gitscheme 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestParse(t *testing.T) { 10 | type testCase struct { 11 | name string 12 | uri string 13 | scheme, user, host, gitPath, branch, filename string 14 | } 15 | 16 | for _, tc := range []testCase{{ 17 | "short", 18 | "git://git.personal.com", 19 | "git", "", "git.personal.com", "", "", "", 20 | }, { 21 | "long", 22 | "git://git.personal.com/path/to/deep/repo", 23 | "git", "", "git.personal.com", "/path/to/deep/repo", "", "", 24 | }, { 25 | "github host", 26 | "git://github.com/openllb/hlb", 27 | "git", "", "github.com", "/openllb/hlb", "", "", 28 | }, { 29 | "branch", 30 | "git://github.com/openllb/hlb@main", 31 | "git", "", "github.com", "/openllb/hlb", "main", "", 32 | }, { 33 | "branch with @", 34 | "git://github.com/openllb/hlb@a@b", 35 | "git", "", "github.com", "/openllb/hlb", "a@b", "", 36 | }, { 37 | "file", 38 | "git://github.com/openllb/hlb:file", 39 | "git", "", "github.com", "/openllb/hlb", "", "file", 40 | }, { 41 | "file in subdir", 42 | "git://github.com/openllb/hlb:/sub/dir/file", 43 | "git", "", "github.com", "/openllb/hlb", "", "/sub/dir/file", 44 | }, { 45 | "branch and file", 46 | "git://github.com/openllb/hlb@develop:/sub/dir/file", 47 | "git", "", "github.com", "/openllb/hlb", "develop", "/sub/dir/file", 48 | }, { 49 | "repo with colons", 50 | "git://git.personal.com:1234/hello:world:/file", 51 | "git", "", "git.personal.com:1234", "/hello:world", "", "/file", 52 | }, { 53 | "git+ssh", 54 | "git+ssh://git@git.personal.com:1234/~user/repo.git@main:/file", 55 | "git+ssh", "git", "git.personal.com:1234", "/~user/repo.git", "main", "/file", 56 | }, { 57 | "git+https", 58 | "git+https://github.com/openllb/hlb.git@main:/file", 59 | "git+https", "", "github.com", "/openllb/hlb.git", "main", "/file", 60 | }} { 61 | tc := tc 62 | t.Run(tc.name, func(t *testing.T) { 63 | t.Parallel() 64 | uri, err := Parse(tc.uri) 65 | require.NoError(t, err) 66 | require.NotNil(t, uri) 67 | require.Equal(t, tc.scheme, uri.Scheme) 68 | require.Equal(t, tc.user, uri.User) 69 | require.Equal(t, tc.host, uri.Host) 70 | require.Equal(t, tc.gitPath, uri.Path) 71 | require.Equal(t, tc.branch, uri.Branch) 72 | require.Equal(t, tc.filename, uri.Filename) 73 | }) 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /cmd/hlb/command/app.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | 8 | _ "github.com/moby/buildkit/client/connhelper/dockercontainer" 9 | _ "github.com/moby/buildkit/client/connhelper/kubepod" 10 | cli "github.com/urfave/cli/v2" 11 | ) 12 | 13 | func App() *cli.App { 14 | app := cli.NewApp() 15 | app.Name = "hlb" 16 | app.Usage = "high-level build language compiler" 17 | 18 | app.Flags = []cli.Flag{ 19 | &cli.StringFlag{ 20 | Name: "addr", 21 | Usage: "buildkitd address", 22 | EnvVars: []string{ 23 | "BUILDKIT_HOST", 24 | }, 25 | }, 26 | } 27 | 28 | app.Commands = []*cli.Command{ 29 | versionCommand, 30 | runCommand, 31 | formatCommand, 32 | lintCommand, 33 | moduleCommand, 34 | langserverCommand, 35 | } 36 | return app 37 | } 38 | 39 | func collectReaders(c *cli.Context) (rs []io.Reader, cleanup func() error, err error) { 40 | cleanup = func() error { return nil } 41 | 42 | var rcs []io.ReadCloser 43 | if c.NArg() == 0 { 44 | rcs = append(rcs, os.Stdin) 45 | } else { 46 | for _, arg := range c.Args().Slice() { 47 | info, err := os.Stat(arg) 48 | if err != nil { 49 | return nil, cleanup, err 50 | } 51 | 52 | if info.IsDir() { 53 | drcs, err := readDir(arg) 54 | if err != nil { 55 | return nil, cleanup, err 56 | } 57 | rcs = append(rcs, drcs...) 58 | } else { 59 | f, err := os.Open(arg) 60 | if err != nil { 61 | return nil, cleanup, err 62 | } 63 | 64 | rcs = append(rcs, f) 65 | } 66 | } 67 | } 68 | 69 | for _, rc := range rcs { 70 | rs = append(rs, rc) 71 | } 72 | 73 | return rs, func() error { 74 | for _, rc := range rcs { 75 | err := rc.Close() 76 | if err != nil { 77 | return err 78 | } 79 | } 80 | return nil 81 | }, nil 82 | } 83 | 84 | func readDir(dir string) ([]io.ReadCloser, error) { 85 | var rcs []io.ReadCloser 86 | err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { 87 | if err != nil { 88 | return err 89 | } 90 | 91 | if info.IsDir() { 92 | return nil 93 | } 94 | 95 | if filepath.Ext(path) != ".hlb" { 96 | return nil 97 | } 98 | 99 | f, err := os.Open(path) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | rcs = append(rcs, f) 105 | return nil 106 | }) 107 | if err != nil { 108 | return nil, err 109 | } 110 | 111 | return rcs, nil 112 | } 113 | -------------------------------------------------------------------------------- /docs/templates/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "src", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "commander": { 8 | "version": "2.20.3", 9 | "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", 10 | "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", 11 | "optional": true 12 | }, 13 | "handlebars": { 14 | "version": "4.7.2", 15 | "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.2.tgz", 16 | "integrity": "sha512-4PwqDL2laXtTWZghzzCtunQUTLbo31pcCJrd/B/9JP8XbhVzpS5ZXuKqlOzsd1rtcaLo4KqAn8nl8mkknS4MHw==", 17 | "requires": { 18 | "neo-async": "^2.6.0", 19 | "optimist": "^0.6.1", 20 | "source-map": "^0.6.1", 21 | "uglify-js": "^3.1.4" 22 | } 23 | }, 24 | "minimist": { 25 | "version": "0.0.10", 26 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", 27 | "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" 28 | }, 29 | "neo-async": { 30 | "version": "2.6.1", 31 | "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", 32 | "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==" 33 | }, 34 | "optimist": { 35 | "version": "0.6.1", 36 | "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", 37 | "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", 38 | "requires": { 39 | "minimist": "~0.0.1", 40 | "wordwrap": "~0.0.2" 41 | } 42 | }, 43 | "source-map": { 44 | "version": "0.6.1", 45 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", 46 | "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" 47 | }, 48 | "uglify-js": { 49 | "version": "3.7.6", 50 | "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.6.tgz", 51 | "integrity": "sha512-yYqjArOYSxvqeeiYH2VGjZOqq6SVmhxzaPjJC1W2F9e+bqvFL9QXQ2osQuKUFjM2hGjKG2YclQnRKWQSt/nOTQ==", 52 | "optional": true, 53 | "requires": { 54 | "commander": "~2.20.3", 55 | "source-map": "~0.6.1" 56 | } 57 | }, 58 | "wordwrap": { 59 | "version": "0.0.3", 60 | "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", 61 | "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /rpc/dapserver/server.go: -------------------------------------------------------------------------------- 1 | package dapserver 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "errors" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | 11 | "github.com/chzyer/readline" 12 | dap "github.com/google/go-dap" 13 | "github.com/openllb/hlb/codegen" 14 | "golang.org/x/sync/errgroup" 15 | ) 16 | 17 | type Server struct { 18 | dbgr codegen.Debugger 19 | } 20 | 21 | func New(dbgr codegen.Debugger) *Server { 22 | return &Server{dbgr} 23 | } 24 | 25 | func (s *Server) Listen(ctx context.Context, output, stdin io.Reader, stdout io.Writer) error { 26 | ctx, cancel := context.WithCancel(ctx) 27 | cancelableStdin := readline.NewCancelableStdin(stdin) 28 | session := Session{ 29 | dbgr: s.dbgr, 30 | rw: bufio.NewReadWriter( 31 | bufio.NewReader(cancelableStdin), 32 | bufio.NewWriter(stdout), 33 | ), 34 | cancel: cancel, 35 | sendQueue: make(chan dap.Message), 36 | caps: make(map[Capability]struct{}), 37 | sourcesHandles: newHandlesMap(), 38 | variablesHandles: newHandlesMap(), 39 | stackFrameHandles: newHandlesMap(), 40 | } 41 | 42 | g, ctx := errgroup.WithContext(ctx) 43 | 44 | g.Go(func() error { 45 | return session.sendFromQueue(ctx) 46 | }) 47 | 48 | if output == nil { 49 | g.Go(func() error { 50 | <-ctx.Done() 51 | return cancelableStdin.Close() 52 | }) 53 | } else { 54 | g.Go(func() error { 55 | defer cancelableStdin.Close() 56 | 57 | scanner := bufio.NewScanner(output) 58 | for scanner.Scan() { 59 | session.send(&dap.OutputEvent{ 60 | Event: newEvent("output"), 61 | Body: dap.OutputEventBody{ 62 | Category: "stdout", 63 | Output: scanner.Text() + "\n", 64 | }, 65 | }) 66 | select { 67 | case <-ctx.Done(): 68 | return nil 69 | default: 70 | } 71 | } 72 | 73 | return scanner.Err() 74 | }) 75 | } 76 | 77 | // f, err := os.Create("/tmp/hlb-dapserver.log") 78 | // if err != nil { 79 | // panic(err) 80 | // } 81 | // defer f.Close() 82 | // log.SetOutput(f) 83 | 84 | log.SetOutput(ioutil.Discard) 85 | 86 | log.Printf("Listening on stdio") 87 | g.Go(func() error { 88 | for { 89 | select { 90 | case <-ctx.Done(): 91 | return nil 92 | default: 93 | } 94 | if err := session.handleRequest(ctx); err != nil { 95 | return err 96 | } 97 | } 98 | }) 99 | 100 | session.sendWg.Wait() 101 | if err := g.Wait(); !errors.Is(err, io.EOF) { 102 | return err 103 | } 104 | return session.err 105 | } 106 | -------------------------------------------------------------------------------- /parser/parse_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | var def = ` 12 | fs foo() { 13 | image "alpine" with option { 14 | resolve 15 | } 16 | run "echo foo" with option { 17 | readonlyRootfs 18 | env "key" "value" 19 | dir "path" 20 | user "name" 21 | network unset 22 | security sandbox 23 | host "name" "ip" 24 | ssh with option { 25 | target "path" 26 | id "cacheid" 27 | uid 1000 28 | gid 1000 29 | mode 0o700 30 | optional 31 | } 32 | secret "target" with option { 33 | id "cacheid" 34 | uid 1000 35 | mode 0o700 36 | optional 37 | } 38 | mount bar "target" with option { 39 | readonly 40 | tmpfs 41 | source "target" 42 | cache "cacheid" shared 43 | } 44 | forward "tcp://localhost:1234" "/tmp/servicee.sock" with option { 45 | uid 1000 46 | gid 1000 47 | mode 0o666 48 | } 49 | } 50 | env "key" "value" 51 | dir "path" 52 | user "name" 53 | mkdir "path" 0o700 with option { 54 | createParents 55 | chown "user:group" 56 | createdTime "time" 57 | } 58 | mkfile "path" 0o700 "content" with option { 59 | chown "user:group" 60 | createdTime "time" 61 | } 62 | rm "path" with option { 63 | allowNotFound 64 | allowWildcard 65 | } 66 | copy bar "src" "dst" with option { 67 | followSymlinks 68 | contentsOnly 69 | unpack 70 | createDestPath 71 | allowWildcard 72 | allowEmptyWildcard 73 | chown "user:group" 74 | createdTime "time" 75 | } 76 | } 77 | 78 | fs bar() { 79 | scratch 80 | copy fs { 81 | http "url" with option { 82 | checksum "digest" 83 | chmod 0o700 84 | filename "name" 85 | } 86 | } "src" "dst" 87 | copy fs { 88 | git "remote" "ref" with option { 89 | keepGitDir 90 | } 91 | } "src" "dst" 92 | } 93 | 94 | string heredocTest() { 95 | value <<-EOM 96 | this 97 | should 98 | dedent 99 | EOM 100 | value <<~EOM 101 | this 102 | should 103 | fold 104 | EOM 105 | value <", 24 | Flags: []cli.Flag{ 25 | &cli.BoolFlag{ 26 | Name: "fix", 27 | Usage: "write module with lint errors fixed and formatted to source file", 28 | }, 29 | }, 30 | Action: func(c *cli.Context) error { 31 | uri, err := GetURI(c) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | cln, ctx, err := hlb.Client(Context(), c.String("addr")) 37 | if err != nil { 38 | return err 39 | } 40 | ctx = hlb.WithDefaultContext(ctx, cln) 41 | 42 | return Lint(ctx, cln, uri, LintInfo{ 43 | Fix: c.Bool("fix"), 44 | }) 45 | }, 46 | } 47 | 48 | type LintInfo struct { 49 | Fix bool 50 | Stdin io.Reader 51 | Stderr io.Writer 52 | } 53 | 54 | func Lint(ctx context.Context, cln *client.Client, uri string, info LintInfo) error { 55 | if info.Stdin == nil { 56 | info.Stdin = os.Stdin 57 | } 58 | if info.Stderr == nil { 59 | info.Stderr = os.Stderr 60 | } 61 | 62 | mod, err := ParseModuleURI(ctx, cln, info.Stdin, uri) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | err = checker.SemanticPass(mod) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | err = linter.Lint(ctx, mod) 73 | if err != nil { 74 | spans := diagnostic.Spans(err) 75 | for _, span := range spans { 76 | if !info.Fix { 77 | fmt.Fprintln(info.Stderr, span.Pretty(ctx)) 78 | continue 79 | } 80 | 81 | var em *errdefs.ErrModule 82 | if errors.As(span, &em) { 83 | filename := em.Module.Pos.Filename 84 | info, err := os.Stat(filename) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | err = ioutil.WriteFile(filename, []byte(em.Module.String()), info.Mode()) 90 | if err != nil { 91 | return err 92 | } 93 | } 94 | } 95 | if info.Fix { 96 | return nil 97 | } 98 | 99 | color := diagnostic.Color(ctx) 100 | fmt.Fprint(info.Stderr, color.Sprintf( 101 | color.Bold("\nRun %s to automatically fix lint errors.\n"), 102 | color.Green(fmt.Sprintf("`hlb lint --fix %s`", mod.Pos.Filename)), 103 | )) 104 | 105 | return errdefs.WithAbort(err, len(spans)) 106 | } 107 | 108 | return checker.Check(mod) 109 | } 110 | -------------------------------------------------------------------------------- /pkg/stargzutil/stargzutil.go: -------------------------------------------------------------------------------- 1 | package stargzutil 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/containerd/containerd/images" 9 | "github.com/containerd/containerd/platforms" 10 | "github.com/containerd/containerd/remotes" 11 | specs "github.com/opencontainers/image-spec/specs-go/v1" 12 | ) 13 | 14 | // HasNonStargzLayer traverses a manifest by resolving the reference and 15 | // walking over its children to see if there's any non-stargz layers. 16 | // 17 | // If ref points to a manifest list, the platform matcher is used to only 18 | // consider the current platform. Although BuildKit supports building manifest 19 | // lists, and multi-platform conversion is technically possible client side, 20 | // it requires pulling blobs locally (not via BuildKit) which is undesirable. 21 | // 22 | // See: https://github.com/containerd/stargz-snapshotter/blob/v0.6.4/nativeconverter/estargz/estargz.go 23 | func HasNonStargzLayer(ctx context.Context, resolver remotes.Resolver, matcher platforms.MatchComparer, ref string) (bool, error) { 24 | _, desc, err := resolver.Resolve(ctx, ref) 25 | if err != nil { 26 | return false, err 27 | } 28 | 29 | fetcher, err := resolver.Fetcher(ctx, ref) 30 | if err != nil { 31 | return false, err 32 | } 33 | 34 | nonStargz := false 35 | err = images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) { 36 | switch desc.MediaType { 37 | case images.MediaTypeDockerSchema2Manifest, specs.MediaTypeImageManifest: 38 | rc, err := fetcher.Fetch(ctx, desc) 39 | if err != nil { 40 | return nil, err 41 | } 42 | defer rc.Close() 43 | 44 | var mfst specs.Manifest 45 | err = json.NewDecoder(rc).Decode(&mfst) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | for _, layer := range mfst.Layers { 51 | _, ok := layer.Annotations["containerd.io/snapshot/stargz/toc.digest"] 52 | if !ok { 53 | nonStargz = true 54 | break 55 | } 56 | } 57 | return nil, nil 58 | case images.MediaTypeDockerSchema2ManifestList, specs.MediaTypeImageIndex: 59 | rc, err := fetcher.Fetch(ctx, desc) 60 | if err != nil { 61 | return nil, err 62 | } 63 | defer rc.Close() 64 | 65 | var idx specs.Index 66 | err = json.NewDecoder(rc).Decode(&idx) 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | for _, d := range idx.Manifests { 72 | if d.Platform == nil || matcher.Match(*d.Platform) { 73 | return []specs.Descriptor{d}, nil 74 | } 75 | } 76 | return nil, fmt.Errorf("failed to find manifest matching platform") 77 | } 78 | return nil, fmt.Errorf("unexpected media type %v for %v", desc.MediaType, desc.Digest) 79 | }), desc) 80 | return nonStargz, err 81 | } 82 | -------------------------------------------------------------------------------- /parser/ast/scope.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "github.com/openllb/hlb/diagnostic" 5 | ) 6 | 7 | type ScopeLevel string 8 | 9 | var ( 10 | BuiltinScope ScopeLevel = "Builtins" 11 | ModuleScope ScopeLevel = "Module" 12 | FunctionScope ScopeLevel = "Function" 13 | BlockScope ScopeLevel = "Block" 14 | ArgsScope ScopeLevel = "Arguments" 15 | ) 16 | 17 | // Scope maintains the set of named language entities declared in the scope 18 | // and a link to the immediately surrounding (outer) scope. 19 | type Scope struct { 20 | Node 21 | Level ScopeLevel 22 | Outer *Scope 23 | Objects map[string]*Object 24 | } 25 | 26 | // NewScope creates a new scope linking to an outer scope. 27 | func NewScope(outer *Scope, level ScopeLevel, node Node) *Scope { 28 | return &Scope{ 29 | Node: node, 30 | Level: level, 31 | Outer: outer, 32 | Objects: make(map[string]*Object), 33 | } 34 | } 35 | 36 | func (s *Scope) Depth() int { 37 | depth := 1 38 | if s.Outer != nil { 39 | depth += s.Outer.Depth() 40 | } 41 | return depth 42 | } 43 | 44 | func (s *Scope) ByLevel(level ScopeLevel) *Scope { 45 | if s.Level == level { 46 | return s 47 | } 48 | if s.Outer != nil { 49 | return s.Outer.ByLevel(level) 50 | } 51 | return nil 52 | } 53 | 54 | // Lookup returns the object with the given name if it is 55 | // found in scope, otherwise it returns nil. 56 | func (s *Scope) Lookup(name string) *Object { 57 | obj, ok := s.Objects[name] 58 | if ok { 59 | return obj 60 | } 61 | 62 | if s.Outer != nil { 63 | return s.Outer.Lookup(name) 64 | } 65 | 66 | return nil 67 | } 68 | 69 | func (s *Scope) Identifiers(kset *KindSet) (idents []string) { 70 | if s.Outer != nil { 71 | idents = s.Outer.Identifiers(kset) 72 | } 73 | for ident, obj := range s.Objects { 74 | if kset == nil || kset.Has(obj.Kind) { 75 | idents = append(idents, ident) 76 | } 77 | } 78 | return idents 79 | } 80 | 81 | func (s *Scope) Suggestion(name string, kset *KindSet) *Object { 82 | return s.Lookup(diagnostic.Suggestion(name, s.Identifiers(kset))) 83 | } 84 | 85 | // Insert inserts a named object obj into the scope. 86 | func (s *Scope) Insert(obj *Object) { 87 | s.Objects[obj.Ident.Text] = obj 88 | } 89 | 90 | func (s *Scope) Locals() []*Object { 91 | var objs []*Object 92 | for _, obj := range s.Objects { 93 | objs = append(objs, obj) 94 | } 95 | return objs 96 | } 97 | 98 | // ObjKind describes what an object represents. 99 | type ObjKind int 100 | 101 | // The list of possible Object types. 102 | const ( 103 | BadKind ObjKind = iota 104 | DeclKind 105 | FieldKind 106 | ) 107 | 108 | // Object represents a named language entity such as a function, or variable. 109 | type Object struct { 110 | Kind Kind 111 | Ident *Ident 112 | Node Node 113 | Data interface{} 114 | Exported bool 115 | } 116 | -------------------------------------------------------------------------------- /linter/linter_test.go: -------------------------------------------------------------------------------- 1 | package linter 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/lithammer/dedent" 9 | "github.com/openllb/hlb/builtin" 10 | "github.com/openllb/hlb/checker" 11 | "github.com/openllb/hlb/diagnostic" 12 | "github.com/openllb/hlb/errdefs" 13 | "github.com/openllb/hlb/parser" 14 | "github.com/openllb/hlb/parser/ast" 15 | "github.com/openllb/hlb/pkg/filebuffer" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | type testCase struct { 20 | name string 21 | input string 22 | fn func(*ast.Module) error 23 | } 24 | 25 | func TestLinter_Lint(t *testing.T) { 26 | t.Parallel() 27 | 28 | for _, tc := range []testCase{{ 29 | "import without from", 30 | ` 31 | import foo "./foo.hlb" 32 | `, 33 | func(mod *ast.Module) error { 34 | return errdefs.WithDeprecated( 35 | mod, ast.Search(mod, `"./foo.hlb"`).(*ast.StringLit), 36 | `import path without keyword "from" is deprecated`, 37 | ) 38 | }, 39 | }, { 40 | "group and parallel", 41 | ` 42 | group default() { 43 | parallel foo bar 44 | } 45 | 46 | fs foo() 47 | fs bar() 48 | `, 49 | func(mod *ast.Module) error { 50 | return &diagnostic.Error{ 51 | Diagnostics: []error{ 52 | errdefs.WithDeprecated( 53 | mod, ast.Search(mod, "group"), 54 | "type `group` is deprecated, use `pipeline` instead", 55 | ), 56 | errdefs.WithDeprecated( 57 | mod, ast.Search(mod, "parallel"), 58 | "function `parallel` is deprecated, use `stage` instead", 59 | ), 60 | }, 61 | } 62 | }, 63 | }} { 64 | tc := tc 65 | t.Run(tc.name, func(t *testing.T) { 66 | ctx := filebuffer.WithBuffers(context.Background(), builtin.Buffers()) 67 | ctx = ast.WithModules(ctx, builtin.Modules()) 68 | 69 | in := strings.NewReader(dedent.Dedent(tc.input)) 70 | mod, err := parser.Parse(ctx, in) 71 | require.NoError(t, err) 72 | 73 | err = checker.SemanticPass(mod) 74 | require.NoError(t, err) 75 | 76 | var expected error 77 | if tc.fn != nil { 78 | expected = tc.fn(mod) 79 | } 80 | err = Lint(ctx, mod) 81 | validateError(t, ctx, expected, err, tc.name) 82 | }) 83 | } 84 | } 85 | 86 | func validateError(t *testing.T, ctx context.Context, expected, actual error, name string) { 87 | switch { 88 | case expected == nil: 89 | require.NoError(t, actual, name) 90 | case actual == nil: 91 | require.NotNil(t, actual, name) 92 | default: 93 | espans := diagnostic.Spans(expected) 94 | aspans := diagnostic.Spans(actual) 95 | require.Equal(t, len(espans), len(aspans)) 96 | 97 | for i := 0; i < len(espans); i++ { 98 | epretty := espans[i].Pretty(ctx) 99 | t.Logf("[Expected]\n%s", epretty) 100 | apretty := aspans[i].Pretty(ctx) 101 | t.Logf("[Actual]\n%s", apretty) 102 | require.Equal(t, epretty, apretty, name) 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /codegen/resolver.go: -------------------------------------------------------------------------------- 1 | package codegen 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/docker/buildx/util/progress" 8 | "github.com/moby/buildkit/client" 9 | "github.com/moby/buildkit/client/llb" 10 | "github.com/moby/buildkit/client/llb/sourceresolver" 11 | gateway "github.com/moby/buildkit/frontend/gateway/client" 12 | digest "github.com/opencontainers/go-digest" 13 | "github.com/openllb/hlb/parser/ast" 14 | "github.com/openllb/hlb/pkg/llbutil" 15 | "github.com/openllb/hlb/solver" 16 | "golang.org/x/sync/errgroup" 17 | ) 18 | 19 | const ( 20 | // ModuleFilename is the filename of the HLB module expected to be in the 21 | // solved filesystem provided to the import declaration. 22 | ModuleFilename = "module.hlb" 23 | ) 24 | 25 | // Resolver resolves imports into a reader ready for parsing and checking. 26 | type Resolver interface { 27 | // Resolve returns a reader for the HLB module and its compiled LLB. 28 | Resolve(ctx context.Context, id *ast.ImportDecl, fs Filesystem) (ast.Directory, error) 29 | } 30 | 31 | func NewCachedImageResolver(cln *client.Client) llb.ImageMetaResolver { 32 | return &cachedImageResolver{ 33 | cln: cln, 34 | cache: make(map[cacheKey]*imageConfig), 35 | } 36 | } 37 | 38 | type cacheKey struct { 39 | ref string 40 | os string 41 | arch string 42 | } 43 | 44 | type cachedImageResolver struct { 45 | cln *client.Client 46 | cache map[cacheKey]*imageConfig 47 | mu sync.RWMutex 48 | } 49 | 50 | type imageConfig struct { 51 | ref string 52 | dgst digest.Digest 53 | config []byte 54 | } 55 | 56 | func (r *cachedImageResolver) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (resolvedRef string, dgst digest.Digest, config []byte, err error) { 57 | key := cacheKey{ref: ref} 58 | if opt.Platform != nil { 59 | key.os = opt.Platform.OS 60 | key.arch = opt.Platform.Architecture 61 | } 62 | r.mu.RLock() 63 | cfg, ok := r.cache[key] 64 | r.mu.RUnlock() 65 | if ok { 66 | return cfg.ref, cfg.dgst, cfg.config, nil 67 | } 68 | 69 | s, err := llbutil.NewSession(ctx) 70 | if err != nil { 71 | return 72 | } 73 | 74 | g, ctx := errgroup.WithContext(ctx) 75 | 76 | g.Go(func() error { 77 | return s.Run(ctx, r.cln.Dialer()) 78 | }) 79 | 80 | g.Go(func() error { 81 | defer s.Close() 82 | var pw progress.Writer 83 | 84 | mw := MultiWriter(ctx) 85 | if mw != nil { 86 | pw = mw.WithPrefix("", false) 87 | } 88 | 89 | return solver.Build(ctx, r.cln, s, pw, func(ctx context.Context, c gateway.Client) (res *gateway.Result, err error) { 90 | resolvedRef, dgst, config, err = c.ResolveImageConfig(ctx, ref, opt) 91 | return gateway.NewResult(), err 92 | }) 93 | }) 94 | 95 | err = g.Wait() 96 | if err != nil { 97 | return 98 | } 99 | 100 | r.mu.Lock() 101 | r.cache[key] = &imageConfig{ 102 | ref: resolvedRef, 103 | dgst: dgst, 104 | config: config, 105 | } 106 | r.mu.Unlock() 107 | return 108 | } 109 | -------------------------------------------------------------------------------- /pkg/llbutil/secret_test.go: -------------------------------------------------------------------------------- 1 | package llbutil 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "sort" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestFilterLocalFiles(t *testing.T) { 13 | localPath, err := os.MkdirTemp("", "test") 14 | require.NoError(t, err) 15 | files := []string{"decrypted/secret", "other/decrypted/secret", "secret", "src/foo"} 16 | for _, f := range files { 17 | err = os.MkdirAll(filepath.Dir(filepath.Join(localPath, f)), 0755) 18 | require.NoError(t, err) 19 | fs, err := os.Create(filepath.Join(localPath, f)) 20 | require.NoError(t, err) 21 | fs.Close() 22 | } 23 | 24 | got, err := FilterLocalFiles(localPath, nil, nil) 25 | require.NoError(t, err) 26 | relativeFiles(localPath, got) 27 | require.Equal(t, files, got) 28 | 29 | got, err = FilterLocalFiles(localPath, []string{"**/nada"}, nil) 30 | require.NoError(t, err) 31 | relativeFiles(localPath, got) 32 | require.Nil(t, got) 33 | 34 | got, err = FilterLocalFiles(localPath, []string{"secret"}, nil) 35 | require.NoError(t, err) 36 | relativeFiles(localPath, got) 37 | require.Equal(t, []string{"secret"}, got) 38 | 39 | got, err = FilterLocalFiles(localPath, []string{"*/secret"}, nil) 40 | require.NoError(t, err) 41 | relativeFiles(localPath, got) 42 | require.Equal(t, []string{"decrypted/secret"}, got) 43 | 44 | got, err = FilterLocalFiles(localPath, []string{"**/secret"}, nil) 45 | require.NoError(t, err) 46 | relativeFiles(localPath, got) 47 | require.Equal(t, []string{"decrypted/secret", "other/decrypted/secret", "secret"}, got) 48 | 49 | got, err = FilterLocalFiles(localPath, []string{"**/decrypted"}, nil) 50 | require.NoError(t, err) 51 | relativeFiles(localPath, got) 52 | require.Equal(t, []string{"decrypted/secret", "other/decrypted/secret"}, got) 53 | 54 | got, err = FilterLocalFiles(localPath, []string{"**/decrypted"}, []string{"other"}) 55 | require.NoError(t, err) 56 | relativeFiles(localPath, got) 57 | require.Equal(t, []string{"decrypted/secret"}, got) 58 | 59 | got, err = FilterLocalFiles(localPath, []string{"**/secret"}, []string{"secret"}) 60 | require.NoError(t, err) 61 | relativeFiles(localPath, got) 62 | require.Equal(t, []string{"decrypted/secret", "other/decrypted/secret"}, got) 63 | 64 | got, err = FilterLocalFiles(localPath, nil, []string{"secret"}) 65 | require.NoError(t, err) 66 | relativeFiles(localPath, got) 67 | require.Equal(t, []string{"decrypted/secret", "other/decrypted/secret", "src/foo"}, got) 68 | 69 | got, err = FilterLocalFiles(localPath, nil, []string{"**/secret"}) 70 | require.NoError(t, err) 71 | relativeFiles(localPath, got) 72 | require.Equal(t, []string{"src/foo"}, got) 73 | 74 | got, err = FilterLocalFiles(localPath+"/secret", nil, nil) 75 | require.NoError(t, err) 76 | relativeFiles(localPath, got) 77 | require.Equal(t, []string{"secret"}, got) 78 | } 79 | 80 | func relativeFiles(localPath string, localFiles []string) { 81 | for i, f := range localFiles { 82 | localFiles[i], _ = filepath.Rel(localPath, f) 83 | } 84 | sort.Strings(localFiles) 85 | } 86 | -------------------------------------------------------------------------------- /docs/specification.md: -------------------------------------------------------------------------------- 1 | This is the draft specification for the High Level Build (HLB) programming language. 2 | 3 | HLB is a functional language to describe a build and its dependencies. It is strongly typed, and implicitly constructs a build graph that is evaluated efficiently and concurrently. Programs are defined in a `.hlb` file, and may consume build graphs produced by other systems (Dockerfiles, Buildpacks, etc). 4 | 5 | The grammar is compact and regular, allowing for SDKs to be implemented for common programming languages to emit HLB. 6 | 7 | ## Notation 8 | 9 | The syntax is specified using Extended Backus-Naur Form (EBNF). 10 | 11 | ### Source code representation 12 | 13 | #### Characters 14 | 15 | ```ebnf 16 | newline = /* the Unicode code point U+000A */ . 17 | unicode_char = /* an arbitrary Unicode code point except newline */ . 18 | ``` 19 | 20 | ##### Letters and digits 21 | 22 | ```ebnf 23 | decimal_digit = "0" … "9" . 24 | octal_digit = "0" … "7" . 25 | ``` 26 | 27 | ### Lexical elements 28 | 29 | #### String literals 30 | 31 | ```ebnf 32 | string_lit = quoted_string_lit | double_quoted_string_lit 33 | quoted_string_lit = `'` { unicode_char } `'` 34 | double_quoted_string_lit = `"` { unicode_char } `"` 35 | ``` 36 | 37 | #### Octal literals 38 | 39 | ```ebnf 40 | octal_lit = octal_digits . 41 | octal_digits = octal_digit { octal_digit } . 42 | ``` 43 | 44 | #### Integer literals 45 | 46 | ```ebnf 47 | int_lit = "0" | ( "1" … "9" ) [ decimal_digits ] . 48 | decimal_digits = decimal_digit { decimal_digit } . 49 | ``` 50 | 51 | #### Bool literals 52 | 53 | ```ebnf 54 | bool_lit = "true" | "false" . 55 | ``` 56 | 57 | ### Types 58 | 59 | #### Function types 60 | 61 | ```ebnf 62 | ReturnType = Type . 63 | Parameters = "(" [ ParameterList [ "," ] ] ")" . 64 | ParameterList = ParameterDecl { "," ParameterDecl } . 65 | ParameterDecl = [ Variadic ] Type ParameterName . 66 | ParameterName = identifier . 67 | Variadic = "variadic" . 68 | ``` 69 | 70 | ### Declarations 71 | 72 | ```ebnf 73 | Declaration = FunctionDecl . 74 | ``` 75 | 76 | #### Function declarations 77 | 78 | ```ebnf 79 | FunctionDecl = ReturnType ( ) FunctionName Parameters [ FunctionBody ] . 80 | FunctionName = identifier . 81 | FunctionBody = Block . 82 | ``` 83 | 84 | #### Alias declarations 85 | 86 | ```ebnf 87 | AliasDecl = "as" FunctionName . 88 | ``` 89 | 90 | ### Expressions 91 | 92 | ```ebnf 93 | ExprList = Expr { Expr } . 94 | Expr = identifier | BasicLit | FuncLit . 95 | ``` 96 | 97 | #### Operands 98 | 99 | ```ebnf 100 | BasicLit = string_lit | octal_lit | int_lit | bool_lit . 101 | FuncLit = ReturnType Block . 102 | ``` 103 | 104 | ### Statements 105 | 106 | ```ebnf 107 | Block = "{" StatementList "}" . 108 | StatementList = { Statement ";" } . 109 | Statement = CallStatement 110 | ``` 111 | 112 | #### Call statements 113 | 114 | ```ebnf 115 | CallStatement = FunctionName [ ExprList ] [ WithOption ] [ AliasDecl ] . 116 | WithOption = "with" Option 117 | Option = identifier | FuncLit . 118 | ``` 119 | -------------------------------------------------------------------------------- /pkg/stargzutil/stargzutil_test.go: -------------------------------------------------------------------------------- 1 | package stargzutil 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "testing" 10 | 11 | _ "embed" 12 | 13 | "github.com/containerd/containerd/platforms" 14 | "github.com/containerd/containerd/remotes" 15 | specs "github.com/opencontainers/image-spec/specs-go/v1" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | var ( 20 | //go:embed fixtures/alpine_desc.json 21 | alpineDesc []byte 22 | 23 | //go:embed fixtures/alpine.json 24 | alpine []byte 25 | 26 | //go:embed fixtures/alpine_index_desc.json 27 | alpineIndexDesc []byte 28 | 29 | //go:embed fixtures/alpine_index.json 30 | alpineIndex []byte 31 | 32 | //go:embed fixtures/alpine_stargz_desc.json 33 | alpineStargzDesc []byte 34 | 35 | //go:embed fixtures/alpine_stargz.json 36 | alpineStargz []byte 37 | ) 38 | 39 | type testResolver struct{} 40 | 41 | func (ts *testResolver) Resolve(ctx context.Context, ref string) (name string, desc specs.Descriptor, err error) { 42 | var dt []byte 43 | switch ref { 44 | case "alpine": 45 | dt = alpineDesc 46 | case "alpine:multiplatform": 47 | dt = alpineIndexDesc 48 | case "alpine:stargz": 49 | dt = alpineStargzDesc 50 | default: 51 | err = fmt.Errorf("unrecognized ref %s", ref) 52 | return 53 | } 54 | return ref, desc, json.Unmarshal(dt, &desc) 55 | } 56 | 57 | func (ts *testResolver) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { 58 | return &testFetcher{}, nil 59 | } 60 | 61 | func (ts *testResolver) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { 62 | return nil, fmt.Errorf("unimplemented") 63 | } 64 | 65 | type testFetcher struct{} 66 | 67 | func (tf *testFetcher) Fetch(ctx context.Context, d specs.Descriptor) (io.ReadCloser, error) { 68 | var dt []byte 69 | switch d.Digest.String() { 70 | // alpine.json 71 | case "sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3": 72 | dt = alpine 73 | // alpine_index.json 74 | case "sha256:21a3deaa0d32a8057914f36584b5288d2e5ecc984380bc0118285c70fa8c9300": 75 | dt = alpineIndex 76 | // alpine_stargz.json 77 | case "sha256:4382407e6f4fab29345722ba819c33f9d158b1bce240839e889d3ff715f0ad93": 78 | dt = alpineStargz 79 | default: 80 | return nil, fmt.Errorf("unrecognized digest %s", d.Digest) 81 | } 82 | return io.NopCloser(bytes.NewReader(dt)), nil 83 | } 84 | 85 | func TestHasNonStargzLayer(t *testing.T) { 86 | ctx := context.Background() 87 | resolver := &testResolver{} 88 | platform := specs.Platform{OS: "linux", Architecture: "amd64"} 89 | 90 | type testCase struct { 91 | ref string 92 | expected bool 93 | } 94 | 95 | for _, tc := range []testCase{{ 96 | "alpine", true, 97 | }, { 98 | "alpine:multiplatform", true, 99 | }, { 100 | "alpine:stargz", false, 101 | }} { 102 | tc := tc 103 | t.Run(tc.ref, func(t *testing.T) { 104 | t.Parallel() 105 | actual, err := HasNonStargzLayer(ctx, resolver, platforms.Only(platform), tc.ref) 106 | require.NoError(t, err) 107 | require.Equal(t, tc.expected, actual) 108 | }) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /module/vendor.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "sync" 8 | 9 | "github.com/moby/buildkit/client" 10 | "github.com/openllb/hlb/codegen" 11 | "github.com/openllb/hlb/parser/ast" 12 | "golang.org/x/sync/errgroup" 13 | ) 14 | 15 | // Vendor resolves the import graph and writes the contents into the modules 16 | // directory of the current working directory. 17 | // 18 | // If tidy mode is enabled, vertices with digests that already exist in the 19 | // modules directory are skipped, and unused modules are pruned. 20 | func Vendor(ctx context.Context, cln *client.Client, mod *ast.Module, targets []string, tidy bool) error { 21 | root := ModulesPath 22 | 23 | var mu sync.Mutex 24 | markedPaths := make(map[string]struct{}) 25 | 26 | var resolver codegen.Resolver 27 | if tidy { 28 | resolver = &tidyResolver{ 29 | cln: cln, 30 | remote: &remoteResolver{cln, root}, 31 | } 32 | } else { 33 | resolver = &targetResolver{ 34 | filename: mod.Pos.Filename, 35 | targets: targets, 36 | cln: cln, 37 | remote: &remoteResolver{cln, root}, 38 | } 39 | } 40 | 41 | g, ctx := errgroup.WithContext(ctx) 42 | 43 | ready := make(chan struct{}) 44 | err := ResolveGraph(ctx, cln, resolver, mod, func(info VisitInfo) error { 45 | g.Go(func() error { 46 | <-ready 47 | 48 | // Local imports have no digest, and they should not be vendored. 49 | if info.Digest == "" { 50 | return nil 51 | } 52 | 53 | // If this is the top-most module, then only deal with modules that are in 54 | // the list of targets. 55 | if info.Parent == mod { 56 | if len(targets) > 0 { 57 | matchTarget := false 58 | for _, target := range targets { 59 | if info.ImportDecl.Name.Text == target { 60 | matchTarget = true 61 | } 62 | } 63 | 64 | if !matchTarget { 65 | return nil 66 | } 67 | } 68 | } 69 | 70 | vp := VendorPath(root, info.Digest) 71 | 72 | // If tidy mode is enabled, then we mark imported modules during graph 73 | // traversal, and then sweep unused vendored modules. 74 | if tidy { 75 | // Mark path for used imports. 76 | mu.Lock() 77 | markedPaths[vp] = struct{}{} 78 | mu.Unlock() 79 | 80 | _, err := os.Stat(vp) 81 | if err == nil { 82 | // Skip modules that have already been vendored. 83 | return nil 84 | } 85 | if !os.IsNotExist(err) { 86 | return err 87 | } 88 | } 89 | 90 | err := os.MkdirAll(vp, 0700) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | f, err := os.Create(filepath.Join(vp, info.Filename)) 96 | if err != nil { 97 | return err 98 | } 99 | defer f.Close() 100 | 101 | _, err = f.WriteString(info.Import.String()) 102 | return err 103 | }) 104 | return nil 105 | }) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | close(ready) 111 | err = g.Wait() 112 | if err != nil { 113 | return err 114 | } 115 | 116 | if tidy { 117 | matches, err := filepath.Glob(filepath.Join(ModulesPath, "*/*/*")) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | for _, match := range matches { 123 | if _, ok := markedPaths[match]; !ok { 124 | err = os.RemoveAll(match) 125 | if err != nil { 126 | return err 127 | } 128 | } 129 | } 130 | } 131 | 132 | return nil 133 | } 134 | -------------------------------------------------------------------------------- /solver/directory.go: -------------------------------------------------------------------------------- 1 | package solver 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/docker/buildx/util/progress" 12 | "github.com/moby/buildkit/client" 13 | "github.com/moby/buildkit/client/llb" 14 | gateway "github.com/moby/buildkit/frontend/gateway/client" 15 | digest "github.com/opencontainers/go-digest" 16 | "github.com/openllb/hlb/parser" 17 | "github.com/openllb/hlb/parser/ast" 18 | "github.com/openllb/hlb/pkg/llbutil" 19 | "golang.org/x/sync/errgroup" 20 | ) 21 | 22 | // NewRemoteDirectory returns an ast.Directory representing a directory backed 23 | // by a BuildKit gateway reference. 24 | func NewRemoteDirectory(ctx context.Context, cln *client.Client, pw progress.Writer, def *llb.Definition, root string, dgst digest.Digest, solveOpts []SolveOption, sessionOpts []llbutil.SessionOption) (ast.Directory, error) { 25 | return &remoteDirectory{ 26 | root: root, 27 | dgst: dgst, 28 | def: def, 29 | cln: cln, 30 | pw: pw, 31 | solveOpts: solveOpts, 32 | sessionOpts: sessionOpts, 33 | ctx: ctx, 34 | }, nil 35 | } 36 | 37 | type remoteDirectory struct { 38 | root string 39 | dgst digest.Digest 40 | def *llb.Definition 41 | cln *client.Client 42 | pw progress.Writer 43 | solveOpts []SolveOption 44 | sessionOpts []llbutil.SessionOption 45 | ctx context.Context 46 | } 47 | 48 | func (r *remoteDirectory) Path() string { 49 | return r.root 50 | } 51 | 52 | func (r *remoteDirectory) Digest() digest.Digest { 53 | return r.dgst 54 | } 55 | 56 | func (r *remoteDirectory) Definition() *llb.Definition { 57 | return r.def 58 | } 59 | 60 | func (r *remoteDirectory) Open(filename string) (io.ReadCloser, error) { 61 | s, err := llbutil.NewSession(r.ctx, r.sessionOpts...) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | g, ctx := errgroup.WithContext(r.ctx) 67 | 68 | g.Go(func() error { 69 | return s.Run(ctx, r.cln.Dialer()) 70 | }) 71 | 72 | var data []byte 73 | g.Go(func() error { 74 | defer s.Close() 75 | return Build(ctx, r.cln, s, r.pw, func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { 76 | dir, err := c.Solve(ctx, gateway.SolveRequest{ 77 | Definition: r.def.ToPB(), 78 | }) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | ref, err := dir.SingleRef() 84 | if err != nil { 85 | return nil, err 86 | } 87 | _, err = ref.StatFile(r.ctx, gateway.StatRequest{ 88 | Path: filename, 89 | }) 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | data, err = ref.ReadFile(r.ctx, gateway.ReadRequest{ 95 | Filename: filename, 96 | }) 97 | if err != nil { 98 | return nil, err 99 | } 100 | return gateway.NewResult(), nil 101 | }, r.solveOpts...) 102 | }) 103 | 104 | if err = g.Wait(); err != nil { 105 | return nil, err 106 | } 107 | 108 | return &parser.NamedReader{ 109 | Reader: bytes.NewReader(data), 110 | Value: filepath.Join(r.root, filename), 111 | }, nil 112 | } 113 | 114 | // Stat is not called for remoteDirectory anywhere in the codebase, so 115 | // here to satisfy the ast.Directory interface. 116 | func (r *remoteDirectory) Stat(filename string) (os.FileInfo, error) { 117 | return nil, fmt.Errorf("stat on remote directory is unimplemented") 118 | } 119 | -------------------------------------------------------------------------------- /diagnostic/error.go: -------------------------------------------------------------------------------- 1 | package diagnostic 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "strings" 9 | 10 | "github.com/moby/buildkit/solver/errdefs" 11 | "github.com/openllb/hlb/pkg/filebuffer" 12 | perrors "github.com/pkg/errors" 13 | ) 14 | 15 | type Error struct { 16 | Err error 17 | Diagnostics []error 18 | } 19 | 20 | func (e *Error) Error() string { 21 | var errs []string 22 | for _, err := range e.Diagnostics { 23 | errs = append(errs, err.Error()) 24 | } 25 | return strings.Join(errs, "\n") 26 | } 27 | 28 | func (e *Error) Unwrap() error { 29 | return e.Err 30 | } 31 | 32 | func Spans(err error) (spans []*SpanError) { 33 | var e *Error 34 | if errors.As(err, &e) { 35 | for _, err := range e.Diagnostics { 36 | var span *SpanError 37 | if errors.As(err, &span) { 38 | spans = append(spans, span) 39 | } 40 | } 41 | } 42 | var span *SpanError 43 | if errors.As(err, &span) { 44 | spans = append(spans, span) 45 | } 46 | return 47 | } 48 | 49 | func DisplayError(ctx context.Context, w io.Writer, spans []*SpanError, err error, printBacktrace bool) { 50 | if len(spans) == 0 { 51 | return 52 | } 53 | 54 | color := Color(ctx) 55 | if err != nil { 56 | fmt.Fprintf(w, color.Sprintf( 57 | "%s: %s\n", 58 | color.Bold(color.Red("error")), 59 | color.Bold(Cause(err)), 60 | )) 61 | } 62 | 63 | for i, span := range spans { 64 | if !printBacktrace && i != len(spans)-1 { 65 | if i == 0 { 66 | color := Color(ctx) 67 | frame := "frame" 68 | if len(spans) > 2 { 69 | frame = "frames" 70 | } 71 | fmt.Fprintf(w, color.Sprintf(color.Cyan(" ⫶ %d %s hidden ⫶\n"), len(spans)-1, frame)) 72 | } 73 | continue 74 | } 75 | 76 | pretty := span.Pretty(ctx, WithNumContext(2)) 77 | lines := strings.Split(pretty, "\n") 78 | for j, line := range lines { 79 | if j == 0 { 80 | lines[j] = fmt.Sprintf(" %d: %s", i+1, line) 81 | } else { 82 | lines[j] = fmt.Sprintf(" %s", line) 83 | } 84 | } 85 | fmt.Fprintf(w, "%s\n", strings.Join(lines, "\n")) 86 | } 87 | } 88 | 89 | func SourcesToSpans(ctx context.Context, srcs []*errdefs.Source, err error) (spans []*SpanError) { 90 | for i, src := range srcs { 91 | fb := filebuffer.Buffers(ctx).Get(src.Info.Filename) 92 | if fb != nil { 93 | var msg string 94 | if i == len(srcs)-1 { 95 | var se *SpanError 96 | if errors.As(err, &se) { 97 | span := &SpanError{ 98 | Pos: se.Pos, 99 | End: se.End, 100 | } 101 | 102 | if len(se.Spans) == 0 { 103 | Spanf(Primary, se.Pos, se.End, se.Err.Error())(span) 104 | } else { 105 | span.Spans = make([]Span, len(se.Spans)) 106 | copy(span.Spans, se.Spans) 107 | } 108 | spans = append(spans, span) 109 | continue 110 | } 111 | 112 | msg = Cause(err) 113 | } 114 | 115 | loc := src.Ranges[0] 116 | start := fb.Position(int(loc.Start.Line), int(loc.Start.Character)) 117 | end := fb.Position(int(loc.End.Line), int(loc.End.Character)) 118 | se := WithError(nil, start, end, Spanf(Primary, start, end, msg)) 119 | 120 | var span *SpanError 121 | if errors.As(se, &span) { 122 | spans = append(spans, span) 123 | } 124 | } 125 | } 126 | return spans 127 | } 128 | 129 | func Cause(err error) string { 130 | if err == nil { 131 | return "" 132 | } 133 | return strings.TrimPrefix(perrors.Cause(err).Error(), "rpc error: code = Unknown desc = ") 134 | } 135 | -------------------------------------------------------------------------------- /builtin/gen/builtins.go: -------------------------------------------------------------------------------- 1 | //go:generate go run ../../cmd/builtingen ../../language/builtin.hlb ../lookup.go 2 | 3 | package gen 4 | 5 | import ( 6 | "bytes" 7 | "context" 8 | "fmt" 9 | "go/format" 10 | "html/template" 11 | "io" 12 | "log" 13 | "os" 14 | "strconv" 15 | "strings" 16 | 17 | "github.com/openllb/hlb/parser" 18 | "github.com/openllb/hlb/parser/ast" 19 | "github.com/openllb/hlb/pkg/filebuffer" 20 | ) 21 | 22 | type BuiltinData struct { 23 | Command string 24 | FuncsByKind map[ast.Kind][]ParsedFunc 25 | Reference string 26 | } 27 | 28 | type ParsedFunc struct { 29 | Name string 30 | Params []*ast.Field 31 | Effects []*ast.Field 32 | } 33 | 34 | func GenerateBuiltins(ctx context.Context, r io.Reader) ([]byte, error) { 35 | files := filebuffer.NewBuffers() 36 | ctx = filebuffer.WithBuffers(ctx, files) 37 | mod, err := parser.Parse(ctx, r) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | funcsByKind := make(map[ast.Kind][]ParsedFunc) 43 | for _, decl := range mod.Decls { 44 | fd := decl.Func 45 | if fd == nil { 46 | continue 47 | } 48 | 49 | var effects []*ast.Field 50 | if fd.Sig.Effects != nil && fd.Sig.Effects.Effects != nil { 51 | effects = fd.Sig.Effects.Effects.Fields() 52 | } 53 | 54 | kind := fd.Kind() 55 | funcsByKind[kind] = append(funcsByKind[kind], ParsedFunc{ 56 | Name: fd.Sig.Name.Text, 57 | Params: fd.Sig.Params.Fields(), 58 | Effects: effects, 59 | }) 60 | } 61 | 62 | fb := files.Get(mod.Pos.Filename) 63 | data := BuiltinData{ 64 | Command: fmt.Sprintf("builtingen %s", strings.Join(os.Args[1:], " ")), 65 | FuncsByKind: funcsByKind, 66 | Reference: fmt.Sprintf("`%s`", string(fb.Bytes())), 67 | } 68 | 69 | var buf bytes.Buffer 70 | err = referenceTmpl.Execute(&buf, &data) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | src, err := format.Source(buf.Bytes()) 76 | if err != nil { 77 | log.Printf("warning: internal error: invalid Go generated: %s", err) 78 | log.Printf("warning: compile the package to analyze the error") 79 | src = buf.Bytes() 80 | } 81 | 82 | return src, nil 83 | } 84 | 85 | var tmplFunctions = template.FuncMap{ 86 | "kind": func(kind ast.Kind) template.HTML { 87 | switch kind { 88 | case ast.String: 89 | return template.HTML("ast.String") 90 | case ast.Int: 91 | return template.HTML("ast.Int") 92 | case ast.Bool: 93 | return template.HTML("ast.Bool") 94 | case ast.Filesystem: 95 | return template.HTML("ast.Filesystem") 96 | default: 97 | return template.HTML(strconv.Quote(string(kind))) 98 | } 99 | }, 100 | } 101 | 102 | var referenceTmpl = template.Must(template.New("reference").Funcs(tmplFunctions).Parse(` 103 | // Code generated by {{.Command}}; DO NOT EDIT. 104 | 105 | package builtin 106 | 107 | import "github.com/openllb/hlb/parser/ast" 108 | 109 | type BuiltinLookup struct { 110 | ByKind map[ast.Kind]LookupByKind 111 | } 112 | 113 | type LookupByKind struct { 114 | Func map[string]FuncLookup 115 | } 116 | 117 | type FuncLookup struct { 118 | Params []*ast.Field 119 | Effects []*ast.Field 120 | } 121 | 122 | var ( 123 | Lookup = BuiltinLookup{ 124 | ByKind: map[ast.Kind]LookupByKind{ 125 | {{range $kind, $funcs := .FuncsByKind}}{{kind $kind}}: { 126 | Func: map[string]FuncLookup{ 127 | {{range $i, $func := $funcs}}"{{$func.Name}}": { 128 | Params: []*ast.Field{ 129 | {{range $i, $param := $func.Params}}ast.NewField({{kind $param.Type.Kind}}, "{{$param.Name}}", {{if $param.Modifier}}true{{else}}false{{end}}), 130 | {{end}} 131 | }, 132 | Effects: []*ast.Field{ 133 | {{range $i, $effect := $func.Effects}}ast.NewField({{kind $effect.Type.Kind}}, "{{$effect.Name}}", false), 134 | {{end}} 135 | }, 136 | }, 137 | {{end}} 138 | }, 139 | }, 140 | {{end}} 141 | }, 142 | } 143 | 144 | Reference = {{.Reference}} 145 | ) 146 | `)) 147 | -------------------------------------------------------------------------------- /builtin/gen/documentation.go: -------------------------------------------------------------------------------- 1 | package gen 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "sort" 8 | "strings" 9 | 10 | "github.com/openllb/doxygen-parser/doxygen" 11 | "github.com/openllb/hlb/parser" 12 | "github.com/openllb/hlb/parser/ast" 13 | ) 14 | 15 | // Documentation contains all the builtin functions defined for HLB. 16 | type Documentation struct { 17 | Builtins []Builtin 18 | } 19 | 20 | type Builtin struct { 21 | Type string 22 | Funcs []*Func 23 | } 24 | 25 | type Func struct { 26 | Doc string 27 | Type string 28 | Name string 29 | Params []Field 30 | Options []*Func 31 | } 32 | 33 | type Field struct { 34 | Doc string 35 | Variadic bool 36 | Type string 37 | Name string 38 | } 39 | 40 | func GenerateDocumentation(ctx context.Context, r io.Reader) (*Documentation, error) { 41 | mod, err := parser.Parse(ctx, r) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | var ( 47 | funcsByKind = make(map[string][]*Func) 48 | optionsByFunc = make(map[string][]*Func) 49 | ) 50 | 51 | for _, decl := range mod.Decls { 52 | fd := decl.Func 53 | if fd == nil { 54 | continue 55 | } 56 | 57 | var ( 58 | group *doxygen.Group 59 | kind string 60 | name string 61 | fields []Field 62 | ) 63 | 64 | if fd.Doc != nil { 65 | var commentBlock []string 66 | for _, comment := range fd.Doc.List { 67 | text := strings.TrimSpace(strings.TrimPrefix(comment.Text, "#")) 68 | commentBlock = append(commentBlock, fmt.Sprintf("%s\n", text)) 69 | } 70 | 71 | group, err = doxygen.Parse(strings.NewReader(strings.Join(commentBlock, ""))) 72 | if err != nil { 73 | return nil, err 74 | } 75 | } 76 | 77 | if fd.Sig.Type != nil { 78 | kind = fd.Sig.Type.String() 79 | } 80 | 81 | if fd.Sig.Name != nil { 82 | name = fd.Sig.Name.String() 83 | } 84 | 85 | if fd.Sig.Params != nil { 86 | for _, param := range fd.Sig.Params.Fields() { 87 | var ( 88 | fieldType string 89 | fieldName string 90 | ) 91 | 92 | if param.Type != nil { 93 | fieldType = param.Type.String() 94 | } 95 | 96 | if param.Name != nil { 97 | fieldName = param.Name.String() 98 | } 99 | 100 | field := Field{ 101 | Variadic: param.Modifier != nil && param.Modifier.Variadic != nil, 102 | Type: fieldType, 103 | Name: fieldName, 104 | } 105 | 106 | if group != nil { 107 | for _, dparam := range group.Params { 108 | if dparam.Name != fieldName { 109 | continue 110 | } 111 | 112 | field.Doc = dparam.Description 113 | } 114 | } 115 | 116 | fields = append(fields, field) 117 | } 118 | } 119 | 120 | funcDoc := &Func{ 121 | Type: kind, 122 | Name: name, 123 | Params: fields, 124 | } 125 | 126 | if group != nil { 127 | funcDoc.Doc = strings.TrimSpace(group.Doc) 128 | } 129 | 130 | if fd.Kind().Primary() == ast.Option { 131 | subtype := string(fd.Sig.Type.Kind.Secondary()) 132 | optionsByFunc[subtype] = append(optionsByFunc[subtype], funcDoc) 133 | } 134 | funcsByKind[kind] = append(funcsByKind[kind], funcDoc) 135 | } 136 | 137 | for _, funcs := range funcsByKind { 138 | for _, fun := range funcs { 139 | options, ok := optionsByFunc[fun.Name] 140 | if !ok { 141 | continue 142 | } 143 | 144 | fun.Options = append(fun.Options, options...) 145 | } 146 | } 147 | 148 | var doc Documentation 149 | 150 | for _, kind := range []string{"fs", "string"} { 151 | funcs := funcsByKind[kind] 152 | for _, fun := range funcs { 153 | fun := fun 154 | sort.SliceStable(fun.Options, func(i, j int) bool { 155 | return fun.Options[i].Name < fun.Options[j].Name 156 | }) 157 | } 158 | 159 | sort.SliceStable(funcs, func(i, j int) bool { 160 | return funcs[i].Name < funcs[j].Name 161 | }) 162 | 163 | builtin := Builtin{ 164 | Type: kind, 165 | } 166 | 167 | builtin.Funcs = append(builtin.Funcs, funcs...) 168 | 169 | doc.Builtins = append(doc.Builtins, builtin) 170 | } 171 | 172 | return &doc, nil 173 | } 174 | -------------------------------------------------------------------------------- /language/hlb-pygments.py: -------------------------------------------------------------------------------- 1 | from pygments.lexer import RegexLexer, bygroups 2 | from pygments.token import * 3 | 4 | import re 5 | 6 | __all__=['HlbLexer'] 7 | 8 | class HlbLexer(RegexLexer): 9 | name = 'Hlb' 10 | aliases = ['hlb'] 11 | filenames = ['*.hlb'] 12 | flags = re.MULTILINE | re.UNICODE 13 | 14 | tokens = { 15 | 'root' : [ 16 | (u'(#.*)', bygroups(Comment.Single)), 17 | (u'((\\b(0(b|B|o|O|x|X)[a-fA-F0-9]+)\\b)|(\\b(0|[1-9][0-9]*)\\b)|(\\b(true|false)\\b))', bygroups(Name.Constant)), 18 | (u'(\")', bygroups(Punctuation), 'common__1'), 19 | (u'(<<[-~]?)([A-Z]+)', bygroups(Punctuation, Name.Constant), 'common__2'), 20 | (u'(\\bstring\\b|\\bint\\b|\\bbool\\b|\\bfs\\b|\\bgroup\\b|\\boption(?!::)\\b|\\boption::(?:copy|frontend|git|http|image|local|mkdir|mkfile|mount|rm|run|secret|ssh|template)\\b)', bygroups(Keyword.Type)), 21 | (u'(\\b(import|export|from|binds|as|with|variadic)\\b)', bygroups(Keyword)), 22 | (u'(\\b[a-zA-Z_][a-zA-Z0-9_]*\\b)(\\()', bygroups(Name.Variable, Punctuation), 'params'), 23 | (u'(\\))', bygroups(Generic.Error)), 24 | (u'((?:[\\t ]+)binds(?:[\\t ]+))(\\()', bygroups(Keyword, Punctuation), 'params'), 25 | (u'(\\))', bygroups(Generic.Error)), 26 | (u'(\\{)', bygroups(Punctuation), 'block'), 27 | (u'(\\})', bygroups(Generic.Error)), 28 | (u'(\\b[a-zA-Z_][a-zA-Z0-9_]*\\b)', bygroups(Name.Variable)), 29 | ('(\n|\r|\r\n)', String), 30 | ('.', String), 31 | ], 32 | 'binding' : [ 33 | (u'(\\b[a-zA-Z_][a-zA-Z0-9_]*\\b)((?:[\\t ]+))(\\b[a-zA-Z_][a-zA-Z0-9_]*\\b)', bygroups(Name.Builtin, Punctuation, Name.Variable)), 34 | ('(\n|\r|\r\n)', String), 35 | ('.', String), 36 | ], 37 | 'block' : [ 38 | (u'(#.*)', bygroups(Comment.Single)), 39 | (u'((\\b(0(b|B|o|O|x|X)[a-fA-F0-9]+)\\b)|(\\b(0|[1-9][0-9]*)\\b)|(\\b(true|false)\\b))', bygroups(Name.Constant)), 40 | (u'(\")', bygroups(Punctuation), 'common__1'), 41 | (u'(<<[-~]?)([A-Z]+)', bygroups(Punctuation, Name.Constant), 'common__2'), 42 | (u'((?:[\\t ]+)with(?:[\\t ]+))', bygroups(Keyword)), 43 | (u'(as)((?:[\\t ]+))(\\b[a-zA-Z_][a-zA-Z0-9_]*\\b)', bygroups(Keyword, Punctuation, Name.Variable)), 44 | (u'(binds)((?:[\\t ]+))(\\()', bygroups(Keyword, Punctuation, Punctuation), 'binding'), 45 | (u'(\\bstring\\b|\\bint\\b|\\bbool\\b|\\bfs\\b|\\bgroup\\b|\\boption(?!::)\\b|\\boption::(?:copy|frontend|git|http|image|local|mkdir|mkfile|mount|rm|run|secret|ssh|template)\\b)((?:[\\t ]+))(\\{)', bygroups(Keyword.Type, Punctuation, Punctuation), 'block'), 46 | (u'(\\b((?!(allowEmptyWildcard|allowNotFound|allowWildcard|cache|checksum|chmod|chown|contentsOnly|copy|createDestPath|createParents|createdTime|dir|dockerLoad|dockerPush|download|downloadDockerTarball|downloadOCITarball|downloadTarball|env|excludePatterns|filename|followPaths|followSymlinks|format|forward|frontend|gid|git|host|http|id|ignoreCache|image|includePatterns|input|insecure|keepGitDir|local|localEnv|localPaths|locked|mkdir|mkfile|mode|mount|network|node|opt|parallel|private|readonly|readonlyRootfs|resolve|rm|run|sandbox|scratch|secret|security|shared|sourcePath|ssh|stringField|target|template|tmpfs|uid|unix|unpack|unset|user|value)\\b)[a-zA-Z_][a-zA-Z0-9]*\\b))', bygroups(Name.Variable)), 47 | (u'(\\b[a-zA-Z_][a-zA-Z0-9_]*\\b)', bygroups(Name.Builtin)), 48 | ('(\n|\r|\r\n)', String), 49 | ('.', String), 50 | ], 51 | 'common__1' : [ 52 | ('(\n|\r|\r\n)', String), 53 | ('.', String), 54 | ], 55 | 'common__2' : [ 56 | ('(\n|\r|\r\n)', String), 57 | ('.', String), 58 | ], 59 | 'params' : [ 60 | (u'(variadic)', bygroups(Keyword)), 61 | (u'(\\bstring\\b|\\bint\\b|\\bbool\\b|\\bfs\\b|\\bgroup\\b|\\boption(?!::)\\b|\\boption::(?:copy|frontend|git|http|image|local|mkdir|mkfile|mount|rm|run|secret|ssh|template)\\b)', bygroups(Keyword.Type)), 62 | (u'(\\b[a-zA-Z_][a-zA-Z0-9_]*\\b)', bygroups(Name.Variable)), 63 | ('(\n|\r|\r\n)', String), 64 | ('.', String), 65 | ] 66 | } 67 | 68 | -------------------------------------------------------------------------------- /parser/ast/match_test.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | type matches []match 12 | 13 | type match []string 14 | 15 | func matched(ifaces ...interface{}) match { 16 | var types []string 17 | for _, iface := range ifaces { 18 | types = append(types, reflect.TypeOf(iface).String()) 19 | } 20 | return match(types) 21 | } 22 | 23 | func TestMatch(t *testing.T) { 24 | type testCase struct { 25 | name string 26 | input string 27 | fun func(Node, chan match) 28 | expected matches 29 | } 30 | 31 | for _, tc := range []testCase{{ 32 | "empty", 33 | ``, 34 | func(root Node, ms chan match) { 35 | Match(root, MatchOpts{}) 36 | }, 37 | nil, 38 | }, { 39 | "root matcher", 40 | ``, 41 | func(root Node, ms chan match) { 42 | Match(root, MatchOpts{}, 43 | func(mod *Module) { 44 | ms <- matched(mod) 45 | }, 46 | ) 47 | }, 48 | matches{ 49 | matched(&Module{}), 50 | }, 51 | }, { 52 | "single matcher", 53 | ` 54 | fs foo() 55 | fs bar() 56 | `, 57 | func(root Node, ms chan match) { 58 | Match(root, MatchOpts{}, 59 | func(fd *FuncDecl) { 60 | ms <- matched(fd) 61 | }, 62 | ) 63 | }, 64 | matches{ 65 | matched(&FuncDecl{}), 66 | matched(&FuncDecl{}), 67 | }, 68 | }, { 69 | "chain matcher", 70 | ` 71 | fs default() { 72 | image "alpine" 73 | run "echo foo" with option { 74 | env "key" "value" 75 | } 76 | } 77 | `, 78 | func(root Node, ms chan match) { 79 | Match(root, MatchOpts{}, 80 | func(parentCall *CallStmt, childCall *CallStmt) { 81 | ms <- matched(parentCall, childCall) 82 | }, 83 | ) 84 | }, 85 | matches{ 86 | matched(&CallStmt{}, &CallStmt{}), 87 | }, 88 | }, { 89 | "chain matcher with nodes between", 90 | ` 91 | fs default() { 92 | image "alpine" 93 | run "echo foo" with option { 94 | env "key" "value" 95 | } 96 | } 97 | `, 98 | func(root Node, ms chan match) { 99 | Match(root, MatchOpts{}, 100 | func(fd *FuncDecl, call *CallStmt) { 101 | ms <- matched(fd, call) 102 | }, 103 | ) 104 | }, 105 | matches{ 106 | // image "alpine" 107 | matched(&FuncDecl{}, &CallStmt{}), 108 | // run "echo foo" 109 | matched(&FuncDecl{}, &CallStmt{}), 110 | }, 111 | }, { 112 | "chain matcher with nodes between but allow duplicates", 113 | ` 114 | fs default() { 115 | image "alpine" 116 | run "echo foo" with option { 117 | env "key" "value" 118 | } 119 | } 120 | `, 121 | func(root Node, ms chan match) { 122 | Match(root, MatchOpts{ 123 | AllowDuplicates: true, 124 | }, func(fd *FuncDecl, call *CallStmt) { 125 | ms <- matched(fd, call) 126 | }, 127 | ) 128 | }, 129 | matches{ 130 | // image "alpine" 131 | matched(&FuncDecl{}, &CallStmt{}), 132 | // run "echo foo" 133 | matched(&FuncDecl{}, &CallStmt{}), 134 | // env "key" "value" 135 | matched(&FuncDecl{}, &CallStmt{}), 136 | }, 137 | }, { 138 | "multiple matchers", 139 | ` 140 | import util from fs { 141 | image "util.hlb" 142 | } 143 | 144 | fs default() { 145 | util.base 146 | run "echo foo" with option { 147 | env "key" "value" 148 | } 149 | } 150 | `, 151 | func(root Node, ms chan match) { 152 | Match(root, MatchOpts{}, 153 | func(id *ImportDecl, lit *FuncLit) { 154 | ms <- matched(id, lit) 155 | }, 156 | func(fd *FuncDecl, lit *FuncLit) { 157 | ms <- matched(fd, lit) 158 | }, 159 | ) 160 | }, 161 | matches{ 162 | // from fs { ... } 163 | matched(&ImportDecl{}, &FuncLit{}), 164 | // with option { ... } 165 | matched(&FuncDecl{}, &FuncLit{}), 166 | }, 167 | }} { 168 | tc := tc 169 | t.Run(tc.name, func(t *testing.T) { 170 | mod := &Module{} 171 | r := strings.NewReader(cleanup(tc.input)) 172 | err := Parser.Parse("", r, mod) 173 | require.NoError(t, err) 174 | 175 | ms := make(chan match) 176 | go func() { 177 | defer close(ms) 178 | tc.fun(mod, ms) 179 | }() 180 | 181 | var actual matches 182 | for m := range ms { 183 | actual = append(actual, m) 184 | } 185 | require.Equal(t, tc.expected, actual) 186 | }) 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /language/hlb-rouge.rb: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- # 2 | 3 | module Rouge 4 | module Lexers 5 | class Hlb < RegexLexer 6 | title "hlb" 7 | tag 'Hlb' 8 | mimetypes 'text/x-hlb' 9 | filenames '*.hlb' 10 | 11 | state:root do 12 | rule /(#.*)/, Comment::Single 13 | rule /((\b(0(b|B|o|O|x|X)[a-fA-F0-9]+)\b)|(\b(0|[1-9][0-9]*)\b)|(\b(true|false)\b))/, Name::Constant 14 | rule /(")/, Punctuation, :common__1 15 | rule /(<<[-~]?)([A-Z]+)/ do 16 | groups Punctuation, Name::Constant 17 | push :common__2 18 | end 19 | rule /(\bstring\b|\bint\b|\bbool\b|\bfs\b|\bgroup\b|\boption(?!::)\b|\boption::(?:copy|frontend|git|http|image|local|mkdir|mkfile|mount|rm|run|secret|ssh|template)\b)/, Keyword::Type 20 | rule /(\b(import|export|from|binds|as|with|variadic)\b)/, Keyword 21 | rule /(\b[a-zA-Z_][a-zA-Z0-9_]*\b)(\()/ do 22 | groups Name::Variable, Punctuation 23 | push :params 24 | end 25 | rule /(\))/, Generic::Error 26 | rule /((?:[\t ]+)binds(?:[\t ]+))(\()/ do 27 | groups Keyword, Punctuation 28 | push :params 29 | end 30 | rule /(\))/, Generic::Error 31 | rule /(\{)/, Punctuation, :block 32 | rule /(\})/, Generic::Error 33 | rule /(\b[a-zA-Z_][a-zA-Z0-9_]*\b)/, Name::Variable 34 | rule /(\n|\r|\r\n)/, String 35 | rule /./, String 36 | end 37 | 38 | state:binding do 39 | rule /(\b[a-zA-Z_][a-zA-Z0-9_]*\b)((?:[\t ]+))(\b[a-zA-Z_][a-zA-Z0-9_]*\b)/ do 40 | groups Name::Builtin, Punctuation, Name::Variable 41 | end 42 | rule /(\n|\r|\r\n)/, String 43 | rule /./, String 44 | end 45 | 46 | state:block do 47 | rule /(#.*)/, Comment::Single 48 | rule /((\b(0(b|B|o|O|x|X)[a-fA-F0-9]+)\b)|(\b(0|[1-9][0-9]*)\b)|(\b(true|false)\b))/, Name::Constant 49 | rule /(")/, Punctuation, :common__1 50 | rule /(<<[-~]?)([A-Z]+)/ do 51 | groups Punctuation, Name::Constant 52 | push :common__2 53 | end 54 | rule /((?:[\t ]+)with(?:[\t ]+))/, Keyword 55 | rule /(as)((?:[\t ]+))(\b[a-zA-Z_][a-zA-Z0-9_]*\b)/ do 56 | groups Keyword, Punctuation, Name::Variable 57 | end 58 | rule /(binds)((?:[\t ]+))(\()/ do 59 | groups Keyword, Punctuation, Punctuation 60 | push :binding 61 | end 62 | rule /(\bstring\b|\bint\b|\bbool\b|\bfs\b|\bgroup\b|\boption(?!::)\b|\boption::(?:copy|frontend|git|http|image|local|mkdir|mkfile|mount|rm|run|secret|ssh|template)\b)((?:[\t ]+))(\{)/ do 63 | groups Keyword::Type, Punctuation, Punctuation 64 | push :block 65 | end 66 | rule /(\b((?!(allowEmptyWildcard|allowNotFound|allowWildcard|cache|checksum|chmod|chown|contentsOnly|copy|createDestPath|createParents|createdTime|dir|dockerLoad|dockerPush|download|downloadDockerTarball|downloadOCITarball|downloadTarball|env|excludePatterns|filename|followPaths|followSymlinks|format|forward|frontend|gid|git|host|http|id|ignoreCache|image|includePatterns|input|insecure|keepGitDir|local|localEnv|localPaths|locked|mkdir|mkfile|mode|mount|network|node|opt|parallel|private|readonly|readonlyRootfs|resolve|rm|run|sandbox|scratch|secret|security|shared|sourcePath|ssh|stringField|target|template|tmpfs|uid|unix|unpack|unset|user|value)\b)[a-zA-Z_][a-zA-Z0-9]*\b))/, Name::Variable 67 | rule /(\b[a-zA-Z_][a-zA-Z0-9_]*\b)/, Name::Builtin 68 | rule /(\n|\r|\r\n)/, String 69 | rule /./, String 70 | end 71 | 72 | state:common__1 do 73 | rule /(\n|\r|\r\n)/, String 74 | rule /./, String 75 | end 76 | 77 | state:common__2 do 78 | rule /(\n|\r|\r\n)/, String 79 | rule /./, String 80 | end 81 | 82 | state:params do 83 | rule /(variadic)/, Keyword 84 | rule /(\bstring\b|\bint\b|\bbool\b|\bfs\b|\bgroup\b|\boption(?!::)\b|\boption::(?:copy|frontend|git|http|image|local|mkdir|mkfile|mount|rm|run|secret|ssh|template)\b)/, Keyword::Type 85 | rule /(\b[a-zA-Z_][a-zA-Z0-9_]*\b)/, Name::Variable 86 | rule /(\n|\r|\r\n)/, String 87 | rule /./, String 88 | end 89 | 90 | end 91 | end 92 | end 93 | 94 | -------------------------------------------------------------------------------- /scripts/mkBuildkitdDroplet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # TODO add flags for --sshkey, --size, --region, --name, --image 5 | # TODO check for curl and jq binaries 6 | 7 | REPO="$1" 8 | SSH="$2" 9 | if [[ -z $REPO || -z $SSH ]]; then 10 | echo "Usage: $0 " >&2 11 | echo >&2 12 | echo "For example to add buildkitd actions runner to openllb/hlb project you would run:" >&2 13 | echo >&2 14 | echo " $0 https://github.com/openllb/hlb 3f:2a:4b:7d:d9:46:9f:43:99:3d:c3:48:bb:62:f3:a4" >&2 15 | echo >&2 16 | exit 1 17 | fi 18 | 19 | DIGITAL_OCEAN_TOKEN=$(cat ~/.digitalocean-token) 20 | 21 | if [[ -z $DIGITAL_OCEAN_TOKEN ]]; then 22 | echo "Missing ~/.digitalocean-token file which must contain an API token for DigitalOcean" >&2 23 | echo "To create a token go here: https://cloud.digitalocean.com/account/api/tokens and select" >&2 24 | echo "\"Generate New Token\", then run:" >&2 25 | echo >&2 26 | echo " echo \"put-token-here\" > $HOME/.digitalocean-token" >&2 27 | echo >&2 28 | exit 1 29 | fi 30 | 31 | GITHUB_TOKEN=$(cat ~/.github-repo-token) 32 | 33 | if [[ -z $GITHUB_TOKEN ]]; then 34 | echo "Missing ~/.github-repo-token file which must contain an API token for Github" >&2 35 | echo "To create a token go here: https://github.com/settings/tokens and select" >&2 36 | echo "\"Generate new token\", then under \"Selected scopes\" check the \"repo\" box." >&2 37 | echo >&2 38 | echo " echo \"put-token-here\" > $HOME/.github-repo-token" >&2 39 | echo >&2 40 | exit 1 41 | fi 42 | 43 | # get short-lived token to allow actions runner to register with Github 44 | REGISTER_TOKEN=$(curl -qsfL -X POST -H 'Authorization: token '$GITHUB_TOKEN'' https://api.github.com/repos/openllb/hlb/actions/runners/registration-token | jq -r .token) 45 | 46 | # Use github api to find latest release tarball location 47 | ACTIONS_RUNNER_TGZ=$(curl -qsLf https://api.github.com/repos/actions/runner/releases/latest | jq -r '.assets[] | select(.name | contains("linux")) | select(.name | contains("x64")) | .browser_download_url') 48 | 49 | USERDATA=$(cat < $5/mo 89 | # "s-1vcpu-2gb" => $10/mo 90 | # "s-3vcpu-1gb" => $15/mo 91 | # "s-2vcpu-2gb" => $15/mo 92 | # "s-1vcpu-3gb" => $15/mo 93 | # "s-2vcpu-4gb" => $20/mo 94 | 95 | PAYLOAD=$(cat <<'EOM' 96 | { 97 | name: "buildkitd", 98 | region: "nyc3", 99 | size: "s-2vcpu-2gb", 100 | image: "ubuntu-18-04-x64", 101 | ssh_keys: [$sshkey], 102 | user_data: $userdata 103 | } 104 | EOM 105 | ) 106 | 107 | jq -n --arg userdata "$USERDATA" --arg sshkey "$SSH" "$PAYLOAD" | \ 108 | curl -qsfL -X POST \ 109 | -H 'Content-Type: application/json' \ 110 | -H 'Authorization: Bearer '$DIGITAL_OCEAN_TOKEN'' \ 111 | "https://api.digitalocean.com/v2/droplets" \ 112 | -d@- | jq . 113 | -------------------------------------------------------------------------------- /pkg/llbutil/id.go: -------------------------------------------------------------------------------- 1 | package llbutil 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/fs" 9 | "net" 10 | "os" 11 | "strings" 12 | "time" 13 | 14 | "github.com/moby/buildkit/client/llb" 15 | digest "github.com/opencontainers/go-digest" 16 | "github.com/pkg/errors" 17 | "github.com/tonistiigi/fsutil" 18 | ) 19 | 20 | // LocalID returns a consistent hash for this local (path + options) so that 21 | // the same content doesn't get transported multiple times when referenced 22 | // repeatedly. 23 | func LocalID(ctx context.Context, absPath string, opts ...llb.LocalOption) (string, error) { 24 | uniqID, err := localUniqueID(absPath, opts...) 25 | if err != nil { 26 | return "", err 27 | } 28 | opts = append(opts, llb.LocalUniqueID(uniqID)) 29 | st := llb.Local("", opts...) 30 | 31 | def, err := st.Marshal(ctx) 32 | if err != nil { 33 | return "", err 34 | } 35 | 36 | // The terminal op of the graph def.Def[len(def.Def)-1] is an empty vertex with 37 | // an input to the last vertex's digest. Since that vertex also has its digests 38 | // of its inputs and so on, the digest of the terminal op is a merkle hash for 39 | // the graph. 40 | return digest.FromBytes(def.Def[len(def.Def)-1]).String(), nil 41 | } 42 | 43 | // localUniqueID returns a consistent string that is unique per host + dir + 44 | // last modified time. 45 | // 46 | // If there is already a solve in progress using the same local dir, we want to 47 | // deduplicate the "local" if the directory hasn't changed, but if there has 48 | // been a change, we must not identify the "local" as a duplicate. Thus, we 49 | // incorporate the last modified timestamp into the result. 50 | func localUniqueID(localPath string, opts ...llb.LocalOption) (string, error) { 51 | mac, err := FirstUpInterface() 52 | if err != nil { 53 | return "", err 54 | } 55 | 56 | fi, err := os.Stat(localPath) 57 | if err != nil { 58 | return "", err 59 | } 60 | 61 | lastModified := fi.ModTime() 62 | if fi.IsDir() { 63 | var localInfo llb.LocalInfo 64 | for _, opt := range opts { 65 | opt.SetLocalOption(&localInfo) 66 | } 67 | 68 | var walkOpts fsutil.FilterOpt 69 | if localInfo.IncludePatterns != "" { 70 | if err := json.Unmarshal([]byte(localInfo.IncludePatterns), &walkOpts.IncludePatterns); err != nil { 71 | return "", errors.Wrap(err, "failed to unmarshal IncludePatterns for localUniqueID") 72 | } 73 | } 74 | if localInfo.ExcludePatterns != "" { 75 | if err := json.Unmarshal([]byte(localInfo.ExcludePatterns), &walkOpts.ExcludePatterns); err != nil { 76 | return "", errors.Wrap(err, "failed to unmarshal ExcludePatterns for localUniqueID") 77 | } 78 | } 79 | if localInfo.FollowPaths != "" { 80 | if err := json.Unmarshal([]byte(localInfo.FollowPaths), &walkOpts.FollowPaths); err != nil { 81 | return "", errors.Wrap(err, "failed to unmarshal FollowPaths for localUniqueID") 82 | } 83 | } 84 | 85 | err := fsutil.Walk(context.Background(), localPath, &walkOpts, func(path string, info fs.FileInfo, err error) error { 86 | if info.ModTime().After(lastModified) { 87 | lastModified = info.ModTime() 88 | } 89 | return nil 90 | }) 91 | if err != nil { 92 | return "", err 93 | } 94 | } 95 | 96 | return fmt.Sprintf("path:%s,mac:%s,modified:%s", localPath, mac, lastModified.Format(time.RFC3339Nano)), nil 97 | } 98 | 99 | // FirstUpInterface returns the mac address for the first "UP" network 100 | // interface. 101 | func FirstUpInterface() (string, error) { 102 | interfaces, err := net.Interfaces() 103 | if err != nil { 104 | return "", err 105 | } 106 | 107 | for _, iface := range interfaces { 108 | if iface.Flags&net.FlagUp == 0 { 109 | continue // not up 110 | } 111 | if iface.HardwareAddr.String() == "" { 112 | continue // no mac 113 | } 114 | return iface.HardwareAddr.String(), nil 115 | 116 | } 117 | return "no-valid-interface", nil 118 | } 119 | 120 | func SecretID(path string) string { 121 | return digest.FromString(path).String() 122 | } 123 | 124 | func SSHID(paths ...string) string { 125 | return digest.FromString(strings.Join(paths, "")).String() 126 | } 127 | 128 | func OutputFromWriter(w io.WriteCloser) func(map[string]string) (io.WriteCloser, error) { 129 | return func(map[string]string) (io.WriteCloser, error) { 130 | return w, nil 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /pkg/llbutil/session.go: -------------------------------------------------------------------------------- 1 | package llbutil 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | 8 | "github.com/docker/cli/cli/config" 9 | "github.com/moby/buildkit/session" 10 | "github.com/moby/buildkit/session/auth/authprovider" 11 | "github.com/moby/buildkit/session/filesync" 12 | "github.com/moby/buildkit/session/secrets/secretsprovider" 13 | "github.com/openllb/hlb/pkg/sockproxy" 14 | "github.com/tonistiigi/fsutil" 15 | ) 16 | 17 | type SessionInfo struct { 18 | SyncTargetDir *string 19 | SyncTarget func(map[string]string) (io.WriteCloser, error) 20 | SyncedDirs filesync.StaticDirSource 21 | FileSourceByID map[string]secretsprovider.Source 22 | AgentConfigByID map[string]sockproxy.AgentConfig 23 | } 24 | 25 | type SessionOption func(*SessionInfo) 26 | 27 | func WithSyncTargetDir(dir string) SessionOption { 28 | return func(si *SessionInfo) { 29 | si.SyncTargetDir = &dir 30 | } 31 | } 32 | 33 | func WithSyncTarget(f func(map[string]string) (io.WriteCloser, error)) SessionOption { 34 | return func(si *SessionInfo) { 35 | si.SyncTarget = f 36 | } 37 | } 38 | 39 | func WithSyncedDir(name string, dir fsutil.FS) SessionOption { 40 | return func(si *SessionInfo) { 41 | si.SyncedDirs[name] = dir 42 | } 43 | } 44 | 45 | func WithSecretSource(id string, source secretsprovider.Source) SessionOption { 46 | return func(si *SessionInfo) { 47 | si.FileSourceByID[id] = source 48 | } 49 | } 50 | 51 | func WithAgentConfig(id string, cfg sockproxy.AgentConfig) SessionOption { 52 | return func(si *SessionInfo) { 53 | si.AgentConfigByID[id] = cfg 54 | } 55 | } 56 | 57 | func NewSession(ctx context.Context, opts ...SessionOption) (*session.Session, error) { 58 | si := SessionInfo{ 59 | SyncedDirs: make(filesync.StaticDirSource), 60 | FileSourceByID: make(map[string]secretsprovider.Source), 61 | AgentConfigByID: make(map[string]sockproxy.AgentConfig), 62 | } 63 | for _, opt := range opts { 64 | opt(&si) 65 | } 66 | 67 | // By default, forward docker authentication through the session. 68 | dockerConfig := config.LoadDefaultConfigFile(os.Stderr) 69 | attachables := []session.Attachable{authprovider.NewDockerAuthProvider(dockerConfig, nil)} 70 | 71 | // Attach local directory the session can write to. 72 | syncIndex := 0 73 | if si.SyncTargetDir != nil { 74 | attachables = append(attachables, filesync.NewFSSyncTarget(filesync.WithFSSyncDir(syncIndex, *si.SyncTargetDir))) 75 | syncIndex++ 76 | } 77 | 78 | // Attach writer the session can write to. 79 | if si.SyncTarget != nil { 80 | attachables = append(attachables, filesync.NewFSSyncTarget(filesync.WithFSSync(syncIndex, si.SyncTarget))) 81 | } 82 | 83 | // Attach local directory providers to the session. 84 | if len(si.SyncedDirs) > 0 { 85 | attachables = append(attachables, filesync.NewFSSyncProvider(si.SyncedDirs)) 86 | } 87 | 88 | // Attach ssh forwarding providers to the session. 89 | var agentConfigs []sockproxy.AgentConfig 90 | for _, cfg := range si.AgentConfigByID { 91 | agentConfigs = append(agentConfigs, cfg) 92 | } 93 | if len(agentConfigs) > 0 { 94 | sp, err := sockproxy.NewProvider(agentConfigs) 95 | if err != nil { 96 | return nil, err 97 | } 98 | attachables = append(attachables, sp) 99 | } 100 | 101 | // Attach secret providers to the session. 102 | var fileSources []secretsprovider.Source 103 | for _, cfg := range si.FileSourceByID { 104 | fileSources = append(fileSources, cfg) 105 | } 106 | if len(fileSources) > 0 { 107 | fileStore, err := secretsprovider.NewStore(fileSources) 108 | if err != nil { 109 | return nil, err 110 | } 111 | attachables = append(attachables, secretsprovider.NewSecretProvider(fileStore)) 112 | } 113 | 114 | // SharedKey is empty because we already use `llb.SharedKeyHint` for locals. 115 | // 116 | // Currently, the only use of SharedKey is in the calculation of the cache key 117 | // for local immutable ref in BuildKit. There isn't any functional difference 118 | // between `llb.SharedKeyHint` and a session's shared key atm. If anything 119 | // needs to start leveraging the session's shared key in the future, we 120 | // should probably use the codegen.Session(ctx) session id. 121 | s, err := session.NewSession(ctx, "hlb", "") 122 | if err != nil { 123 | return s, err 124 | } 125 | 126 | for _, a := range attachables { 127 | s.Allow(a) 128 | } 129 | 130 | return s, nil 131 | } 132 | -------------------------------------------------------------------------------- /parser/ast/match.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | ) 7 | 8 | // MatchOpts provides options while walking the CST. 9 | type MatchOpts struct { 10 | // Filter is called to see if the node should be walked. If nil, then all 11 | // nodes are walked. 12 | Filter func(Node) bool 13 | 14 | // AllowDuplicates is enabled to allow duplicate CST structures between 15 | // arguments in the functions provided. 16 | // 17 | // For example, if a function is defined as `func(*FuncDecl, *CallStmt)` 18 | // sequences like `[*FuncDecl, ..., *CallStmt, ..., *CallStmt]` are not 19 | // matched by default. Allowing duplicates will match instead. 20 | AllowDuplicates bool 21 | } 22 | 23 | type matcher struct { 24 | opts MatchOpts 25 | vs []reflect.Value 26 | expects [][]reflect.Type 27 | actuals [][]reflect.Value 28 | indices []int 29 | } 30 | 31 | // Match walks a CST and invokes given functions if their arguments match a 32 | // non-contiguous sequence of current path walked. This is useful when you want 33 | // to walk to a specific type of Node, while having access to specific parents 34 | // of the Node. 35 | // 36 | // The function arguments must all implement the Node interface, and may be 37 | // a non-contiguous sequence. That is, you don't have to specify every CST 38 | // structure. 39 | // 40 | // The sequence is matched right to left, from the deepest node first. The 41 | // final argument will always be the current node being visited. 42 | // 43 | // When multiple functions are matched, they are invoked in the order given 44 | // to Match. That way, you can write functions that positively match, and then 45 | // provide a more general function as a catch all without walking the CST a 46 | // second time. 47 | // 48 | // For example, you can invoke Match to find CallStmts inside FuncLits: 49 | // 50 | // ```go 51 | // Match(root, MatchOpts{}, 52 | // func(lit *FuncLit, call *CallStmt) { 53 | // fmt.Println(lit.Pos, call.Pos) 54 | // }, 55 | // ) 56 | // ``` 57 | func Match(root Node, opts MatchOpts, funs ...interface{}) { 58 | m := &matcher{ 59 | opts: opts, 60 | vs: make([]reflect.Value, len(funs)), 61 | expects: make([][]reflect.Type, len(funs)), 62 | actuals: make([][]reflect.Value, len(funs)), 63 | indices: make([]int, len(funs)), 64 | } 65 | 66 | for i, fun := range funs { 67 | m.vs[i] = reflect.ValueOf(fun) 68 | } 69 | 70 | node := reflect.TypeOf((*Node)(nil)).Elem() 71 | for i, v := range m.vs { 72 | t := v.Type() 73 | for j := 0; j < t.NumIn(); j++ { 74 | arg := t.In(j) 75 | if !arg.Implements(node) { 76 | panic(fmt.Sprintf("%s has bad signature: %s does not implement Node", t, arg)) 77 | } 78 | 79 | m.expects[i] = append(m.expects[i], arg) 80 | } 81 | } 82 | 83 | for i, expect := range m.expects { 84 | m.actuals[i] = make([]reflect.Value, len(expect)) 85 | } 86 | 87 | Walk(root, m) 88 | } 89 | 90 | func (m *matcher) Visit(in Introspector, n Node) Visitor { 91 | if n == nil { 92 | return nil 93 | } 94 | 95 | if m.opts.Filter != nil { 96 | if !m.opts.Filter(n) { 97 | return nil 98 | } 99 | } 100 | 101 | // Clear out indices from a previous visit. 102 | for i := 0; i < len(m.expects); i++ { 103 | m.indices[i] = len(m.expects[i]) - 1 104 | } 105 | 106 | for i := len(in.Path()) - 1; i >= 0; i-- { 107 | p := in.Path()[i] 108 | v := reflect.ValueOf(p) 109 | 110 | for j, expect := range m.expects { 111 | k := m.indices[j] 112 | 113 | // Either the function has been matched or will never match. 114 | if k < 0 { 115 | continue 116 | } 117 | 118 | if v.Type() != expect[k] { 119 | if i == len(in.Path())-1 { 120 | // The final argument must always match the deepest node. 121 | m.indices[j] = -2 122 | } else if !m.opts.AllowDuplicates && v.Type() == expect[k+1] { 123 | // Unless duplicates are allowed, the current node cannot be the same 124 | // type as the previous matched node. 125 | m.indices[j] = -2 126 | } 127 | 128 | continue 129 | } 130 | 131 | m.actuals[j][k] = v 132 | m.indices[j] -= 1 133 | } 134 | } 135 | 136 | // Invoke matched functions in the order they were given. 137 | for i := 0; i < len(m.vs); i++ { 138 | // Functions that will never match have an index of -2. 139 | // Functions that matched have an index of -1. 140 | if m.indices[i] == -1 { 141 | m.vs[i].Call(m.actuals[i]) 142 | } 143 | } 144 | 145 | return m 146 | } 147 | -------------------------------------------------------------------------------- /mkdocs.hlb: -------------------------------------------------------------------------------- 1 | export generatedBuiltin 2 | 3 | export generatedMarkdown 4 | 5 | export build 6 | 7 | export publish 8 | 9 | import go from image("openllb/go.hlb") 10 | 11 | fs mkdocsMaterial() { 12 | image "python:alpine" 13 | run "apk add -U git git-fast-import openssh-client build-base" 14 | sshKeyScan "github.com" 15 | run "pip install --upgrade pip" 16 | run "pip install mkdocs-material pymdown-extensions pygments" 17 | } 18 | 19 | fs _runMkdocsBuild() { 20 | mkdocsMaterial 21 | run "mkdocs build -d /site" with option { 22 | dir "/mkdocs" 23 | mount fs { 24 | local "." with includePatterns("mkdocs.yml", "docs/", ".git") 25 | } "/mkdocs" with readonly 26 | mount generatedMarkdown "/mkdocs/docs/reference.md" with option { 27 | sourcePath "reference.md" 28 | readonly 29 | } 30 | mount scratch "/site" as build 31 | } 32 | } 33 | 34 | # Note this only publishes master on github, it does 35 | # not publish local files 36 | fs publish() { 37 | mkdocsMaterial 38 | run "mkdocs gh-deploy" with option { 39 | dir "/src" 40 | mount gitSource "/src" 41 | ssh 42 | } 43 | } 44 | 45 | fs _runHandleBars() { 46 | image "node:alpine" 47 | run "node src/compile.js" with option { 48 | dir "src" 49 | mount fs { 50 | local "docs/templates" with includePatterns("src", "reference") 51 | } "/src" with readonly 52 | mount fs { 53 | nodeModules fs { 54 | local "docs/templates" with includePatterns("package.json", "package-lock.json") 55 | } 56 | } "/src/node_modules" with readonly 57 | mount referenceJson "/src/data" with readonly 58 | mount scratch "/src/dist" as generatedMarkdown 59 | } 60 | } 61 | 62 | fs npmInstall(fs src) { 63 | image "node:alpine" 64 | run "npm install" with option { 65 | dir "/src" 66 | mount src "/src" 67 | mount scratch "/src/node_modules" as nodeModules 68 | } 69 | } 70 | 71 | fs _runDocGen() { 72 | scratch 73 | run "/docgen" "/language/builtin.hlb" "/out/reference.json" with option { 74 | mount fs { 75 | staticGoBuild "./cmd/docgen" fs { 76 | local "." with includePatterns("**/*.go", "go.mod", "go.sum") 77 | } 78 | } "/" with readonly 79 | mount fs { 80 | local "language" with includePatterns("builtin.hlb") 81 | } "language" with readonly 82 | mount scratch "/out" as referenceJson 83 | } 84 | } 85 | 86 | fs _runBuiltinGen() { 87 | scratch 88 | run "/builtingen" "/language/builtin.hlb" "/out/lookup.go" with option { 89 | mount fs { 90 | staticGoBuild "./cmd/builtingen" fs { 91 | local "." with includePatterns("**/*.go", "go.mod", "go.sum") 92 | } 93 | } "/" with readonly 94 | mount fs { 95 | local "language" with includePatterns("builtin.hlb") 96 | } "language" with readonly 97 | mount scratch "/out" as generatedBuiltin 98 | } 99 | } 100 | 101 | fs staticGoBuild(string package, fs src) { 102 | go.buildWithOptions src package option::template { 103 | stringField "base" "docker.elastic.co/beats-dev/golang-crossbuild" 104 | stringField "goBuildFlags" <<~EOM 105 | -ldflags "-extldflags -static" 106 | EOM 107 | stringField "goVersion" "1.21.3" 108 | stringField "platform" "linux" 109 | stringField "arch" "amd64" 110 | } option::run { 111 | env "CGO_ENABLED" "0" 112 | } 113 | } 114 | 115 | # TODO add this to a generic util module? 116 | fs testSSH() { 117 | image "alpine" 118 | run "apk add -U openssh-client" 119 | sshKeyScan "github.com" 120 | run "ssh -q -T git@github.com" with ssh 121 | } 122 | 123 | # TODO add this to a generic util moduile 124 | fs sshKeyScan(string host) { 125 | mkdir "/root/.ssh" 0o700 126 | run "ssh-keyscan ${host} >> ~/.ssh/known_hosts" 127 | } 128 | 129 | # TODO can we add this logic to a generic util module for publishing gh-pages? 130 | fs _fetchGhPagesBranch() { 131 | image "alpine/git" 132 | sshKeyScan "github.com" 133 | run "git fetch origin gh-pages" with option { 134 | dir "/src" 135 | mount fs { 136 | git "git://github.com/openllb/hlb.git" "master" with keepGitDir 137 | # we have to recreate the .git/config because the one that 138 | # comes from buildkit has invalid remote.origin.url and 139 | # no branch.master properties 140 | mkfile ".git/config" 0o644 <<-EOM 141 | [core] 142 | repositoryformatversion = 0 143 | filemode = true 144 | bare = false 145 | logallrefupdates = true 146 | [remote "origin"] 147 | url = git@github.com:openllb/hlb.git 148 | fetch = +refs/heads/*:refs/remotes/origin/* 149 | [branch "master"] 150 | remote = origin 151 | merge = refs/heads/master 152 | EOM 153 | } "/src" as gitSource 154 | ssh 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /module/resolve_test.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/lithammer/dedent" 13 | "github.com/moby/buildkit/client/llb" 14 | digest "github.com/opencontainers/go-digest" 15 | "github.com/openllb/hlb/builtin" 16 | "github.com/openllb/hlb/checker" 17 | "github.com/openllb/hlb/diagnostic" 18 | "github.com/openllb/hlb/errdefs" 19 | "github.com/openllb/hlb/parser" 20 | "github.com/openllb/hlb/parser/ast" 21 | "github.com/openllb/hlb/pkg/filebuffer" 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | type testCase struct { 26 | name string 27 | input string 28 | fn func(mod *ast.Module, imods map[string]*ast.Module) error 29 | } 30 | 31 | type testDirectory struct { 32 | fixtures map[string]string 33 | } 34 | 35 | func (r *testDirectory) Path() string { 36 | return "" 37 | } 38 | 39 | func (r *testDirectory) Digest() digest.Digest { 40 | return "" 41 | } 42 | 43 | func (r *testDirectory) Definition() *llb.Definition { 44 | return nil 45 | } 46 | 47 | func (r *testDirectory) Open(filename string) (io.ReadCloser, error) { 48 | fixture, ok := r.fixtures[filename] 49 | if !ok { 50 | return nil, os.ErrNotExist 51 | } 52 | return ioutil.NopCloser(strings.NewReader(fixture)), nil 53 | } 54 | 55 | func (r *testDirectory) Stat(filename string) (os.FileInfo, error) { 56 | return nil, fmt.Errorf("unimplemented") 57 | } 58 | 59 | func TestResolveGraph(t *testing.T) { 60 | t.Parallel() 61 | 62 | dir := &testDirectory{map[string]string{ 63 | "simple.hlb": ` 64 | export build 65 | fs build() {} 66 | `, 67 | "transitive.hlb": ` 68 | import simple from "simple.hlb" 69 | `, 70 | "transitive-deprecated.hlb": ` 71 | import simple "simple.hlb" 72 | `, 73 | "transitive-deprecated-unknown.hlb": ` 74 | import unknown "unknown.hlb" 75 | `, 76 | }} 77 | 78 | for _, tc := range []testCase{{ 79 | "simple import", 80 | ` 81 | import simple from "simple.hlb" 82 | `, 83 | nil, 84 | }, { 85 | "transitive import", 86 | ` 87 | import transitive from "transitive.hlb" 88 | `, 89 | nil, 90 | }, { 91 | "transitive deprecated import", 92 | ` 93 | import transitive from "transitive-deprecated.hlb" 94 | `, 95 | nil, 96 | }, { 97 | "import path not exist", 98 | ` 99 | import unknown from "unknown.hlb" 100 | `, 101 | func(mod *ast.Module, imods map[string]*ast.Module) error { 102 | return errdefs.WithImportPathNotExist( 103 | os.ErrNotExist, 104 | ast.Search(mod, `"unknown.hlb"`), 105 | "unknown.hlb", 106 | ) 107 | }, 108 | }, { 109 | "import path not exist deprecated", 110 | ` 111 | import transitive from "transitive-deprecated-unknown.hlb" 112 | `, 113 | func(mod *ast.Module, imods map[string]*ast.Module) error { 114 | return errdefs.WithImportPathNotExist( 115 | os.ErrNotExist, 116 | ast.Search(imods["transitive"], `"unknown.hlb"`), 117 | "unknown.hlb", 118 | ) 119 | }, 120 | }} { 121 | tc := tc 122 | t.Run(tc.name, func(t *testing.T) { 123 | ctx := filebuffer.WithBuffers(context.Background(), builtin.Buffers()) 124 | ctx = ast.WithModules(ctx, builtin.Modules()) 125 | 126 | in := strings.NewReader(dedent.Dedent(tc.input)) 127 | mod, err := parser.Parse(ctx, in) 128 | require.NoError(t, err) 129 | mod.Directory = dir 130 | 131 | err = checker.SemanticPass(mod) 132 | require.NoError(t, err) 133 | 134 | err = checker.Check(mod) 135 | require.NoError(t, err) 136 | 137 | imods := make(map[string]*ast.Module) 138 | err = ResolveGraph(ctx, nil, nil, mod, func(info VisitInfo) error { 139 | imods[info.ImportDecl.Name.Text] = info.Import 140 | return nil 141 | }) 142 | var expected error 143 | if tc.fn != nil { 144 | expected = tc.fn(mod, imods) 145 | } 146 | validateError(t, ctx, expected, err, tc.name) 147 | }) 148 | } 149 | } 150 | 151 | func validateError(t *testing.T, ctx context.Context, expected, actual error, name string) { 152 | switch { 153 | case expected == nil: 154 | require.NoError(t, actual, name) 155 | case actual == nil: 156 | require.NotNil(t, actual, name) 157 | default: 158 | espans := diagnostic.Spans(expected) 159 | aspans := diagnostic.Spans(actual) 160 | require.Equal(t, len(espans), len(aspans)) 161 | 162 | for i := 0; i < len(espans); i++ { 163 | epretty := espans[i].Pretty(ctx) 164 | t.Logf("[Expected]\n%s", epretty) 165 | apretty := aspans[i].Pretty(ctx) 166 | t.Logf("[Actual]\n%s", apretty) 167 | require.Equal(t, epretty, apretty, name) 168 | } 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /solver/request.go: -------------------------------------------------------------------------------- 1 | package solver 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/docker/buildx/util/progress" 7 | "github.com/moby/buildkit/client" 8 | "github.com/moby/buildkit/client/llb" 9 | "github.com/openllb/hlb/pkg/llbutil" 10 | "github.com/xlab/treeprint" 11 | "golang.org/x/sync/errgroup" 12 | ) 13 | 14 | // Request is a node in the solve request tree produced by the compiler. The 15 | // solve request tree has peer nodes that should be executed in parallel, and 16 | // next nodes that should be executed sequentially. These can be intermingled 17 | // to produce a complex build pipeline. 18 | type Request interface { 19 | // Solve sends the request and its children to BuildKit. The request passes 20 | // down the progress.Writer for them to spawn their own progress writers 21 | // for each independent solve. 22 | Solve(ctx context.Context, cln *client.Client, mw *MultiWriter, opts ...SolveOption) error 23 | 24 | Tree(tree treeprint.Tree) error 25 | } 26 | 27 | type nilRequest struct{} 28 | 29 | func NilRequest() Request { 30 | return &nilRequest{} 31 | } 32 | 33 | func (r *nilRequest) Solve(ctx context.Context, cln *client.Client, mw *MultiWriter, opts ...SolveOption) error { 34 | return nil 35 | } 36 | 37 | func (r *nilRequest) Tree(tree treeprint.Tree) error { 38 | return nil 39 | } 40 | 41 | type Params struct { 42 | Def *llb.Definition 43 | SolveOpts []SolveOption 44 | SessionOpts []llbutil.SessionOption 45 | } 46 | 47 | type singleRequest struct { 48 | params *Params 49 | } 50 | 51 | // Single returns a single solve request. 52 | func Single(params *Params) Request { 53 | return &singleRequest{params: params} 54 | } 55 | 56 | func (r *singleRequest) Solve(ctx context.Context, cln *client.Client, mw *MultiWriter, opts ...SolveOption) error { 57 | var pw progress.Writer 58 | if mw != nil { 59 | pw = mw.WithPrefix("", false) 60 | } 61 | 62 | s, err := llbutil.NewSession(ctx, r.params.SessionOpts...) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | g, ctx := errgroup.WithContext(ctx) 68 | 69 | g.Go(func() error { 70 | return s.Run(ctx, cln.Dialer()) 71 | }) 72 | 73 | g.Go(func() error { 74 | defer s.Close() 75 | return Solve(ctx, cln, s, pw, r.params.Def, append(r.params.SolveOpts, opts...)...) 76 | }) 77 | 78 | return g.Wait() 79 | } 80 | 81 | func (r *singleRequest) Tree(tree treeprint.Tree) error { 82 | return TreeFromDef(tree, r.params.Def, r.params.SolveOpts) 83 | } 84 | 85 | type parallelRequest struct { 86 | reqs []Request 87 | } 88 | 89 | func Parallel(candidates ...Request) Request { 90 | var reqs []Request 91 | for _, req := range candidates { 92 | switch r := req.(type) { 93 | case *nilRequest: 94 | continue 95 | case *parallelRequest: 96 | reqs = append(reqs, r.reqs...) 97 | continue 98 | } 99 | reqs = append(reqs, req) 100 | } 101 | if len(reqs) == 0 { 102 | return NilRequest() 103 | } else if len(reqs) == 1 { 104 | return reqs[0] 105 | } 106 | return ¶llelRequest{reqs: reqs} 107 | } 108 | 109 | func (r *parallelRequest) Solve(ctx context.Context, cln *client.Client, mw *MultiWriter, opts ...SolveOption) error { 110 | g, ctx := errgroup.WithContext(ctx) 111 | for _, req := range r.reqs { 112 | req := req 113 | g.Go(func() error { 114 | return req.Solve(ctx, cln, mw, opts...) 115 | }) 116 | } 117 | return g.Wait() 118 | } 119 | 120 | func (r *parallelRequest) Tree(tree treeprint.Tree) error { 121 | branch := tree.AddBranch("parallel") 122 | for _, req := range r.reqs { 123 | err := req.Tree(branch) 124 | if err != nil { 125 | return err 126 | } 127 | } 128 | return nil 129 | } 130 | 131 | type sequentialRequest struct { 132 | reqs []Request 133 | } 134 | 135 | func Sequential(candidates ...Request) Request { 136 | var reqs []Request 137 | for _, req := range candidates { 138 | switch r := req.(type) { 139 | case *nilRequest: 140 | continue 141 | case *sequentialRequest: 142 | reqs = append(reqs, r.reqs...) 143 | continue 144 | } 145 | reqs = append(reqs, req) 146 | } 147 | if len(reqs) == 0 { 148 | return NilRequest() 149 | } else if len(reqs) == 1 { 150 | return reqs[0] 151 | } 152 | return &sequentialRequest{reqs: reqs} 153 | } 154 | 155 | func (r *sequentialRequest) Solve(ctx context.Context, cln *client.Client, mw *MultiWriter, opts ...SolveOption) error { 156 | for _, req := range r.reqs { 157 | err := req.Solve(ctx, cln, mw, opts...) 158 | if err != nil { 159 | return err 160 | } 161 | } 162 | return nil 163 | } 164 | 165 | func (r *sequentialRequest) Tree(tree treeprint.Tree) error { 166 | branch := tree.AddBranch("sequential") 167 | for _, req := range r.reqs { 168 | err := req.Tree(branch) 169 | if err != nil { 170 | return err 171 | } 172 | } 173 | return nil 174 | } 175 | -------------------------------------------------------------------------------- /docs/tutorial/lets-begin.md: -------------------------------------------------------------------------------- 1 | Welcome to the `hlb` tutorial! 2 | 3 | In this tutorial, we will write a `hlb` program to fetch the `node_modules` of a node project. 4 | 5 | Along the way, you will learn the basics of creating and debugging a build graph. If you ever get stuck at any point of the tutorial, feel free to clone [hlb-tutorial]() to get a complete working example. 6 | 7 | ## Defining a function 8 | 9 | Let's start by creating a new directory and a file `node.hlb`. This is where we will write our `hlb` program. We will begin by defining a function which will become our target to build later. 10 | 11 | #!hlb 12 | fs npmInstall() { 13 | image "node:alpine" 14 | } 15 | 16 | A function begins with a `return type`, an `identifier`, an optional list of `arguments`, and then followed by a body enclosed in braces. The body must be non-empty, and in this example we are starting from a filesystem of a Docker image `node:alpine`. 17 | 18 | Since we haven't executed anything, we aren't done yet. Let's add a few more instructions to complete our program: 19 | 20 | #!hlb 21 | fs npmInstall() { 22 | image "node:alpine" 23 | run "apk add -U git" 24 | run "git clone https://github.com/left-pad/left-pad.git /src" 25 | dir "/src" 26 | run "npm install" 27 | } 28 | 29 | If you are thinking, "Hey, that looks like Dockerfile!", then you would be right! `hlb` is a superset of the features from Dockerfiles, but designed to leverage the full power of BuildKit. Let's go over what we did. 30 | 31 | 1. Fetched `git` using alpine's package manager `apk` 32 | 2. Cloned a simple node project from stash 33 | 3. Changed the current working directory to `/src` 34 | 4. Run `npm install`, which should produce a directory at `/src/node_modules` containing all the dependencies for the node project 35 | 36 | When you are ready, save the `node.hlb` file and run the build by using the `hlb` binary we previously installed. 37 | 38 | ```sh 39 | hlb run --target npmInstall node.hlb 40 | ``` 41 | 42 | You generated a `node_modules` directory, but since nothing was exported it is still with the BuildKit daemon. Of course, that is what we will be learning next. 43 | 44 | ## Exporting a directory 45 | 46 | Now that our build graph produces a `/src/node_modules`, one thing we might want to do is to export it to our system. However, if we export the target `npmInstall`, we'll not only get the `node_modules` directory, but also the rest of the alpine filesystem. In order to isolate the directory we want, we need to copy it to a new filesystem. 47 | 48 | #!hlb 49 | fs nodeModules() { 50 | scratch 51 | copy npmInstall "/src/node_modules" "/" 52 | } 53 | 54 | As we learned earlier, we can define functions which we can later target when running the build. In this new function, we are starting from a `scratch` filesystem (an empty one), and then copying the `/src/node_modules` from `npmInstall`. 55 | 56 | Since `hlb` is a functional language, variables and functions cannot be modified dynamically. When we copy from `npmInstall`, it is always referring to a snapshot of its filesystem after all its instructions have been executed. If we want to modify `npmInstall`, we will have to write a new function that starts from `npmInstall` but it will have to be defined with a new identifier. 57 | 58 | Now that we have isolated the directory, we can download the filesystem (containing only the `node_modules`) by specifying `--target ,download=`: 59 | 60 | ```sh 61 | hlb run --target nodeModules,download=. node.hlb 62 | ``` 63 | 64 | After the build have finished, you should see the `node_modules` in your working directory. 65 | 66 | ```sh 67 | $ ls 68 | node_modules node.hlb 69 | 70 | $ tree node_modules | tail -n 1 71 | 307 directories, 4014 files 72 | 73 | $ rm -rf node_modules 74 | ``` 75 | 76 | Once you have verified the directory is correct, remove it so we can keep our workspace clean. 77 | 78 | !!! tip "Not just to produce images" 79 | Although we are running a containerized build, it doesn't have to result in a container image. We can leverage the sandboxed environment in containers as a way to have a repeatable workflow. 80 | 81 | ## Going further 82 | 83 | Well done! We've now defined two functions `npmInstall` and `nodeModules` in our `hlb` program, and we've successfully downloaded just the `node_modules` directory to our system. We can still run `npmInstall` independently, because unused nodes in the build graph will be pruned if they're not a dependency of the target. 84 | 85 | If you've noticed, we didn't explicitly declare that `npmInstall` must be run before `nodeModules`. The superpower of `hlb` comes from the implicit build graph constructed by the instructions that invoke other functions. You don't need to think about what is safe to parallelize, and the more you decompose your build into smaller functions, the more concurrent your build! 86 | 87 | However, what we achieved so far is also possible with multi-stage Dockerfiles today. In the next chapter, we'll find out about hidden powers in BuildKit which we can start using in `hlb`. 88 | -------------------------------------------------------------------------------- /docs/tutorial/arguments.md: -------------------------------------------------------------------------------- 1 | At the end of the previous chapter, we were left with this: 2 | 3 | #!hlb 4 | fs src() { 5 | git "https://github.com/left-pad/left-pad.git" "master" 6 | } 7 | 8 | fs npmInstall() { 9 | image "node:alpine" 10 | dir "/src" 11 | run "npm install" with option { 12 | mount src "/src" 13 | mount fs { scratch; } "/src/node_modules" as nodeModules 14 | } 15 | } 16 | 17 | Currently our git repository is hardcoded into our program. Let's refactor it out so we can build `nodeModules` for other node projects. 18 | 19 | ## Refactoring our program 20 | 21 | When considering what to change to variables, we want to aim for extensibility. One approach may be to change the git remote into a variable, but that would mean we're stuck with git sources. Instead, let's refactor `src` to `npmInstall`'s function arguments so that we are opaque to where the source comes from. 22 | 23 | #!hlb 24 | fs npmInstall(fs src) { 25 | image "node:alpine" 26 | dir "/src" 27 | run "npm install" with option { 28 | mount src "/src" 29 | mount fs { scratch; } "/src/node_modules" as nodeModules 30 | } 31 | } 32 | 33 | Great! Now we can write a new function named `remoteModules` to pass in our git source. However, we don't want to invoke `npmInstall` because that will return the alpine filesystem, what we rather want is `nodeModules`. 34 | 35 | When an alias is declared in the body of a `fs` block, it also inherits the signature of the parent function. This is because `nodeModules` depends on the value of `src`, so when we defined a signature for `npmInstall`, `nodeModules` also inherited the signature: 36 | 37 | #!hlb 38 | fs nodeModules(fs src) 39 | 40 | We know how to invoke `nodeModules`, so let's pass the same git source we used earlier. 41 | 42 | #!hlb 43 | fs remoteModules() { 44 | nodeModules fs { git "https://github.com/left-pad/left-pad.git" "master"; } 45 | } 46 | 47 | There we have it. A reusable `nodeModules` function that is opaque to where the source code comes from. Let's take a look at one more type of source filesystem. 48 | 49 | ## Local sources 50 | 51 | So far we have been dealing with only remote sources like `image` and `git`, but what if you wanted to provide your working directory as a source filesystem? 52 | 53 | Turns out there is a source `local` that provides just that ability. Here's the signature: 54 | 55 | #!hlb 56 | # Rsyncs a local directory at path to a scratch filesystem. 57 | fs local(string path) 58 | 59 | We don't have a local node project at the moment, so let's write a function to initialize a node project and add `left-pad` as a dependency. We learnt how to use arguments just now, so let's apply our learnings and write a generic function. 60 | 61 | #!hlb 62 | fs npmInit(string package) { 63 | image "node:alpine" 64 | dir "/src" 65 | run string { 66 | format "npm init -y && npm install --package-lock-only %s" package 67 | } with option { 68 | mount fs { scratch; } "/src" as nodeProject 69 | } 70 | } 71 | 72 | fs nodeProjectWithLeftPad() { 73 | nodeProject "left-pad" 74 | } 75 | 76 | This time, instead of passing a string literal, we can pass a string block literal where we have access to string functions like `format`. This allows us to interpolate values into string to install an arbitrary package. 77 | 78 | When you're ready, run a build targetting `nodeProjectWithLeftPad` and download the initialized node project. 79 | 80 | ```sh 81 | hlb run --target nodeProjectWithLeftPad --download . node.hlb 82 | ``` 83 | 84 | You should see two new files `package.json` and `package-lock.json` in your working directory. 85 | 86 | ```sh 87 | $ ls 88 | node.hlb package.json package-lock.json 89 | ``` 90 | 91 | Now we can use the `local` source to download `node_modules`, but let's also use a `includePatterns` option to specify exactly what files we should sync up. 92 | 93 | #!hlb 94 | fs localModules() { 95 | nodeModules fs { 96 | local "." with option { 97 | includePatterns "package.json" "package-lock.json" 98 | } 99 | } 100 | } 101 | 102 | And finally, we can run `npm install` remotely using our working directory and transfer back the `node_modules`. 103 | 104 | ```sh 105 | hlb run --target localModules --download node_modules node.hlb 106 | ``` 107 | 108 | ## Advanced concepts 109 | 110 | You've made it to the end of basic concepts! Throughout the last few chapters, you wrote a few `hlb` programs to run `npm install` downloaded just the `node_modules` directory back to your system. You refactored it so that the function can be opaque to what the source comes from, and then you provided your working directory as a source filesystem. 111 | 112 | Next up, you can start writing your own `hlb` programs with the help of the [Reference](../reference.md). 113 | 114 | As your build graphs grow in complexity, it will be crucial to be able to introspect and diagonose issues as they come up. The next chapter will walk through the debugger that comes with the `hlb` CLI. 115 | -------------------------------------------------------------------------------- /language/hlb-sublime3.yaml: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: hlb 4 | scope: source.hlb 5 | file_extensions: [ hlb ] 6 | 7 | contexts: 8 | main: 9 | - include: common 10 | - match: '(\bstring\b|\bint\b|\bbool\b|\bfs\b|\bgroup\b|\boption(?!::)\b|\boption::(?:copy|frontend|git|http|image|local|mkdir|mkfile|mount|rm|run|secret|ssh|template)\b)' 11 | captures: 12 | 0: entity.name.type.hlb 13 | - match: '(\b(import|export|from|binds|as|with|variadic)\b)' 14 | captures: 15 | 0: keyword.hlb 16 | - match: '(\b[a-zA-Z_][a-zA-Z0-9_]*\b)(\()' 17 | push: params 18 | captures: 19 | 0: variable.hlb 20 | 1: punctuation.hlb 21 | - match: '(\))' 22 | captures: 23 | 0: invalid.hlb 24 | - match: '((?:[\t\x{0020}]+)binds(?:[\t\x{0020}]+))(\()' 25 | push: params 26 | captures: 27 | 0: keyword.hlb 28 | 1: punctuation.hlb 29 | - match: '(\))' 30 | captures: 31 | 0: invalid.hlb 32 | - match: '(\{)' 33 | push: block 34 | captures: 35 | 0: punctuation.hlb 36 | - match: '(\})' 37 | captures: 38 | 0: invalid.hlb 39 | - match: '(\b[a-zA-Z_][a-zA-Z0-9_]*\b)' 40 | captures: 41 | 0: variable.hlb 42 | - match: '(.)' 43 | captures: 44 | 0: text.hlb 45 | common: 46 | - match: '(#.*)' 47 | captures: 48 | 0: comment.hlb 49 | - match: '((\b(0(b|B|o|O|x|X)[a-fA-F0-9]+)\b)|(\b(0|[1-9][0-9]*)\b)|(\b(true|false)\b))' 50 | captures: 51 | 0: constant.hlb 52 | - match: '(")' 53 | captures: 54 | 0: punctuation.hlb 55 | push: 56 | - match: '(")' 57 | pop: true 58 | captures: 59 | 0: punctuation.hlb 60 | - match: '(.)' 61 | captures: 62 | 0: string.hlb 63 | - match: '(<<[-\x{007e}]?)([A-Z]+)' 64 | captures: 65 | 0: punctuation.hlb 66 | 1: constant.hlb 67 | push: 68 | - match: '(\2)' 69 | pop: true 70 | captures: 71 | 0: constant.hlb 72 | - match: '(.)' 73 | captures: 74 | 0: string.hlb 75 | - match: '(.)' 76 | captures: 77 | 0: text.hlb 78 | params: 79 | - match: '(\))' 80 | pop: true 81 | captures: 82 | 0: punctuation.hlb 83 | - match: '(variadic)' 84 | captures: 85 | 0: keyword.hlb 86 | - match: '(\bstring\b|\bint\b|\bbool\b|\bfs\b|\bgroup\b|\boption(?!::)\b|\boption::(?:copy|frontend|git|http|image|local|mkdir|mkfile|mount|rm|run|secret|ssh|template)\b)' 87 | captures: 88 | 0: entity.name.type.hlb 89 | - match: '(\b[a-zA-Z_][a-zA-Z0-9_]*\b)' 90 | captures: 91 | 0: variable.hlb 92 | - match: '(.)' 93 | captures: 94 | 0: text.hlb 95 | binding: 96 | - match: '(\))' 97 | pop: true 98 | captures: 99 | 0: punctuation.hlb 100 | - match: '(\b[a-zA-Z_][a-zA-Z0-9_]*\b)((?:[\t\x{0020}]+))(\b[a-zA-Z_][a-zA-Z0-9_]*\b)' 101 | captures: 102 | 0: variable.language.hlb 103 | 1: punctuation.hlb 104 | 2: variable.hlb 105 | - match: '(.)' 106 | captures: 107 | 0: text.hlb 108 | block: 109 | - match: '(\})' 110 | pop: true 111 | captures: 112 | 0: punctuation.hlb 113 | - include: common 114 | - match: '((?:[\t\x{0020}]+)with(?:[\t\x{0020}]+))' 115 | captures: 116 | 0: keyword.hlb 117 | - match: '(as)((?:[\t\x{0020}]+))(\b[a-zA-Z_][a-zA-Z0-9_]*\b)' 118 | captures: 119 | 0: keyword.hlb 120 | 1: punctuation.hlb 121 | 2: variable.hlb 122 | - match: '(binds)((?:[\t\x{0020}]+))(\()' 123 | push: binding 124 | captures: 125 | 0: keyword.hlb 126 | 1: punctuation.hlb 127 | 2: punctuation.hlb 128 | - match: '(\bstring\b|\bint\b|\bbool\b|\bfs\b|\bgroup\b|\boption(?!::)\b|\boption::(?:copy|frontend|git|http|image|local|mkdir|mkfile|mount|rm|run|secret|ssh|template)\b)((?:[\t\x{0020}]+))(\{)' 129 | push: block 130 | captures: 131 | 0: entity.name.type.hlb 132 | 1: punctuation.hlb 133 | 2: punctuation.hlb 134 | - match: '(\b((?!(allowEmptyWildcard|allowNotFound|allowWildcard|cache|checksum|chmod|chown|contentsOnly|copy|createDestPath|createParents|createdTime|dir|dockerLoad|dockerPush|download|downloadDockerTarball|downloadOCITarball|downloadTarball|env|excludePatterns|filename|followPaths|followSymlinks|format|forward|frontend|gid|git|host|http|id|ignoreCache|image|includePatterns|input|insecure|keepGitDir|local|localEnv|localPaths|locked|mkdir|mkfile|mode|mount|network|node|opt|parallel|private|readonly|readonlyRootfs|resolve|rm|run|sandbox|scratch|secret|security|shared|sourcePath|ssh|stringField|target|template|tmpfs|uid|unix|unpack|unset|user|value)\b)[a-zA-Z_][a-zA-Z0-9]*\b))' 135 | captures: 136 | 0: variable.hlb 137 | - match: '(\b[a-zA-Z_][a-zA-Z0-9_]*\b)' 138 | captures: 139 | 0: variable.language.hlb 140 | - match: '(.)' 141 | captures: 142 | 0: text.hlb 143 | -------------------------------------------------------------------------------- /codegen/debug/linespec.go: -------------------------------------------------------------------------------- 1 | package debug 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/openllb/hlb/checker" 12 | "github.com/openllb/hlb/linter" 13 | "github.com/openllb/hlb/parser" 14 | "github.com/openllb/hlb/parser/ast" 15 | ) 16 | 17 | // ParseLinespec returns an ast.StopNode that matches one of the location 18 | // specifiers supported by linespec. 19 | // 20 | // `` Specifies the line *line* in the current file. 21 | // 22 | // `+` Specifies the line *offset* lines after the current one. 23 | // 24 | // `-` Specifies the line *offset* lines before the current one. 25 | // 26 | // `[:] Specifies the line *line* inside *function* in the 27 | // current file. 28 | // 29 | // `:` Specifies the line *line* in *filename*, *filename* 30 | // can be the partial path to a file or even just the base name as long as the 31 | // expression remains unambiguous. 32 | // 33 | // `:[:] Specifies the line *line* inside *function* 34 | // in *filename*, *filename* can be the partial path toa file or eve njust the 35 | // base name as long as the expression remains unambiguous. 36 | // 37 | // `//` Specifies the location of all functions matching *regex*. 38 | func ParseLinespec(ctx context.Context, scope *ast.Scope, node ast.Node, linespec string) (ast.StopNode, error) { 39 | parts := strings.Split(linespec, ":") 40 | 41 | var ( 42 | root ast.Node 43 | line int 44 | err error 45 | ) 46 | switch len(parts) { 47 | case 1: // Either ``, `+`, `-`, `` or `//` 48 | mod := scope.ByLevel(ast.ModuleScope).Node.(*ast.Module) 49 | if strings.HasPrefix(parts[0], "+") { 50 | offset, err := strconv.Atoi(parts[0][1:]) 51 | if err != nil { 52 | return nil, fmt.Errorf("offset is not an int: %w", err) 53 | } 54 | root = mod 55 | line = node.Position().Line + offset 56 | } else if strings.HasPrefix(parts[0], "-") { 57 | offset, err := strconv.Atoi(parts[0][1:]) 58 | if err != nil { 59 | return nil, fmt.Errorf("offset is not an int: %w", err) 60 | } 61 | root = mod 62 | line = node.Position().Line - offset 63 | } else if strings.HasPrefix(parts[0], "/") && strings.HasSuffix(parts[0], "/") { 64 | return nil, fmt.Errorf("regex is unimplemented") 65 | } else { 66 | line, err = strconv.Atoi(parts[0]) 67 | if err == nil { 68 | root = mod 69 | } else { 70 | fd, err := findFunction(mod, parts[0]) 71 | if err != nil { 72 | return nil, err 73 | } 74 | root = fd 75 | line = fd.Position().Line 76 | } 77 | } 78 | case 2: // Either `:` or `:` or `: 79 | mod, err := findModule(ctx, parts[0]) 80 | if err == nil { // : or : 81 | root = mod 82 | line, err = strconv.Atoi(parts[1]) 83 | if err != nil { // : 84 | fd, err := findFunction(mod, parts[1]) 85 | if err != nil { 86 | return nil, err 87 | } 88 | root = fd 89 | line = fd.Pos.Line 90 | } 91 | } else if errors.Is(err, os.ErrNotExist) { // : 92 | mod := scope.ByLevel(ast.ModuleScope).Node.(*ast.Module) 93 | fd, err := findFunction(mod, parts[0]) 94 | if err != nil { 95 | return nil, err 96 | } 97 | root = fd 98 | 99 | line, err = strconv.Atoi(parts[1]) 100 | if err != nil { 101 | return nil, fmt.Errorf("line is not an int: %w", err) 102 | } 103 | 104 | line = fd.Pos.Line + line 105 | } else { 106 | return nil, err 107 | } 108 | case 3: // :: 109 | filename, function := parts[0], parts[1] 110 | line, err = strconv.Atoi(parts[2]) 111 | if err != nil { 112 | return nil, fmt.Errorf("line is not an int: %w", err) 113 | } 114 | 115 | mod, err := findModule(ctx, filename) 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | fd, err := findFunction(mod, function) 121 | if err != nil { 122 | return nil, err 123 | } 124 | root = fd 125 | line = fd.Pos.Line + line 126 | default: 127 | return nil, fmt.Errorf("invalid linespec %s", linespec) 128 | } 129 | 130 | node = ast.Find(root, line, 0, ast.StopNodeFilter) 131 | if node == nil { 132 | return nil, fmt.Errorf("%s:%d is not a valid line to breakpoint", root.Position().Filename, line) 133 | } 134 | return node.(ast.StopNode), nil 135 | } 136 | 137 | func findModule(ctx context.Context, filename string) (*ast.Module, error) { 138 | mod := ast.Modules(ctx).Get(filename) 139 | if mod == nil { 140 | f, err := os.Open(filename) 141 | if err != nil { 142 | return nil, err 143 | } 144 | defer f.Close() 145 | 146 | mod, err = parser.Parse(ctx, f) 147 | if err != nil { 148 | return nil, err 149 | } 150 | 151 | err = checker.SemanticPass(mod) 152 | if err != nil { 153 | return nil, err 154 | } 155 | 156 | _ = linter.Lint(ctx, mod) 157 | 158 | err = checker.Check(mod) 159 | if err != nil { 160 | return nil, err 161 | } 162 | } 163 | return mod, nil 164 | } 165 | 166 | func findFunction(mod *ast.Module, function string) (*ast.FuncDecl, error) { 167 | obj := mod.Scope.Lookup(function) 168 | if obj == nil { 169 | return nil, fmt.Errorf("no function %s in %s", function, mod.Pos.Filename) 170 | } 171 | fd, ok := obj.Node.(*ast.FuncDecl) 172 | if !ok { 173 | return nil, fmt.Errorf("%s is not a function but %T", function, obj.Node) 174 | } 175 | return fd, nil 176 | } 177 | -------------------------------------------------------------------------------- /pkg/filebuffer/filebuffer.go: -------------------------------------------------------------------------------- 1 | package filebuffer 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "sort" 9 | "sync" 10 | 11 | "github.com/alecthomas/participle/v2/lexer" 12 | "github.com/moby/buildkit/client/llb" 13 | ) 14 | 15 | type buffersKey struct{} 16 | 17 | func WithBuffers(ctx context.Context, buffers *BufferLookup) context.Context { 18 | return context.WithValue(ctx, buffersKey{}, buffers) 19 | } 20 | 21 | func Buffers(ctx context.Context) *BufferLookup { 22 | buffers, ok := ctx.Value(buffersKey{}).(*BufferLookup) 23 | if !ok { 24 | return NewBuffers() 25 | } 26 | return buffers 27 | } 28 | 29 | type BufferLookup struct { 30 | fbs map[string]*FileBuffer 31 | mu sync.Mutex 32 | } 33 | 34 | func NewBuffers() *BufferLookup { 35 | return &BufferLookup{ 36 | fbs: make(map[string]*FileBuffer), 37 | } 38 | } 39 | 40 | func (b *BufferLookup) Get(filename string) *FileBuffer { 41 | b.mu.Lock() 42 | defer b.mu.Unlock() 43 | return b.fbs[filename] 44 | } 45 | 46 | func (b *BufferLookup) Set(filename string, fb *FileBuffer) { 47 | b.mu.Lock() 48 | defer b.mu.Unlock() 49 | b.fbs[filename] = fb 50 | } 51 | 52 | func (b *BufferLookup) All() []*FileBuffer { 53 | var filenames []string 54 | for filename := range b.fbs { 55 | filenames = append(filenames, filename) 56 | } 57 | sort.Strings(filenames) 58 | var fbs []*FileBuffer 59 | for _, filename := range filenames { 60 | fbs = append(fbs, b.Get(filename)) 61 | } 62 | return fbs 63 | } 64 | 65 | type FileBuffer struct { 66 | filename string 67 | buf bytes.Buffer 68 | offset int 69 | offsets []int 70 | mu sync.Mutex 71 | onDisk bool 72 | sourceMap *llb.SourceMap 73 | } 74 | 75 | type Option func(*FileBuffer) 76 | 77 | func WithEphemeral() Option { 78 | return func(fb *FileBuffer) { 79 | fb.onDisk = false 80 | } 81 | } 82 | 83 | func New(filename string, opts ...Option) *FileBuffer { 84 | fb := &FileBuffer{ 85 | filename: filename, 86 | onDisk: true, 87 | } 88 | for _, opt := range opts { 89 | opt(fb) 90 | } 91 | return fb 92 | } 93 | 94 | func (fb *FileBuffer) Filename() string { 95 | return fb.filename 96 | } 97 | 98 | func (fb *FileBuffer) OnDisk() bool { 99 | return fb.onDisk 100 | } 101 | 102 | func (fb *FileBuffer) SourceMap() *llb.SourceMap { 103 | fb.mu.Lock() 104 | defer fb.mu.Unlock() 105 | if fb.sourceMap == nil { 106 | // This caching is important - BuildKit dedups SourceMaps based 107 | // on pointer address, so returning a fresh SourceMap each time 108 | // would blow up the size of the solve request. 109 | fb.sourceMap = llb.NewSourceMap(nil, fb.filename, "HLB", fb.buf.Bytes()) 110 | } 111 | return fb.sourceMap 112 | } 113 | 114 | func (fb *FileBuffer) Len() int { 115 | return len(fb.offsets) 116 | } 117 | 118 | func (fb *FileBuffer) Bytes() []byte { 119 | return fb.buf.Bytes() 120 | } 121 | 122 | func (fb *FileBuffer) Write(p []byte) (n int, err error) { 123 | fb.mu.Lock() 124 | defer fb.mu.Unlock() 125 | 126 | fb.sourceMap = nil 127 | 128 | n, err = fb.buf.Write(p) 129 | 130 | start := 0 131 | index := bytes.IndexByte(p[:n], byte('\n')) 132 | for index >= 0 { 133 | fb.offsets = append(fb.offsets, fb.offset+start+index) 134 | start += index + 1 135 | index = bytes.IndexByte(p[start:n], byte('\n')) 136 | } 137 | fb.offset += n 138 | 139 | return n, err 140 | } 141 | 142 | func (fb *FileBuffer) Position(line, column int) lexer.Position { 143 | var offset int 144 | if line-2 < 0 { 145 | offset = column - 1 146 | } else { 147 | offset = fb.offsets[line-2] + column - 1 148 | } 149 | return lexer.Position{ 150 | Filename: fb.filename, 151 | Offset: offset, 152 | Line: line, 153 | Column: column, 154 | } 155 | } 156 | 157 | func (fb *FileBuffer) Segment(offset int) ([]byte, error) { 158 | if len(fb.offsets) == 0 { 159 | return fb.buf.Bytes(), nil 160 | } 161 | 162 | index := fb.findNearestLineIndex(offset) 163 | start := 0 164 | if index >= 0 { 165 | start = fb.offsets[index] + 1 166 | } 167 | 168 | if start > fb.buf.Len()-1 { 169 | return nil, io.EOF 170 | } 171 | 172 | var end int 173 | if offset < fb.offsets[len(fb.offsets)-1] { 174 | end = fb.offsets[index+1] 175 | } else { 176 | end = fb.buf.Len() 177 | } 178 | 179 | return fb.read(start, end) 180 | } 181 | 182 | func (fb *FileBuffer) Line(ln int) ([]byte, error) { 183 | if ln > len(fb.offsets) { 184 | return nil, fmt.Errorf("line %d outside of offsets", ln) 185 | } 186 | 187 | start := 0 188 | if ln > 0 { 189 | start = fb.offsets[ln-1] + 1 190 | } 191 | 192 | end := fb.offsets[0] 193 | if ln > 0 { 194 | end = fb.offsets[ln] 195 | } 196 | 197 | return fb.read(start, end) 198 | } 199 | 200 | func (fb *FileBuffer) findNearestLineIndex(offset int) int { 201 | index := sort.Search(len(fb.offsets), func(i int) bool { 202 | return fb.offsets[i] >= offset 203 | }) 204 | 205 | if index < len(fb.offsets) { 206 | if fb.offsets[index] < offset { 207 | return index 208 | } 209 | return index - 1 210 | } else { 211 | // If offset is further than any newline, then the last newline is the 212 | // nearest. 213 | return index - 1 214 | } 215 | } 216 | 217 | func (fb *FileBuffer) read(start, end int) ([]byte, error) { 218 | r := bytes.NewReader(fb.buf.Bytes()) 219 | 220 | _, err := r.Seek(int64(start), io.SeekStart) 221 | if err != nil { 222 | return nil, err 223 | } 224 | 225 | line := make([]byte, end-start) 226 | n, err := r.Read(line) 227 | if err != nil && err != io.EOF { 228 | return nil, err 229 | } 230 | 231 | return line[:n], nil 232 | } 233 | -------------------------------------------------------------------------------- /parser/ast/walk_test.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/alecthomas/participle/v2/lexer" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestSearch(t *testing.T) { 13 | type testCase struct { 14 | name string 15 | input string 16 | query string 17 | opts []SearchOption 18 | expected string 19 | } 20 | 21 | for _, tc := range []testCase{{ 22 | "empty", 23 | ``, 24 | "", 25 | nil, 26 | "1:1", 27 | }, { 28 | "specific match", 29 | ` 30 | fs foo() 31 | `, 32 | "foo", 33 | nil, 34 | "1:4", 35 | }, { 36 | "partial match", 37 | ` 38 | fs foo() 39 | `, 40 | "foo()", 41 | nil, 42 | "1:1", 43 | }, { 44 | "skip match", 45 | ` 46 | fs default() { 47 | image "alpine" 48 | run "echo hello" 49 | run "echo world" 50 | } 51 | `, 52 | `run`, 53 | []SearchOption{WithSkip(1)}, 54 | "4:2", 55 | }} { 56 | tc := tc 57 | t.Run(tc.name, func(t *testing.T) { 58 | mod := &Module{} 59 | r := strings.NewReader(cleanup(tc.input)) 60 | err := Parser.Parse("", r, mod) 61 | require.NoError(t, err) 62 | 63 | node := Search(mod, tc.query, tc.opts...) 64 | actual := "" 65 | if node != nil { 66 | actual = formatPos(node.Position()) 67 | } 68 | require.Equal(t, tc.expected, actual) 69 | }) 70 | } 71 | } 72 | 73 | func TestFind(t *testing.T) { 74 | type testCase struct { 75 | name string 76 | input string 77 | line, column int 78 | filter func(Node) bool 79 | expectedStart, expectedEnd string 80 | } 81 | 82 | for _, tc := range []testCase{{ 83 | "empty", 84 | ``, 85 | 0, 0, 86 | nil, 87 | "", "", 88 | }, { 89 | "no column match", 90 | ` 91 | fs default() { 92 | image "alpine" 93 | } 94 | `, 95 | 2, 0, 96 | nil, 97 | "2:2", "3:1", // image "alpine" 98 | }, { 99 | "no column match multiline", 100 | ` 101 | fs default() { 102 | image "alpine" with option { 103 | resolve 104 | } 105 | } 106 | `, 107 | 2, 0, 108 | nil, 109 | "2:2", "5:1", // image "alpine" with option { ... } 110 | }, { 111 | "column match node", 112 | ` 113 | fs default() { 114 | image "alpine" 115 | } 116 | `, 117 | 2, 3, 118 | nil, 119 | "2:2", "2:7", // image 120 | }, { 121 | "column match parent", 122 | ` 123 | fs default() { 124 | image "alpine" 125 | } 126 | `, 127 | 2, 1, 128 | nil, 129 | "1:14", "3:2", // parent block stmt { ... } 130 | }, { 131 | "filter match", 132 | ` 133 | fs default() { 134 | image "alpine" 135 | } 136 | `, 137 | 1, 0, 138 | StopNodeFilter, 139 | "1:1", "1:13", // fs default() 140 | }, { 141 | "", 142 | ` 143 | fs default() { 144 | breakpoint 145 | image "alpine" 146 | run "echo hello" with breakpoint 147 | run "echo world" with option { 148 | breakpoint 149 | mount fs { 150 | breakpoint 151 | } "/in" 152 | } 153 | } 154 | `, 155 | 5, 0, 156 | StopNodeFilter, 157 | "5:2", "11:1", // run "echo world" with option { ... } 158 | }} { 159 | tc := tc 160 | t.Run(tc.name, func(t *testing.T) { 161 | mod := &Module{} 162 | r := strings.NewReader(cleanup(tc.input)) 163 | err := Parser.Parse("", r, mod) 164 | require.NoError(t, err) 165 | 166 | node := Find(mod, tc.line, tc.column, tc.filter) 167 | actualStart := "" 168 | actualEnd := "" 169 | if node != nil { 170 | actualStart = formatPos(node.Position()) 171 | actualEnd = formatPos(node.End()) 172 | } 173 | require.Equal(t, tc.expectedStart, actualStart) 174 | require.Equal(t, tc.expectedEnd, actualEnd) 175 | }) 176 | } 177 | } 178 | 179 | func formatPos(pos lexer.Position) string { 180 | return fmt.Sprintf("%d:%d", pos.Line, pos.Column) 181 | } 182 | 183 | func TestIsPositionWithinNode(t *testing.T) { 184 | type testCase struct { 185 | name string 186 | input string 187 | match func(root Node) Node 188 | line, column int 189 | expected bool 190 | } 191 | 192 | for _, tc := range []testCase{{ 193 | "empty", 194 | ``, 195 | func(root Node) Node { 196 | return root 197 | }, 198 | 1, 1, 199 | true, 200 | }, { 201 | "within", 202 | ` 203 | fs default() { 204 | image "alpine" 205 | } 206 | `, 207 | func(root Node) Node { 208 | return Search(root, `image "alpine"`) 209 | }, 210 | 2, 3, 211 | true, 212 | }, { 213 | "within whitespace", 214 | ` 215 | fs default() { 216 | image "alpine" 217 | } 218 | `, 219 | func(root Node) Node { 220 | return Search(root, `image "alpine"`) 221 | }, 222 | 2, 7, 223 | true, 224 | }, { 225 | "not within", 226 | ` 227 | fs default() { 228 | image "alpine" 229 | } 230 | `, 231 | func(root Node) Node { 232 | return Search(root, `image "alpine"`) 233 | }, 234 | 2, 1, 235 | false, 236 | }, { 237 | "not within beyond EOL", 238 | ` 239 | fs default() { 240 | image "alpine" 241 | } 242 | `, 243 | func(root Node) Node { 244 | return Search(root, `image "alpine"`) 245 | }, 246 | 2, 99, // image "alpine" contains newline 2:2 -> 3:1 247 | true, 248 | }} { 249 | tc := tc 250 | t.Run(tc.name, func(t *testing.T) { 251 | mod := &Module{} 252 | r := strings.NewReader(cleanup(tc.input)) 253 | err := Parser.Parse("", r, mod) 254 | require.NoError(t, err) 255 | 256 | n := tc.match(mod) 257 | require.NotNil(t, n) 258 | 259 | actual := IsPositionWithinNode(n, tc.line, tc.column) 260 | require.Equal(t, tc.expected, actual) 261 | }) 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /solver/tree.go: -------------------------------------------------------------------------------- 1 | package solver 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "sort" 7 | 8 | "github.com/kballard/go-shellquote" 9 | "github.com/moby/buildkit/client/llb" 10 | "github.com/moby/buildkit/solver/pb" 11 | "github.com/opencontainers/go-digest" 12 | "github.com/pkg/errors" 13 | "github.com/xlab/treeprint" 14 | ) 15 | 16 | func TreeFromDef(tree treeprint.Tree, def *llb.Definition, opts []SolveOption) error { 17 | var info SolveInfo 18 | for _, opt := range opts { 19 | err := opt(&info) 20 | if err != nil { 21 | return err 22 | } 23 | } 24 | 25 | ops := make(map[digest.Digest]*pb.Op) 26 | 27 | var dgst digest.Digest 28 | for _, dt := range def.Def { 29 | var op pb.Op 30 | if err := (&op).Unmarshal(dt); err != nil { 31 | return err 32 | } 33 | dgst = digest.FromBytes(dt) 34 | ops[dgst] = &op 35 | } 36 | 37 | if dgst == "" { 38 | return nil 39 | } 40 | 41 | terminal := ops[dgst] 42 | child := op{dgst: terminal.Inputs[0].Digest, ops: ops, meta: def.Metadata, info: info} 43 | return child.Tree(tree) 44 | } 45 | 46 | type op struct { 47 | dgst digest.Digest 48 | ops map[digest.Digest]*pb.Op 49 | meta map[digest.Digest]pb.OpMetadata 50 | info SolveInfo 51 | } 52 | 53 | func (o op) Tree(tree treeprint.Tree) error { 54 | pbOp := o.ops[o.dgst] 55 | 56 | var branch treeprint.Tree 57 | 58 | reportedInputs := map[digest.Digest]struct{}{} 59 | 60 | switch v := pbOp.Op.(type) { 61 | case *pb.Op_Source: 62 | var keys []string 63 | for key := range v.Source.Attrs { 64 | keys = append(keys, key) 65 | } 66 | sort.Strings(keys) 67 | branch = tree.AddMetaBranch("source", v.Source.Identifier) 68 | for _, key := range keys { 69 | branch.AddMetaNode(key, v.Source.Attrs[key]) 70 | } 71 | case *pb.Op_Exec: 72 | meta := v.Exec.Meta 73 | cmd := shellquote.Join(meta.Args...) 74 | if o.meta[o.dgst].IgnoreCache { 75 | cmd += " [ignoreCache]" 76 | } 77 | branch = tree.AddMetaBranch("exec", cmd) 78 | if len(meta.Env) > 0 { 79 | for _, env := range meta.Env { 80 | branch.AddMetaNode("env", env) 81 | } 82 | } 83 | 84 | if meta.Cwd != "" { 85 | branch.AddMetaNode("cwd", meta.Cwd) 86 | } 87 | if meta.User != "" { 88 | branch.AddMetaNode("user", meta.User) 89 | } 90 | 91 | for _, input := range pbOp.Inputs { 92 | reportedInputs[input.Digest] = struct{}{} 93 | } 94 | 95 | for _, mnt := range v.Exec.Mounts { 96 | opts := fmt.Sprintf("type=%s", mnt.MountType) 97 | if mnt.Readonly { 98 | opts += ",ro" 99 | } 100 | if mnt.CacheOpt != nil { 101 | opts += fmt.Sprintf(",cache-id=%s", mnt.CacheOpt.ID) 102 | opts += fmt.Sprintf(",sharing=%s", mnt.CacheOpt.Sharing) 103 | } 104 | if mnt.SecretOpt != nil { 105 | opts += fmt.Sprintf(",secret=%s", mnt.SecretOpt.ID) 106 | } 107 | if mnt.SSHOpt != nil { 108 | opts += fmt.Sprintf(",ssh=%s", mnt.SSHOpt.ID) 109 | } 110 | 111 | mountBranch := branch.AddMetaBranch("mount", fmt.Sprintf("%s [%s]", mnt.Dest, opts)) 112 | if mnt.Input >= 0 && int(mnt.Input) < len(pbOp.Inputs) { 113 | child := op{dgst: pbOp.Inputs[mnt.Input].Digest, ops: o.ops, meta: o.meta} 114 | err := child.Tree(mountBranch) 115 | if err != nil { 116 | return err 117 | } 118 | } else { 119 | mountBranch.AddNode("scratch") 120 | } 121 | } 122 | 123 | case *pb.Op_File: 124 | branch = tree.AddMetaBranch("file", v.File) 125 | case *pb.Op_Build: 126 | branch = tree.AddMetaBranch("build", v.Build) 127 | case *pb.Op_Merge: 128 | branch = tree.AddMetaBranch("merge", v.Merge) 129 | case *pb.Op_Diff: 130 | branch = tree.AddMetaBranch("diff", v.Diff) 131 | default: 132 | return errors.Errorf("unrecognized op %T", pbOp.Op) 133 | } 134 | 135 | var solve treeprint.Tree 136 | initSolve := func() { 137 | if solve == nil { 138 | solve = branch.AddBranch("solve options") 139 | } 140 | } 141 | if o.info.OutputDockerRef != "" { 142 | initSolve() 143 | solve.AddMetaNode("dockerRef", o.info.OutputDockerRef) 144 | } 145 | if o.info.OutputPushImage != "" { 146 | initSolve() 147 | solve.AddMetaNode("pushImage", o.info.OutputPushImage) 148 | } 149 | if o.info.OutputLocal != "" { 150 | initSolve() 151 | solve.AddMetaNode("download", o.info.OutputLocal) 152 | } 153 | if o.info.OutputLocalTarball { 154 | initSolve() 155 | solve.AddNode("downloadTarball") 156 | } 157 | if o.info.OutputLocalOCITarball { 158 | initSolve() 159 | solve.AddNode("downloadOCITarball") 160 | } 161 | if o.info.ImageSpec != nil { 162 | initSolve() 163 | dt, err := json.Marshal(o.info.ImageSpec) 164 | if err != nil { 165 | return err 166 | } 167 | solve.AddMetaNode("imageSpec", string(dt)) 168 | } 169 | if len(o.info.Entitlements) > 0 { 170 | initSolve() 171 | ent := solve.AddBranch("entitlements") 172 | for _, entitlements := range o.info.Entitlements { 173 | ent.AddNode(string(entitlements)) 174 | } 175 | } 176 | 177 | if pbOp.Platform != nil { 178 | branch.AddMetaNode("platform", fmt.Sprintf("%s,%s", pbOp.Platform.OS, pbOp.Platform.Architecture)) 179 | } 180 | 181 | if pbOp.Constraints != nil && len(pbOp.Constraints.Filter) > 0 { 182 | constraints := branch.AddBranch("constraints") 183 | for _, filter := range pbOp.Constraints.Filter { 184 | constraints.AddNode(filter) 185 | } 186 | } 187 | 188 | for _, input := range pbOp.Inputs { 189 | if _, ok := reportedInputs[input.Digest]; ok { 190 | continue 191 | } 192 | child := op{dgst: input.Digest, ops: o.ops, meta: o.meta} 193 | err := child.Tree(branch) 194 | if err != nil { 195 | return err 196 | } 197 | } 198 | 199 | return nil 200 | } 201 | -------------------------------------------------------------------------------- /codegen/builtin_string.go: -------------------------------------------------------------------------------- 1 | package codegen 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "os/exec" 9 | "strings" 10 | "text/template" 11 | 12 | "github.com/containerd/containerd/images" 13 | "github.com/containerd/containerd/platforms" 14 | "github.com/docker/distribution/reference" 15 | "github.com/moby/buildkit/client" 16 | "github.com/moby/buildkit/client/llb/sourceresolver" 17 | specs "github.com/opencontainers/image-spec/specs-go/v1" 18 | "github.com/openllb/hlb/errdefs" 19 | "github.com/openllb/hlb/local" 20 | "github.com/openllb/hlb/pkg/imageutil" 21 | ) 22 | 23 | type Format struct{} 24 | 25 | func (f Format) Call(ctx context.Context, cln *client.Client, val Value, opts Option, formatStr string, values ...string) (Value, error) { 26 | var a []interface{} 27 | for _, value := range values { 28 | a = append(a, value) 29 | } 30 | return NewValue(ctx, fmt.Sprintf(formatStr, a...)) 31 | } 32 | 33 | type Template struct{} 34 | 35 | func (t Template) Call(ctx context.Context, cln *client.Client, val Value, opts Option, text string) (Value, error) { 36 | tmpl, err := template.New("").Parse(text) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | data := map[string]interface{}{} 42 | for _, opt := range opts { 43 | switch o := opt.(type) { 44 | case *TemplateField: 45 | data[o.Name] = o.Value 46 | } 47 | } 48 | 49 | buf := bytes.NewBufferString("") 50 | err = tmpl.Execute(buf, data) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | return NewValue(ctx, buf.String()) 56 | } 57 | 58 | type LocalArch struct{} 59 | 60 | func (la LocalArch) Call(ctx context.Context, cln *client.Client, val Value, opts Option) (Value, error) { 61 | return NewValue(ctx, local.Arch(ctx)) 62 | } 63 | 64 | type LocalCwd struct{} 65 | 66 | func (lc LocalCwd) Call(ctx context.Context, cln *client.Client, val Value, opts Option) (Value, error) { 67 | cwd, err := local.Cwd(ctx) 68 | if err != nil { 69 | return nil, err 70 | } 71 | return NewValue(ctx, cwd) 72 | } 73 | 74 | type LocalOS struct{} 75 | 76 | func (lo LocalOS) Call(ctx context.Context, cln *client.Client, val Value, opts Option) (Value, error) { 77 | return NewValue(ctx, local.Os(ctx)) 78 | } 79 | 80 | type LocalEnv struct{} 81 | 82 | func (le LocalEnv) Call(ctx context.Context, cln *client.Client, val Value, opts Option, key string) (Value, error) { 83 | return NewValue(ctx, local.Env(ctx, key)) 84 | } 85 | 86 | type LocalRun struct{} 87 | 88 | func (lr LocalRun) Call(ctx context.Context, cln *client.Client, val Value, opts Option, args ...string) (Value, error) { 89 | var ( 90 | localRunOpts = &LocalRunOption{} 91 | shlex = false 92 | ) 93 | for _, opt := range opts { 94 | switch o := opt.(type) { 95 | case func(*LocalRunOption): 96 | o(localRunOpts) 97 | case *Shlex: 98 | shlex = true 99 | } 100 | } 101 | 102 | runArgs, err := ShlexArgs(args, shlex) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | cmd := exec.CommandContext(ctx, runArgs[0], runArgs[1:]...) 108 | cmd.Env = local.Environ(ctx) 109 | cmd.Dir = ModuleDir(ctx) 110 | 111 | var buf strings.Builder 112 | if localRunOpts.OnlyStderr { 113 | cmd.Stderr = &buf 114 | } else { 115 | cmd.Stdout = &buf 116 | } 117 | if localRunOpts.IncludeStderr { 118 | cmd.Stderr = &buf 119 | } 120 | 121 | err = cmd.Run() 122 | if err != nil && !localRunOpts.IgnoreError { 123 | return nil, err 124 | } 125 | 126 | return NewValue(ctx, strings.TrimRight(buf.String(), "\n")) 127 | } 128 | 129 | type Manifest struct{} 130 | 131 | func (m Manifest) Call(ctx context.Context, cln *client.Client, val Value, opts Option, ref string) (Value, error) { 132 | named, err := reference.ParseNormalizedNamed(ref) 133 | if err != nil { 134 | return nil, errdefs.WithInvalidImageRef(err, Arg(ctx, 0), ref) 135 | } 136 | ref = reference.TagNameOnly(named).String() 137 | 138 | var ( 139 | resolver = imageutil.NewBufferedImageResolver() 140 | matcher = resolver.MatchDefaultPlatform() 141 | ) 142 | var platform *specs.Platform 143 | for _, opt := range opts { 144 | if p, ok := opt.(*specs.Platform); ok { 145 | matcher = platforms.Only(*p) 146 | platform = p 147 | } 148 | } 149 | 150 | dgst, config, err := resolver.ResolveImageConfig(ctx, ref, sourceresolver.Opt{Platform: platform}) 151 | if err != nil { 152 | return nil, err 153 | } 154 | if dgst == "" { 155 | return nil, fmt.Errorf("no digest available for ref %q", ref) 156 | } 157 | 158 | desc, err := resolver.DigestDescriptor(ctx, dgst) 159 | if err != nil { 160 | return nil, err 161 | } 162 | 163 | switch Binding(ctx).Binds() { 164 | case "digest": 165 | return NewValue(ctx, dgst.String()) 166 | case "config": 167 | return NewValue(ctx, string(config)) 168 | case "index": 169 | switch desc.MediaType { 170 | case images.MediaTypeDockerSchema2ManifestList, 171 | specs.MediaTypeImageIndex: 172 | ra, err := resolver.ReaderAt(ctx, desc) 173 | if err != nil { 174 | return nil, err 175 | } 176 | defer ra.Close() 177 | 178 | dt := make([]byte, ra.Size()) 179 | _, err = ra.ReadAt(dt, 0) 180 | if err != nil { 181 | return nil, err 182 | } 183 | 184 | return NewValue(ctx, string(dt)) 185 | 186 | default: 187 | return nil, Arg(ctx, 0).WithError(fmt.Errorf("has no manifest index")) 188 | } 189 | } 190 | 191 | manifest, err := images.Manifest(ctx, resolver, desc, matcher) 192 | if err != nil { 193 | return nil, err 194 | } 195 | 196 | p, err := json.Marshal(manifest) 197 | if err != nil { 198 | return nil, err 199 | } 200 | 201 | return NewValue(ctx, string(p)) 202 | } 203 | -------------------------------------------------------------------------------- /pkg/llbutil/readonly_mountpoints.go: -------------------------------------------------------------------------------- 1 | package llbutil 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/moby/buildkit/client/llb" 9 | ) 10 | 11 | // ShimReadonlyMountpoints will modify the source for readonly mounts so 12 | // subsequent mounts that mount onto the readonly-mounts will have the 13 | // mountpoint present. 14 | // 15 | // For example if we have this code: 16 | // 17 | // ```hlb 18 | // run "make" with option { 19 | // dir "/src" 20 | // mount fs { 21 | // local "." 22 | // } "/src" with readonly 23 | // mount scratch "/src/output" as buildOutput 24 | // # ^^^^^ FAIL cannot create `output` directory for mount on readonly fs 25 | // secret "./secret/foo.pem" "/src/secret/foo.pem" 26 | // # ^^^^^ FAIL cannot create `./secret/foo.pm` for secret on readonly fs 27 | // } 28 | // } 29 | // ``` 30 | // 31 | // When containerd tries to mount /src/output on top of the /src mountpoint it 32 | // will fail because /src is mounted as readonly. The work around for this is 33 | // to inline create the mountpoints so that they pre-exist and containerd will 34 | // not have to create them. 35 | // 36 | // It can be done with HLB like: 37 | // 38 | // ```hlb 39 | // 40 | // run "make" with option { 41 | // dir "/src" 42 | // mount fs { 43 | // local "." 44 | // mkdir "output" 0o755 # <-- this is added to ensure mountpoint exists 45 | // mkdir "secret" 0o755 # <-- added so the secret can be mounted 46 | // mkfile "secret/foo.pm" 0o644 "" # <-- added so the secret can be mounted 47 | // } "/src" with readonly 48 | // mount scratch "/src/output" as buildOutput 49 | // } 50 | // } 51 | // 52 | // ``` 53 | // 54 | // So this function is effectively automatically adding the `mkdir` and `mkfile` 55 | // instructions when it detects that a mountpoint is required to be on a 56 | // readonly fs. 57 | func ShimReadonlyMountpoints(opts []llb.RunOption) error { 58 | // Short-circuit if we don't have any readonly mounts. 59 | haveReadonly := false 60 | for _, opt := range opts { 61 | if mnt, ok := opt.(*MountRunOption); ok { 62 | haveReadonly = mnt.IsReadonly() 63 | if haveReadonly { 64 | break 65 | } 66 | } 67 | } 68 | if !haveReadonly { 69 | return nil 70 | } 71 | 72 | // Collecting run options to look for targets (secrets, mounts) so we can 73 | // determine if there are overlapping mounts with readonly attributes. 74 | mountDetails := make([]struct { 75 | Target string 76 | Mount *MountRunOption 77 | }, len(opts)) 78 | 79 | for i, opt := range opts { 80 | switch runOpt := opt.(type) { 81 | case *MountRunOption: 82 | mountDetails[i].Target = runOpt.Target 83 | mountDetails[i].Mount = runOpt 84 | case llb.RunOption: 85 | ei := llb.ExecInfo{} 86 | runOpt.SetRunOption(&ei) 87 | if len(ei.Secrets) > 0 { 88 | // We only processed one option, so can have at most one secret. 89 | mountDetails[i].Target = ei.Secrets[0].Target 90 | continue 91 | } 92 | } 93 | } 94 | 95 | // madeDirs will keep track of directories we have had to create 96 | // so we don't duplicate instructions. 97 | madeDirs := map[string]struct{}{} 98 | 99 | // If we have readonly mounts and then secrets or other mounts on top of the 100 | // readonly mounts then we have to run a mkdir or mkfile on the mount first 101 | // before it become readonly. 102 | 103 | // Now walk the mountDetails backwards and look for common target paths 104 | // in prior mounts (mount ordering is significant). 105 | for i := len(mountDetails) - 1; i >= 0; i-- { 106 | src := mountDetails[i] 107 | if src.Target == "" { 108 | // Not a target option, like `dir "foo"`, so just skip. 109 | continue 110 | } 111 | for j := i - 1; j >= 0; j-- { 112 | dest := mountDetails[j] 113 | if !strings.HasPrefix(src.Target, dest.Target) { 114 | // Paths not common, skip. 115 | continue 116 | } 117 | if dest.Mount == nil { 118 | // Dest is not a mount, so skip. 119 | continue 120 | } 121 | if !dest.Mount.IsReadonly() { 122 | // Not mounting into readonly fs, so we are good with this mount. 123 | break 124 | } 125 | 126 | // We need to rewrite the mount at opts[j] so that that we mkdir and/or 127 | // mkfile. 128 | st := dest.Mount.Source 129 | if src.Mount != nil { 130 | // This is a mount, so we need to ensure the mount point 131 | // directory has been created. 132 | if _, ok := madeDirs[src.Target]; ok { 133 | // Already created the dir. 134 | break 135 | } 136 | 137 | // Update local cache so we don't make this dir again. 138 | madeDirs[dest.Target] = struct{}{} 139 | 140 | relativeDir, err := filepath.Rel(dest.Target, src.Target) 141 | if err != nil { 142 | return err 143 | } 144 | st = st.File( 145 | llb.Mkdir(relativeDir, os.FileMode(0755), llb.WithParents(true)), 146 | ) 147 | } else { 148 | // Not a mount, so must be a `secret` which will be a path to a file, we 149 | // will need to make the directory for the secret as well as an empty file 150 | // to be mounted over. 151 | dir := filepath.Dir(src.Target) 152 | relativeDir := strings.TrimPrefix(dir, dest.Target) 153 | 154 | if _, ok := madeDirs[dir]; !ok { 155 | // Update local cache so we don't make this dir again. 156 | madeDirs[dir] = struct{}{} 157 | 158 | st = st.File( 159 | llb.Mkdir(relativeDir, os.FileMode(0755), llb.WithParents(true)), 160 | ) 161 | } 162 | 163 | relativeFile, err := filepath.Rel(dest.Target, src.Target) 164 | if err != nil { 165 | return err 166 | } 167 | 168 | st = st.File( 169 | llb.Mkfile(relativeFile, os.FileMode(0644), []byte{}), 170 | ) 171 | } 172 | 173 | // Reset the mount option to include our state with mkdir/mkfile actions. 174 | opts[j] = &MountRunOption{ 175 | Target: dest.Target, 176 | Source: st, 177 | Opts: dest.Mount.Opts, 178 | } 179 | 180 | // Save the state for later in case we need to add more mkdir/mkfile actions. 181 | mountDetails[j].Mount.Source = st 182 | break 183 | } 184 | } 185 | 186 | return nil 187 | } 188 | --------------------------------------------------------------------------------