├── .compose ├── .env ├── grafana │ └── custom.ini ├── loki │ └── config.yaml ├── prometheus │ └── config.yaml ├── promtail │ └── config.yaml └── tempo │ └── config.yaml ├── .drone.yml ├── .gitignore ├── .gitmodules ├── CONTRIBUTING.md ├── LICENSE ├── Magefile.go ├── NOTICE ├── README.md ├── args ├── args.go ├── docs.go ├── flag_map_arg.go └── flag_optional_int.go ├── ci ├── main.go └── version.go ├── cmd ├── commands │ ├── args.go │ └── run.go ├── docs.go └── main.go ├── cmdutil ├── command.go ├── doc.go └── signals.go ├── config.go ├── counter.go ├── demo ├── README.md ├── basic │ ├── build.go │ ├── gen_drone.sh │ └── gen_drone.yml ├── complex │ ├── gen_drone.yml │ ├── log.go │ ├── logs │ │ ├── buildbackend.log │ │ ├── builddocker.log │ │ ├── builddocs.log │ │ ├── buildfrontend.log │ │ ├── initialize.log │ │ ├── integrationtests.log │ │ ├── integrationtests_mssql.log │ │ ├── integrationtests_mysql.log │ │ ├── integrationtests_pg.log │ │ ├── integrationtests_sqlite.log │ │ ├── notifyslack.log │ │ ├── package.log │ │ ├── publishdockerimage.log │ │ ├── publishdocs.log │ │ ├── publishpackage.log │ │ ├── testbackend.log │ │ └── testfrontend.log │ └── main.go ├── custom-client │ ├── gen_drone.yml │ └── main.go ├── generate.sh ├── multi-sub │ ├── build.go │ └── gen_drone.yml ├── multi │ ├── build.go │ ├── dependencies.go │ ├── gen_drone.yml │ ├── main.go │ ├── package.go │ ├── publish.go │ └── test.go └── state │ ├── example-directory │ ├── a.txt │ └── sub-folder │ │ └── b.txt │ ├── example-state-file.txt │ ├── gen_drone.yml │ └── main.go ├── docs.go ├── echo-test ├── go.mod ├── go.sum ├── go.work └── main.go ├── errors └── errors.go ├── exec ├── cmd.go └── docs.go ├── execute.go ├── fs ├── cachers.go └── replace.go ├── git ├── client.go ├── clone.go ├── describe.go ├── events.go └── x │ └── describe.go ├── go.mod ├── go.sum ├── go.work ├── go.work.sum ├── golang ├── build.go ├── modules.go ├── test.go └── x │ └── build.go ├── initializers.go ├── jsonnet └── jsonnet.go ├── makefile └── target.go ├── out.log ├── pipeline.go ├── pipeline ├── arguments_known.go ├── build.go ├── cacher.go ├── client.go ├── clients │ ├── cli │ │ ├── client.go │ │ ├── known_args.go │ │ ├── state.go │ │ └── state_writer.go │ ├── common │ │ └── step_compile.go │ ├── dagger │ │ ├── client.go │ │ └── compile.go │ ├── drone │ │ ├── client.go │ │ ├── client_test.go │ │ ├── config.go │ │ ├── doc.go │ │ ├── events.go │ │ ├── initializer.go │ │ └── schema.go │ ├── graphviz │ │ └── client.go │ └── opts.go ├── collection.go ├── collection_test.go ├── configurer.go ├── dag │ ├── dag.go │ ├── dag_test.go │ └── test_helpers.go ├── doc.go ├── errors.go ├── event.go ├── event_test.go ├── opts.go ├── pipeline.go ├── pipeline_test.go ├── print.go ├── step.go ├── step_env.go ├── step_test.go └── walker.go ├── pipelineutil ├── build.go └── doc.go ├── plog ├── docs.go ├── fields.go ├── helpers.go └── logger.go ├── scribe.go ├── scribe_client_test.go ├── scribe_multi.go ├── scribe_multi_test.go ├── scribe_test.go ├── state ├── arguments.go ├── default.go ├── json.go ├── object_storage.go ├── object_storage_gcs.go ├── object_storage_s3.go ├── observer.go ├── state.go ├── state_args.go ├── state_filesystem.go ├── state_gcs.go ├── state_gcs_test.go ├── state_helpers.go ├── state_log_wrapper.go ├── state_noop.go ├── state_object_store.go ├── state_object_store_test.go ├── state_s3.go ├── state_stdin.go ├── without.go └── without_test.go ├── stringutil ├── doc.go ├── random.go └── slugify.go ├── swfs ├── copy.go ├── copy_test.go ├── equal.go ├── extract.go ├── extract_test.go ├── hash.go ├── hash_test.go └── testdata │ ├── a.json │ ├── b.json │ └── c │ └── c.json ├── swhttp ├── default_client.go ├── download.go ├── download_test.go ├── request.go ├── response.go └── response_test.go ├── syncutil ├── docs.go ├── pipelinewaitgroup.go ├── stepwaitgroup.go └── waitgroup.go ├── tarfs ├── testdir │ ├── a.txt │ ├── folder-1 │ │ └── folder-4 │ │ │ └── b.txt │ └── folder-3 │ │ └── c.txt ├── untar.go ├── untar_test.go ├── write.go └── write_test.go ├── testutil ├── errors.go ├── io.go ├── pipeline.go ├── scribe.go ├── slices.go └── timeout.go ├── wrappers ├── log_wrapper.go └── trace_wrapper.go └── yarn └── run.go /.compose/.env: -------------------------------------------------------------------------------- 1 | JAEGER_SERVICE_NAME=groan 2 | JAEGER_ENDPOINT=http://localhost:14268/api/traces 3 | JAEGER_SAMPLER_TYPE=const 4 | JAEGER_SAMPLER_PARAM=1 5 | LOKI_ADDR=http://localhost:3100 6 | -------------------------------------------------------------------------------- /.compose/grafana/custom.ini: -------------------------------------------------------------------------------- 1 | app_mode = development 2 | 3 | [log] 4 | level=debug 5 | -------------------------------------------------------------------------------- /.compose/loki/config.yaml: -------------------------------------------------------------------------------- 1 | auth_enabled: false 2 | 3 | server: 4 | http_listen_port: 3100 5 | grpc_listen_port: 9096 6 | 7 | common: 8 | path_prefix: /tmp/loki 9 | storage: 10 | filesystem: 11 | chunks_directory: /tmp/loki/chunks 12 | rules_directory: /tmp/loki/rules 13 | replication_factor: 1 14 | ring: 15 | instance_addr: 127.0.0.1 16 | kvstore: 17 | store: inmemory 18 | 19 | schema_config: 20 | configs: 21 | - from: 2020-10-24 22 | store: boltdb-shipper 23 | object_store: filesystem 24 | schema: v11 25 | index: 26 | prefix: index_ 27 | period: 24h 28 | 29 | ruler: 30 | alertmanager_url: http://localhost:9093 31 | -------------------------------------------------------------------------------- /.compose/prometheus/config.yaml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 10s 3 | scrape_timeout: 10s 4 | 5 | scrape_configs: 6 | - job_name: services 7 | metrics_path: /metrics 8 | static_configs: 9 | - targets: 10 | - 'tempo:3200' 11 | - 'loki:3100' 12 | - 'grafana:3000' 13 | -------------------------------------------------------------------------------- /.compose/promtail/config.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | http_listen_port: 9080 3 | grpc_listen_port: 0 4 | 5 | positions: 6 | filename: /tmp/positions.yaml 7 | 8 | clients: 9 | - url: http://localhost:3100/loki/api/v1/push 10 | 11 | scrape_configs: 12 | - job_name: system 13 | static_configs: 14 | - targets: 15 | - localhost 16 | labels: 17 | job: varlogs 18 | __path__: /var/log/*log 19 | -------------------------------------------------------------------------------- /.compose/tempo/config.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | http_listen_port: 3200 3 | 4 | distributor: 5 | receivers: # this configuration will listen on all ports and protocols that tempo is capable of. 6 | jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can 7 | protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver 8 | thrift_http: # 9 | grpc: # for a production deployment you should only enable the receivers you need! 10 | thrift_binary: 11 | thrift_compact: 12 | zipkin: 13 | otlp: 14 | protocols: 15 | http: 16 | grpc: 17 | opencensus: 18 | 19 | ingester: 20 | trace_idle_period: 10s # the length of time after a trace has not received spans to consider it complete and flush it 21 | max_block_bytes: 1_000_000 # cut the head block when it hits this size or ... 22 | max_block_duration: 5m # this much time passes 23 | 24 | compactor: 25 | compaction: 26 | compaction_window: 1h # blocks in this time window will be compacted together 27 | max_block_bytes: 100_000_000 # maximum size of compacted blocks 28 | block_retention: 1h 29 | compacted_block_retention: 10m 30 | 31 | storage: 32 | trace: 33 | backend: local # backend configuration to use 34 | block: 35 | bloom_filter_false_positive: .05 # bloom filter false positive rate. lower values create larger filters but fewer false positives 36 | index_downsample_bytes: 1000 # number of bytes per index record 37 | encoding: zstd # block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 38 | wal: 39 | path: /tmp/tempo/wal # where to store the the wal locally 40 | encoding: snappy # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 41 | local: 42 | path: /tmp/tempo/blocks 43 | pool: 44 | max_workers: 100 # worker pool determines the number of parallel requests to the object store backend 45 | queue_depth: 10000 46 | -------------------------------------------------------------------------------- /.drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | type: docker 4 | name: test_and_build 5 | 6 | platform: 7 | os: linux 8 | arch: amd64 9 | 10 | steps: 11 | - name: builtin-compile-pipeline 12 | image: golang:1.19 13 | command: 14 | - go 15 | - build 16 | - -o 17 | - /var/scribe/pipeline 18 | - ./ci 19 | environment: 20 | CGO_ENABLED: 0 21 | GOARCH: amd64 22 | GOOS: linux 23 | volumes: 24 | - name: scribe 25 | path: /var/scribe 26 | 27 | - name: test_and_build 28 | image: golang:1.19 29 | commands: 30 | - /var/scribe/pipeline --pipeline="test and build" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=v0.10.0-5-ge3d2c2b-dirty ./ci 31 | volumes: 32 | - name: scribe 33 | path: /var/scribe 34 | - name: scribe-state 35 | path: /var/scribe-state 36 | depends_on: 37 | - builtin-compile-pipeline 38 | 39 | volumes: 40 | - name: scribe 41 | temp: {} 42 | - name: scribe-state 43 | temp: {} 44 | - name: docker_socket 45 | host: 46 | path: /var/run/docker.sock 47 | 48 | --- 49 | kind: pipeline 50 | type: docker 51 | name: create_github_release 52 | 53 | platform: 54 | os: linux 55 | arch: amd64 56 | 57 | steps: 58 | - name: builtin-compile-pipeline 59 | image: golang:1.19 60 | command: 61 | - go 62 | - build 63 | - -o 64 | - /var/scribe/pipeline 65 | - ./ci 66 | environment: 67 | CGO_ENABLED: 0 68 | GOARCH: amd64 69 | GOOS: linux 70 | volumes: 71 | - name: scribe 72 | path: /var/scribe 73 | 74 | - name: create_github_release 75 | image: golang:1.19 76 | commands: 77 | - /var/scribe/pipeline --pipeline="create github release" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=v0.10.0-5-ge3d2c2b-dirty ./ci 78 | volumes: 79 | - name: scribe 80 | path: /var/scribe 81 | - name: scribe-state 82 | path: /var/scribe-state 83 | depends_on: 84 | - builtin-compile-pipeline 85 | 86 | volumes: 87 | - name: scribe 88 | temp: {} 89 | - name: scribe-state 90 | temp: {} 91 | - name: docker_socket 92 | host: 93 | path: /var/run/docker.sock 94 | 95 | trigger: 96 | event: 97 | - tag 98 | 99 | depends_on: 100 | - test_and_build 101 | 102 | ... 103 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | node_modules/ 3 | frontend/node_modules/ 4 | ./shipwright 5 | ./pipeline 6 | echo-test/echo-test 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule ".compose/grafana/plugins/shipwright-app"] 2 | path = .compose/grafana/plugins/shipwright-app 3 | url = git@github.com:grafana/shipwright-app.git 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Development guide 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /Magefile.go: -------------------------------------------------------------------------------- 1 | //go:build mage 2 | // +build mage 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | 9 | "github.com/magefile/mage/sh" 10 | ) 11 | 12 | func Version() (string, error) { 13 | return sh.Output("git", "describe", "--tags", "--dirty", "--always") 14 | } 15 | 16 | func Build() error { 17 | version, err := Version() 18 | if err != nil { 19 | return err 20 | } 21 | 22 | // go build \ 23 | // -ldflags \ 24 | // "-X main.Version=$(git describe --tags --dirty --always)" \ 25 | // -o bin/scribe ./plumbing/cmd 26 | 27 | fmt.Println("building version", version) 28 | return sh.Run("go", 29 | "build", 30 | "-ldflags", fmt.Sprintf("-X main.Version=%s", version), 31 | "-o", "./bin/scribe", 32 | "./cmd", 33 | ) 34 | } 35 | 36 | var Default = Build 37 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Grafana Labs 2 | -------------------------------------------------------------------------------- /args/docs.go: -------------------------------------------------------------------------------- 1 | // Package args just holds the type definitions for pipeline / cmd arguments to avoid cyclic imports 2 | package args 3 | -------------------------------------------------------------------------------- /args/flag_map_arg.go: -------------------------------------------------------------------------------- 1 | package args 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | type ArgMap map[string]string 10 | 11 | func (a *ArgMap) String() string { 12 | return fmt.Sprintf("%v", *a) 13 | } 14 | 15 | func (a *ArgMap) Set(val string) error { 16 | p := strings.Split(val, "=") 17 | 18 | if len(p) < 2 { 19 | return errors.New("invalid value") 20 | } 21 | 22 | var ( 23 | k = p[0] 24 | v = p[1:] 25 | ) 26 | 27 | (*a)[k] = strings.Join(v, "=") 28 | 29 | return nil 30 | } 31 | 32 | func (a *ArgMap) Get(key string) (string, error) { 33 | if v, ok := (*a)[key]; ok { 34 | return v, nil 35 | } 36 | 37 | return "", errors.New("value not found") 38 | } 39 | 40 | func (a *ArgMap) Type() string { 41 | return "string" 42 | } 43 | -------------------------------------------------------------------------------- /args/flag_optional_int.go: -------------------------------------------------------------------------------- 1 | package args 2 | 3 | import "strconv" 4 | 5 | type OptionalInt struct { 6 | Value int64 7 | Valid bool 8 | } 9 | 10 | func (o *OptionalInt) String() string { 11 | if o.Valid { 12 | return strconv.FormatInt(o.Value, 10) 13 | } 14 | 15 | return "" 16 | } 17 | 18 | func (o *OptionalInt) Set(v string) error { 19 | if v == "" { 20 | return nil 21 | } 22 | 23 | i, err := strconv.ParseInt(v, 10, 64) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | o.Value = i 29 | o.Valid = true 30 | 31 | return nil 32 | } 33 | 34 | func (o *OptionalInt) Type() string { 35 | return "integer" 36 | } 37 | -------------------------------------------------------------------------------- /ci/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe" 7 | "github.com/grafana/scribe/golang" 8 | "github.com/grafana/scribe/pipeline" 9 | ) 10 | 11 | // "main" defines our program pipeline. 12 | // Every pipeline step should be instantiated using the scribe client (sw). 13 | // This allows the various client modes to work properly in different scenarios, like in a CI environment or locally. 14 | // Logic and processing done outside of the `sw.*` family of functions may not be included in the resulting pipeline. 15 | func main() { 16 | sw := scribe.NewMulti() 17 | defer sw.Done() 18 | 19 | sw.Add( 20 | sw.New("test and build", func(sw *scribe.Scribe) { 21 | sw.Add(golang.Test(sw, "./...").WithName("test")) 22 | }), 23 | ) 24 | 25 | sw.Add( 26 | sw.New("create github release", func(sw *scribe.Scribe) { 27 | sw.When( 28 | pipeline.GitTagEvent(pipeline.GitTagFilters{}), 29 | ) 30 | 31 | sw.Add(pipeline.NamedStep("am I on a tag event?", func(ctx context.Context, opts pipeline.ActionOpts) error { 32 | opts.Logger.Infoln("1. I'm on a tag event.") 33 | opts.Logger.Infoln("2. I'm on a tag event.") 34 | opts.Logger.Infoln("3. I'm on a tag event.") 35 | return nil 36 | }).WithImage("alpine:latest")) 37 | }), 38 | ) 39 | } 40 | -------------------------------------------------------------------------------- /ci/version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os/exec" 7 | "strings" 8 | 9 | "github.com/grafana/scribe/pipeline" 10 | "github.com/grafana/scribe/state" 11 | ) 12 | 13 | var ( 14 | ArgumentVersion = state.NewStringArgument("version") 15 | ) 16 | 17 | func version() (string, error) { 18 | // git config --global --add safe.directory * is needed to resolve the restriction introduced by CVE-2022-24765. 19 | out, err := exec.Command("git", "config", "--global", "--add", "safe.directory", "*").CombinedOutput() 20 | if err != nil { 21 | return "", fmt.Errorf("running command 'git config --global --add safe.directory *' resulted in error '%w'. Output: '%s'", err, string(out)) 22 | } 23 | 24 | version, err := exec.Command("git", "describe", "--tags", "--dirty", "--always").CombinedOutput() 25 | if err != nil { 26 | return "", fmt.Errorf("running command 'git describe --tags --dirty --always' resulted in the error '%w'. Output: '%s'", err, string(version)) 27 | } 28 | 29 | return strings.TrimSpace(string(version)), nil 30 | } 31 | 32 | func getVersion(ctx context.Context, opts pipeline.ActionOpts) error { 33 | v, err := version() 34 | if err != nil { 35 | return err 36 | } 37 | 38 | return opts.State.SetString(ctx, ArgumentVersion, v) 39 | } 40 | 41 | func StepGetVersion(version string) pipeline.Step { 42 | return pipeline.NewStep(getVersion). 43 | Requires( 44 | pipeline.ArgumentSourceFS, 45 | ). 46 | Provides(ArgumentVersion). 47 | WithImage("alpine/git:2.36.3") 48 | } 49 | -------------------------------------------------------------------------------- /cmd/commands/args.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import "github.com/grafana/scribe/args" 4 | 5 | // MustParseRunArgs parses the "run" arguments from the args slice. These options are provided by the scribe command and are typically not user-specified 6 | func MustParseArgs(pargs []string) *args.PipelineArgs { 7 | v, err := args.ParseArguments(pargs) 8 | if err != nil { 9 | panic(err) 10 | } 11 | 12 | return v 13 | } 14 | -------------------------------------------------------------------------------- /cmd/commands/run.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "strconv" 10 | 11 | "github.com/grafana/scribe/args" 12 | "github.com/grafana/scribe/plog" 13 | ) 14 | 15 | type RunOpts struct { 16 | // Version specifies the scribe version to run. 17 | // This value is used in generating the scribe image. A value will be provided if using the scribe CLI. 18 | // If no value is provided, then it will be replaced with "latest". 19 | Version string 20 | 21 | // Path specifies the path to the scribe pipeline. 22 | // This value is assumed to be "." if not provided. 23 | // This is not the same as the "Path" argument for the pipeline itself, which is required and used for code / config generation. 24 | Path string 25 | 26 | State string 27 | // Stdout is the stdout stream of the "go run" command that runs the pipeline 28 | // If it is not provided, it defaults to "os.Stdout" 29 | Stdout io.Writer 30 | // Stderr is the stderr stream of the "go run" command that runs the pipeline. 31 | // The stderr stream contains mostly logging info and is particularly useful if a problem is encountered. 32 | // If it is not provided, it defaults to "os.Stderr" 33 | Stderr io.Writer 34 | // Stdin is the stdin stream of the "go run" command that runs the pipeline. 35 | // The stdin stream is used to accept arguments in docker or cli client that were not provided in command-line arguments. 36 | // If it is not provided, it defaults to "os.Stdin" 37 | Stdin io.Reader 38 | 39 | // Args are arguments that are passed to the scribe pipeline 40 | Args *args.PipelineArgs 41 | } 42 | 43 | // Run handles the default scribe command, "scribe run". 44 | // The run command attempts to run the pipeline by using "go run ...". 45 | // This function will exit the program if it encounters an error. 46 | // TODO: there is a function in `cmdutil` that should be able to create this command to run. 47 | func Run(ctx context.Context, opts *RunOpts) *exec.Cmd { 48 | var ( 49 | path = opts.Path 50 | args = opts.Args 51 | state = opts.State 52 | 53 | stdout = opts.Stdout 54 | stderr = opts.Stderr 55 | stdin = opts.Stdin 56 | version = opts.Version 57 | ) 58 | 59 | if stdout == nil { 60 | stdout = os.Stdout 61 | } 62 | 63 | if stderr == nil { 64 | stderr = os.Stderr 65 | } 66 | 67 | if stdin == nil { 68 | stdin = os.Stdin 69 | } 70 | 71 | if version == "" { 72 | version = "latest" 73 | } 74 | 75 | logger := plog.New(opts.Args.LogLevel) 76 | 77 | // This will run a weird looking command, like this: 78 | // go run ./demo/basic -client drone -path ./demo/basic 79 | // But it's important to note that a lot happens before it actually reaches the pipeline code and produces a command like this: 80 | // /tmp/random-string -client drone -path ./demo/basic 81 | // So the path to the pipeline is not preserved, which is why we have to provide the path as an argument 82 | cmdArgs := []string{"run", path, "--client", args.Client, "--log-level", args.LogLevel.String(), "--path", args.Path, "--version", version, "--build-id", args.BuildID, "--event", args.Event, "--state", state} 83 | 84 | for k, v := range args.ArgMap { 85 | cmdArgs = append(cmdArgs, "--arg", fmt.Sprintf("%s=%s", k, v)) 86 | } 87 | 88 | if args.PipelineName != nil { 89 | for _, v := range args.PipelineName { 90 | cmdArgs = append(cmdArgs, fmt.Sprintf("--pipeline=\"%s\"", v)) 91 | } 92 | } else if args.Step != nil { 93 | cmdArgs = append(cmdArgs, "--step", strconv.FormatInt(*args.Step, 10)) 94 | } 95 | 96 | logger.Infoln("Running scribe pipeline with command", append([]string{"go"}, cmdArgs...)) 97 | 98 | cmd := exec.CommandContext(ctx, "go", cmdArgs...) 99 | cmd.Stdout = stdout 100 | cmd.Stderr = stderr 101 | cmd.Stdin = stdin 102 | 103 | return cmd 104 | } 105 | -------------------------------------------------------------------------------- /cmd/docs.go: -------------------------------------------------------------------------------- 1 | // Package main contains the logic for the `scribe` CLI 2 | package main 3 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | // Package main contains the CLI logic for the `scribe` command 2 | // The scribe command's main responsibility is to run a pipeline. 3 | package main 4 | 5 | import ( 6 | "context" 7 | "math/rand" 8 | "os" 9 | "os/exec" 10 | "time" 11 | 12 | "github.com/grafana/scribe/cmd/commands" 13 | "github.com/grafana/scribe/cmdutil" 14 | "github.com/grafana/scribe/plog" 15 | "github.com/sirupsen/logrus" 16 | ) 17 | 18 | // Arguments provided at compile-time 19 | var ( 20 | Version = "latest" 21 | ) 22 | 23 | func init() { 24 | rand.Seed(time.Now().Unix()) 25 | } 26 | 27 | func handleSignal(log *logrus.Logger, cmd *exec.Cmd, sig os.Signal) int { 28 | log.Debugln("Received OS signal", sig.String()) 29 | 30 | log.Debugf("Sending pipeline '%s' signal...", sig.String()) 31 | cmd.Process.Signal(sig) 32 | 33 | log.Debugln("Waiting for pipeline to exit...") 34 | p, err := cmd.Process.Wait() 35 | if err != nil { 36 | log.Error(err) 37 | return 0 38 | } 39 | 40 | return p.ExitCode() 41 | } 42 | 43 | func main() { 44 | log := plog.New(logrus.InfoLevel) 45 | 46 | log.Debugln("Running version", Version) 47 | var ( 48 | ctx = context.Background() 49 | ) 50 | 51 | args := commands.MustParseArgs(os.Args[1:]) 52 | 53 | cmd := commands.Run(ctx, &commands.RunOpts{ 54 | Version: Version, 55 | State: args.State, 56 | Path: args.Path, 57 | Args: args, 58 | Stdout: os.Stdout, 59 | Stderr: os.Stderr, 60 | Stdin: os.Stdin, 61 | }) 62 | 63 | var ( 64 | c = make(chan os.Signal, 1) 65 | errChan = make(chan error) 66 | doneChan = make(chan bool) 67 | ) 68 | 69 | go func(cmd *exec.Cmd) { 70 | if err := cmd.Run(); err != nil { 71 | errChan <- err 72 | return 73 | } 74 | doneChan <- true 75 | }(cmd) 76 | 77 | log.Debugln("Watching for OS signals...") 78 | cmdutil.NotifySignals(c) 79 | 80 | select { 81 | case sig := <-c: 82 | os.Exit(handleSignal(log, cmd, sig)) 83 | case err := <-errChan: 84 | log.Error(err) 85 | os.Exit(1) 86 | case <-doneChan: 87 | os.Exit(0) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /cmdutil/command.go: -------------------------------------------------------------------------------- 1 | package cmdutil 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/grafana/scribe/args" 7 | "github.com/grafana/scribe/pipeline" 8 | ) 9 | 10 | // CommandOpts is a list of arguments that can be provided to the StepCommand function. 11 | type CommandOpts struct { 12 | args.PipelineArgs 13 | 14 | // Step is the pipeline step this command is being generated for. The step contains a lot of necessary information for generating a command, mostly around arguments. 15 | Step pipeline.Step 16 | CompiledPipeline string 17 | } 18 | 19 | // StepCommand returns the command string for running a single step. 20 | // The path argument can be omitted, which is particularly helpful if the current directory is a pipeline. 21 | func StepCommand(opts CommandOpts) ([]string, error) { 22 | args := []string{"--client", "cli"} 23 | 24 | if opts.BuildID != "" { 25 | args = append(args, fmt.Sprintf("--build-id=%s", opts.BuildID)) 26 | } 27 | 28 | if opts.State != "" { 29 | args = append(args, fmt.Sprintf("--state=%s", opts.State)) 30 | } 31 | 32 | if opts.LogLevel != 0 { 33 | args = append(args, fmt.Sprintf("--log-level=%s", opts.LogLevel.String())) 34 | } 35 | 36 | if opts.Version != "" { 37 | args = append(args, fmt.Sprintf("--version=%s", opts.Version)) 38 | } 39 | 40 | if len(opts.ArgMap) != 0 { 41 | for k, v := range opts.ArgMap { 42 | args = append(args, fmt.Sprintf("--arg=%s=%s", k, v)) 43 | } 44 | } 45 | 46 | name := "scribe" 47 | 48 | if p := opts.CompiledPipeline; p != "" { 49 | name = p 50 | } 51 | 52 | cmd := append([]string{name, fmt.Sprintf("--step=%d", opts.Step.ID)}, args...) 53 | if opts.Path != "" { 54 | cmd = append(cmd, opts.Path) 55 | } 56 | 57 | return cmd, nil 58 | } 59 | 60 | type PipelineCommandOpts struct { 61 | CommandOpts 62 | Pipeline pipeline.Pipeline 63 | } 64 | 65 | func PipelineCommand(opts PipelineCommandOpts) ([]string, error) { 66 | args := []string{"--client", "cli"} 67 | 68 | if opts.BuildID != "" { 69 | args = append(args, fmt.Sprintf("--build-id=%s", opts.BuildID)) 70 | } 71 | 72 | if opts.State != "" { 73 | args = append(args, fmt.Sprintf("--state=%s", opts.State)) 74 | } 75 | 76 | if opts.LogLevel != 0 { 77 | args = append(args, fmt.Sprintf("--log-level=%s", opts.LogLevel)) 78 | } 79 | 80 | if opts.Version != "" { 81 | args = append(args, fmt.Sprintf("--version=%s", opts.Version)) 82 | } 83 | 84 | if opts.Event != "" { 85 | args = append(args, fmt.Sprintf("--event=%s", opts.Event)) 86 | } 87 | 88 | if len(opts.ArgMap) != 0 { 89 | for k, v := range opts.ArgMap { 90 | args = append(args, fmt.Sprintf("--arg=%s=%s", k, v)) 91 | } 92 | } 93 | 94 | name := "scribe" 95 | 96 | if p := opts.CompiledPipeline; p != "" { 97 | name = p 98 | } 99 | 100 | cmd := append([]string{name, fmt.Sprintf("--pipeline=\"%s\"", opts.Pipeline.Name)}, args...) 101 | if opts.Path != "" { 102 | cmd = append(cmd, opts.Path) 103 | } 104 | 105 | return cmd, nil 106 | 107 | } 108 | -------------------------------------------------------------------------------- /cmdutil/doc.go: -------------------------------------------------------------------------------- 1 | // Package cmdutil provides utility functions and types for working with the 'scribe' CLI. 2 | package cmdutil 3 | -------------------------------------------------------------------------------- /cmdutil/signals.go: -------------------------------------------------------------------------------- 1 | package cmdutil 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "syscall" 7 | ) 8 | 9 | // WatchSignals blocks the current goroutine / thread and waits for any signal and returns an error 10 | func WatchSignals() os.Signal { 11 | // Set up channel on which to send signal notifications. 12 | // We must use a buffered channel or risk missing the signal 13 | // if we're not ready to receive when the signal is sent. 14 | c := make(chan os.Signal, 1) 15 | 16 | NotifySignals(c) 17 | 18 | sig := <-c 19 | return sig 20 | } 21 | 22 | func NotifySignals(c chan os.Signal) { 23 | // Passing no signals to Notify means that 24 | // all signals will be sent to the channel. 25 | signal.Notify(c, 26 | os.Interrupt, 27 | syscall.SIGINT, 28 | syscall.SIGTERM, 29 | syscall.SIGQUIT, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package scribe 2 | 3 | type GitConfig struct{} 4 | 5 | // Config defines the typical options that are provided in a pipeline. 6 | // These options can be provided many ways, and often depend on the execution environment. 7 | // They are retrieved at pipeline-time. 8 | type PipelineConfig struct{} 9 | -------------------------------------------------------------------------------- /counter.go: -------------------------------------------------------------------------------- 1 | package scribe 2 | 3 | import "sync/atomic" 4 | 5 | type counter struct { 6 | n int64 7 | } 8 | 9 | func (c *counter) Next() int64 { 10 | n := atomic.LoadInt64(&c.n) 11 | atomic.AddInt64(&c.n, 1) 12 | return n 13 | } 14 | -------------------------------------------------------------------------------- /demo/basic/build.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe" 7 | "github.com/grafana/scribe/fs" 8 | gitx "github.com/grafana/scribe/git/x" 9 | "github.com/grafana/scribe/golang" 10 | "github.com/grafana/scribe/makefile" 11 | "github.com/grafana/scribe/pipeline" 12 | "github.com/grafana/scribe/state" 13 | "github.com/grafana/scribe/yarn" 14 | ) 15 | 16 | func writeVersion(sw *scribe.Scribe) pipeline.Step { 17 | action := func(ctx context.Context, opts pipeline.ActionOpts) error { 18 | 19 | // equivalent of `git describe --tags --dirty --always` 20 | version, err := gitx.Describe(ctx, ".", true, true, true) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | // write the version string in the `.version` file. 26 | return fs.ReplaceString(".version", version)(ctx, opts) 27 | } 28 | 29 | return pipeline.NewStep(action).WithImage("alpine:latest") 30 | } 31 | 32 | // "main" defines our program pipeline. 33 | // Every pipeline step should be instantiated using the scribe client (sw). 34 | // This allows the various clients to work properly in different scenarios, like in a CI environment or locally. 35 | // Logic and processing done outside of the `sw.*` family of functions may not be included in the resulting pipeline. 36 | func main() { 37 | sw := scribe.New("basic pipeline") 38 | defer sw.Done() 39 | 40 | sw.When( 41 | pipeline.GitCommitEvent(pipeline.GitCommitFilters{ 42 | Branch: pipeline.StringFilter("main"), 43 | }), 44 | pipeline.GitTagEvent(pipeline.GitTagFilters{ 45 | Name: pipeline.GlobFilter("v*"), 46 | }), 47 | ) 48 | 49 | // In parallel, install the yarn and go dependencies, and cache the node_modules and $GOPATH/pkg folders. 50 | // The cache should invalidate if the yarn.lock or go.sum files have changed 51 | sw.Add( 52 | pipeline.NamedStep("install frontend dependencies", sw.Cache( 53 | yarn.InstallAction(), 54 | fs.Cache("node_modules", fs.FileHasChanged("yarn.lock")), 55 | )).WithImage("node:latest"), 56 | pipeline.NamedStep("install backend dependencies", sw.Cache( 57 | golang.ModDownload(), 58 | fs.Cache("$GOPATH/pkg", fs.FileHasChanged("go.sum")), 59 | )).WithImage("node:latest"), 60 | writeVersion(sw).WithName("write-version-file"), 61 | ) 62 | 63 | sw.Add( 64 | pipeline.NamedStep("compile backend", makefile.Target("build")).WithImage("alpine:latest"), 65 | pipeline.NamedStep("compile frontend", makefile.Target("package")).WithImage("alpine:latest"), 66 | pipeline.NamedStep("build docker image", makefile.Target("build")).Requires(pipeline.ArgumentDockerSocketFS).WithImage("alpine:latest"), 67 | ) 68 | 69 | sw.Add( 70 | pipeline.NamedStep("publish", makefile.Target("publish")). 71 | Requires( 72 | state.NewSecretArgument("gcs-publish-key"), 73 | ), 74 | ) 75 | } 76 | -------------------------------------------------------------------------------- /demo/basic/gen_drone.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | go run ./demo/basic -path=./demo/basic -client=drone -log-level=info -version=latest > ./demo/basic/gen_drone.yml 4 | -------------------------------------------------------------------------------- /demo/basic/gen_drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | type: docker 4 | name: basic_pipeline 5 | 6 | platform: 7 | os: linux 8 | arch: amd64 9 | 10 | steps: 11 | - name: builtin-compile-pipeline 12 | image: golang:1.19 13 | command: 14 | - go 15 | - build 16 | - -o 17 | - /var/scribe/pipeline 18 | - ./demo/basic 19 | environment: 20 | CGO_ENABLED: 0 21 | GOARCH: amd64 22 | GOOS: linux 23 | volumes: 24 | - name: scribe 25 | path: /var/scribe 26 | 27 | - name: basic_pipeline 28 | image: golang:1.19 29 | commands: 30 | - /var/scribe/pipeline --pipeline="basic pipeline" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=latest ./demo/basic 31 | volumes: 32 | - name: scribe 33 | path: /var/scribe 34 | - name: scribe-state 35 | path: /var/scribe-state 36 | depends_on: 37 | - builtin-compile-pipeline 38 | 39 | volumes: 40 | - name: scribe 41 | temp: {} 42 | - name: scribe-state 43 | temp: {} 44 | - name: docker_socket 45 | host: 46 | path: /var/run/docker.sock 47 | 48 | trigger: 49 | branch: 50 | - main 51 | event: 52 | - branch 53 | - tag 54 | ref: 55 | - refs/tags/v* 56 | 57 | ... 58 | -------------------------------------------------------------------------------- /demo/complex/gen_drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | type: docker 4 | name: complex_pipeline 5 | 6 | platform: 7 | os: linux 8 | arch: amd64 9 | 10 | steps: 11 | - name: builtin-compile-pipeline 12 | image: golang:1.19 13 | command: 14 | - go 15 | - build 16 | - -o 17 | - /var/scribe/pipeline 18 | - ./demo/complex 19 | environment: 20 | CGO_ENABLED: 0 21 | GOARCH: amd64 22 | GOOS: linux 23 | volumes: 24 | - name: scribe 25 | path: /var/scribe 26 | 27 | - name: complex_pipeline 28 | image: golang:1.19 29 | commands: 30 | - /var/scribe/pipeline --pipeline="complex-pipeline" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=latest ./demo/complex 31 | volumes: 32 | - name: scribe 33 | path: /var/scribe 34 | - name: scribe-state 35 | path: /var/scribe-state 36 | depends_on: 37 | - builtin-compile-pipeline 38 | 39 | volumes: 40 | - name: scribe 41 | temp: {} 42 | - name: scribe-state 43 | temp: {} 44 | - name: docker_socket 45 | host: 46 | path: /var/run/docker.sock 47 | 48 | ... 49 | -------------------------------------------------------------------------------- /demo/complex/log.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "log" 7 | "math/rand" 8 | "os" 9 | "path/filepath" 10 | "time" 11 | 12 | "github.com/grafana/scribe/pipeline" 13 | "github.com/opentracing/opentracing-go" 14 | ) 15 | 16 | func NoOpAction(name string, duration time.Duration) pipeline.Action { 17 | return func(ctx context.Context, opts pipeline.ActionOpts) error { 18 | f, err := os.Open(filepath.Join("demo", "complex", "logs", name+".log")) 19 | if err != nil { 20 | return err 21 | } 22 | 23 | time.Sleep(duration) 24 | 25 | io.ReadAll(io.TeeReader(f, opts.Stdout)) 26 | 27 | return nil 28 | } 29 | } 30 | 31 | func IntegrationTest(variant string, duration time.Duration) pipeline.Action { 32 | return func(ctx context.Context, opts pipeline.ActionOpts) error { 33 | d := int64(duration.Seconds()) / 2 34 | tests := []string{"fs", "docker", "exec", "git", "golang", "makefile", "yarn"} 35 | parent := opentracing.SpanFromContext(ctx) 36 | 37 | for _, test := range tests { 38 | span, _ := opentracing.StartSpanFromContextWithTracer(ctx, opts.Tracer, test, opentracing.ChildOf(parent.Context())) 39 | span.SetTag("job", "scribe") 40 | l := log.New(opts.Stdout, variant, 0) 41 | l.Printf("Testing '%s' package with '%s'...", test, variant) 42 | 43 | r := rand.Int63n(d) 44 | d -= r 45 | time.Sleep(time.Duration(r) * time.Second) 46 | span.Finish() 47 | } 48 | 49 | return nil 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /demo/complex/logs/buildbackend.log: -------------------------------------------------------------------------------- 1 | building binary for 'linux-amd64'... 2 | done building binary for 'linux-amd64' 3 | building binary for 'linux-arm64'... 4 | done building binary for 'linux-arm64' 5 | building binary for 'linux-arm7'... 6 | done building binary for 'linux-arm7' 7 | building binary for 'linux-arm6'... 8 | done building binary for 'linux-arm6' 9 | building binary for 'darwin-arm64'... 10 | done building binary for 'darwin-arm64' 11 | building binary for 'darwin-amd64'... 12 | done building binary for 'darwin-amd64' 13 | building binary for 'windows-amd64'... 14 | done building binary for 'windows-amd64' 15 | -------------------------------------------------------------------------------- /demo/complex/logs/builddocker.log: -------------------------------------------------------------------------------- 1 | Sending build context to Docker daemon 7.456MB 2 | Step 1/8 : FROM golang:1.17 as builder 3 | ---> 73fa8a9cf041 4 | Step 2/8 : WORKDIR /app 5 | ---> Using cache 6 | ---> 8abdefa311fb 7 | Step 3/8 : COPY . . 8 | ---> c648b646437f 9 | Step 4/8 : RUN go build -ldflags "-X main.Version=$(git describe --tags --dirty --always)" -o bin/scribe ./plumbing/cmd 10 | ---> Running in 212fc0bbc4dd 11 | go: downloading github.com/sirupsen/logrus v1.8.1 12 | go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible 13 | go: downloading github.com/opentracing/opentracing-go v1.2.0 14 | go: downloading golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 15 | go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible 16 | go: downloading go.uber.org/atomic v1.9.0 17 | Removing intermediate container 212fc0bbc4dd 18 | ---> 7bfc36522f16 19 | Step 5/8 : FROM alpine:3 20 | ---> c059bfaa849c 21 | Step 6/8 : COPY --from=builder /app/bin/scribe /bin/scribe 22 | ---> c01e626b56a0 23 | Step 7/8 : RUN apk add --no-cache bash go 24 | ---> Running in 0c53f17b1d7d 25 | fetch https://dl-cdn.alpinelinux.org/alpine/v3.15/main/x86_64/APKINDEX.tar.gz 26 | fetch https://dl-cdn.alpinelinux.org/alpine/v3.15/community/x86_64/APKINDEX.tar.gz 27 | (1/17) Installing ncurses-terminfo-base (6.3_p20211120-r0) 28 | (2/17) Installing ncurses-libs (6.3_p20211120-r0) 29 | (3/17) Installing readline (8.1.1-r0) 30 | (4/17) Installing bash (5.1.16-r0) 31 | Executing bash-5.1.16-r0.post-install 32 | (5/17) Installing libgcc (10.3.1_git20211027-r0) 33 | (6/17) Installing libstdc++ (10.3.1_git20211027-r0) 34 | (7/17) Installing binutils (2.37-r3) 35 | (8/17) Installing libgomp (10.3.1_git20211027-r0) 36 | (9/17) Installing libatomic (10.3.1_git20211027-r0) 37 | (10/17) Installing libgphobos (10.3.1_git20211027-r0) 38 | (11/17) Installing gmp (6.2.1-r1) 39 | (12/17) Installing isl22 (0.22-r0) 40 | (13/17) Installing mpfr4 (4.1.0-r0) 41 | (14/17) Installing mpc1 (1.2.1-r0) 42 | (15/17) Installing gcc (10.3.1_git20211027-r0) 43 | (16/17) Installing musl-dev (1.2.2-r7) 44 | (17/17) Installing go (1.17.4-r0) 45 | Executing busybox-1.34.1-r3.trigger 46 | OK: 498 MiB in 31 packages 47 | Removing intermediate container 0c53f17b1d7d 48 | ---> 8b8d51dbd88f 49 | Step 8/8 : WORKDIR /var/scribe 50 | ---> Running in cfe0789e1e8b 51 | Removing intermediate container cfe0789e1e8b 52 | ---> 8e9f1ee9e525 53 | Successfully built 8e9f1ee9e525 54 | -------------------------------------------------------------------------------- /demo/complex/logs/builddocs.log: -------------------------------------------------------------------------------- 1 | Building documentation... 2 | Built docs/index.html 3 | Built docs/scribe.html 4 | Built docs/groan.html 5 | Built docs/exec.html 6 | Built docs/fs.html 7 | Built docs/plumbing.html 8 | Built docs/makefile.html 9 | Built docs/mage.html 10 | Built docs/config.html 11 | Built docs/why-88-miles-per-hour.html 12 | Done building documentation 13 | 14 | -------------------------------------------------------------------------------- /demo/complex/logs/buildfrontend.log: -------------------------------------------------------------------------------- 1 | Compiling frontend... 2 | 3 | Compiled successfully! 4 | 5 | Hash: 14c7fcd42833366d331a 6 | Version: webpack 4.41.5 7 | Time: 148ms 8 | Built at: 03/09/2022 1:55:23 PM 9 | Asset Size Chunks Chunk Names 10 | index.html 370 bytes [emitted] 11 | module.js 9.79 KiB module module 12 | module.js.map 7.76 KiB module [dev] module 13 | panels/scribe-build-list-panel/module.js 12.7 KiB panels/scribe-build-list-panel/module [emitted] panels/scribe-build-list-panel/module 14 | panels/scribe-build-list-panel/module.js.map 12.8 KiB panels/scribe-build-list-panel/module [emitted] [dev] panels/scribe-build-list-panel/module 15 | panels/scribe-pipeline-list-panel/module.js 26.4 KiB panels/scribe-pipeline-list-panel/module [emitted] panels/scribe-pipeline-list-panel/module 16 | panels/scribe-pipeline-list-panel/module.js.map 26.1 KiB panels/scribe-pipeline-list-panel/module [emitted] [dev] panels/scribe-pipeline-list-panel/module 17 | Entrypoint module = module.js module.js.map 18 | Entrypoint panels/scribe-build-list-panel/module = panels/scribe-build-list-panel/module.js panels/scribe-build-list-panel/module.js.map 19 | Entrypoint panels/scribe-pipeline-list-panel/module = panels/scribe-pipeline-list-panel/module.js panels/scribe-pipeline-list-panel/module.js.map 20 | [../node_modules/tslib/tslib.es6.js] 10 KiB {panels/scribe-pipeline-list-panel/module} 21 | [./components/BuildCard.tsx] 1.46 KiB {panels/scribe-build-list-panel/module} [built] 22 | [./components/StatusIndicators.tsx] 1.41 KiB {panels/scribe-pipeline-list-panel/module} 23 | [./config/Loki.tsx] 735 bytes {module} 24 | [./configctrl/config.ts] 1.08 KiB {module} 25 | [./module.ts] 311 bytes {module} 26 | [./panels/scribe-build-list-panel/Panel.tsx] 2.87 KiB {panels/scribe-build-list-panel/module} 27 | [./panels/scribe-build-list-panel/module.ts] 475 bytes {panels/scribe-build-list-panel/module} 28 | [./panels/scribe-pipeline-list-panel/Panel.tsx] 3.23 KiB {panels/scribe-pipeline-list-panel/module} [built] 29 | [./panels/scribe-pipeline-list-panel/module.ts] 179 bytes {panels/scribe-pipeline-list-panel/module} 30 | [@grafana/data] external "@grafana/data" 42 bytes {module} {panels/scribe-build-list-panel/module} {panels/scribe-pipeline-list-panel/module} 31 | [@grafana/runtime] external "@grafana/runtime" 42 bytes {module} 32 | [@grafana/ui] external "@grafana/ui" 42 bytes {module} {panels/scribe-build-list-panel/module} {panels/scribe-pipeline-list-panel/module} 33 | [react] external "react" 42 bytes {module} {panels/scribe-build-list-panel/module} {panels/scribe-pipeline-list-panel/module} 34 | Child html-webpack-plugin for "index.html": 35 | Asset Size Chunks Chunk Names 36 | index.html 538 KiB 1 37 | Entrypoint undefined = index.html 38 | [../node_modules/html-webpack-plugin/lib/loader.js!../node_modules/html-webpack-plugin/default_index.ejs] 376 bytes {1} 39 | [../node_modules/lodash/lodash.js] 531 KiB {1} 40 | [../node_modules/webpack/buildin/global.js] 472 bytes {1} 41 | [../node_modules/webpack/buildin/module.js] 497 bytes {1} 42 | -------------------------------------------------------------------------------- /demo/complex/logs/initialize.log: -------------------------------------------------------------------------------- 1 | starting ci program... 2 | starting flux capacitor... 3 | reaching 88 miles per hour... 4 | -------------------------------------------------------------------------------- /demo/complex/logs/integrationtests.log: -------------------------------------------------------------------------------- 1 | Running integration tests... 2 | 3 | ok github.com/grafana/scribe 0.003s 4 | ? github.com/grafana/scribe/ci [no test files] 5 | ? github.com/grafana/scribe/ci/docker [no test files] 6 | ? github.com/grafana/scribe/demo/basic [no test files] 7 | ? github.com/grafana/scribe/demo/complex [no test files] 8 | ? github.com/grafana/scribe/docker [no test files] 9 | ? github.com/grafana/scribe/exec [no test files] 10 | ? github.com/grafana/scribe/fs [no test files] 11 | ? github.com/grafana/scribe/git [no test files] 12 | ? github.com/grafana/scribe/git/x [no test files] 13 | ? github.com/grafana/scribe/golang [no test files] 14 | ? github.com/grafana/scribe/golang/x [no test files] 15 | ? github.com/grafana/scribe/makefile [no test files] 16 | ? github.com/grafana/scribe/plumbing [no test files] 17 | ? github.com/grafana/scribe/plumbing/cmd [no test files] 18 | ? github.com/grafana/scribe/plumbing/cmd/commands [no test files] 19 | ? github.com/grafana/scribe/plumbing/cmdutil [no test files] 20 | ? github.com/grafana/scribe/plumbing/pipeline [no test files] 21 | ? github.com/grafana/scribe/plumbing/pipeline/clients/cli [no test files] 22 | ok github.com/grafana/scribe/plumbing/pipeline/clients/docker 0.009s 23 | ok github.com/grafana/scribe/plumbing/pipeline/clients/drone 0.329s 24 | ? github.com/grafana/scribe/plumbing/plog [no test files] 25 | ? github.com/grafana/scribe/plumbing/schemas/drone [no test files] 26 | ? github.com/grafana/scribe/plumbing/stringutil [no test files] 27 | ? github.com/grafana/scribe/plumbing/syncutil [no test files] 28 | ? github.com/grafana/scribe/plumbing/testutil [no test files] 29 | ? github.com/grafana/scribe/plumbing/wrappers [no test files] 30 | ? github.com/grafana/scribe/yarn [no test files] 31 | 32 | -------------------------------------------------------------------------------- /demo/complex/logs/integrationtests_mssql.log: -------------------------------------------------------------------------------- 1 | Running integration tests for mssql... 2 | ok github.com/grafana/scribe 0.003s 3 | ? github.com/grafana/scribe/ci [no test files] 4 | ? github.com/grafana/scribe/ci/docker [no test files] 5 | ? github.com/grafana/scribe/demo/basic [no test files] 6 | ? github.com/grafana/scribe/demo/complex [no test files] 7 | ? github.com/grafana/scribe/docker [no test files] 8 | ? github.com/grafana/scribe/exec [no test files] 9 | ? github.com/grafana/scribe/fs [no test files] 10 | ? github.com/grafana/scribe/git [no test files] 11 | ? github.com/grafana/scribe/git/x [no test files] 12 | ? github.com/grafana/scribe/golang [no test files] 13 | ? github.com/grafana/scribe/golang/x [no test files] 14 | ? github.com/grafana/scribe/makefile [no test files] 15 | ? github.com/grafana/scribe/plumbing [no test files] 16 | ? github.com/grafana/scribe/plumbing/cmd [no test files] 17 | ? github.com/grafana/scribe/plumbing/cmd/commands [no test files] 18 | ? github.com/grafana/scribe/plumbing/cmdutil [no test files] 19 | ? github.com/grafana/scribe/plumbing/pipeline [no test files] 20 | ? github.com/grafana/scribe/plumbing/pipeline/clients/cli [no test files] 21 | ok github.com/grafana/scribe/plumbing/pipeline/clients/docker 0.009s 22 | ok github.com/grafana/scribe/plumbing/pipeline/clients/drone 0.329s 23 | ? github.com/grafana/scribe/plumbing/plog [no test files] 24 | ? github.com/grafana/scribe/plumbing/schemas/drone [no test files] 25 | ? github.com/grafana/scribe/plumbing/stringutil [no test files] 26 | ? github.com/grafana/scribe/plumbing/syncutil [no test files] 27 | ? github.com/grafana/scribe/plumbing/testutil [no test files] 28 | ? github.com/grafana/scribe/plumbing/wrappers [no test files] 29 | ? github.com/grafana/scribe/yarn [no test files] 30 | Integration tests done 31 | -------------------------------------------------------------------------------- /demo/complex/logs/integrationtests_mysql.log: -------------------------------------------------------------------------------- 1 | Running integration tests for mysql... 2 | ok github.com/grafana/scribe 0.003s 3 | ? github.com/grafana/scribe/ci [no test files] 4 | ? github.com/grafana/scribe/ci/docker [no test files] 5 | ? github.com/grafana/scribe/demo/basic [no test files] 6 | ? github.com/grafana/scribe/demo/complex [no test files] 7 | ? github.com/grafana/scribe/docker [no test files] 8 | ? github.com/grafana/scribe/exec [no test files] 9 | ? github.com/grafana/scribe/fs [no test files] 10 | ? github.com/grafana/scribe/git [no test files] 11 | ? github.com/grafana/scribe/git/x [no test files] 12 | ? github.com/grafana/scribe/golang [no test files] 13 | ? github.com/grafana/scribe/golang/x [no test files] 14 | ? github.com/grafana/scribe/makefile [no test files] 15 | ? github.com/grafana/scribe/plumbing [no test files] 16 | ? github.com/grafana/scribe/plumbing/cmd [no test files] 17 | ? github.com/grafana/scribe/plumbing/cmd/commands [no test files] 18 | ? github.com/grafana/scribe/plumbing/cmdutil [no test files] 19 | ? github.com/grafana/scribe/plumbing/pipeline [no test files] 20 | ? github.com/grafana/scribe/plumbing/pipeline/clients/cli [no test files] 21 | ok github.com/grafana/scribe/plumbing/pipeline/clients/docker 0.009s 22 | ok github.com/grafana/scribe/plumbing/pipeline/clients/drone 0.329s 23 | ? github.com/grafana/scribe/plumbing/plog [no test files] 24 | ? github.com/grafana/scribe/plumbing/schemas/drone [no test files] 25 | ? github.com/grafana/scribe/plumbing/stringutil [no test files] 26 | ? github.com/grafana/scribe/plumbing/syncutil [no test files] 27 | ? github.com/grafana/scribe/plumbing/testutil [no test files] 28 | ? github.com/grafana/scribe/plumbing/wrappers [no test files] 29 | ? github.com/grafana/scribe/yarn [no test files] 30 | Integration tests done 31 | -------------------------------------------------------------------------------- /demo/complex/logs/integrationtests_pg.log: -------------------------------------------------------------------------------- 1 | Running integration tests for postgres... 2 | ok github.com/grafana/scribe 0.003s 3 | ? github.com/grafana/scribe/ci [no test files] 4 | ? github.com/grafana/scribe/ci/docker [no test files] 5 | ? github.com/grafana/scribe/demo/basic [no test files] 6 | ? github.com/grafana/scribe/demo/complex [no test files] 7 | ? github.com/grafana/scribe/docker [no test files] 8 | ? github.com/grafana/scribe/exec [no test files] 9 | ? github.com/grafana/scribe/fs [no test files] 10 | ? github.com/grafana/scribe/git [no test files] 11 | ? github.com/grafana/scribe/git/x [no test files] 12 | ? github.com/grafana/scribe/golang [no test files] 13 | ? github.com/grafana/scribe/golang/x [no test files] 14 | ? github.com/grafana/scribe/makefile [no test files] 15 | ? github.com/grafana/scribe/plumbing [no test files] 16 | ? github.com/grafana/scribe/plumbing/cmd [no test files] 17 | ? github.com/grafana/scribe/plumbing/cmd/commands [no test files] 18 | ? github.com/grafana/scribe/plumbing/cmdutil [no test files] 19 | ? github.com/grafana/scribe/plumbing/pipeline [no test files] 20 | ? github.com/grafana/scribe/plumbing/pipeline/clients/cli [no test files] 21 | ok github.com/grafana/scribe/plumbing/pipeline/clients/docker 0.009s 22 | ok github.com/grafana/scribe/plumbing/pipeline/clients/drone 0.329s 23 | ? github.com/grafana/scribe/plumbing/plog [no test files] 24 | ? github.com/grafana/scribe/plumbing/schemas/drone [no test files] 25 | ? github.com/grafana/scribe/plumbing/stringutil [no test files] 26 | ? github.com/grafana/scribe/plumbing/syncutil [no test files] 27 | ? github.com/grafana/scribe/plumbing/testutil [no test files] 28 | ? github.com/grafana/scribe/plumbing/wrappers [no test files] 29 | ? github.com/grafana/scribe/yarn [no test files] 30 | Integration tests done 31 | -------------------------------------------------------------------------------- /demo/complex/logs/integrationtests_sqlite.log: -------------------------------------------------------------------------------- 1 | Running integration tests for sqlite... 2 | ok github.com/grafana/scribe 0.003s 3 | ? github.com/grafana/scribe/ci [no test files] 4 | ? github.com/grafana/scribe/ci/docker [no test files] 5 | ? github.com/grafana/scribe/demo/basic [no test files] 6 | ? github.com/grafana/scribe/demo/complex [no test files] 7 | ? github.com/grafana/scribe/docker [no test files] 8 | ? github.com/grafana/scribe/exec [no test files] 9 | ? github.com/grafana/scribe/fs [no test files] 10 | ? github.com/grafana/scribe/git [no test files] 11 | ? github.com/grafana/scribe/git/x [no test files] 12 | ? github.com/grafana/scribe/golang [no test files] 13 | ? github.com/grafana/scribe/golang/x [no test files] 14 | ? github.com/grafana/scribe/makefile [no test files] 15 | ? github.com/grafana/scribe/plumbing [no test files] 16 | ? github.com/grafana/scribe/plumbing/cmd [no test files] 17 | ? github.com/grafana/scribe/plumbing/cmd/commands [no test files] 18 | ? github.com/grafana/scribe/plumbing/cmdutil [no test files] 19 | ? github.com/grafana/scribe/plumbing/pipeline [no test files] 20 | ? github.com/grafana/scribe/plumbing/pipeline/clients/cli [no test files] 21 | ok github.com/grafana/scribe/plumbing/pipeline/clients/docker 0.009s 22 | ok github.com/grafana/scribe/plumbing/pipeline/clients/drone 0.329s 23 | ? github.com/grafana/scribe/plumbing/plog [no test files] 24 | ? github.com/grafana/scribe/plumbing/schemas/drone [no test files] 25 | ? github.com/grafana/scribe/plumbing/stringutil [no test files] 26 | ? github.com/grafana/scribe/plumbing/syncutil [no test files] 27 | ? github.com/grafana/scribe/plumbing/testutil [no test files] 28 | ? github.com/grafana/scribe/plumbing/wrappers [no test files] 29 | ? github.com/grafana/scribe/yarn [no test files] 30 | Integration tests done 31 | -------------------------------------------------------------------------------- /demo/complex/logs/notifyslack.log: -------------------------------------------------------------------------------- 1 | Notifying slack that this pipeline has succeeded in #very-ignored-channel... 2 | Slack notified! 3 | -------------------------------------------------------------------------------- /demo/complex/logs/package.log: -------------------------------------------------------------------------------- 1 | Loading binary from state... 2 | Loading frontend from state... 3 | Packaging binary and frontend... 4 | Done packaging binary and frontend 5 | -------------------------------------------------------------------------------- /demo/complex/logs/publishdockerimage.log: -------------------------------------------------------------------------------- 1 | The push refers to repository [docker.io/example/example] 2 | fecab273b6a0: Pushed 3 | a3efa5a9cf62: Pushed 4 | 92e86890c45b: Pushed 5 | 8d3ac3489996: Layer already exists 6 | latest: digest: sha256:908263a0488df507c78fe6c2fb1577c1b96f693256d056ac9ac1790345c62831 size: 1159 7 | -------------------------------------------------------------------------------- /demo/complex/logs/publishdocs.log: -------------------------------------------------------------------------------- 1 | Packaging documentation... 2 | Done packaging documentation 3 | Uploading documentation to Google Cloud Storage... 4 | Done uploading documentation 5 | Enabling the thing... 6 | Done enabling 7 | -------------------------------------------------------------------------------- /demo/complex/logs/publishpackage.log: -------------------------------------------------------------------------------- 1 | Uploading debian package... 2 | Done uploading debain package 3 | Uploading RPM package... 4 | Done uploading RPM package 5 | Upading package index... 6 | Done updating package index 7 | -------------------------------------------------------------------------------- /demo/complex/logs/testbackend.log: -------------------------------------------------------------------------------- 1 | Testing backend... 2 | ok github.com/grafana/scribe (cached) 3 | ? github.com/grafana/scribe/ci [no test files] 4 | ? github.com/grafana/scribe/ci/docker [no test files] 5 | ? github.com/grafana/scribe/demo/basic [no test files] 6 | ? github.com/grafana/scribe/demo/complex [no test files] 7 | ? github.com/grafana/scribe/docker [no test files] 8 | ? github.com/grafana/scribe/exec [no test files] 9 | ? github.com/grafana/scribe/fs [no test files] 10 | ? github.com/grafana/scribe/git [no test files] 11 | ? github.com/grafana/scribe/git/x [no test files] 12 | ? github.com/grafana/scribe/golang [no test files] 13 | ? github.com/grafana/scribe/golang/x [no test files] 14 | ? github.com/grafana/scribe/makefile [no test files] 15 | ? github.com/grafana/scribe/plumbing [no test files] 16 | ? github.com/grafana/scribe/plumbing/cmd [no test files] 17 | ? github.com/grafana/scribe/plumbing/cmd/commands [no test files] 18 | ? github.com/grafana/scribe/plumbing/cmdutil [no test files] 19 | ? github.com/grafana/scribe/plumbing/pipeline [no test files] 20 | ? github.com/grafana/scribe/plumbing/pipeline/clients/cli [no test files] 21 | ok github.com/grafana/scribe/plumbing/pipeline/clients/docker (cached) 22 | ok github.com/grafana/scribe/plumbing/pipeline/clients/drone (cached) 23 | ? github.com/grafana/scribe/plumbing/plog [no test files] 24 | ? github.com/grafana/scribe/plumbing/schemas/drone [no test files] 25 | ? github.com/grafana/scribe/plumbing/stringutil [no test files] 26 | ? github.com/grafana/scribe/plumbing/syncutil [no test files] 27 | ? github.com/grafana/scribe/plumbing/testutil [no test files] 28 | ? github.com/grafana/scribe/plumbing/wrappers [no test files] 29 | ? github.com/grafana/scribe/yarn [no test files] 30 | Done testing backend 31 | -------------------------------------------------------------------------------- /demo/complex/logs/testfrontend.log: -------------------------------------------------------------------------------- 1 | yarn run v1.22.17 2 | $ grafana-toolkit plugin:test 3 | ⠋ Running tests Using standard jest plugin config /home/kminehart/Grafana/scribe/.compose/grafana/plugins/scribe-app/node_modules/@grafana/toolkit/src/config/jest.plugin.config.local.js 4 | ts-jest[config] (WARN) The option `tsConfig` is deprecated and will be removed in ts-jest 27, use `tsconfig` instead 5 | ts-jest[config] (WARN) The option `tsConfig` is deprecated and will be removed in ts-jest 27, use `tsconfig` instead 6 | PASS src/panels/scribe-build-list-panel/module.test.ts 7 | PASS src/panels/scribe-pipeline-list-panel/module.test.ts 8 | 9 | Test Suites: 2 passed, 2 total 10 | Tests: 2 passed, 2 total 11 | Snapshots: 0 total 12 | Time: 2.13 s 13 | Ran all test suites with tests matching "". 14 | ✔ Running tests 15 | Done in 3.58s. 16 | 17 | -------------------------------------------------------------------------------- /demo/complex/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/grafana/scribe" 7 | "github.com/grafana/scribe/pipeline" 8 | ) 9 | 10 | func main() { 11 | sw := scribe.New("complex-pipeline") 12 | defer sw.Done() 13 | 14 | sw.Background(pipeline.NamedStep("redis", pipeline.DefaultAction).WithImage("redis:6")) 15 | 16 | sw.Add(pipeline.NamedStep("initalize", NoOpAction("initialize", time.Second*22))) 17 | 18 | sw.Add( 19 | pipeline.NamedStep("build backend", NoOpAction("buildbackend", time.Second*39)), 20 | pipeline.NamedStep("build frontend", NoOpAction("buildfrontend", time.Minute)), 21 | pipeline.NamedStep("build documentation", NoOpAction("builddocs", time.Second*9)), 22 | ) 23 | 24 | sw.Add( 25 | pipeline.NamedStep("test backend", NoOpAction("testbackend", time.Second*27)), 26 | pipeline.NamedStep("test frontend", NoOpAction("testfrontend", time.Second*32)), 27 | ) 28 | 29 | sw.Add( 30 | pipeline.NamedStep("integration tests: sqlite", IntegrationTest("integrationtests_sqlite", time.Minute)), 31 | pipeline.NamedStep("integration tests: postgres", IntegrationTest("integrationtests_pg", time.Second*42)), 32 | pipeline.NamedStep("integration tests: mysql", IntegrationTest("integrationtests_mysql", time.Second*32)), 33 | pipeline.NamedStep("integration tests: mssql", IntegrationTest("integrationtests_mssql", time.Second*55)), 34 | ) 35 | 36 | sw.Add( 37 | pipeline.NamedStep("package", NoOpAction("package", time.Second*13)), 38 | pipeline.NamedStep("build docker image", NoOpAction("builddocker", time.Second*44)), 39 | ) 40 | 41 | sw.Add( 42 | pipeline.NamedStep("publish documentation", NoOpAction("publishdocs", time.Second*13)), 43 | pipeline.NamedStep("publish package", NoOpAction("publishpackage", time.Second*12)), 44 | pipeline.NamedStep("publish docker image", NoOpAction("publishdockerimage", time.Second*23)), 45 | ) 46 | 47 | sw.Add( 48 | pipeline.NamedStep("notify slack", NoOpAction("notifyslack", time.Second*3)), 49 | ) 50 | } 51 | -------------------------------------------------------------------------------- /demo/custom-client/gen_drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | type: docker 4 | name: custom_client 5 | 6 | platform: 7 | os: linux 8 | arch: amd64 9 | 10 | steps: 11 | - name: builtin-compile-pipeline 12 | image: golang:1.19 13 | command: 14 | - go 15 | - build 16 | - -o 17 | - /var/scribe/pipeline 18 | - ./demo/custom-client 19 | environment: 20 | CGO_ENABLED: 0 21 | GOARCH: amd64 22 | GOOS: linux 23 | volumes: 24 | - name: scribe 25 | path: /var/scribe 26 | 27 | - name: custom_client 28 | image: golang:1.19 29 | commands: 30 | - /var/scribe/pipeline --pipeline="custom-client" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=latest ./demo/custom-client 31 | volumes: 32 | - name: scribe 33 | path: /var/scribe 34 | - name: scribe-state 35 | path: /var/scribe-state 36 | depends_on: 37 | - builtin-compile-pipeline 38 | 39 | volumes: 40 | - name: scribe 41 | temp: {} 42 | - name: scribe-state 43 | temp: {} 44 | - name: docker_socket 45 | host: 46 | path: /var/run/docker.sock 47 | 48 | ... 49 | -------------------------------------------------------------------------------- /demo/custom-client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe" 7 | "github.com/grafana/scribe/pipeline" 8 | "github.com/grafana/scribe/pipeline/clients" 9 | "github.com/grafana/scribe/state" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | type MyClient struct { 14 | Log logrus.FieldLogger 15 | } 16 | 17 | func (c *MyClient) Validate(step pipeline.Step) error { 18 | return nil 19 | } 20 | 21 | func (c *MyClient) Provides() []state.Argument { 22 | return nil 23 | } 24 | 25 | func (c *MyClient) Done(ctx context.Context, w *pipeline.Collection) error { 26 | return w.WalkPipelines(ctx, func(ctx context.Context, p pipeline.Pipeline) error { 27 | c.Log.Infoln("pipeline:", p.Name) 28 | return w.WalkSteps(ctx, p.ID, func(ctx context.Context, step pipeline.Step) error { 29 | c.Log.Infoln("step:", step.Name) 30 | return nil 31 | }) 32 | }) 33 | } 34 | 35 | func init() { 36 | scribe.RegisterClient("my-custom-client", func(ctx context.Context, opts clients.CommonOpts) (pipeline.Client, error) { 37 | return &MyClient{ 38 | Log: opts.Log, 39 | }, nil 40 | }) 41 | } 42 | 43 | func main() { 44 | sw := scribe.New("custom-client") 45 | defer sw.Done() 46 | 47 | sw.Add( 48 | pipeline.NoOpStep.WithName("step 1"), 49 | pipeline.NoOpStep.WithName("step 2"), 50 | ) 51 | sw.Add( 52 | pipeline.NoOpStep.WithName("step 3"), 53 | pipeline.NoOpStep.WithName("step 4"), 54 | ) 55 | } 56 | -------------------------------------------------------------------------------- /demo/generate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | for demo in ./demo/* ; do 4 | if [ -d "$demo" ]; then 5 | echo "go run $demo -path=$demo -client=drone > $demo/gen_drone.yml" 6 | go run $demo --path=$demo --client=drone > $demo/gen_drone.yml 7 | fi 8 | done 9 | -------------------------------------------------------------------------------- /demo/multi-sub/build.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe" 7 | "github.com/grafana/scribe/fs" 8 | gitx "github.com/grafana/scribe/git/x" 9 | "github.com/grafana/scribe/golang" 10 | "github.com/grafana/scribe/makefile" 11 | "github.com/grafana/scribe/pipeline" 12 | "github.com/grafana/scribe/state" 13 | "github.com/grafana/scribe/yarn" 14 | ) 15 | 16 | var ( 17 | ArgumentTestResult = state.NewBoolArgument("test-results") 18 | ) 19 | 20 | func writeVersion(sw *scribe.Scribe) pipeline.Step { 21 | action := func(ctx context.Context, opts pipeline.ActionOpts) error { 22 | 23 | // equivalent of `git describe --tags --dirty --always` 24 | version, err := gitx.Describe(ctx, ".", true, true, true) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | // write the version string in the `.version` file. 30 | return fs.ReplaceString(".version", version)(ctx, opts) 31 | } 32 | 33 | return pipeline.NewStep(action) 34 | } 35 | 36 | func installDependencies(sw *scribe.Scribe) { 37 | sw.Add( 38 | pipeline.NamedStep("install frontend dependencies", sw.Cache( 39 | yarn.InstallAction(), 40 | fs.Cache("node_modules", fs.FileHasChanged("yarn.lock")), 41 | )), 42 | pipeline.NamedStep("install backend dependencies", sw.Cache( 43 | golang.ModDownload(), 44 | fs.Cache("$GOPATH/pkg", fs.FileHasChanged("go.sum")), 45 | )), 46 | ) 47 | } 48 | 49 | func testPipeline(sw *scribe.Scribe) { 50 | installDependencies(sw) 51 | 52 | sw.Add( 53 | golang.Test(sw, "./...").WithName("test backend"), 54 | pipeline.NamedStep("test frontend", makefile.Target("test-frontend")), 55 | ) 56 | } 57 | 58 | func publishPipeline(sw *scribe.Scribe) { 59 | sw.When( 60 | pipeline.GitCommitEvent(pipeline.GitCommitFilters{ 61 | Branch: pipeline.StringFilter("main"), 62 | }), 63 | pipeline.GitTagEvent(pipeline.GitTagFilters{ 64 | Name: pipeline.GlobFilter("v*"), 65 | }), 66 | ) 67 | 68 | installDependencies(sw) 69 | 70 | sw.Add( 71 | pipeline.NamedStep("compile backend", makefile.Target("build")), 72 | pipeline.NamedStep("compile frontend", makefile.Target("package")), 73 | ) 74 | 75 | sw.Add( 76 | pipeline.NamedStep("publish", makefile.Target("publish")).Requires(state.NewSecretArgument("gcp-publish-key")), 77 | ) 78 | } 79 | 80 | func codeqlPipeline(sw *scribe.Scribe) { 81 | sw.Add( 82 | pipeline.NoOpStep.WithName("codeql"), 83 | pipeline.NoOpStep.WithName("notify-slack"), 84 | ) 85 | } 86 | 87 | // "main" defines our program pipeline. 88 | // Every pipeline step should be instantiated using the scribe client (sw). 89 | // This allows the various clients to work properly in different scenarios, like in a CI environment or locally. 90 | // Logic and processing done outside of the `sw.*` family of functions may not be included in the resulting pipeline. 91 | func main() { 92 | sw := scribe.NewMulti() 93 | defer sw.Done() 94 | 95 | sw.Add( 96 | sw.New("code quality check", codeqlPipeline), 97 | sw.New("test", testPipeline).Provides(ArgumentTestResult), 98 | sw.New("publish", publishPipeline).Requires(ArgumentTestResult), 99 | ) 100 | } 101 | -------------------------------------------------------------------------------- /demo/multi-sub/gen_drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | type: docker 4 | name: code_quality_check 5 | 6 | platform: 7 | os: linux 8 | arch: amd64 9 | 10 | steps: 11 | - name: builtin-compile-pipeline 12 | image: golang:1.19 13 | command: 14 | - go 15 | - build 16 | - -o 17 | - /var/scribe/pipeline 18 | - ./demo/multi-sub 19 | environment: 20 | CGO_ENABLED: 0 21 | GOARCH: amd64 22 | GOOS: linux 23 | volumes: 24 | - name: scribe 25 | path: /var/scribe 26 | 27 | - name: code_quality_check 28 | image: golang:1.19 29 | commands: 30 | - /var/scribe/pipeline --pipeline="code quality check" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=latest ./demo/multi-sub 31 | volumes: 32 | - name: scribe 33 | path: /var/scribe 34 | - name: scribe-state 35 | path: /var/scribe-state 36 | depends_on: 37 | - builtin-compile-pipeline 38 | 39 | volumes: 40 | - name: scribe 41 | temp: {} 42 | - name: scribe-state 43 | temp: {} 44 | - name: docker_socket 45 | host: 46 | path: /var/run/docker.sock 47 | 48 | --- 49 | kind: pipeline 50 | type: docker 51 | name: test 52 | 53 | platform: 54 | os: linux 55 | arch: amd64 56 | 57 | steps: 58 | - name: builtin-compile-pipeline 59 | image: golang:1.19 60 | command: 61 | - go 62 | - build 63 | - -o 64 | - /var/scribe/pipeline 65 | - ./demo/multi-sub 66 | environment: 67 | CGO_ENABLED: 0 68 | GOARCH: amd64 69 | GOOS: linux 70 | volumes: 71 | - name: scribe 72 | path: /var/scribe 73 | 74 | - name: test 75 | image: golang:1.19 76 | commands: 77 | - /var/scribe/pipeline --pipeline="test" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=latest ./demo/multi-sub 78 | volumes: 79 | - name: scribe 80 | path: /var/scribe 81 | - name: scribe-state 82 | path: /var/scribe-state 83 | depends_on: 84 | - builtin-compile-pipeline 85 | 86 | volumes: 87 | - name: scribe 88 | temp: {} 89 | - name: scribe-state 90 | temp: {} 91 | - name: docker_socket 92 | host: 93 | path: /var/run/docker.sock 94 | 95 | --- 96 | kind: pipeline 97 | type: docker 98 | name: publish 99 | 100 | platform: 101 | os: linux 102 | arch: amd64 103 | 104 | steps: 105 | - name: builtin-compile-pipeline 106 | image: golang:1.19 107 | command: 108 | - go 109 | - build 110 | - -o 111 | - /var/scribe/pipeline 112 | - ./demo/multi-sub 113 | environment: 114 | CGO_ENABLED: 0 115 | GOARCH: amd64 116 | GOOS: linux 117 | volumes: 118 | - name: scribe 119 | path: /var/scribe 120 | 121 | - name: publish 122 | image: golang:1.19 123 | commands: 124 | - /var/scribe/pipeline --pipeline="publish" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=latest ./demo/multi-sub 125 | volumes: 126 | - name: scribe 127 | path: /var/scribe 128 | - name: scribe-state 129 | path: /var/scribe-state 130 | depends_on: 131 | - builtin-compile-pipeline 132 | 133 | volumes: 134 | - name: scribe 135 | temp: {} 136 | - name: scribe-state 137 | temp: {} 138 | - name: docker_socket 139 | host: 140 | path: /var/run/docker.sock 141 | 142 | trigger: 143 | branch: 144 | - main 145 | event: 146 | - branch 147 | - tag 148 | ref: 149 | - refs/tags/v* 150 | 151 | depends_on: 152 | - test 153 | 154 | ... 155 | -------------------------------------------------------------------------------- /demo/multi/build.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/grafana/scribe" 9 | "github.com/grafana/scribe/pipeline" 10 | "github.com/grafana/scribe/state" 11 | ) 12 | 13 | var ( 14 | ArgumentCompiledBackend = state.NewFileArgument("compiled-backend") 15 | ArgumentCompiledFrontend = state.NewDirectoryArgument("compiled-frontend") 16 | ) 17 | 18 | func actionBuildBackend(ctx context.Context, opts pipeline.ActionOpts) error { 19 | f, err := os.CreateTemp("", "*") 20 | if err != nil { 21 | return err 22 | } 23 | 24 | defer f.Close() 25 | 26 | if _, err := opts.State.SetFileReader(ctx, ArgumentCompiledBackend, f); err != nil { 27 | return err 28 | } 29 | 30 | return nil 31 | } 32 | 33 | func actionBuildFrontend(ctx context.Context, opts pipeline.ActionOpts) error { 34 | path := filepath.Join(os.TempDir(), "frontend") 35 | if err := os.MkdirAll(path, 0755); err != nil { 36 | return err 37 | } 38 | 39 | return opts.State.SetDirectory(ctx, ArgumentCompiledFrontend, path) 40 | } 41 | 42 | var stepBuildBackend = pipeline.NamedStep("build-backend", actionBuildBackend). 43 | Requires(ArgumentNodeDependencies, pipeline.ArgumentSourceFS). 44 | Provides(ArgumentCompiledBackend) 45 | 46 | var stepBuildFrontend = pipeline.NamedStep("build-backend", actionBuildFrontend). 47 | Requires(ArgumentGoDependencies, pipeline.ArgumentSourceFS). 48 | Provides(ArgumentCompiledFrontend) 49 | 50 | var PipelineBuild = scribe.Pipeline{ 51 | Name: "build", 52 | Provides: []state.Argument{ArgumentCompiledBackend, ArgumentCompiledFrontend}, 53 | Requires: []state.Argument{ArgumentGoDependencies, ArgumentNodeDependencies}, 54 | Steps: []pipeline.Step{stepBuildBackend, stepBuildFrontend}, 55 | } 56 | -------------------------------------------------------------------------------- /demo/multi/dependencies.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "time" 9 | 10 | "github.com/grafana/scribe" 11 | "github.com/grafana/scribe/pipeline" 12 | "github.com/grafana/scribe/state" 13 | ) 14 | 15 | func actionInstallFrontendDeps(ctx context.Context, opts pipeline.ActionOpts) error { 16 | opts.Logger.Infoln("Installing frontend dependencies...") 17 | time.Sleep(1 * time.Second) 18 | opts.Logger.Infoln("Done installing frontend dependencies") 19 | 20 | // yarn install... 21 | // for demo purposes just creating an empty dir... 22 | path := filepath.Join(os.TempDir(), "frontend-deps") 23 | if err := os.MkdirAll(path, 0755); err != nil { 24 | return err 25 | } 26 | 27 | b, err := exec.Command("touch", filepath.Join(path, "a")).CombinedOutput() 28 | opts.Logger.Infoln(string(b), "error", err) 29 | b, err = exec.Command("ls", "-al", path).CombinedOutput() 30 | opts.Logger.Infoln(string(b), "error", err) 31 | return opts.State.SetDirectory(ctx, ArgumentNodeDependencies, path) 32 | } 33 | 34 | func actionInstallBackendDeps(ctx context.Context, opts pipeline.ActionOpts) error { 35 | opts.Logger.Infoln("Installing backend dependencies...") 36 | time.Sleep(1 * time.Second) 37 | opts.Logger.Infoln("Done installing backend dependencies") 38 | 39 | path := filepath.Join(os.TempDir(), "backend-deps") 40 | if err := os.MkdirAll(path, 0755); err != nil { 41 | return err 42 | } 43 | 44 | b, err := exec.Command("touch", filepath.Join(path, "a")).CombinedOutput() 45 | opts.Logger.Infoln(string(b), "error", err) 46 | b, err = exec.Command("ls", "-al", path).CombinedOutput() 47 | opts.Logger.Infoln(string(b), "error", err) 48 | return opts.State.SetDirectory(ctx, ArgumentGoDependencies, path) 49 | } 50 | 51 | var ( 52 | ArgumentGoDependencies = state.NewDirectoryArgument("go-dependencies") 53 | ArgumentNodeDependencies = state.NewDirectoryArgument("node-dependencies") 54 | ) 55 | 56 | var stepInstallFrontendDeps = pipeline.NamedStep("install frontend dependencies", actionInstallFrontendDeps). 57 | Provides(ArgumentNodeDependencies). 58 | Requires(pipeline.ArgumentSourceFS) 59 | 60 | var stepInstallBackendDeps = pipeline.NamedStep("install backend dependencies", actionInstallBackendDeps). 61 | Provides(ArgumentGoDependencies). 62 | Requires(pipeline.ArgumentSourceFS) 63 | 64 | var PipelineDependencies = scribe.Pipeline{ 65 | Name: "dependencies", 66 | Provides: []state.Argument{ArgumentNodeDependencies, ArgumentGoDependencies}, 67 | Steps: []pipeline.Step{ 68 | stepInstallFrontendDeps, 69 | stepInstallBackendDeps, 70 | }, 71 | } 72 | 73 | // ExtractBackendDependencies retrieves the backend dependencies from the state handler and places them in the appropriate place on the filesystem. 74 | // This function assumes that your step requires the "ArgumentGoDependencies" argument 75 | func ExtractBackendDependencies(st state.Handler) error { 76 | return nil 77 | } 78 | 79 | // ExtractBackendDependencies retrieves the backend dependencies from the state handler and places them in the appropriate place on the filesystem. 80 | // This function assumes that your step requires the "ArgumentGoDependencies" argument 81 | func ExtractFrontendDependencies(st state.Handler) error { 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /demo/multi/gen_drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | type: docker 4 | name: test 5 | 6 | platform: 7 | os: linux 8 | arch: amd64 9 | 10 | steps: 11 | - name: builtin-compile-pipeline 12 | image: golang:1.19 13 | command: 14 | - go 15 | - build 16 | - -o 17 | - /var/scribe/pipeline 18 | - ./demo/multi 19 | environment: 20 | CGO_ENABLED: 0 21 | GOARCH: amd64 22 | GOOS: linux 23 | volumes: 24 | - name: scribe 25 | path: /var/scribe 26 | 27 | - name: test 28 | image: golang:1.19 29 | commands: 30 | - /var/scribe/pipeline --pipeline="test" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=latest ./demo/multi 31 | volumes: 32 | - name: scribe 33 | path: /var/scribe 34 | - name: scribe-state 35 | path: /var/scribe-state 36 | depends_on: 37 | - builtin-compile-pipeline 38 | 39 | volumes: 40 | - name: scribe 41 | temp: {} 42 | - name: scribe-state 43 | temp: {} 44 | - name: docker_socket 45 | host: 46 | path: /var/run/docker.sock 47 | 48 | --- 49 | kind: pipeline 50 | type: docker 51 | name: publish 52 | 53 | platform: 54 | os: linux 55 | arch: amd64 56 | 57 | steps: 58 | - name: builtin-compile-pipeline 59 | image: golang:1.19 60 | command: 61 | - go 62 | - build 63 | - -o 64 | - /var/scribe/pipeline 65 | - ./demo/multi 66 | environment: 67 | CGO_ENABLED: 0 68 | GOARCH: amd64 69 | GOOS: linux 70 | volumes: 71 | - name: scribe 72 | path: /var/scribe 73 | 74 | - name: publish 75 | image: golang:1.19 76 | commands: 77 | - /var/scribe/pipeline --pipeline="publish" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=latest ./demo/multi 78 | volumes: 79 | - name: scribe 80 | path: /var/scribe 81 | - name: scribe-state 82 | path: /var/scribe-state 83 | depends_on: 84 | - builtin-compile-pipeline 85 | 86 | volumes: 87 | - name: scribe 88 | temp: {} 89 | - name: scribe-state 90 | temp: {} 91 | - name: docker_socket 92 | host: 93 | path: /var/run/docker.sock 94 | 95 | trigger: 96 | branch: 97 | - main 98 | event: 99 | - branch 100 | - tag 101 | ref: 102 | - refs/tags/v* 103 | 104 | depends_on: 105 | - test 106 | 107 | ... 108 | -------------------------------------------------------------------------------- /demo/multi/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/grafana/scribe" 4 | 5 | var Pipelines = []scribe.Pipeline{ 6 | PipelineDependencies, 7 | PipelineBuild, 8 | PipelineTest, 9 | PipelinePublish, 10 | } 11 | 12 | // "main" defines our program pipeline. 13 | // Every pipeline step should be instantiated using the scribe client (sw). 14 | // This allows the various clients to work properly in different scenarios, like in a CI environment or locally. 15 | // Logic and processing done outside of the `sw.*` family of functions may not be included in the resulting pipeline. 16 | func main() { 17 | sw := scribe.NewMulti() 18 | defer sw.Done() 19 | 20 | sw.AddPipelines(Pipelines...) 21 | } 22 | -------------------------------------------------------------------------------- /demo/multi/package.go: -------------------------------------------------------------------------------- 1 | package main 2 | -------------------------------------------------------------------------------- /demo/multi/publish.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/grafana/scribe" 8 | "github.com/grafana/scribe/pipeline" 9 | "github.com/grafana/scribe/state" 10 | ) 11 | 12 | var ( 13 | ArgumentTarPackage = state.NewFileArgument("package-tarball") 14 | ) 15 | 16 | func actionPackage(ctx context.Context, opts pipeline.ActionOpts) error { 17 | f, err := os.CreateTemp("", "*.tar.gz") 18 | if err != nil { 19 | return err 20 | } 21 | 22 | opts.Logger.Infoln("Created file:", f.Name()) 23 | defer f.Close() 24 | path, err := opts.State.SetFileReader(ctx, ArgumentTarPackage, f) 25 | opts.Logger.Infoln("Stored in state as:", path) 26 | return err 27 | } 28 | 29 | func actionPublish(ctx context.Context, opts pipeline.ActionOpts) error { 30 | opts.Logger.Warnln("Pipeline done!") 31 | opts.Logger.Warnln("Pipeline done!") 32 | opts.Logger.Warnln("Pipeline done!") 33 | 34 | return nil 35 | } 36 | 37 | var stepPackage = pipeline.NamedStep("package", actionPackage). 38 | Provides(ArgumentTarPackage). 39 | Requires(ArgumentCompiledBackend, ArgumentCompiledFrontend) 40 | 41 | var stepPublish = pipeline.NamedStep("publish", actionPublish). 42 | Requires( 43 | state.NewSecretArgument("gcp-publish-key"), 44 | ArgumentTarPackage, 45 | ) 46 | 47 | var PipelinePublish = scribe.Pipeline{ 48 | Name: "publish", 49 | Steps: []pipeline.Step{ 50 | stepPackage, 51 | stepPublish, 52 | }, 53 | Requires: []state.Argument{ArgumentCompiledBackend, ArgumentCompiledFrontend}, 54 | Provides: []state.Argument{ArgumentTarPackage}, 55 | When: []pipeline.Event{ 56 | pipeline.GitCommitEvent(pipeline.GitCommitFilters{ 57 | Branch: pipeline.StringFilter("main"), 58 | }), 59 | pipeline.GitTagEvent(pipeline.GitTagFilters{ 60 | Name: pipeline.GlobFilter("v*"), 61 | }), 62 | }, 63 | } 64 | -------------------------------------------------------------------------------- /demo/multi/test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/grafana/scribe" 8 | "github.com/grafana/scribe/pipeline" 9 | "github.com/grafana/scribe/state" 10 | ) 11 | 12 | var ( 13 | ArgumentTestResultBackend = state.NewBoolArgument("backend-test-result") 14 | ArgumentTestResultFrontend = state.NewBoolArgument("frontend-test-result") 15 | ) 16 | 17 | func actionTestFrontend(ctx context.Context, opts pipeline.ActionOpts) error { 18 | opts.Logger.Infoln("Testing frontend...") 19 | time.Sleep(time.Second * 1) 20 | opts.Logger.Infoln("Done testing frontend") 21 | // make test-frontend 22 | // assume it passed... 23 | return opts.State.SetBool(ctx, ArgumentTestResultFrontend, true) 24 | } 25 | 26 | func actionTestBackend(ctx context.Context, opts pipeline.ActionOpts) error { 27 | opts.Logger.Infoln("Testing backend...") 28 | time.Sleep(time.Second * 1) 29 | opts.Logger.Infoln("Done testing backend") 30 | // go test ./... 31 | // assume it passed... 32 | return opts.State.SetBool(ctx, ArgumentTestResultBackend, true) 33 | } 34 | 35 | var stepTestBackend = pipeline.NamedStep("test backend", actionTestBackend). 36 | Provides(ArgumentTestResultBackend). 37 | Requires(ArgumentGoDependencies) 38 | 39 | var stepTestFrontend = pipeline.NamedStep("test frontend", actionTestFrontend). 40 | Provides(ArgumentTestResultFrontend). 41 | Requires(ArgumentNodeDependencies) 42 | 43 | var PipelineTest = scribe.Pipeline{ 44 | Name: "test", 45 | Steps: []pipeline.Step{ 46 | stepTestBackend, 47 | stepTestFrontend, 48 | }, 49 | Requires: []state.Argument{ArgumentNodeDependencies, ArgumentGoDependencies}, 50 | Provides: []state.Argument{ArgumentTestResultFrontend, ArgumentTestResultBackend}, 51 | } 52 | -------------------------------------------------------------------------------- /demo/state/example-directory/a.txt: -------------------------------------------------------------------------------- 1 | file a.txt 2 | -------------------------------------------------------------------------------- /demo/state/example-directory/sub-folder/b.txt: -------------------------------------------------------------------------------- 1 | file b.txt 2 | -------------------------------------------------------------------------------- /demo/state/example-state-file.txt: -------------------------------------------------------------------------------- 1 | This is an example file that will be stored in the state. 2 | 3 | 1. this is a demo. 4 | 2. this is a demo. 5 | 3. this is a demo. 6 | 4. this is a demo. 7 | 5. this is a demo. 8 | 6. this is a demo. 9 | 7. this is a demo. 10 | 8. this is a demo. 11 | 9. this is a demo. 12 | -------------------------------------------------------------------------------- /demo/state/gen_drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | type: docker 4 | name: state_example 5 | 6 | platform: 7 | os: linux 8 | arch: amd64 9 | 10 | steps: 11 | - name: builtin-compile-pipeline 12 | image: golang:1.19 13 | command: 14 | - go 15 | - build 16 | - -o 17 | - /var/scribe/pipeline 18 | - ./demo/state 19 | environment: 20 | CGO_ENABLED: 0 21 | GOARCH: amd64 22 | GOOS: linux 23 | volumes: 24 | - name: scribe 25 | path: /var/scribe 26 | 27 | - name: state_example 28 | image: golang:1.19 29 | commands: 30 | - /var/scribe/pipeline --pipeline="state-example" --client cli --build-id=$DRONE_BUILD_NUMBER --state=file:///var/scribe-state/state.json --log-level=debug --version=latest ./demo/state 31 | volumes: 32 | - name: scribe 33 | path: /var/scribe 34 | - name: scribe-state 35 | path: /var/scribe-state 36 | depends_on: 37 | - builtin-compile-pipeline 38 | 39 | volumes: 40 | - name: scribe 41 | temp: {} 42 | - name: scribe-state 43 | temp: {} 44 | - name: docker_socket 45 | host: 46 | path: /var/run/docker.sock 47 | 48 | ... 49 | -------------------------------------------------------------------------------- /docs.go: -------------------------------------------------------------------------------- 1 | package scribe 2 | -------------------------------------------------------------------------------- /echo-test/go.mod: -------------------------------------------------------------------------------- 1 | module echo-test 2 | 3 | go 1.18 4 | 5 | require github.com/grafana/scribe v0.9.13 6 | 7 | require ( 8 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 9 | github.com/Microsoft/go-winio v0.5.2 // indirect 10 | github.com/Microsoft/hcsshim v0.9.3 // indirect 11 | github.com/bmatcuk/doublestar v1.3.4 // indirect 12 | github.com/buildkite/yaml v2.1.0+incompatible // indirect 13 | github.com/containerd/cgroups v1.0.4 // indirect 14 | github.com/containerd/containerd v1.6.6 // indirect 15 | github.com/docker/docker v20.10.17+incompatible // indirect 16 | github.com/docker/go-connections v0.4.0 // indirect 17 | github.com/docker/go-units v0.4.0 // indirect 18 | github.com/drone/drone-yaml v1.2.3 // indirect 19 | github.com/fsouza/go-dockerclient v1.8.1 // indirect 20 | github.com/gogo/protobuf v1.3.2 // indirect 21 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 22 | github.com/magefile/mage v1.13.0 // indirect 23 | github.com/moby/sys/mount v0.3.3 // indirect 24 | github.com/moby/sys/mountinfo v0.6.2 // indirect 25 | github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect 26 | github.com/morikuni/aec v1.0.0 // indirect 27 | github.com/opencontainers/go-digest v1.0.0 // indirect 28 | github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect 29 | github.com/opencontainers/runc v1.1.3 // indirect 30 | github.com/opentracing/opentracing-go v1.2.0 // indirect 31 | github.com/pkg/errors v0.9.1 // indirect 32 | github.com/sirupsen/logrus v1.8.1 // indirect 33 | github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect 34 | github.com/uber/jaeger-lib v2.4.1+incompatible // indirect 35 | go.opencensus.io v0.23.0 // indirect 36 | go.uber.org/atomic v1.9.0 // indirect 37 | golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect 38 | ) 39 | -------------------------------------------------------------------------------- /echo-test/go.work: -------------------------------------------------------------------------------- 1 | go 1.18 2 | 3 | use ../ 4 | -------------------------------------------------------------------------------- /echo-test/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe" 7 | "github.com/grafana/scribe/exec" 8 | "github.com/grafana/scribe/pipeline" 9 | ) 10 | 11 | func echo(ctx context.Context, opts pipeline.ActionOpts) error { 12 | return exec.RunCommandWithOpts(ctx, exec.RunOpts{ 13 | Name: "/bin/sh", 14 | Args: []string{"-c", `sleep 10; echo "hello ?"`}, 15 | Stdout: opts.Stdout, 16 | Stderr: opts.Stderr, 17 | }) 18 | } 19 | 20 | func StepEcho() pipeline.Step { 21 | return pipeline.NewStep(echo).WithImage("ubuntu:latest") 22 | } 23 | 24 | func main() { 25 | sw := scribe.New("test-pipeline") 26 | defer sw.Done() 27 | 28 | sw.Add(StepEcho()) 29 | } 30 | -------------------------------------------------------------------------------- /errors/errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | var ( 9 | New = errors.New 10 | Is = errors.Is 11 | ) 12 | 13 | // ErrorSkipValidation can be returned in the Client's Validate interface to prevent the error from stopping the pipeline execution 14 | var ErrorSkipValidation = errors.New("skipping step validation") 15 | 16 | var ( 17 | ErrorMissingArgument = errors.New("argument requested but not provided") 18 | ) 19 | 20 | type PipelineError struct { 21 | Err string 22 | Description string 23 | } 24 | 25 | func (p *PipelineError) Error() string { 26 | return fmt.Sprintf("%s: %s", p.Err, p.Description) 27 | } 28 | 29 | func NewPipelineError(err string, desc string) *PipelineError { 30 | return &PipelineError{ 31 | Err: err, 32 | Description: desc, 33 | } 34 | } 35 | 36 | type ErrorStack struct { 37 | Errors []error 38 | } 39 | 40 | func (e *ErrorStack) Push(err error) { 41 | e.Errors = append(e.Errors, err) 42 | } 43 | 44 | // Peek returns the error at the end of the stack without removing it. 45 | func (e *ErrorStack) Peek() error { 46 | if len(e.Errors) == 0 { 47 | return nil 48 | } 49 | 50 | return e.Errors[len(e.Errors)-1] 51 | } 52 | 53 | // Pop returns the error at the end of the stack and removes it. 54 | func (e *ErrorStack) Pop() error { 55 | if len(e.Errors) == 0 { 56 | return nil 57 | } 58 | 59 | err := e.Errors[len(e.Errors)-1] 60 | 61 | e.Errors = e.Errors[:len(e.Errors)-1] 62 | return err 63 | } 64 | -------------------------------------------------------------------------------- /exec/cmd.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | "os/exec" 8 | 9 | "github.com/grafana/scribe/pipeline" 10 | ) 11 | 12 | type RunOpts struct { 13 | Path string 14 | Stdout io.Writer 15 | Stderr io.Writer 16 | Name string 17 | Args []string 18 | Env []string 19 | } 20 | 21 | // CommandWithOpts returns the equivalent *exec.Cmd that matches the RunOpts provided (opts). 22 | func CommandWithOpts(ctx context.Context, opts RunOpts) *exec.Cmd { 23 | c := exec.CommandContext(ctx, opts.Name, opts.Args...) 24 | c.Dir = opts.Path 25 | 26 | if opts.Stdout != nil { 27 | c.Stdout = opts.Stdout 28 | } 29 | 30 | if opts.Stderr != nil { 31 | c.Stderr = opts.Stderr 32 | } 33 | 34 | c.Env = append(os.Environ(), opts.Env...) 35 | 36 | return c 37 | } 38 | 39 | // RunCommandWithOpts runs the command defined by the RunOpts provided (opts). 40 | // Be warned that the stdout and stderr are not captured by this function and are instead written to opts.Stdout/opts.Stderr. 41 | func RunCommandWithOpts(ctx context.Context, opts RunOpts) error { 42 | return CommandWithOpts(ctx, opts).Run() 43 | } 44 | 45 | // RunCommandAt runs a given command and set of arguments at the given location 46 | // The command's stdout and stderr are assigned the systems' stdout/stderr streams. 47 | func RunCommandAt(ctx context.Context, stdout, stderr io.Writer, path string, name string, arg ...string) error { 48 | return RunCommandWithOpts(ctx, RunOpts{ 49 | Path: path, 50 | Name: name, 51 | Args: arg, 52 | Stderr: stderr, 53 | Stdout: stdout, 54 | }) 55 | } 56 | 57 | // RunCommand runs a given command and set of arguments. 58 | // The command's stdout and stderr are assigned the systems' stdout/stderr streams. 59 | func RunCommand(ctx context.Context, stdout, stderr io.Writer, name string, arg ...string) error { 60 | return RunCommandAt(ctx, stdout, stderr, ".", name, arg...) 61 | } 62 | 63 | // RunAction returns an action that runs a given command and set of arguments. 64 | // The command's stdout and stderr are assigned the systems' stdout/stderr streams. 65 | func RunAction(name string, arg ...string) pipeline.Action { 66 | return func(ctx context.Context, opts pipeline.ActionOpts) error { 67 | return RunCommand(ctx, opts.Stdout, opts.Stderr, name, arg...) 68 | } 69 | } 70 | 71 | // Run returns an action that runs a given command and set of arguments. 72 | // The command's stdout and stderr are assigned the systems' stdout/stderr streams. 73 | func RunAt(path string, name string, arg ...string) pipeline.Action { 74 | return func(ctx context.Context, opts pipeline.ActionOpts) error { 75 | return RunCommandAt(ctx, opts.Stdout, opts.Stderr, path, name, arg...) 76 | } 77 | } 78 | 79 | func Run(ctx context.Context, opts pipeline.ActionOpts, name string, args ...string) error { 80 | return RunCommandWithOpts(ctx, RunOpts{ 81 | Name: name, 82 | Args: args, 83 | Stdout: opts.Stdout, 84 | Stderr: opts.Stderr, 85 | }) 86 | } 87 | -------------------------------------------------------------------------------- /exec/docs.go: -------------------------------------------------------------------------------- 1 | // Package exec provides helper functions and Actions for executing shell commands. 2 | package exec 3 | -------------------------------------------------------------------------------- /fs/cachers.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import "github.com/grafana/scribe/pipeline" 4 | 5 | // FileHasChanged creates a checksum for the file "file" and stores it. 6 | // If the checksum does not exist in the scribe key store, then it will return false. 7 | func FileHasChanged(file string) pipeline.CacheCondition { 8 | return func() bool { 9 | return false 10 | } 11 | } 12 | 13 | // Cache will store the directory or file located at `path` if the conditions return true. 14 | // If all of the conditions return true, then the step is skipped and the directory is added to the local filesystem. 15 | func Cache(path string, conditions ...pipeline.CacheCondition) pipeline.Cacher { 16 | return func(pipeline.Step) {} 17 | } 18 | -------------------------------------------------------------------------------- /fs/replace.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | ) 8 | 9 | func Replace(file string, content string) pipeline.Action { 10 | return func(context.Context, pipeline.ActionOpts) error { 11 | return nil 12 | } 13 | } 14 | 15 | func ReplaceString(file string, content string) pipeline.Action { 16 | return func(context.Context, pipeline.ActionOpts) error { 17 | return nil 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /git/client.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | type CloneOpts struct { 4 | URL string 5 | Folder string 6 | Ref string 7 | } 8 | -------------------------------------------------------------------------------- /git/clone.go: -------------------------------------------------------------------------------- 1 | package git 2 | -------------------------------------------------------------------------------- /git/describe.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | "github.com/grafana/scribe/state" 8 | ) 9 | 10 | var ( 11 | ArgGitDescription = state.NewStringArgument("git-description") 12 | ) 13 | 14 | type DescribeOpts struct { 15 | Tags bool 16 | Dirty bool 17 | Always bool 18 | } 19 | 20 | func DescribeAction(opts DescribeOpts) pipeline.Action { 21 | return func(context.Context, pipeline.ActionOpts) error { 22 | return nil 23 | } 24 | } 25 | 26 | func Describe(opts DescribeOpts) pipeline.Step { 27 | return pipeline.NewStep(DescribeAction(opts)).Provides(ArgGitDescription) 28 | } 29 | -------------------------------------------------------------------------------- /git/events.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | type EventCommit struct{} 4 | -------------------------------------------------------------------------------- /git/x/describe.go: -------------------------------------------------------------------------------- 1 | package x 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/grafana/scribe/exec" 10 | ) 11 | 12 | func Describe(ctx context.Context, dir string, tags bool, dirty bool, always bool) (string, error) { 13 | var ( 14 | stdout = bytes.NewBuffer(nil) 15 | stderr = bytes.NewBuffer(nil) 16 | ) 17 | 18 | args := []string{"describe"} 19 | if tags { 20 | args = append(args, "--tags") 21 | } 22 | if dirty { 23 | args = append(args, "--dirty") 24 | } 25 | if always { 26 | args = append(args, "--always") 27 | } 28 | 29 | if err := exec.RunCommandAt(ctx, stdout, stderr, dir, "git", args...); err != nil { 30 | return "", fmt.Errorf("%w\n%s", err, stderr.String()) 31 | } 32 | 33 | return strings.TrimSpace(stdout.String()), nil 34 | } 35 | -------------------------------------------------------------------------------- /go.work: -------------------------------------------------------------------------------- 1 | go 1.18 2 | 3 | use ( 4 | . 5 | ./echo-test 6 | ) 7 | -------------------------------------------------------------------------------- /golang/build.go: -------------------------------------------------------------------------------- 1 | package golang 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/golang/x" 7 | "github.com/grafana/scribe/pipeline" 8 | ) 9 | 10 | func BuildStep(pkg, output string, args, env []string) pipeline.Step { 11 | return pipeline.NewStep(func(ctx context.Context, opts pipeline.ActionOpts) error { 12 | return x.RunBuild(ctx, x.BuildOpts{ 13 | Pkg: pkg, 14 | Output: output, 15 | Stdout: opts.Stdout, 16 | Stderr: opts.Stderr, 17 | Env: env, 18 | Args: args, 19 | }) 20 | }) 21 | } 22 | 23 | func BuildAction(pkg, output string, args, env []string) pipeline.Action { 24 | return func(ctx context.Context, opts pipeline.ActionOpts) error { 25 | opts.Logger.Infoln("args: ", args) 26 | return x.RunBuild(ctx, x.BuildOpts{ 27 | Pkg: pkg, 28 | Output: output, 29 | Stdout: opts.Stdout, 30 | Stderr: opts.Stderr, 31 | Env: env, 32 | Args: args, 33 | }) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /golang/modules.go: -------------------------------------------------------------------------------- 1 | package golang 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | ) 8 | 9 | func ModDownload() pipeline.Action { 10 | return func(context.Context, pipeline.ActionOpts) error { 11 | return nil 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /golang/test.go: -------------------------------------------------------------------------------- 1 | package golang 2 | 3 | import ( 4 | "github.com/grafana/scribe" 5 | "github.com/grafana/scribe/exec" 6 | "github.com/grafana/scribe/pipeline" 7 | ) 8 | 9 | func Test(sw *scribe.Scribe, pkg string) pipeline.Step { 10 | return pipeline.NewStep(exec.RunAction("go", "test", pkg)). 11 | WithImage("golang:1.19"). 12 | Requires(pipeline.ArgumentSourceFS) 13 | } 14 | -------------------------------------------------------------------------------- /golang/x/build.go: -------------------------------------------------------------------------------- 1 | package x 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os/exec" 7 | 8 | swexec "github.com/grafana/scribe/exec" 9 | ) 10 | 11 | type BuildOpts struct { 12 | Env []string 13 | Args []string 14 | Pkg string 15 | Output string 16 | Module string 17 | LDFlags string 18 | 19 | Stdout io.Writer 20 | Stderr io.Writer 21 | } 22 | 23 | func Build(ctx context.Context, opts BuildOpts) *exec.Cmd { 24 | // for the go build command optional arguments have to come before the -o output and package name we are building 25 | fullArgs := append([]string{"build"}, opts.Args...) 26 | fullArgs = append(fullArgs, "-o", opts.Output) 27 | if opts.LDFlags != "" { 28 | fullArgs = append(fullArgs, "-ldflags", opts.LDFlags) 29 | } 30 | 31 | fullArgs = append(fullArgs, opts.Pkg) 32 | return swexec.CommandWithOpts(ctx, swexec.RunOpts{ 33 | Stdout: opts.Stdout, 34 | Stderr: opts.Stderr, 35 | Path: opts.Module, 36 | Name: "go", 37 | Args: fullArgs, 38 | Env: opts.Env, 39 | }) 40 | } 41 | 42 | func RunBuild(ctx context.Context, opts BuildOpts) error { 43 | return Build(ctx, opts).Run() 44 | } 45 | -------------------------------------------------------------------------------- /initializers.go: -------------------------------------------------------------------------------- 1 | package scribe 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | "github.com/grafana/scribe/pipeline/clients" 8 | "github.com/grafana/scribe/pipeline/clients/cli" 9 | "github.com/grafana/scribe/pipeline/clients/dagger" 10 | "github.com/grafana/scribe/pipeline/clients/drone" 11 | "github.com/grafana/scribe/pipeline/clients/graphviz" 12 | ) 13 | 14 | var ( 15 | ClientCLI string = "cli" 16 | ClientDrone = "drone" 17 | ClientDagger = "dagger" 18 | ClientGraphviz = "graphviz" 19 | ) 20 | 21 | func NewDefaultCollection(opts clients.CommonOpts) *pipeline.Collection { 22 | p := pipeline.NewCollection() 23 | if err := p.AddPipelines(pipeline.New(opts.Name, DefaultPipelineID)); err != nil { 24 | panic(err) 25 | } 26 | p.Root = []int64{DefaultPipelineID} 27 | 28 | return p 29 | } 30 | 31 | func NewMultiCollection() *pipeline.Collection { 32 | return pipeline.NewCollection() 33 | } 34 | 35 | type InitializerFunc func(context.Context, clients.CommonOpts) (pipeline.Client, error) 36 | 37 | // The ClientInitializers define how different RunModes initialize the Scribe client 38 | var ClientInitializers = map[string]InitializerFunc{ 39 | ClientCLI: cli.New, 40 | ClientDrone: drone.New, 41 | ClientDagger: dagger.New, 42 | ClientGraphviz: graphviz.New, 43 | } 44 | 45 | func RegisterClient(name string, initializer InitializerFunc) { 46 | ClientInitializers[name] = initializer 47 | } 48 | -------------------------------------------------------------------------------- /jsonnet/jsonnet.go: -------------------------------------------------------------------------------- 1 | package jsonnet 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/fs" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/google/go-jsonnet" 13 | "github.com/google/go-jsonnet/formatter" 14 | "github.com/google/go-jsonnet/linter" 15 | "github.com/grafana/scribe/pipeline" 16 | "github.com/grafana/scribe/state" 17 | "github.com/grafana/tanka/pkg/kubernetes/util" 18 | ) 19 | 20 | func Lint(path string) pipeline.Step { 21 | var errFiles []string 22 | vm := jsonnet.MakeVM() 23 | 24 | return pipeline.NewStep( 25 | func(ctx context.Context, opts pipeline.ActionOpts) error { 26 | path := filepath.Join(state.MustGetDirectoryString(opts.State, ctx, pipeline.ArgumentSourceFS), path) 27 | err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { 28 | if err != nil { 29 | return err 30 | } 31 | if d.IsDir() || (!strings.Contains(d.Name(), ".jsonnet") && !strings.Contains(d.Name(), ".libsonnet")) { 32 | return nil 33 | } 34 | f, err := os.Open(path) 35 | if err != nil { 36 | return err 37 | } 38 | data, err := ioutil.ReadAll(f) 39 | if err != nil { 40 | return err 41 | } 42 | err = f.Close() 43 | if err != nil { 44 | return err 45 | } 46 | snippet := linter.Snippet{FileName: path, Code: string(data)} 47 | if !linter.LintSnippet(vm, opts.Stderr, []linter.Snippet{snippet}) { 48 | errFiles = append(errFiles, path) 49 | } 50 | return nil 51 | }) 52 | if err != nil { 53 | return err 54 | } 55 | if len(errFiles) != 0 { 56 | return fmt.Errorf("jsonnetfmt found lint errors in files: %s", errFiles) 57 | } 58 | return nil 59 | }, 60 | ).Requires(pipeline.ArgumentSourceFS) 61 | } 62 | 63 | func Format(path string) pipeline.Step { 64 | return pipeline.NewStep( 65 | func(ctx context.Context, opts pipeline.ActionOpts) error { 66 | var errFiles []string 67 | 68 | path := filepath.Join(state.MustGetDirectoryString(opts.State, ctx, pipeline.ArgumentSourceFS), path) 69 | err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { 70 | if err != nil { 71 | return err 72 | } 73 | if d.IsDir() || (!strings.Contains(d.Name(), ".jsonnet") && !strings.Contains(d.Name(), ".libsonnet")) { 74 | return nil 75 | } 76 | f, err := os.Open(path) 77 | if err != nil { 78 | return err 79 | } 80 | data, err := ioutil.ReadAll(f) 81 | if err != nil { 82 | return err 83 | } 84 | err = f.Close() 85 | if err != nil { 86 | return err 87 | } 88 | // snippet := linter.Snippet{FileName: path, Code: string(data)} 89 | out, err := formatter.Format(d.Name(), string(data), formatter.DefaultOptions()) 90 | if err != nil { 91 | return fmt.Errorf("jsonnet linting failed for file: %s", path) 92 | } 93 | if out == string(data) { 94 | return nil 95 | } 96 | s, err := util.DiffStr(d.Name(), string(data), out) 97 | fmt.Printf("diff: %s\n", s) 98 | if err != nil { 99 | return err 100 | } 101 | errFiles = append(errFiles, path) 102 | return nil 103 | }) 104 | if err != nil { 105 | return err 106 | } 107 | if len(errFiles) != 0 { 108 | return fmt.Errorf("jsonnetfmt found lint errors in files: %s", errFiles) 109 | } 110 | return nil 111 | }, 112 | ).Requires(pipeline.ArgumentSourceFS) 113 | } 114 | -------------------------------------------------------------------------------- /makefile/target.go: -------------------------------------------------------------------------------- 1 | package makefile 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | ) 8 | 9 | func Target(name string) pipeline.Action { 10 | return func(ctx context.Context, opts pipeline.ActionOpts) error { 11 | return nil 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /out.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grafana/scribe/0ed01dde233c9a5250bb3341131d365e596ceabd/out.log -------------------------------------------------------------------------------- /pipeline.go: -------------------------------------------------------------------------------- 1 | package scribe 2 | 3 | import ( 4 | "github.com/grafana/scribe/pipeline" 5 | "github.com/grafana/scribe/state" 6 | ) 7 | 8 | // Pipeline is a more user-friendly, declarative representation of a Pipeline in the 'pipeline' package. 9 | // This is only used when defining a pipeline in a declarative manner using the 'AddPipelines' function. 10 | type Pipeline struct { 11 | Name string 12 | Requires []state.Argument 13 | Steps []pipeline.Step 14 | Provides []state.Argument 15 | When []pipeline.Event 16 | } 17 | 18 | // AddPipelines adds a list of pipelines into the DAG. The order in which they are defined or added is not important; the order in which 19 | // they run depends on what they require and what they provide. 20 | // This function can be ran multiple times; every new item added with 'AddPipelines' will be appended to the dag. 21 | func (s *ScribeMulti) AddPipelines(pipelines ...Pipeline) { 22 | for _, v := range pipelines { 23 | p := s.New(v.Name, func(s *Scribe) { 24 | if v.When != nil { 25 | s.When(v.When...) 26 | } 27 | s.Add(v.Steps...) 28 | }) 29 | 30 | p = p.Requires(v.Requires...) 31 | p = p.Provides(v.Provides...) 32 | 33 | s.Add(p) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pipeline/arguments_known.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "github.com/grafana/scribe/state" 4 | 5 | // These arguments are the pre-defined ones and are mostly used in events. 6 | var ( 7 | // Git arguments 8 | ArgumentCommitSHA = state.NewStringArgument("git-commit-sha") 9 | ArgumentCommitRef = state.NewStringArgument("git-commit-ref") 10 | ArgumentBranch = state.NewStringArgument("git-branch") 11 | ArgumentRemoteURL = state.NewStringArgument("remote-url") 12 | ArgumentTagName = state.NewStringArgument("git-tag") 13 | 14 | ArgumentWorkingDir = state.NewStringArgument("workdir") 15 | // ArgumentSourceFS is the path to the root of the source code for this project. 16 | ArgumentSourceFS = state.NewUnpackagedDirectoryArgument("source") 17 | ArgumentPipelineGoModFS = state.NewUnpackagedDirectoryArgument("pipeline-go-mod") 18 | ArgumentDockerSocketFS = state.NewUnpackagedDirectoryArgument("docker-socket") 19 | 20 | // CI service arguments 21 | ArgumentBuildID = state.NewStringArgument("build-id") 22 | ) 23 | 24 | // ClientProvidedArguments are argumnets that must be provided by the Client and not another step. 25 | var ClientProvidedArguments = []state.Argument{ArgumentBuildID, ArgumentSourceFS, ArgumentPipelineGoModFS, ArgumentDockerSocketFS, ArgumentWorkingDir} 26 | -------------------------------------------------------------------------------- /pipeline/build.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | -------------------------------------------------------------------------------- /pipeline/cacher.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | // a CacheCondition should return true if the cacher should cache the step. 4 | // In other words, returning false means that the step must run. 5 | type CacheCondition func() bool 6 | 7 | // A Cacher defines behavior for caching data. 8 | // Some behaviors that can happen when dealing with a cacheable step: 9 | // * The provided Step, using an expensive process, produces some consistent output, possibly on the filesystem. If nothing changes in between runs, then we can re-use the output in the current step and skip this one. 10 | // * The most common example of this is `npm install` producing the `node_modules` folder, which can be re-used if `package-lock.json` is unchanged. 11 | // * The provided Step, using an expensive process, calculates a value. If nothing changes in between runs, then this value can be reused. 12 | type Cacher func(Step) 13 | -------------------------------------------------------------------------------- /pipeline/client.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type Client interface { 8 | // Validate is ran internally before calling Run or Parallel and allows the client to effectively configure per-step requirements 9 | // For example, Drone steps MUST have an image so the Drone client returns an error in this function when the provided step does not have an image. 10 | // If the error encountered is not critical but should still be logged, then return a plumbing.ErrorSkipValidation. 11 | // The error is checked with `errors.Is` so the error can be wrapped with fmt.Errorf. 12 | Validate(Step) error 13 | 14 | // Done must be ran at the end of the pipeline. 15 | // This is typically what takes the defined pipeline steps, runs them in the order defined, and produces some kind of output. 16 | Done(context.Context, *Collection) error 17 | } 18 | -------------------------------------------------------------------------------- /pipeline/clients/cli/client.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "os" 9 | 10 | "github.com/grafana/scribe/pipeline" 11 | "github.com/grafana/scribe/pipeline/clients" 12 | "github.com/grafana/scribe/state" 13 | "github.com/grafana/scribe/syncutil" 14 | "github.com/grafana/scribe/wrappers" 15 | "github.com/sirupsen/logrus" 16 | ) 17 | 18 | // The Client is used when interacting with a scribe pipeline using the scribe CLI. It is used to run only one step. 19 | // The CLI client simply runs the anonymous function defined in the step. 20 | type Client struct { 21 | Opts clients.CommonOpts 22 | Log *logrus.Logger 23 | State *StateWrapper 24 | } 25 | 26 | func New(ctx context.Context, opts clients.CommonOpts) (pipeline.Client, error) { 27 | if opts.Args.Step == nil || *opts.Args.Step == 0 { 28 | return nil, errors.New("--step argument can not be empty or 0 when using the CLI client") 29 | } 30 | 31 | return &Client{ 32 | Opts: opts, 33 | Log: opts.Log, 34 | State: NewStateWrapper( 35 | state.ReaderWithLogs(opts.Log, state.NewArgMapReader(opts.Args.ArgMap)), 36 | &StateHandler{}, 37 | ), 38 | }, nil 39 | } 40 | 41 | // PipelineWalkFunc walks through the pipelines that the collection provides. Each pipeline is a pipeline of steps, so each will walk through the list of steps using the StepWalkFunc. 42 | func (c *Client) HandlePipeline(ctx context.Context, p pipeline.Pipeline) error { 43 | var ( 44 | wg = syncutil.NewStepWaitGroup() 45 | ) 46 | 47 | for _, node := range p.Graph.Nodes { 48 | // Skip the root step that's always present on every pipeline. 49 | if node.ID == 0 { 50 | continue 51 | } 52 | log := c.Opts.Log 53 | logWrapper := &wrappers.LogWrapper{ 54 | Opts: c.Opts, 55 | Log: log.WithField("step", node.Value.Name), 56 | } 57 | traceWrapper := &wrappers.TraceWrapper{ 58 | Opts: c.Opts, 59 | Tracer: c.Opts.Tracer, 60 | } 61 | 62 | step := logWrapper.WrapStep(node.Value) 63 | step = traceWrapper.WrapStep(step) 64 | 65 | // Otherwise, add this pipeline to the set that needs to complete before moving on to the next set of pipelines. 66 | wg.Add(step, pipeline.ActionOpts{ 67 | Path: c.Opts.Args.Path, 68 | State: c.State, 69 | Tracer: c.Opts.Tracer, 70 | Version: c.Opts.Version, 71 | Logger: log, 72 | }) 73 | } 74 | 75 | if err := wg.Wait(ctx); err != nil { 76 | return err 77 | } 78 | 79 | if err := json.NewEncoder(os.Stdout).Encode(c.State.data); err != nil { 80 | return fmt.Errorf("error encoding JSON for CLI client state updates: %w", err) 81 | } 82 | 83 | return nil 84 | } 85 | 86 | func (c *Client) Validate(step pipeline.Step) error { 87 | return nil 88 | } 89 | 90 | func (c *Client) HandleEvents(events []pipeline.Event) error { 91 | return nil 92 | } 93 | 94 | func (c *Client) Done(ctx context.Context, w *pipeline.Collection) error { 95 | for _, node := range w.Graph.Nodes { 96 | // Skip the root node because there's always a root node that just exists as a starting point. 97 | if node.ID == 0 { 98 | continue 99 | } 100 | pipeline := node.Value 101 | // Not counting the root pipeline, there should really only be 1 pipeline here with 1 step since we've filtered by step ID. 102 | if err := c.HandlePipeline(ctx, pipeline); err != nil { 103 | return err 104 | } 105 | } 106 | 107 | return nil 108 | } 109 | 110 | func (c *Client) prepopulateState(ctx context.Context, s state.Handler) error { 111 | log := c.Log 112 | for k, v := range KnownValues { 113 | exists, err := s.Exists(ctx, k) 114 | if err != nil { 115 | // Even if we encounter an error, we still want to attempt to set the state. 116 | // One error that could happen here is if the state is empty. 117 | log.WithError(err).Debugln("Failed to read state") 118 | } 119 | 120 | if !exists { 121 | log.Debugln("State not found for", k.Key, "preopulating value") 122 | if err := v(ctx, s); err != nil { 123 | log.WithError(err).Debugln("Failed to pre-populate state for argument", k.Key) 124 | } 125 | } 126 | } 127 | 128 | return nil 129 | } 130 | -------------------------------------------------------------------------------- /pipeline/clients/cli/known_args.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "strings" 9 | 10 | "github.com/grafana/scribe/pipeline" 11 | "github.com/grafana/scribe/state" 12 | "github.com/grafana/scribe/stringutil" 13 | ) 14 | 15 | // This function effectively runs 'git remote get-url $(git remote)' 16 | func setCurrentRemote(ctx context.Context, s state.Writer) error { 17 | remote, err := exec.Command("git", "remote").CombinedOutput() 18 | if err != nil { 19 | return fmt.Errorf("%w. output: %s", err, string(remote)) 20 | } 21 | 22 | v, err := exec.Command("git", "remote", "get-url", strings.TrimSpace(string(remote))).CombinedOutput() 23 | if err != nil { 24 | return fmt.Errorf("%w. output: %s", err, string(v)) 25 | } 26 | 27 | return s.SetString(ctx, pipeline.ArgumentRemoteURL, string(v)) 28 | } 29 | 30 | // This function effectively runs 'git rev-parse HEAD' 31 | func setCurrentCommit(ctx context.Context, s state.Writer) error { 32 | v, err := exec.Command("git", "rev-parse", "HEAD").CombinedOutput() 33 | if err != nil { 34 | return fmt.Errorf("%w. output: %s", err, string(v)) 35 | } 36 | 37 | return s.SetString(ctx, pipeline.ArgumentCommitRef, string(v)) 38 | } 39 | 40 | // This function effectively runs 'git rev-parse --abrev-ref HEAD' 41 | func setCurrentBranch(ctx context.Context, s state.Writer) error { 42 | v, err := exec.Command("git", "rev-parse", "--abrev-ref", "HEAD").CombinedOutput() 43 | if err != nil { 44 | return fmt.Errorf("%w. output: %s", err, string(v)) 45 | } 46 | 47 | return s.SetString(ctx, pipeline.ArgumentBranch, string(v)) 48 | } 49 | 50 | func setWorkingDir(ctx context.Context, s state.Writer) error { 51 | wd, err := os.Getwd() 52 | if err != nil { 53 | return err 54 | } 55 | 56 | return s.SetString(ctx, pipeline.ArgumentWorkingDir, wd) 57 | } 58 | 59 | func setSourceFS(ctx context.Context, s state.Writer) error { 60 | wd, err := os.Getwd() 61 | if err != nil { 62 | return err 63 | } 64 | 65 | return s.SetDirectory(ctx, pipeline.ArgumentSourceFS, wd) 66 | } 67 | 68 | func setBuildID(ctx context.Context, s state.Writer) error { 69 | r := stringutil.Random(8) 70 | return s.SetString(ctx, pipeline.ArgumentBuildID, r) 71 | } 72 | 73 | // KnownValues are URL values that we know how to retrieve using the command line. 74 | var KnownValues = map[state.Argument]func(context.Context, state.Writer) error{ 75 | pipeline.ArgumentRemoteURL: setCurrentRemote, 76 | pipeline.ArgumentCommitRef: setCurrentCommit, 77 | pipeline.ArgumentBranch: setCurrentBranch, 78 | pipeline.ArgumentWorkingDir: setWorkingDir, 79 | pipeline.ArgumentSourceFS: setSourceFS, 80 | pipeline.ArgumentBuildID: setBuildID, 81 | } 82 | -------------------------------------------------------------------------------- /pipeline/clients/cli/state.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | 8 | "github.com/grafana/scribe/state" 9 | ) 10 | 11 | type StateHandler struct{} 12 | 13 | func (n *StateHandler) SetString(ctx context.Context, arg state.Argument, val string) error { 14 | return nil 15 | } 16 | func (n *StateHandler) SetInt64(ctx context.Context, arg state.Argument, val int64) error { return nil } 17 | func (n *StateHandler) SetFloat64(ctx context.Context, arg state.Argument, val float64) error { 18 | return nil 19 | } 20 | func (n *StateHandler) SetBool(ctx context.Context, arg state.Argument, val bool) error { return nil } 21 | func (n *StateHandler) SetFile(ctx context.Context, arg state.Argument, path string) error { 22 | return nil 23 | } 24 | func (n *StateHandler) SetFileReader(ctx context.Context, arg state.Argument, r io.Reader) (string, error) { 25 | file, err := os.CreateTemp("", "*") 26 | if err != nil { 27 | return "", err 28 | } 29 | 30 | defer file.Close() 31 | if _, err := io.Copy(file, r); err != nil { 32 | return "", err 33 | } 34 | 35 | return file.Name(), nil 36 | } 37 | 38 | func (n *StateHandler) SetDirectory(ctx context.Context, arg state.Argument, dir string) error { 39 | return nil 40 | } 41 | -------------------------------------------------------------------------------- /pipeline/clients/cli/state_writer.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "io/fs" 7 | "os" 8 | 9 | "github.com/grafana/scribe/state" 10 | ) 11 | 12 | type StateWrapper struct { 13 | state.Reader 14 | state.Writer 15 | data map[string]state.StateValueJSON 16 | } 17 | 18 | func (w *StateWrapper) SetString(ctx context.Context, key state.Argument, val string) error { 19 | w.data[key.Key] = state.StateValueJSON{ 20 | Argument: key, 21 | Value: val, 22 | } 23 | return w.Writer.SetString(ctx, key, val) 24 | } 25 | 26 | func (w *StateWrapper) SetInt64(ctx context.Context, key state.Argument, val int64) error { 27 | w.data[key.Key] = state.StateValueJSON{ 28 | Argument: key, 29 | Value: val, 30 | } 31 | return w.Writer.SetInt64(ctx, key, val) 32 | } 33 | 34 | func (w *StateWrapper) SetFloat64(ctx context.Context, key state.Argument, val float64) error { 35 | w.data[key.Key] = state.StateValueJSON{ 36 | Argument: key, 37 | Value: val, 38 | } 39 | return w.Writer.SetFloat64(ctx, key, val) 40 | } 41 | 42 | func (w *StateWrapper) SetBool(ctx context.Context, key state.Argument, val bool) error { 43 | w.data[key.Key] = state.StateValueJSON{ 44 | Argument: key, 45 | Value: val, 46 | } 47 | return w.Writer.SetBool(ctx, key, val) 48 | } 49 | 50 | func (w *StateWrapper) SetFile(ctx context.Context, key state.Argument, val string) error { 51 | w.data[key.Key] = state.StateValueJSON{ 52 | Argument: key, 53 | Value: val, 54 | } 55 | return w.Writer.SetFile(ctx, key, val) 56 | } 57 | 58 | func (w *StateWrapper) SetFileReader(ctx context.Context, key state.Argument, r io.Reader) (string, error) { 59 | path, err := w.Writer.SetFileReader(ctx, key, r) 60 | w.data[key.Key] = state.StateValueJSON{ 61 | Argument: key, 62 | Value: path, 63 | } 64 | return path, err 65 | } 66 | 67 | func (w *StateWrapper) SetDirectory(ctx context.Context, key state.Argument, val string) error { 68 | w.data[key.Key] = state.StateValueJSON{ 69 | Argument: key, 70 | Value: val, 71 | } 72 | return w.Writer.SetDirectory(ctx, key, val) 73 | } 74 | 75 | func (w *StateWrapper) Exists(ctx context.Context, arg state.Argument) (bool, error) { 76 | return w.Reader.Exists(ctx, arg) 77 | } 78 | 79 | func (w *StateWrapper) GetString(ctx context.Context, arg state.Argument) (string, error) { 80 | return w.Reader.GetString(ctx, arg) 81 | } 82 | 83 | func (w *StateWrapper) GetInt64(ctx context.Context, arg state.Argument) (int64, error) { 84 | return w.Reader.GetInt64(ctx, arg) 85 | } 86 | 87 | func (w *StateWrapper) GetFloat64(ctx context.Context, arg state.Argument) (float64, error) { 88 | return w.Reader.GetFloat64(ctx, arg) 89 | } 90 | 91 | func (w *StateWrapper) GetBool(ctx context.Context, arg state.Argument) (bool, error) { 92 | return w.Reader.GetBool(ctx, arg) 93 | } 94 | 95 | func (w *StateWrapper) GetFile(ctx context.Context, arg state.Argument) (*os.File, error) { 96 | return w.Reader.GetFile(ctx, arg) 97 | } 98 | 99 | func (w *StateWrapper) GetDirectory(ctx context.Context, arg state.Argument) (fs.FS, error) { 100 | return w.Reader.GetDirectory(ctx, arg) 101 | } 102 | 103 | func (w *StateWrapper) GetDirectoryString(ctx context.Context, arg state.Argument) (string, error) { 104 | return w.Reader.GetDirectoryString(ctx, arg) 105 | } 106 | 107 | func NewStateWrapper(r state.Reader, w state.Writer) *StateWrapper { 108 | return &StateWrapper{ 109 | Reader: r, 110 | Writer: w, 111 | data: make(map[string]state.StateValueJSON), 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /pipeline/clients/common/step_compile.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | ) 8 | 9 | func compilePipeline(ctx context.Context, opts pipeline.ActionOpts) error { 10 | return nil 11 | } 12 | -------------------------------------------------------------------------------- /pipeline/clients/dagger/compile.go: -------------------------------------------------------------------------------- 1 | package dagger 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | 7 | "dagger.io/dagger" 8 | "github.com/grafana/scribe/pipelineutil" 9 | ) 10 | 11 | func CompilePipeline(ctx context.Context, d *dagger.Client, name, src, gomod, pipeline string) (*dagger.Directory, error) { 12 | var ( 13 | dir = d.Host().Directory(src) 14 | builder = d.Container().From("golang:1.19").WithMountedDirectory("/src", dir) 15 | ) 16 | 17 | path, err := filepath.Rel(src, gomod) 18 | if err != nil { 19 | return nil, err 20 | } 21 | cmd := pipelineutil.GoBuild(ctx, pipelineutil.GoBuildOpts{ 22 | Pipeline: pipeline, 23 | Module: path, 24 | Output: "/opt/scribe/pipeline", 25 | LDFlags: `-extldflags "-static"`, 26 | }) 27 | 28 | builder = builder.WithEnvVariable("GOOS", "linux") 29 | builder = builder.WithEnvVariable("GOARCH", "amd64") 30 | builder = builder.WithEnvVariable("CGO_ENABLED", "0") 31 | // Set the pipeline name to prevent cache collisions. 32 | // Some pipelines with the exact same name and path will sometimes reuse the compiled pipeline from the cache. 33 | // In those scenarios, until we find a more permanent fix, it's best to just change the name. 34 | builder = builder.WithEnvVariable("PIPELINE_NAME", name) 35 | builder = builder.WithWorkdir("/src") 36 | 37 | builder = builder.Exec(dagger.ContainerExecOpts{ 38 | Args: cmd.Args, 39 | }) 40 | 41 | return builder.Directory("/opt/scribe"), nil 42 | } 43 | -------------------------------------------------------------------------------- /pipeline/clients/drone/client_test.go: -------------------------------------------------------------------------------- 1 | package drone_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | "time" 12 | 13 | "github.com/google/go-cmp/cmp" 14 | "github.com/grafana/scribe/args" 15 | "github.com/grafana/scribe/pipeline/clients/drone" 16 | "github.com/grafana/scribe/testutil" 17 | "github.com/sirupsen/logrus" 18 | ) 19 | 20 | // testDemoPipeline tests a pipeline located in "demo" folder. the "path" argument should be relative to the demo folder in the root of the project. 21 | // This function will do a basic equivalency check on what is generated by running the pipeline with the drone mode and what is in the "gen_drone.yml" file in the provided folder. 22 | // Some standard arguments will be provided, like "-mode=drone", '-build-id="test"', "-path={path}", -log-level="debug". 23 | func testDemoPipeline(t *testing.T, path string) { 24 | t.Helper() 25 | 26 | var ( 27 | buf = bytes.NewBuffer(nil) 28 | stderr = bytes.NewBuffer(nil) 29 | ctx = context.Background() 30 | pipelinePath = filepath.Join("../../../demo", path) 31 | ) 32 | 33 | testutil.RunPipeline(ctx, t, pipelinePath, io.MultiWriter(buf, os.Stdout), stderr, &args.PipelineArgs{ 34 | BuildID: "test", 35 | Client: "drone", 36 | Path: fmt.Sprintf("./demo/%s", path), // Note that we're intentionally using ./demo/ instead of filepath because this path is used in a Go command. 37 | LogLevel: logrus.DebugLevel, 38 | }) 39 | 40 | t.Log(stderr.String()) 41 | 42 | expected, err := os.Open(filepath.Join(pipelinePath, "gen_drone.yml")) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | 47 | testutil.ReadersEqual(t, buf, expected) 48 | } 49 | 50 | func TestDroneClient(t *testing.T) { 51 | t.Run("It should generate a simple Drone pipeline", 52 | testutil.WithTimeout(time.Second*10, func(t *testing.T) { 53 | testDemoPipeline(t, "basic") 54 | }), 55 | ) 56 | t.Run("It should generate a more complex multi Drone pipeline", 57 | testutil.WithTimeout(time.Second*10, func(t *testing.T) { 58 | testDemoPipeline(t, "multi") 59 | }), 60 | ) 61 | t.Run("It should generate a multi-drone pipeline with a sub-pipeline", 62 | testutil.WithTimeout(time.Second*10, func(t *testing.T) { 63 | testDemoPipeline(t, "multi-sub") 64 | }), 65 | ) 66 | } 67 | 68 | func TestDroneRun(t *testing.T) { 69 | t.Run("It should run sequential steps sequentially", 70 | testutil.WithTimeout(time.Second*5, func(t *testing.T) { 71 | t.SkipNow() 72 | 73 | t.Log("Creating new drone client...") 74 | sw := testutil.NewScribe(drone.New) 75 | 76 | t.Log("Creating new test steps...") 77 | var ( 78 | step1Chan = make(chan bool) 79 | step1 = testutil.NewTestStep(step1Chan) 80 | 81 | step2Chan = make(chan bool) 82 | step2 = testutil.NewTestStep(step2Chan) 83 | 84 | step3Chan = make(chan bool) 85 | step3 = testutil.NewTestStep(step3Chan) 86 | ) 87 | 88 | t.Log("Running steps...") 89 | sw.Add(step1, step2, step3) 90 | 91 | go func() { 92 | t.Log("Done()") 93 | sw.Done() 94 | t.Log("done with Done()") 95 | }() 96 | 97 | var ( 98 | expectedOrder = []int{1, 2, 3} 99 | order = []int{} 100 | ) 101 | 102 | t.Log("Waiting for order...") 103 | // Only watch for 3 channels 104 | for i := 0; i < 3; i++ { 105 | select { 106 | case <-step1Chan: 107 | order = append(order, 1) 108 | case <-step2Chan: 109 | order = append(order, 2) 110 | case <-step3Chan: 111 | order = append(order, 3) 112 | } 113 | } 114 | 115 | if !cmp.Equal(order, expectedOrder) { 116 | t.Fatal("Steps ran in unexpected order:", cmp.Diff(order, expectedOrder)) 117 | } 118 | })) 119 | } 120 | -------------------------------------------------------------------------------- /pipeline/clients/drone/config.go: -------------------------------------------------------------------------------- 1 | package drone 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/grafana/scribe/errors" 7 | "github.com/grafana/scribe/pipeline" 8 | "github.com/grafana/scribe/state" 9 | ) 10 | 11 | type DroneLanguage int 12 | 13 | const ( 14 | // The languages that are available when generating a Drone config. 15 | LanguageYAML DroneLanguage = iota 16 | LanguageStarlark 17 | ) 18 | 19 | var argVolumeMap = map[state.Argument]string{ 20 | pipeline.ArgumentDockerSocketFS: "/var/run/docker.sock", 21 | } 22 | 23 | var argEnvMap = map[state.Argument]string{ 24 | pipeline.ArgumentCommitSHA: "$DRONE_COMMIT", 25 | pipeline.ArgumentCommitRef: "$DRONE_COMMIT_REF", 26 | pipeline.ArgumentRemoteURL: "$DRONE_GIT_SSH_URL", 27 | pipeline.ArgumentWorkingDir: "$DRONE_REPO_NAME", 28 | } 29 | 30 | // The configurer for the Drone client returns equivalent environment variables for different arguments. 31 | func (c *Client) Value(arg state.Argument) (string, error) { 32 | switch arg.Type { 33 | case state.ArgumentTypeSecret: 34 | return secretEnv(arg.Key), nil 35 | case state.ArgumentTypeUnpackagedFS: 36 | if val, ok := argVolumeMap[arg]; ok { 37 | return val, nil 38 | } 39 | return "", errors.ErrorMissingArgument 40 | } 41 | 42 | if val, ok := argEnvMap[arg]; ok { 43 | return val, nil 44 | } 45 | 46 | return "", fmt.Errorf("could not find equivalent of '%s': %w", arg.Key, errors.ErrorMissingArgument) 47 | } 48 | -------------------------------------------------------------------------------- /pipeline/clients/drone/doc.go: -------------------------------------------------------------------------------- 1 | // Package drone contians the drone client implementation for generating a Drone pipeline config. 2 | package drone 3 | -------------------------------------------------------------------------------- /pipeline/clients/drone/events.go: -------------------------------------------------------------------------------- 1 | package drone 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/drone/drone-yaml/yaml" 7 | "github.com/grafana/scribe/pipeline" 8 | ) 9 | 10 | // TODO: I'm lazy at the moment and haven't implemented reverse filters (exlude). 11 | func addEvent(c yaml.Conditions, e pipeline.Event) (yaml.Conditions, error) { 12 | if branch, ok := e.Filters["branch"]; ok { 13 | c.Event.Include = append(c.Event.Include, "branch") 14 | c.Branch.Include = append(c.Branch.Include, branch.String()) 15 | } 16 | 17 | if tag, ok := e.Filters["tag"]; ok { 18 | c.Event.Include = append(c.Event.Include, "tag") 19 | if tag != nil { 20 | c.Ref.Include = append(c.Ref.Include, fmt.Sprintf("refs/tags/%s", tag.String())) 21 | } 22 | } 23 | 24 | return c, nil 25 | } 26 | 27 | // Events converts the list of pipeline.Events to a list of drone 'Conditions'. 28 | // Drone conditions are what prevents pipelines from running whenever certain certain conditions are met, or what runs pipelines only when certain conditions are met. 29 | func Events(events []pipeline.Event) (yaml.Conditions, error) { 30 | conditions := yaml.Conditions{} 31 | for _, event := range events { 32 | c, err := addEvent(conditions, event) 33 | if err != nil { 34 | return yaml.Conditions{}, err 35 | } 36 | 37 | conditions = c 38 | } 39 | 40 | return conditions, nil 41 | } 42 | -------------------------------------------------------------------------------- /pipeline/clients/drone/initializer.go: -------------------------------------------------------------------------------- 1 | package drone 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | "github.com/grafana/scribe/pipeline/clients" 8 | ) 9 | 10 | func New(ctx context.Context, opts clients.CommonOpts) (pipeline.Client, error) { 11 | return &Client{ 12 | Opts: opts, 13 | Log: opts.Log, 14 | }, nil 15 | } 16 | -------------------------------------------------------------------------------- /pipeline/clients/drone/schema.go: -------------------------------------------------------------------------------- 1 | package drone 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/drone/drone-yaml/yaml" 8 | "github.com/grafana/scribe/args" 9 | "github.com/grafana/scribe/cmdutil" 10 | "github.com/grafana/scribe/pipeline" 11 | "github.com/grafana/scribe/state" 12 | "github.com/grafana/scribe/stringutil" 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | func combineVariables(a map[string]*yaml.Variable, b map[string]*yaml.Variable) map[string]*yaml.Variable { 17 | c := a 18 | 19 | for k, v := range b { 20 | c[k] = v 21 | } 22 | 23 | return c 24 | } 25 | 26 | func secretEnv(key string) string { 27 | return stringutil.Slugify(fmt.Sprintf("secret_%s", key)) 28 | } 29 | 30 | // HandleSecrets handles the different 'Secret' arguments that are defined in the pipeline step. 31 | // Secrets are given a generated value and placed in the 'environment', not a user-defined one. That value is then used when the pipeline attempts to retrieve the value in the argument. 32 | // String arguments are already provided in the command line arguments when `cmdutil.StepCommand' 33 | func HandleSecrets(c pipeline.Configurer, step pipeline.Step) (map[string]*yaml.Variable, map[string]string) { 34 | var ( 35 | env = make(map[string]*yaml.Variable) 36 | args = make(map[string]string) 37 | ) 38 | 39 | for _, arg := range step.RequiredArgs { 40 | name := secretEnv(arg.Key) 41 | switch arg.Type { 42 | case state.ArgumentTypeSecret: 43 | env[name] = &yaml.Variable{ 44 | Secret: arg.Key, 45 | } 46 | args[arg.Key] = "$" + name 47 | } 48 | } 49 | 50 | return env, args 51 | } 52 | 53 | func stepVolumes(c pipeline.Configurer, step pipeline.Step) []*yaml.VolumeMount { 54 | volumes := []*yaml.VolumeMount{} 55 | // TODO: It's unlikely that we want to actually associate volume mounts with "FS" type arguments. 56 | // We will probably want to zip those up and place them in the state volume or something... 57 | for _, v := range step.RequiredArgs { 58 | if v.Type != state.ArgumentTypeFS && v.Type != state.ArgumentTypeUnpackagedFS { 59 | continue 60 | } 61 | 62 | // Explicitely skip ArgumentSouceFS because it's available in every pipeline. 63 | if v == pipeline.ArgumentSourceFS { 64 | continue 65 | } 66 | 67 | // If it's a known argument... 68 | value, _ := c.Value(v) 69 | //if err != nil { 70 | // Skip this then because it's not known. It should be provided by a different step ran previously. 71 | // TODO: handle FS type arguments here? 72 | //} 73 | 74 | volumes = append(volumes, &yaml.VolumeMount{ 75 | Name: stringutil.Slugify(v.Key), 76 | MountPath: value, 77 | }) 78 | } 79 | 80 | return volumes 81 | } 82 | 83 | func NewDaggerStep(c pipeline.Configurer, path, state, version string, p pipeline.Pipeline) (*yaml.Container, error) { 84 | var ( 85 | name = stringutil.Slugify(p.Name) 86 | image = "golang:1.19" 87 | //volumes = stepVolumes(c, step) 88 | ) 89 | //env, args := HandleSecrets(c, p) 90 | 91 | //for i, v := range step.Dependencies { 92 | // deps[i] = stringutil.Slugify(v.Name) 93 | //} 94 | 95 | cmd, err := cmdutil.PipelineCommand(cmdutil.PipelineCommandOpts{ 96 | Pipeline: p, 97 | CommandOpts: cmdutil.CommandOpts{ 98 | CompiledPipeline: PipelinePath, 99 | PipelineArgs: args.PipelineArgs{ 100 | Path: path, 101 | BuildID: "$DRONE_BUILD_NUMBER", 102 | State: state, 103 | //ArgMap: args, 104 | Client: "cli", 105 | LogLevel: logrus.DebugLevel, 106 | Version: version, 107 | }, 108 | }, 109 | }) 110 | 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | return &yaml.Container{ 116 | Name: name, 117 | Image: image, 118 | Commands: []string{strings.Join(cmd, " ")}, 119 | }, nil 120 | } 121 | -------------------------------------------------------------------------------- /pipeline/clients/graphviz/client.go: -------------------------------------------------------------------------------- 1 | package graphviz 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/grafana/scribe/pipeline" 8 | "github.com/grafana/scribe/pipeline/clients" 9 | ) 10 | 11 | type Client struct { 12 | Stdout io.Writer 13 | } 14 | 15 | func New(ctx context.Context, opts clients.CommonOpts) (pipeline.Client, error) { 16 | return &Client{ 17 | Stdout: opts.Output, 18 | }, nil 19 | } 20 | 21 | func (c *Client) Done(ctx context.Context, w *pipeline.Collection) error { 22 | pipelines := []pipeline.Pipeline{} 23 | if err := w.WalkPipelines(ctx, func(ctx context.Context, p pipeline.Pipeline) error { 24 | pipelines = append(pipelines, p) 25 | return nil 26 | }); err != nil { 27 | return err 28 | } 29 | 30 | return nil 31 | } 32 | 33 | func (c *Client) Validate(step pipeline.Step) error { 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /pipeline/clients/opts.go: -------------------------------------------------------------------------------- 1 | package clients 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/grafana/scribe/args" 7 | "github.com/opentracing/opentracing-go" 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | // CommonOpts are provided in the Client's Init function, which includes options that are common to all clients, like 12 | // logging, output, and debug options 13 | type CommonOpts struct { 14 | Name string 15 | Version string 16 | Output io.Writer 17 | Args *args.PipelineArgs 18 | Log *logrus.Logger 19 | Tracer opentracing.Tracer 20 | } 21 | -------------------------------------------------------------------------------- /pipeline/configurer.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "github.com/grafana/scribe/state" 4 | 5 | // Configurer defines how clients can retrieve configuration values for use in pipelines. 6 | // For example, a `clone` step might require a remote URL and branch, but how that data is retrieved can change depending on the environment. 7 | // * In Drone, the value is retrieved from an environment variable at runtime. So the only thing this funcion will likely return is the name of the environment variable. 8 | // * With the Dagger and CLI clients, the remote URL might be provided as a CLI argument or requested via stdin, or even already available with the `git remote` command. 9 | type Configurer interface { 10 | // Value returns the implementation-specific pipeline config. 11 | Value(state.Argument) (string, error) 12 | } 13 | -------------------------------------------------------------------------------- /pipeline/dag/test_helpers.go: -------------------------------------------------------------------------------- 1 | package dag 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func EdgesToMap[T any](e map[int64][]Edge[T]) map[int64][]int64 { 9 | m := map[int64][]int64{} 10 | for k, v := range e { 11 | ids := make([]int64, len(v)) 12 | for i, node := range v { 13 | ids[i] = node.To.ID 14 | } 15 | 16 | m[k] = ids 17 | } 18 | 19 | return m 20 | } 21 | 22 | func NodeIDs[T any](nodes []Node[T]) []int64 { 23 | ids := make([]int64, len(nodes)) 24 | for i, v := range nodes { 25 | ids[i] = v.ID 26 | } 27 | 28 | return ids 29 | } 30 | 31 | func ensureGraphEdges[T any](expected map[int64][]int64, graphEdges map[int64][]Edge[T]) error { 32 | if len(expected) != len(graphEdges) { 33 | return fmt.Errorf("Unexpected number of graph edges. Expected '%d' but received '%d'", len(expected), len(graphEdges)) 34 | } 35 | 36 | for id, expect := range expected { 37 | edges := graphEdges[id] 38 | if _, ok := expected[id]; !ok { 39 | return fmt.Errorf("Found unexpected node ID '%d' in graph edges", id) 40 | } 41 | 42 | // Calculate the IDs since this is a []dag.Node and not []int64 43 | ids := make([]int64, len(edges)) 44 | for i, edge := range edges { 45 | ids[i] = edge.To.ID 46 | } 47 | if len(expect) != len(ids) { 48 | return fmt.Errorf("Unequal number of edges for node '%d'. Expected '%d' edges (%+v), received '%d' edges (%+v)", id, len(expect), expect, len(ids), ids) 49 | } 50 | for i, id := range ids { 51 | if expect[i] != id { 52 | return fmt.Errorf("Expected node '%d' to have edges '%+v' but has '%+v'", id, expect, ids) 53 | } 54 | } 55 | } 56 | return nil 57 | } 58 | 59 | // EnsureGraphEdges is a test helper function that is used inside the dag tests and outside in other implementations 60 | // to easily ensure that we are using the dag properly. 61 | func EnsureGraphEdges[T any](t *testing.T, expected map[int64][]int64, graphEdges map[int64][]Edge[T]) { 62 | t.Helper() 63 | 64 | if err := ensureGraphEdges(expected, graphEdges); err != nil { 65 | t.Fatalf("Unexpected graph edges received.\nError: %s\nEdges: %+v\nExpected: %+v", err.Error(), EdgesToMap(graphEdges), expected) 66 | } 67 | } 68 | 69 | func ensureGraphNodes(expected []int64, nodes []int64) error { 70 | if len(expected) != len(nodes) { 71 | return fmt.Errorf("Unexpected number of graph nodes. Expected '%d' but received '%d'", len(expected), len(nodes)) 72 | } 73 | 74 | for i, expect := range expected { 75 | if nodes[i] != expect { 76 | return fmt.Errorf("Found unexpected node ID '%d' in graph edges", nodes[i]) 77 | } 78 | } 79 | return nil 80 | } 81 | 82 | // EnsureGraphNodes is a test helper function that is used inside the dag tests and outside in other implementations 83 | // to easily ensure that we are using the dag properly. 84 | func EnsureGraphNodes[T any](t *testing.T, expected []int64, nodes []Node[T]) { 85 | t.Helper() 86 | nodeIDs := make([]int64, len(nodes)) 87 | for i, v := range nodes { 88 | nodeIDs[i] = v.ID 89 | } 90 | 91 | if err := ensureGraphNodes(expected, nodeIDs); err != nil { 92 | t.Fatalf("Unexpected graph nodes received.\nError: %s\nNodes: %+v\nExpected: %+v", err.Error(), nodeIDs, expected) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pipeline/doc.go: -------------------------------------------------------------------------------- 1 | // Package pipeline contains the meta types and interfaces that define a Scribe pipeline. 2 | package pipeline 3 | -------------------------------------------------------------------------------- /pipeline/errors.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "errors" 4 | 5 | var ErrorStepNotFound = errors.New("step not found") 6 | -------------------------------------------------------------------------------- /pipeline/event.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | 7 | "github.com/grafana/scribe/state" 8 | ) 9 | 10 | type Stringer string 11 | 12 | func (s Stringer) String() string { 13 | return string(s) 14 | } 15 | 16 | // FilterValueType is the metadata type that identifies the type present event filter. 17 | type FilterValueType int 18 | 19 | const ( 20 | FilterValueString FilterValueType = iota 21 | FilterValueRegex 22 | FilterValueGlob 23 | ) 24 | 25 | type FilterValue struct { 26 | Type FilterValueType 27 | Value fmt.Stringer 28 | } 29 | 30 | func (f *FilterValue) String() string { 31 | return f.Value.String() 32 | } 33 | 34 | func StringFilter(v string) *FilterValue { 35 | return &FilterValue{ 36 | Type: FilterValueString, 37 | Value: Stringer(v), 38 | } 39 | } 40 | 41 | func RegexpFilter(v *regexp.Regexp) *FilterValue { 42 | return &FilterValue{ 43 | Type: FilterValueRegex, 44 | Value: v, 45 | } 46 | } 47 | 48 | func GlobFilter(v string) *FilterValue { 49 | return &FilterValue{ 50 | Type: FilterValueGlob, 51 | Value: Stringer(v), 52 | } 53 | } 54 | 55 | // Event is provided when defining a Scribe pipeline to define the events that cause the pipeline to be ran. 56 | // Some example events that might cause pipelines to be created: 57 | // * Manual events with user input, like 'Promotions' in Drone. In this scenario, the user may have the ability to supply any keys/values as arguments, however, pipeline developers in Scribe should be able to specifically define what fields are accepted. See https://docs.drone.io/promote/. 58 | // * git and SCM-related events like 'Pull Reuqest', 'Commit', 'Tag'. Each one of these events has a unique set of arguments / filters. `Commit` may allow pipeline developers to filter by branch or message. Tags may allow developers to filter by name. 59 | // * cron events, which may allow the pipeline in the CI service to be ran on a schedule. 60 | // The Event type stores both the filters and a list of values that it provides to the pipeline. 61 | // Client implementations of the pipeline (type Client) are expected to handle events that they are capable of handling. 62 | // 'Handling' events means that the the arguments in the `Provides` key should be available before any first steps are ran. It will not typically be up to pipeline developers to decide what arguments an event provides. 63 | // The only case where this may happen is if the event is a manual one, where users are able to submit the event with any arbitrary set of keys/values. 64 | // The 'Filters' key is provided in the pipeline code and should not be populated when pre-defined in the Scribe package. 65 | type Event struct { 66 | Name string 67 | Filters map[string]*FilterValue 68 | Provides []state.Argument 69 | } 70 | 71 | type GitCommitFilters struct { 72 | Branch *FilterValue 73 | } 74 | 75 | // GitCommitEventArgs are arguments that should provide in the pipeline state when a pipeline was created from a git commit event. 76 | var GitCommitEventArgs = []state.Argument{ 77 | ArgumentCommitSHA, 78 | ArgumentBranch, 79 | ArgumentRemoteURL, 80 | } 81 | 82 | func GitCommitEvent(filters GitCommitFilters) Event { 83 | f := map[string]*FilterValue{} 84 | 85 | if filters.Branch != nil { 86 | f["branch"] = filters.Branch 87 | } 88 | 89 | return Event{ 90 | Name: "git-commit", 91 | Filters: f, 92 | Provides: GitCommitEventArgs, 93 | } 94 | } 95 | 96 | type GitTagFilters struct { 97 | Name *FilterValue 98 | } 99 | 100 | // GitTagEventArgs are arguments that should provide in the pipeline state when a pipeline was created from a git tag event. 101 | var GitTagEventArgs = []state.Argument{ 102 | ArgumentCommitSHA, 103 | ArgumentCommitRef, 104 | ArgumentRemoteURL, 105 | } 106 | 107 | func GitTagEvent(filters GitTagFilters) Event { 108 | f := map[string]*FilterValue{} 109 | f["tag"] = filters.Name 110 | 111 | return Event{ 112 | Name: "git-tag", 113 | Filters: f, 114 | Provides: GitTagEventArgs, 115 | } 116 | } 117 | 118 | type PullRequestFilters struct{} 119 | 120 | // PullRequestEventArgs are arguments that should provide in the pipeline state when a pipeline was created from a pull request. 121 | var PullRequestEventArgs = []state.Argument{} 122 | 123 | func PullRequestEvent(filters PullRequestFilters) Event { 124 | f := map[string]*FilterValue{} 125 | 126 | return Event{ 127 | Name: "pull-request", 128 | Filters: f, 129 | Provides: PullRequestEventArgs, 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /pipeline/event_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestManualEvents(t *testing.T) { 8 | // t.Run("A single manual event", func(t *testing.T) { 9 | // // A single manual event should define a filter. 10 | // pipelineEvent := pipeline.NewManualEvent( 11 | // pipeline.NewStringFilter("branch", "main", "dev"), 12 | // ) 13 | 14 | // event := events.NewEvent("commit", map[string][]string{ 15 | // "branch": []string{"main"}, 16 | // }) 17 | 18 | // if !pipelineEvent.Matches(event) { 19 | // t.Fatal("The pipeline event should match the event") 20 | // } 21 | // }) 22 | } 23 | -------------------------------------------------------------------------------- /pipeline/opts.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | -------------------------------------------------------------------------------- /pipeline/pipeline_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | "github.com/grafana/scribe/state" 8 | ) 9 | 10 | func TestBuildEdges(t *testing.T) { 11 | t.Run("An edge from the root node should be created if the pipeline and step depend on the same argument", func(t *testing.T) { 12 | arg := state.NewStringArgument("shared-arg") 13 | p := pipeline.New("test-pipeline", 1).Requires(arg) 14 | step1 := pipeline.NoOpStep.WithName("step 1").Requires(arg) 15 | step1.ID = 5 16 | if err := p.AddSteps(step1); err != nil { 17 | t.Fatal(err) 18 | } 19 | 20 | rootArgs := []state.Argument{} 21 | if err := p.BuildEdges(rootArgs...); err != nil { 22 | t.Fatal(err) 23 | } 24 | }) 25 | } 26 | -------------------------------------------------------------------------------- /pipeline/print.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "log" 4 | 5 | func PrintCollection(col *Collection) { 6 | for _, v := range col.Graph.Nodes { 7 | log.Println("node:", v.Value.Name) 8 | for _, v := range v.Value.Graph.Nodes { 9 | log.Println(" node:", v.Value.Name) 10 | } 11 | for _, e := range v.Value.Graph.Edges { 12 | for _, v := range e { 13 | log.Println(" edge:", v.From.Value.Name, "->", v.To.Value.Name) 14 | } 15 | } 16 | } 17 | for _, e := range col.Graph.Edges { 18 | for _, v := range e { 19 | log.Println("edge:", v.From.Value.Name, "->", v.To.Value.Name) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /pipeline/step_env.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "github.com/grafana/scribe/state" 4 | 5 | type EnvVarType int 6 | 7 | const ( 8 | // EnvVarString should be used whenever the value is known and static. 9 | EnvVarString EnvVarType = iota 10 | 11 | // EnvVarArgument means that the environment variable will be populated by an argument from the state at run-time. 12 | // Most (all?) CI services will then leave these values out of the configuration and will be injected when the step runs. 13 | EnvVarArgument 14 | ) 15 | 16 | type EnvVar struct { 17 | Type EnvVarType 18 | 19 | // argument will be populated if this EnvVar is created using the NewEnvArgument function. 20 | argument state.Argument 21 | 22 | // str will be populated if this EnvVar is created using the NewEnvString function. 23 | str string 24 | } 25 | 26 | type StepEnv map[string]EnvVar 27 | 28 | // NewEnvArgument creates a new EnvVar that will be populated based on an Argument found in the state when the step runs. 29 | func NewEnvArgument(arg state.Argument) EnvVar { 30 | return EnvVar{ 31 | Type: EnvVarArgument, 32 | argument: arg, 33 | } 34 | } 35 | 36 | // NewEnvString creates a new EnvVar that will be populated with a static string value. 37 | func NewEnvString(val string) EnvVar { 38 | return EnvVar{ 39 | Type: EnvVarArgument, 40 | str: val, 41 | } 42 | } 43 | 44 | // String retrieves the static string value set when using the NewEnvString function. 45 | // If the EnvVar's Type property is not "EnvVarString" then it will panic. 46 | func (e EnvVar) String() string { 47 | if e.Type != EnvVarString { 48 | panic("envvar is not a string type, but String() was called") 49 | } 50 | 51 | return e.str 52 | } 53 | 54 | // Argument retrieves the argument value set when using the NewEnvArgument function. 55 | // If the EnvVar's Type property is not "EnvVarString" then it will panic. 56 | func (e EnvVar) Argument() state.Argument { 57 | if e.Type != EnvVarArgument { 58 | panic("envvar is not an argument type, but Argument() was called") 59 | } 60 | 61 | return e.argument 62 | } 63 | -------------------------------------------------------------------------------- /pipeline/step_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | ) 8 | 9 | func TestStepIsBackground(t *testing.T) { 10 | step := pipeline.NamedStep("test step", pipeline.DefaultAction) 11 | step.Type = pipeline.StepTypeBackground 12 | 13 | if step.IsBackground() != true { 14 | t.Fatal("step.IsBackground should return true if the step.Type is pipeline.StepTypeBackground") 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /pipeline/walker.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/grafana/scribe/pipeline/dag" 8 | ) 9 | 10 | // WalkFunc is implemented by the pipeline 'Clients'. This function is executed for each Step. 11 | type StepWalkFunc func(context.Context, Step) error 12 | 13 | // PipelineWalkFunc is implemented by the pipeline 'Clients'. This function is executed for each pipeline. 14 | // This function follows the same rules for pipelines as the StepWalker func does for pipelines. If multiple pipelines are provided in the steps argument, 15 | // then those pipelines are intended to be executed in parallel. 16 | type PipelineWalkFunc func(context.Context, Pipeline) error 17 | 18 | func (c *Collection) WalkPipelines(ctx context.Context, wf PipelineWalkFunc) error { 19 | if err := c.Graph.BreadthFirstSearch(0, c.pipelineVisitFunc(ctx, wf)); err != nil { 20 | return err 21 | } 22 | return nil 23 | } 24 | 25 | func (c *Collection) WalkSteps(ctx context.Context, pipelineID int64, wf StepWalkFunc) error { 26 | node, err := c.Graph.Node(pipelineID) 27 | if err != nil { 28 | return fmt.Errorf("could not find pipeline '%d'. %w", pipelineID, err) 29 | } 30 | 31 | pipeline := node.Value 32 | return pipeline.Graph.BreadthFirstSearch(0, func(n *dag.Node[Step]) error { 33 | if n.ID == 0 { 34 | return nil 35 | } 36 | 37 | return wf(ctx, n.Value) 38 | }) 39 | } 40 | -------------------------------------------------------------------------------- /pipelineutil/build.go: -------------------------------------------------------------------------------- 1 | package pipelineutil 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | 11 | golangx "github.com/grafana/scribe/golang/x" 12 | ) 13 | 14 | // GoBuildOpts is the list of (mostly) optional arguments that can be provided when building a pipeline into a static binary. 15 | // The goal of compiling the pipeline into a binary is that it will be mounted into a container and used in that container. 16 | type GoBuildOpts struct { 17 | // Pipeline is the path to the pipeline that you want to compile. 18 | // This path should be a `go build` compatible path. 19 | Pipeline string 20 | // Module is the path to the root module of the project that defines the go.mod/go.sum for the pipeline. 21 | // If this value is not provided, then 'GoBuild' will assume this is the value of 'os.Getwd'. 22 | Module string 23 | // GoOS sets the "GOOS" environment variable. 24 | // if not set, will not be supplied to the command, defaulting it to the current OS. 25 | GoOS string // GoArch sets the "GOARCH" environment variable. 26 | // If not set, will not be supplied to the command, defaulting it to the current architecture. 27 | GoArch string 28 | // GoModCache sets the "GOMODCACHE" environment variable. 29 | // 'go build' requires a location to search for the go module cache. 30 | // if this is not set, then it uses the value available from the current environment using 'os.Getenv'. 31 | GoModCache string 32 | // GoPath sets the "GOPATH" environment variable. 33 | // 'go build' requires a $GOPATH to be set. 34 | // if this is not set, then it uses the value available from the current environment using 'os.Getenv'. 35 | GoPath string 36 | // Output is used as the '-o' argument in the go build command. 37 | // If this is not set, then we do not provide it, causing the compiled pipeline to be built in the 'os.Getwd', with a potentially confusing or ambiguous (or even colliding) name. 38 | Output string 39 | LDFlags string 40 | 41 | Stdout io.Writer 42 | Stderr io.Writer 43 | } 44 | 45 | func goBuildEnv(opts GoBuildOpts) []string { 46 | env := []string{ 47 | "CGO_ENABLED=0", 48 | } 49 | var ( 50 | goOS = opts.GoOS 51 | goArch = opts.GoArch 52 | goModCache = opts.GoModCache 53 | ) 54 | 55 | if goOS != "" { 56 | env = append(env, fmt.Sprintf("GOOS=%s", goOS)) 57 | } 58 | if goArch != "" { 59 | env = append(env, fmt.Sprintf("GOARCH=%s", goArch)) 60 | } 61 | 62 | if goModCache == "" { 63 | goModCache = os.Getenv("GOMODCACHE") 64 | } 65 | 66 | env = append(env, fmt.Sprintf("GOMODCACHE=%s", goModCache)) 67 | return env 68 | } 69 | 70 | // GoBuild returns the *exec.Cmd will, if ran, statically compile the pipeline provided in the arguments. 71 | // This function shells out to the 'go' process, so ensure that 'go' is installed and available in the current environment. 72 | // We have to shell out because Go does not provide a stdlib function for running 'go build' without the 'go' command. 73 | func GoBuild(ctx context.Context, opts GoBuildOpts) *exec.Cmd { 74 | var ( 75 | wd = filepath.Clean(opts.Module) 76 | env = goBuildEnv(opts) 77 | ) 78 | 79 | return golangx.Build(ctx, golangx.BuildOpts{ 80 | Pkg: opts.Pipeline, 81 | Module: wd, 82 | Env: env, 83 | Output: opts.Output, 84 | LDFlags: opts.LDFlags, 85 | }) 86 | } 87 | -------------------------------------------------------------------------------- /pipelineutil/doc.go: -------------------------------------------------------------------------------- 1 | // Package pipelineutil defines utilities for working with Pipelines and is separated as it may also import packages that import pipeline. 2 | package pipelineutil 3 | -------------------------------------------------------------------------------- /plog/docs.go: -------------------------------------------------------------------------------- 1 | // Package plog (or plumbig log) provides a logging initializer and utility functions for working with a logging library. 2 | package plog 3 | -------------------------------------------------------------------------------- /plog/fields.go: -------------------------------------------------------------------------------- 1 | package plog 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | "github.com/grafana/scribe/pipeline/clients" 8 | "github.com/grafana/scribe/stringutil" 9 | "github.com/opentracing/opentracing-go" 10 | "github.com/sirupsen/logrus" 11 | "github.com/uber/jaeger-client-go" 12 | ) 13 | 14 | // TracingFields adds fields that are derived from the context.Context. 15 | // Tracing has handled using context.Context, and if tracing is enabled, then everything happening should be within a tracing span / trace 16 | func TracingFields(ctx context.Context) logrus.Fields { 17 | fields := logrus.Fields{} 18 | 19 | span := opentracing.SpanFromContext(ctx) 20 | if span != nil { 21 | if jaegerCtx, ok := span.Context().(jaeger.SpanContext); ok { 22 | fields["trace_id"] = jaegerCtx.TraceID().String() 23 | fields["span_id"] = jaegerCtx.SpanID().String() 24 | } 25 | } 26 | 27 | return fields 28 | } 29 | 30 | func StepFields(step pipeline.Step) logrus.Fields { 31 | return logrus.Fields{ 32 | "step": step.Name, 33 | "serial": step.ID, 34 | } 35 | } 36 | 37 | func PipelineFields(opts clients.CommonOpts) logrus.Fields { 38 | return logrus.Fields{ 39 | "build_id": opts.Args.BuildID, 40 | "pipeline": stringutil.Slugify(opts.Name), 41 | } 42 | } 43 | 44 | func Combine(field ...logrus.Fields) logrus.Fields { 45 | fields := logrus.Fields{} 46 | 47 | for _, m := range field { 48 | for k, v := range m { 49 | fields[k] = v 50 | } 51 | } 52 | 53 | return fields 54 | } 55 | 56 | func DefaultFields(ctx context.Context, step pipeline.Step, opts clients.CommonOpts) logrus.Fields { 57 | return Combine(TracingFields(ctx), StepFields(step), PipelineFields(opts)) 58 | } 59 | -------------------------------------------------------------------------------- /plog/helpers.go: -------------------------------------------------------------------------------- 1 | package plog 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | func LogSteps(logger logrus.FieldLogger, steps []pipeline.Step) { 11 | s := make([]string, len(steps)) 12 | for i, v := range steps { 13 | s[i] = v.Name 14 | } 15 | logger.Infof("[%d] step(s) %s", len(steps), strings.Join(s, " | ")) 16 | } 17 | 18 | func LogPipelines(logger logrus.FieldLogger, pipelines []pipeline.Pipeline) { 19 | s := make([]string, len(pipelines)) 20 | for i, v := range pipelines { 21 | s[i] = v.Name 22 | } 23 | logger.Infof("[%d] pipelines(s) %s", len(pipelines), strings.Join(s, " | ")) 24 | } 25 | -------------------------------------------------------------------------------- /plog/logger.go: -------------------------------------------------------------------------------- 1 | package plog 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | ) 6 | 7 | func New(level logrus.Level) *logrus.Logger { 8 | logger := logrus.New() 9 | 10 | logger.SetLevel(level) 11 | logger.SetFormatter(&logrus.TextFormatter{}) 12 | 13 | return logger 14 | } 15 | -------------------------------------------------------------------------------- /scribe_client_test.go: -------------------------------------------------------------------------------- 1 | package scribe_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/grafana/scribe/pipeline" 9 | ) 10 | 11 | // ensurer provides a pipeline.StepWalkFunc that ensures that the steps that it receives are ran in the order provided. 12 | type ensurer struct { 13 | i int 14 | seen []string 15 | steps []string 16 | } 17 | 18 | func (e *ensurer) WalkPipelines(w *pipeline.Collection) func(context.Context, pipeline.Pipeline) error { 19 | return func(ctx context.Context, p pipeline.Pipeline) error { 20 | if err := w.WalkSteps(ctx, p.ID, e.WalkSteps); err != nil { 21 | return err 22 | } 23 | return nil 24 | } 25 | } 26 | 27 | func (e *ensurer) WalkSteps(ctx context.Context, step pipeline.Step) error { 28 | expect := e.steps[e.i] 29 | 30 | if !strings.EqualFold(step.Name, expect) { 31 | return fmt.Errorf("unexpected step at '%d'. expected step '%s', got '%s'", e.i, expect, step.Name) 32 | } 33 | 34 | e.seen[e.i] = step.Name 35 | e.i++ 36 | 37 | return nil 38 | } 39 | 40 | // Validate is ran internally before calling Run or Parallel and allows the client to effectively configure per-step requirements 41 | // For example, Drone steps MUST have an image so the Drone client returns an error in this function when the provided step does not have an image. 42 | // If the error encountered is not critical but should still be logged, then return a plumbing.ErrorSkipValidation. 43 | // The error is checked with `errors.Is` so the error can be wrapped with fmt.Errorf. 44 | func (e *ensurer) Validate(pipeline.Step) error { 45 | return nil 46 | } 47 | 48 | func (e *ensurer) Diff() string { 49 | return fmt.Sprintf("Seen: %+v\nExpected: %+v", e.seen, e.steps) 50 | } 51 | 52 | // Done must be ran at the end of the pipeline. 53 | // This is typically what takes the defined pipeline steps, runs them in the order defined, and produces some kind of output. 54 | func (e *ensurer) Done(ctx context.Context, w *pipeline.Collection) error { 55 | if err := w.WalkPipelines(ctx, e.WalkPipelines(w)); err != nil { 56 | return err 57 | } 58 | 59 | if len(e.seen) != len(e.steps) { 60 | return fmt.Errorf("walked unequal amount of steps. expected '%d', walked '%d'\n%s", len(e.steps), len(e.seen), e.Diff()) 61 | } 62 | 63 | for i, step := range e.steps { 64 | if e.seen[i] != step { 65 | return fmt.Errorf("step seen at '%d' does not match expected. Expected '%s', found '%s'\n%s", i, e.seen[i], step, e.Diff()) 66 | } 67 | } 68 | 69 | return nil 70 | } 71 | 72 | func newEnsurer(steps ...string) *ensurer { 73 | return &ensurer{ 74 | steps: steps, 75 | seen: make([]string, len(steps)), 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /scribe_multi_test.go: -------------------------------------------------------------------------------- 1 | package scribe_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/grafana/scribe" 8 | "github.com/grafana/scribe/pipeline" 9 | "github.com/grafana/scribe/state" 10 | ) 11 | 12 | func TestMulti(t *testing.T) { 13 | t.Run("Multi pipelines should have a root node with an ID of zero", func(t *testing.T) { 14 | // In this test case we're not providing ensurer data because we are not running 'Done'. 15 | sw := scribe.NewMultiWithClient(testOpts, newEnsurer()) 16 | 17 | if sw.Collection == nil { 18 | t.Fatal("Collection is nil") 19 | } 20 | 21 | _, err := sw.Collection.Graph.Node(0) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | }) 26 | 27 | t.Run("Creating a multi-pipeline with steps", func(t *testing.T) { 28 | // This is a potentially flaky test because I think 4 could sometimes show up before 5. 29 | ens := newEnsurer( 30 | "step 1", "step 2", "step 3", "step 5", "step 4", 31 | "step 1", "step 2", "step 3", "step 5", "step 4", 32 | ) 33 | 34 | // In this test case we're not providing ensurer data because we are not running 'Done'. 35 | var ( 36 | sw = scribe.NewMultiWithClient(testOpts, ens) 37 | argA = state.NewStringArgument("a") 38 | argB = state.NewStringArgument("b") 39 | argC = state.NewStringArgument("c") 40 | ) 41 | mf := func(s *scribe.Scribe) { 42 | s.Add( 43 | pipeline.NoOpStep.WithName("step 1").Provides(argA), 44 | pipeline.NoOpStep.WithName("step 2").Provides(argC), 45 | ) 46 | s.Add( 47 | pipeline.NoOpStep.WithName("step 3").Requires(argA).Provides(argB), 48 | ) 49 | s.Add( 50 | pipeline.NoOpStep.WithName("step 4").Requires(argB, argC), 51 | pipeline.NoOpStep.WithName("step 5").Requires(argA, argB, argC), 52 | ) 53 | } 54 | 55 | // each multi-func adds 5 new steps, and each new sub-pipeline adds an additional root step. 56 | // These pipelines are processed after all of the others are, so they will have the highest IDs (23 and 24). 57 | sw.Add( 58 | sw.New("test 1", mf), 59 | sw.New("test 2", mf), 60 | ) 61 | 62 | if err := sw.Execute(context.Background(), sw.Collection); err != nil { 63 | t.Fatal(err) 64 | } 65 | }) 66 | } 67 | 68 | func TestMultiWithEvent(t *testing.T) { 69 | t.Run("Once adding an event, it should be present in the collection", func(t *testing.T) { 70 | ens := newEnsurer() 71 | sw := scribe.NewMultiWithClient(testOpts, ens) 72 | 73 | mf := func(sw *scribe.Scribe) { 74 | sw.When( 75 | pipeline.GitTagEvent(pipeline.GitTagFilters{}), 76 | ) 77 | 78 | sw.Add(pipeline.NoOpStep.WithName("step 1")) 79 | } 80 | 81 | sw.Add( 82 | sw.New("test 1", mf), 83 | ) 84 | 85 | sw.Collection.WalkPipelines(context.Background(), func(ctx context.Context, p pipeline.Pipeline) error { 86 | if len(p.Events) != 1 { 87 | t.Fatal("Expected 1 pipeline event, but found", len(p.Events)) 88 | } 89 | 90 | return nil 91 | }) 92 | }) 93 | } 94 | -------------------------------------------------------------------------------- /state/arguments.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import "strings" 4 | 5 | type ArgumentType int 6 | 7 | const ( 8 | ArgumentTypeString ArgumentType = iota 9 | ArgumentTypeInt64 10 | ArgumentTypeFloat64 11 | ArgumentTypeBool 12 | ArgumentTypeSecret 13 | ArgumentTypeFile 14 | ArgumentTypeFS 15 | // An ArgumentTypeUnpackagedFS is used for filesystems that are invariably consistent regardless of operating system. 16 | // Developers can get around packaging and unpackaging of large directories using this argument type. 17 | // Filesystems and directories used with this argument should always exist on every machine. This basically means that they should be available within the source tree. 18 | // If this argument type is used for directories outside of the source tree, then expect divergeant behavior between operating systems. 19 | ArgumentTypeUnpackagedFS 20 | ) 21 | 22 | var argumentTypeStr = []string{"string", "int", "float", "bool", "secret", "file", "directory", "unpackaged-directory"} 23 | 24 | func (a ArgumentType) String() string { 25 | i := int(a) 26 | return argumentTypeStr[i] 27 | } 28 | 29 | func ArgumentTypesEqual(arg Argument, argTypes ...ArgumentType) bool { 30 | for _, v := range argTypes { 31 | if arg.Type == v { 32 | return true 33 | } 34 | } 35 | return false 36 | } 37 | 38 | // An Argument is a pre-defined argument that is used in a typical CI pipeline. 39 | // This allows the scribe library to define different methods of retrieving the same information on various clients. 40 | // For example, when running in CLI or Dagger client, getting the git ref might be as simple as running `git rev-parse HEAD`. 41 | // But in a Drone pipeline, that information may be available before the repository has been cloned in an environment variable. 42 | // Other arguments may require the user to be prompted if they have not been provided. 43 | // These arguments can be provided to the CLI by using the flag `-arg`, for example `-arg=workdir=./example` will set the "workdir" argument to "example" in the CLI client. 44 | // By default, all steps expect a WorkingDir and Repository. 45 | type Argument struct { 46 | Type ArgumentType 47 | Key string 48 | } 49 | 50 | func NewStringArgument(key string) Argument { 51 | return Argument{ 52 | Type: ArgumentTypeString, 53 | Key: key, 54 | } 55 | } 56 | 57 | func NewInt64Argument(key string) Argument { 58 | return Argument{ 59 | Type: ArgumentTypeInt64, 60 | Key: key, 61 | } 62 | } 63 | 64 | func NewFloat64Argument(key string) Argument { 65 | return Argument{ 66 | Type: ArgumentTypeFloat64, 67 | Key: key, 68 | } 69 | } 70 | 71 | func NewBoolArgument(key string) Argument { 72 | return Argument{ 73 | Type: ArgumentTypeBool, 74 | Key: key, 75 | } 76 | } 77 | 78 | func NewFileArgument(key string) Argument { 79 | return Argument{ 80 | Type: ArgumentTypeFile, 81 | Key: key, 82 | } 83 | } 84 | 85 | func NewDirectoryArgument(key string) Argument { 86 | return Argument{ 87 | Type: ArgumentTypeFS, 88 | Key: key, 89 | } 90 | } 91 | 92 | func NewUnpackagedDirectoryArgument(key string) Argument { 93 | return Argument{ 94 | Type: ArgumentTypeUnpackagedFS, 95 | Key: key, 96 | } 97 | } 98 | 99 | func NewSecretArgument(key string) Argument { 100 | return Argument{ 101 | Type: ArgumentTypeSecret, 102 | Key: key, 103 | } 104 | } 105 | 106 | type Arguments []Argument 107 | 108 | func (a *Arguments) String() string { 109 | str := make([]string, len(*a)) 110 | for i, v := range *a { 111 | str[i] = v.Key 112 | } 113 | 114 | return strings.Join(str, ", ") 115 | } 116 | -------------------------------------------------------------------------------- /state/default.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/url" 8 | "os" 9 | 10 | "cloud.google.com/go/storage" 11 | "github.com/aws/aws-sdk-go-v2/config" 12 | "github.com/aws/aws-sdk-go-v2/service/s3" 13 | "github.com/grafana/scribe/args" 14 | "github.com/sirupsen/logrus" 15 | ) 16 | 17 | // newFilesystemState creates a new filesystem state handler. 18 | // If the directory provided doesn't exist, it will be created. 19 | func newFilesystemState(ctx context.Context, u *url.URL) (Handler, error) { 20 | var ( 21 | dir = u.Path 22 | ) 23 | 24 | if info, err := os.Stat(dir); err == nil { 25 | if !info.IsDir() { 26 | return nil, fmt.Errorf("state argument '%s' must be a directory. example: '/var/scribe-state'", dir) 27 | } 28 | } else { 29 | if errors.Is(err, os.ErrNotExist) { 30 | if err := os.MkdirAll(dir, 0755); err != nil { 31 | return nil, fmt.Errorf("state directory '%s' does not exist. Error attempting to create it: %w", dir, err) 32 | } 33 | } 34 | } 35 | 36 | return NewFilesystemState(dir) 37 | } 38 | 39 | func newGCSState(ctx context.Context, u *url.URL) (Handler, error) { 40 | client, err := storage.NewClient(ctx) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return NewGCSHandler(client, u) 46 | } 47 | 48 | func newS3State(ctx context.Context, u *url.URL) (Handler, error) { 49 | sdkConfig, err := config.LoadDefaultConfig(ctx) 50 | if err != nil { 51 | return nil, err 52 | } 53 | client := s3.NewFromConfig(sdkConfig) 54 | return NewS3Handler(client, u) 55 | 56 | } 57 | 58 | var states = map[string]func(context.Context, *url.URL) (Handler, error){ 59 | "file": newFilesystemState, 60 | "fs": newFilesystemState, 61 | "gs": newGCSState, 62 | "gcs": newGCSState, 63 | "s3": newS3State, 64 | } 65 | 66 | // NewDefaultState creates a new default state given the arguments provided. 67 | // The --no-stdin flag will prevent the State object from using the stdin to populate the state for ClientProvidedArguments. (See `pipeline/arguments_known.go` for those). 68 | // The --state flag defines where the state JSON and state data will be stored. 69 | // If the value for a key is not available in the primary state (defined by the --state flag), then the state object will attempt to retrieve it from the fallback. Currently, the fallback options are the `--arg` flags (--arg={key}={value}), or, if `--no-stdin` is not set, then from the stdin. 70 | func NewDefaultState(ctx context.Context, log logrus.FieldLogger, pargs *args.PipelineArgs) (*State, error) { 71 | u, err := url.Parse(pargs.State) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | fallback := []Reader{ 77 | ReaderWithLogs(log.WithField("state", "arguments"), NewArgMapReader(pargs.ArgMap)), 78 | } 79 | 80 | if pargs.CanStdinPrompt { 81 | fallback = append(fallback, ReaderWithLogs(log.WithField("state", "stdin"), NewStdinReader(os.Stdin, os.Stdout))) 82 | } 83 | 84 | if v, ok := states[u.Scheme]; ok { 85 | handler, err := v(ctx, u) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | return &State{ 91 | Handler: HandlerWithLogs(log.WithField("state", u.Scheme), handler), 92 | Fallback: fallback, 93 | Log: log, 94 | }, nil 95 | } 96 | 97 | return nil, fmt.Errorf("state URL scheme '%s' not recognized", pargs.State) 98 | } 99 | -------------------------------------------------------------------------------- /state/json.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import "context" 4 | 5 | type JSONState map[string]StateValueJSON 6 | 7 | type StateValueJSON struct { 8 | Argument Argument `json:"argument"` 9 | Value any `json:"value"` 10 | } 11 | 12 | func SetValueFromJSON(ctx context.Context, w Writer, value StateValueJSON) error { 13 | switch value.Argument.Type { 14 | case ArgumentTypeString: 15 | return w.SetString(ctx, value.Argument, value.Value.(string)) 16 | case ArgumentTypeInt64: 17 | return w.SetInt64(ctx, value.Argument, int64(value.Value.(float64))) 18 | case ArgumentTypeFloat64: 19 | return w.SetFloat64(ctx, value.Argument, value.Value.(float64)) 20 | case ArgumentTypeBool: 21 | return w.SetBool(ctx, value.Argument, value.Value.(bool)) 22 | //case ArgumentTypeSecret: 23 | //return w.SetSecret(value.Argument, value.Value.(bool)) 24 | case ArgumentTypeFile: 25 | return w.SetFile(ctx, value.Argument, value.Value.(string)) 26 | case ArgumentTypeFS: 27 | return w.SetDirectory(ctx, value.Argument, value.Value.(string)) 28 | case ArgumentTypeUnpackagedFS: 29 | return w.SetDirectory(ctx, value.Argument, value.Value.(string)) 30 | default: 31 | } 32 | 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /state/object_storage.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "io" 6 | ) 7 | 8 | type GetObjectResponse struct { 9 | Body io.ReadCloser 10 | } 11 | 12 | type ObjectStorage interface { 13 | GetObject(ctx context.Context, bucket, key string) (*GetObjectResponse, error) 14 | PutObject(ctx context.Context, bucket, key string, body io.Reader) error 15 | } 16 | -------------------------------------------------------------------------------- /state/object_storage_gcs.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | 8 | "cloud.google.com/go/storage" 9 | ) 10 | 11 | type GCSObjectStorage struct { 12 | Client *storage.Client 13 | } 14 | 15 | func (s *GCSObjectStorage) GetObject(ctx context.Context, bucket, key string) (*GetObjectResponse, error) { 16 | obj := s.Client.Bucket(bucket).Object(key) 17 | r, err := obj.NewReader(ctx) 18 | if err != nil { 19 | if errors.Is(err, storage.ErrObjectNotExist) { 20 | return nil, ErrorFileNotFound 21 | } 22 | return nil, err 23 | } 24 | 25 | return &GetObjectResponse{ 26 | Body: r, 27 | }, nil 28 | } 29 | 30 | func (s *GCSObjectStorage) PutObject(ctx context.Context, bucket, key string, body io.Reader) error { 31 | obj := s.Client.Bucket(bucket).Object(key) 32 | w := obj.NewWriter(ctx) 33 | if _, err := io.Copy(w, body); err != nil { 34 | return err 35 | } 36 | 37 | return w.Close() 38 | } 39 | -------------------------------------------------------------------------------- /state/object_storage_s3.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/aws/aws-sdk-go-v2/aws" 8 | "github.com/aws/aws-sdk-go-v2/service/s3" 9 | ) 10 | 11 | type S3ObjectStorage struct { 12 | Client *s3.Client 13 | } 14 | 15 | func (s *S3ObjectStorage) GetObject(ctx context.Context, bucket, key string) (*GetObjectResponse, error) { 16 | res, err := s.Client.GetObject(ctx, &s3.GetObjectInput{ 17 | Bucket: aws.String(bucket), 18 | Key: aws.String(key), 19 | }) 20 | 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | return &GetObjectResponse{ 26 | Body: res.Body, 27 | }, nil 28 | } 29 | 30 | func (s *S3ObjectStorage) PutObject(ctx context.Context, bucket, key string, body io.Reader) error { 31 | if _, err := s.Client.PutObject(ctx, &s3.PutObjectInput{ 32 | Bucket: aws.String(bucket), 33 | Key: aws.String(key), 34 | Body: body, 35 | }); err != nil { 36 | return err 37 | } 38 | 39 | return nil 40 | } 41 | -------------------------------------------------------------------------------- /state/observer.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "io/fs" 7 | "os" 8 | "sync" 9 | ) 10 | 11 | type Observer struct { 12 | h Handler 13 | conds map[Argument]*sync.Cond 14 | mtx *sync.Mutex 15 | } 16 | 17 | func NewObserver(h Handler) *Observer { 18 | return &Observer{ 19 | h: h, 20 | conds: make(map[Argument]*sync.Cond), 21 | mtx: &sync.Mutex{}, 22 | } 23 | } 24 | 25 | func (o *Observer) CondFor(ctx context.Context, arg Argument) *sync.Cond { 26 | o.mtx.Lock() 27 | defer o.mtx.Unlock() 28 | if val, ok := o.conds[arg]; ok { 29 | return val 30 | } 31 | 32 | o.conds[arg] = sync.NewCond(&sync.Mutex{}) 33 | return o.conds[arg] 34 | } 35 | 36 | func (o *Observer) Notify(ctx context.Context, arg Argument) { 37 | o.mtx.Lock() 38 | defer o.mtx.Unlock() 39 | if v, ok := o.conds[arg]; ok { 40 | v.L.Lock() 41 | v.Broadcast() 42 | v.L.Unlock() 43 | } 44 | } 45 | 46 | // Reader functions 47 | func (o *Observer) Exists(ctx context.Context, arg Argument) (bool, error) { 48 | return o.h.Exists(ctx, arg) 49 | } 50 | func (o *Observer) GetString(ctx context.Context, arg Argument) (string, error) { 51 | return o.h.GetString(ctx, arg) 52 | } 53 | func (o *Observer) GetInt64(ctx context.Context, arg Argument) (int64, error) { 54 | return o.h.GetInt64(ctx, arg) 55 | } 56 | func (o *Observer) GetFloat64(ctx context.Context, arg Argument) (float64, error) { 57 | return o.h.GetFloat64(ctx, arg) 58 | } 59 | func (o *Observer) GetBool(ctx context.Context, arg Argument) (bool, error) { 60 | return o.h.GetBool(ctx, arg) 61 | } 62 | func (o *Observer) GetFile(ctx context.Context, arg Argument) (*os.File, error) { 63 | return o.h.GetFile(ctx, arg) 64 | } 65 | func (o *Observer) GetDirectory(ctx context.Context, arg Argument) (fs.FS, error) { 66 | return o.h.GetDirectory(ctx, arg) 67 | } 68 | func (o *Observer) GetDirectoryString(ctx context.Context, arg Argument) (string, error) { 69 | return o.h.GetDirectoryString(ctx, arg) 70 | } 71 | 72 | // Writer functions 73 | func (o *Observer) SetString(ctx context.Context, arg Argument, val string) error { 74 | defer o.Notify(ctx, arg) 75 | return o.h.SetString(ctx, arg, val) 76 | } 77 | func (o *Observer) SetInt64(ctx context.Context, arg Argument, val int64) error { 78 | defer o.Notify(ctx, arg) 79 | return o.h.SetInt64(ctx, arg, val) 80 | } 81 | func (o *Observer) SetFloat64(ctx context.Context, arg Argument, val float64) error { 82 | defer o.Notify(ctx, arg) 83 | return o.h.SetFloat64(ctx, arg, val) 84 | } 85 | func (o *Observer) SetBool(ctx context.Context, arg Argument, val bool) error { 86 | defer o.Notify(ctx, arg) 87 | return o.h.SetBool(ctx, arg, val) 88 | } 89 | func (o *Observer) SetFile(ctx context.Context, arg Argument, path string) error { 90 | defer o.Notify(ctx, arg) 91 | return o.h.SetFile(ctx, arg, path) 92 | } 93 | func (o *Observer) SetFileReader(ctx context.Context, arg Argument, r io.Reader) (string, error) { 94 | defer o.Notify(ctx, arg) 95 | return o.h.SetFileReader(ctx, arg, r) 96 | } 97 | func (o *Observer) SetDirectory(ctx context.Context, arg Argument, dir string) error { 98 | defer o.Notify(ctx, arg) 99 | return o.h.SetDirectory(ctx, arg, dir) 100 | } 101 | -------------------------------------------------------------------------------- /state/state_args.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "io/fs" 6 | "os" 7 | "strconv" 8 | 9 | "github.com/grafana/scribe/args" 10 | ) 11 | 12 | // ArgMapReader attempts to read state values from the provided "ArgMap". 13 | // The ArgMap is provided by the user by using the '-arg={key}={value}' argument. 14 | // This is typically only used in local executions where some values will not be provided. 15 | type ArgMapReader struct { 16 | defaults args.ArgMap 17 | } 18 | 19 | func NewArgMapReader(defaults args.ArgMap) *ArgMapReader { 20 | return &ArgMapReader{ 21 | defaults: defaults, 22 | } 23 | } 24 | 25 | func (s *ArgMapReader) GetString(ctx context.Context, arg Argument) (string, error) { 26 | val, err := s.defaults.Get(arg.Key) 27 | if err != nil { 28 | return "", err 29 | } 30 | 31 | return val, nil 32 | } 33 | 34 | func (s *ArgMapReader) GetInt64(ctx context.Context, arg Argument) (int64, error) { 35 | val, err := s.defaults.Get(arg.Key) 36 | if err != nil { 37 | return 0, err 38 | } 39 | 40 | return strconv.ParseInt(val, 10, 64) 41 | } 42 | 43 | func (s *ArgMapReader) GetFloat64(ctx context.Context, arg Argument) (float64, error) { 44 | val, err := s.defaults.Get(arg.Key) 45 | if err != nil { 46 | return 0, err 47 | } 48 | 49 | return strconv.ParseFloat(val, 64) 50 | } 51 | 52 | func (s *ArgMapReader) GetBool(ctx context.Context, arg Argument) (bool, error) { 53 | val, err := s.defaults.Get(arg.Key) 54 | if err != nil { 55 | return false, err 56 | } 57 | 58 | return strconv.ParseBool(val) 59 | } 60 | 61 | func (s *ArgMapReader) GetFile(ctx context.Context, arg Argument) (*os.File, error) { 62 | val, err := s.defaults.Get(arg.Key) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | return os.Open(val) 68 | } 69 | 70 | func (s *ArgMapReader) GetDirectory(ctx context.Context, arg Argument) (fs.FS, error) { 71 | val, err := s.defaults.Get(arg.Key) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | return os.DirFS(val), nil 77 | } 78 | 79 | func (s *ArgMapReader) GetDirectoryString(ctx context.Context, arg Argument) (string, error) { 80 | val, err := s.defaults.Get(arg.Key) 81 | if err != nil { 82 | return "", err 83 | } 84 | 85 | return val, nil 86 | } 87 | 88 | func (s *ArgMapReader) Exists(ctx context.Context, arg Argument) (bool, error) { 89 | // defaults.Get only returns an error if no value was found. 90 | _, err := s.defaults.Get(arg.Key) 91 | if err != nil { 92 | return false, nil 93 | } 94 | 95 | return true, nil 96 | } 97 | -------------------------------------------------------------------------------- /state/state_gcs.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "net/url" 5 | "strings" 6 | 7 | "cloud.google.com/go/storage" 8 | ) 9 | 10 | // GCSHandler uses the S3 API but with a round-tripper that makes the API client compatible with the S3 api 11 | type GCSHandler struct { 12 | *ObjectStorageHandler 13 | } 14 | 15 | func BucketAndPath(u *url.URL) (string, string) { 16 | return u.Host, strings.TrimPrefix(u.Path, "/") 17 | } 18 | 19 | func NewGCSHandler(client *storage.Client, u *url.URL) (*GCSHandler, error) { 20 | bucket, path := BucketAndPath(u) 21 | 22 | h := NewObjectStorageHandler( 23 | &GCSObjectStorage{ 24 | Client: client, 25 | }, 26 | bucket, 27 | path, 28 | ) 29 | 30 | return &GCSHandler{ 31 | ObjectStorageHandler: h, 32 | }, nil 33 | 34 | } 35 | -------------------------------------------------------------------------------- /state/state_gcs_test.go: -------------------------------------------------------------------------------- 1 | package state_test 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | 7 | "github.com/grafana/scribe/state" 8 | ) 9 | 10 | func TestBucketAndPath(t *testing.T) { 11 | type result struct { 12 | bucket, path string 13 | } 14 | res := map[string]result{ 15 | "gs://bucket/path": {"bucket", "path"}, 16 | "gs://bucket/path/1/2/3": {"bucket", "path/1/2/3"}, 17 | "gs://the-bucket/path/1/2/3": {"the-bucket", "path/1/2/3"}, 18 | } 19 | 20 | for k, v := range res { 21 | u, _ := url.Parse(k) 22 | bucket, path := state.BucketAndPath(u) 23 | if bucket != v.bucket { 24 | t.Errorf("got: '%s', expected: '%s'", bucket, v.bucket) 25 | } 26 | if path != v.path { 27 | t.Errorf("got: '%s', expected: '%s'", path, v.path) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /state/state_helpers.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | ) 8 | 9 | func GetValueAsString(ctx context.Context, r Reader, arg Argument) (string, error) { 10 | switch arg.Type { 11 | case ArgumentTypeString, ArgumentTypeSecret: 12 | return r.GetString(ctx, arg) 13 | case ArgumentTypeInt64: 14 | val, err := r.GetInt64(ctx, arg) 15 | if err != nil { 16 | return "", err 17 | } 18 | 19 | return strconv.FormatInt(val, 10), nil 20 | case ArgumentTypeFloat64: 21 | val, err := r.GetFloat64(ctx, arg) 22 | if err != nil { 23 | return "", err 24 | } 25 | 26 | return strconv.FormatFloat(val, 'f', 8, 64), nil 27 | case ArgumentTypeBool: 28 | val, err := r.GetBool(ctx, arg) 29 | if err != nil { 30 | return "", err 31 | } 32 | 33 | return strconv.FormatBool(val), nil 34 | case ArgumentTypeFile: 35 | file, err := r.GetFile(ctx, arg) 36 | if err != nil { 37 | return "", err 38 | } 39 | 40 | return file.Name(), nil 41 | 42 | case ArgumentTypeUnpackagedFS, ArgumentTypeFS: 43 | return r.GetDirectoryString(ctx, arg) 44 | 45 | default: 46 | } 47 | 48 | return "", fmt.Errorf("unsupported or unrecognized argument type: %s", arg.Type) 49 | } 50 | 51 | func ArgListContains(args Arguments, arg Argument) bool { 52 | for _, v := range args { 53 | if v == arg { 54 | return true 55 | } 56 | } 57 | 58 | return false 59 | } 60 | -------------------------------------------------------------------------------- /state/state_noop.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "io/fs" 7 | "os" 8 | ) 9 | 10 | // NoOpHandler is a Handler that does nothing and is only used in tests where state is ignored but should never be nil. 11 | type NoOpHandler struct{} 12 | 13 | func NewNoOpHandler() *NoOpHandler { return &NoOpHandler{} } 14 | 15 | // Reader functions 16 | func (n *NoOpHandler) Exists(ctx context.Context, arg Argument) (bool, error) { return false, nil } 17 | func (n *NoOpHandler) GetString(ctx context.Context, arg Argument) (string, error) { return "", nil } 18 | func (n *NoOpHandler) GetInt64(ctx context.Context, arg Argument) (int64, error) { return 0, nil } 19 | func (n *NoOpHandler) GetFloat64(ctx context.Context, arg Argument) (float64, error) { return 0.0, nil } 20 | func (n *NoOpHandler) GetBool(ctx context.Context, arg Argument) (bool, error) { return false, nil } 21 | func (n *NoOpHandler) GetFile(ctx context.Context, arg Argument) (*os.File, error) { return nil, nil } 22 | func (n *NoOpHandler) GetDirectory(ctx context.Context, arg Argument) (fs.FS, error) { return nil, nil } 23 | func (n *NoOpHandler) GetDirectoryString(ctx context.Context, arg Argument) (string, error) { 24 | return "", nil 25 | } 26 | 27 | // Writer functions 28 | func (n *NoOpHandler) SetString(ctx context.Context, arg Argument, val string) error { return nil } 29 | func (n *NoOpHandler) SetInt64(ctx context.Context, arg Argument, val int64) error { return nil } 30 | func (n *NoOpHandler) SetFloat64(ctx context.Context, arg Argument, val float64) error { return nil } 31 | func (n *NoOpHandler) SetBool(ctx context.Context, arg Argument, val bool) error { return nil } 32 | func (n *NoOpHandler) SetFile(ctx context.Context, arg Argument, path string) error { return nil } 33 | func (n *NoOpHandler) SetFileReader(ctx context.Context, arg Argument, r io.Reader) (string, error) { 34 | return "", nil 35 | } 36 | func (n *NoOpHandler) SetDirectory(ctx context.Context, arg Argument, dir string) error { return nil } 37 | -------------------------------------------------------------------------------- /state/state_object_store_test.go: -------------------------------------------------------------------------------- 1 | package state_test 2 | -------------------------------------------------------------------------------- /state/state_s3.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "net/url" 5 | 6 | "github.com/aws/aws-sdk-go-v2/service/s3" 7 | ) 8 | 9 | type S3Handler struct { 10 | *ObjectStorageHandler 11 | } 12 | 13 | func NewS3Handler(client *s3.Client, u *url.URL) (*S3Handler, error) { 14 | h := NewObjectStorageHandler( 15 | &S3ObjectStorage{ 16 | Client: client, 17 | }, 18 | u.Host, 19 | u.Path, 20 | ) 21 | 22 | return &S3Handler{ 23 | ObjectStorageHandler: h, 24 | }, nil 25 | } 26 | -------------------------------------------------------------------------------- /state/state_stdin.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "io/fs" 9 | "os" 10 | "strconv" 11 | ) 12 | 13 | var argTypeExamples = map[ArgumentType]string{ 14 | ArgumentTypeString: "some-value", 15 | ArgumentTypeInt64: "13400", 16 | ArgumentTypeFloat64: "13.4", 17 | ArgumentTypeSecret: "some-value", 18 | ArgumentTypeFile: "./path/to/file.txt", 19 | ArgumentTypeFS: "./path/to/folder", 20 | } 21 | 22 | type StdinReader struct { 23 | out io.Writer 24 | in io.Reader 25 | } 26 | 27 | func NewStdinReader(in io.Reader, out io.Writer) *StdinReader { 28 | return &StdinReader{ 29 | out: out, 30 | in: in, 31 | } 32 | } 33 | 34 | func (s *StdinReader) Get(arg Argument) (string, error) { 35 | fmt.Fprintf(s.out, "Argument '%[1]s' requested but not found. Please provide a value for '%[1]s' of type '%s'. Example: '%s': ", arg.Key, arg.Type.String(), argTypeExamples[arg.Type]) 36 | // Prompt for the value via stdin since it was not found 37 | scanner := bufio.NewScanner(s.in) 38 | scanner.Scan() 39 | 40 | if err := scanner.Err(); err != nil { 41 | return "", err 42 | } 43 | 44 | value := scanner.Text() 45 | fmt.Fprintf(s.out, "In the future, you can provide this value with the '-arg=%s=%s' argument\n", arg.Key, value) 46 | return value, nil 47 | } 48 | 49 | func (s *StdinReader) GetString(ctx context.Context, arg Argument) (string, error) { 50 | val, err := s.Get(arg) 51 | if err != nil { 52 | return "", err 53 | } 54 | 55 | return val, nil 56 | } 57 | 58 | func (s *StdinReader) GetDirectoryString(ctx context.Context, arg Argument) (string, error) { 59 | val, err := s.Get(arg) 60 | if err != nil { 61 | return "", err 62 | } 63 | 64 | return val, nil 65 | } 66 | 67 | func (s *StdinReader) GetInt64(ctx context.Context, arg Argument) (int64, error) { 68 | val, err := s.Get(arg) 69 | if err != nil { 70 | return 0, err 71 | } 72 | 73 | return strconv.ParseInt(val, 10, 64) 74 | } 75 | 76 | func (s *StdinReader) GetFloat64(ctx context.Context, arg Argument) (float64, error) { 77 | val, err := s.Get(arg) 78 | if err != nil { 79 | return 0, err 80 | } 81 | 82 | return strconv.ParseFloat(val, 10) 83 | } 84 | 85 | func (s *StdinReader) GetBool(ctx context.Context, arg Argument) (bool, error) { 86 | val, err := s.Get(arg) 87 | if err != nil { 88 | return false, err 89 | } 90 | 91 | return strconv.ParseBool(val) 92 | } 93 | 94 | func (s *StdinReader) GetFile(ctx context.Context, arg Argument) (*os.File, error) { 95 | val, err := s.Get(arg) 96 | if err != nil { 97 | return nil, err 98 | } 99 | 100 | return os.Open(val) 101 | } 102 | 103 | func (s *StdinReader) GetDirectory(ctx context.Context, arg Argument) (fs.FS, error) { 104 | val, err := s.Get(arg) 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | return os.DirFS(val), nil 110 | } 111 | 112 | // Since the StdinReader can read any state value, it's better if we assume that if it's being used, then it wasn't found in other reasonable state managers. 113 | func (s *StdinReader) Exists(ctx context.Context, arg Argument) (bool, error) { 114 | return false, nil 115 | } 116 | -------------------------------------------------------------------------------- /state/without.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | // Without returns a list of Arguments equal to `args` without the args in the `exclude` list. 4 | // complexity o(n^2) 5 | func Without(args []Argument, exclude []Argument) []Argument { 6 | ret := []Argument{} 7 | for _, v := range args { 8 | found := false 9 | for _, excl := range exclude { 10 | if v == excl { 11 | found = true 12 | } 13 | } 14 | if !found { 15 | ret = append(ret, v) 16 | } 17 | } 18 | 19 | return ret 20 | } 21 | 22 | // EqualArgs checks that the argument list a and b are equal. 23 | // We go out of our way to check equality despite the order. 24 | // complexity o(n^2) 25 | func EqualArgs(a []Argument, b []Argument) bool { 26 | if len(a) != len(b) { 27 | return false 28 | } 29 | 30 | for i := range a { 31 | found := false 32 | for n := range b { 33 | if a[i] == b[n] { 34 | found = true 35 | } 36 | } 37 | if !found { 38 | return false 39 | } 40 | } 41 | 42 | return true 43 | } 44 | -------------------------------------------------------------------------------- /state/without_test.go: -------------------------------------------------------------------------------- 1 | package state_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | "github.com/grafana/scribe/state" 8 | ) 9 | 10 | func TestEqual(t *testing.T) { 11 | var ( 12 | arg1 = state.NewInt64Argument("1") 13 | arg2 = state.NewStringArgument("2") 14 | arg3 = state.NewDirectoryArgument("3") 15 | arg4 = state.NewFileArgument("4") 16 | ) 17 | t.Run("simpel equality check", func(t *testing.T) { 18 | a := []state.Argument{arg1, arg2, arg3, arg4} 19 | b := []state.Argument{arg1, arg2, arg3, arg4} 20 | 21 | if !state.EqualArgs(a, b) { 22 | t.Errorf("%+v != %+v", a, b) 23 | } 24 | }) 25 | t.Run("unequal ordering, same elements", func(t *testing.T) { 26 | a := []state.Argument{arg4, arg3, arg2, arg1} 27 | b := []state.Argument{arg1, arg2, arg3, arg4} 28 | 29 | if !state.EqualArgs(a, b) { 30 | t.Errorf("%+v != %+v", a, b) 31 | } 32 | }) 33 | } 34 | 35 | func TestWithout(t *testing.T) { 36 | var ( 37 | arg1 = state.NewInt64Argument("1") 38 | arg2 = state.NewInt64Argument("2") 39 | arg3 = state.NewInt64Argument("3") 40 | arg4 = state.NewInt64Argument("4") 41 | arg5 = state.NewInt64Argument("5") 42 | arg6 = state.NewInt64Argument("6") 43 | 44 | in = []state.Argument{arg1, arg2, arg3, arg4, arg5, arg6} 45 | ) 46 | 47 | t.Run("simple removal", func(t *testing.T) { 48 | res := state.Without(in, []state.Argument{arg4, arg5, arg6}) 49 | ex := []state.Argument{arg1, arg2, arg3} 50 | if !state.EqualArgs(res, ex) { 51 | t.Errorf("%v != %v", res, ex) 52 | } 53 | }) 54 | 55 | t.Run("complex removal", func(t *testing.T) { 56 | res := state.Without(in, []state.Argument{arg1, arg3, arg5}) 57 | ex := []state.Argument{arg2, arg4, arg6} 58 | if !state.EqualArgs(res, ex) { 59 | t.Errorf("%v != %v", res, ex) 60 | } 61 | }) 62 | 63 | t.Run("removal with more exclusions than in the list", func(t *testing.T) { 64 | res := state.Without(in, []state.Argument{arg1, arg2, arg3, arg5, arg6}) 65 | ex := []state.Argument{arg4} 66 | if !state.EqualArgs(res, ex) { 67 | t.Errorf("%v != %v", res, ex) 68 | } 69 | }) 70 | 71 | t.Run("2022-10-12 bug, ArgumentSourceFS not being excluded", func(t *testing.T) { 72 | // '[{... Key:source}]' without '[{... Key:build-id} {... Key:source} {... Key:pipeline-go-mod} {... Key:docker-socket} {... Key:workdir}]' is '[{... Key:source}]' 73 | in := []state.Argument{pipeline.ArgumentSourceFS} 74 | ex := []state.Argument{} 75 | res := state.Without(in, []state.Argument{pipeline.ArgumentBuildID, pipeline.ArgumentSourceFS, pipeline.ArgumentPipelineGoModFS, pipeline.ArgumentDockerSocketFS}) 76 | if !state.EqualArgs(ex, res) { 77 | t.Errorf("%v != %v", res, ex) 78 | } 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /stringutil/doc.go: -------------------------------------------------------------------------------- 1 | // Package stringutil contains general string utilities used throughout this project. 2 | // This package should not import any other packages in this repository. 3 | package stringutil 4 | -------------------------------------------------------------------------------- /stringutil/random.go: -------------------------------------------------------------------------------- 1 | package stringutil 2 | 3 | import "math/rand" 4 | 5 | // AllowedCharacters is the list of characters that can be used in the 'Random' function. 6 | var AllowedCharacters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 7 | 8 | // Random returns a string of a random length using only characters in the package-level variable 'AllowedCharacters'. 9 | func Random(length int) string { 10 | b := make([]rune, length) 11 | for i := range b { 12 | b[i] = AllowedCharacters[rand.Intn(len(AllowedCharacters))] 13 | } 14 | 15 | return string(b) 16 | } 17 | -------------------------------------------------------------------------------- /stringutil/slugify.go: -------------------------------------------------------------------------------- 1 | package stringutil 2 | 3 | import "strings" 4 | 5 | // Slugify removes typically illegal characters for use in identifiers 6 | func Slugify(s string) string { 7 | s = strings.TrimSpace(s) 8 | s = strings.ReplaceAll(s, " ", "-") 9 | s = strings.ReplaceAll(s, ".", "") 10 | s = strings.ReplaceAll(s, ",", "") 11 | s = strings.ReplaceAll(s, "-", "_") 12 | 13 | return s 14 | } 15 | -------------------------------------------------------------------------------- /swfs/copy.go: -------------------------------------------------------------------------------- 1 | package swfs 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "io/fs" 7 | "os" 8 | "path/filepath" 9 | ) 10 | 11 | func CopyFile(from, to string) error { 12 | r, err := os.Open(from) 13 | if err != nil { 14 | return err 15 | } 16 | defer r.Close() 17 | 18 | return CopyFileReader(r, to) 19 | } 20 | 21 | func CopyFileReader(r io.Reader, to string) error { 22 | info, err := os.Stat(filepath.Dir(to)) 23 | if err != nil { 24 | if errors.Is(err, fs.ErrNotExist) { 25 | if err := os.MkdirAll(filepath.Dir(to), 0755); err != nil { 26 | return err 27 | } 28 | } else { 29 | return err 30 | } 31 | } 32 | 33 | if info != nil { 34 | if !info.IsDir() { 35 | return errors.New("not a directory") 36 | } 37 | } 38 | 39 | w, err := os.OpenFile(to, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) 40 | if err != nil { 41 | return err 42 | } 43 | defer w.Close() 44 | 45 | if _, err := io.Copy(w, r); err != nil { 46 | return err 47 | } 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /swfs/copy_test.go: -------------------------------------------------------------------------------- 1 | package swfs_test 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/grafana/scribe/swfs" 11 | ) 12 | 13 | func TestCopy(t *testing.T) { 14 | var ( 15 | content = `test file` 16 | tmp = t.TempDir() 17 | from = filepath.Join(tmp, "test-from.txt") 18 | to = filepath.Join(tmp, "test-to.txt") 19 | ) 20 | 21 | f, err := os.Create(from) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | defer f.Close() 26 | 27 | if _, err := io.WriteString(f, content); err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | if err := swfs.CopyFile(from, to); err != nil { 32 | t.Fatal(err) 33 | } 34 | 35 | r, err := os.Open(to) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | 40 | b, err := ioutil.ReadAll(r) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | if content != string(b) { 46 | t.Fatalf("Copied file did not have expected content.\nExpected: '%s'\nReceived: '%s'", content, string(b)) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /swfs/equal.go: -------------------------------------------------------------------------------- 1 | package swfs 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "io/fs" 8 | ) 9 | 10 | func equalFiles(a fs.FS, b fs.FS, path string) error { 11 | fileA, err := a.Open(path) 12 | if err != nil { 13 | return err 14 | } 15 | defer fileA.Close() 16 | 17 | fileAInfo, err := fileA.Stat() 18 | if err != nil { 19 | return err 20 | } 21 | 22 | fileB, err := b.Open(path) 23 | if err != nil { 24 | return err 25 | } 26 | defer fileB.Close() 27 | 28 | fileBInfo, err := fileB.Stat() 29 | if err != nil { 30 | return err 31 | } 32 | 33 | if fileAInfo.Size() != fileBInfo.Size() { 34 | return errors.New("file sizes not equal") 35 | } 36 | 37 | scannerA := bufio.NewScanner(fileA) 38 | scannerB := bufio.NewScanner(fileB) 39 | 40 | for scannerA.Scan() { 41 | if !scannerB.Scan() { 42 | return fmt.Errorf("reached end of file") 43 | } 44 | 45 | textA := scannerA.Text() 46 | textB := scannerB.Text() 47 | if textA != textB { 48 | return fmt.Errorf("lines not equal: '%s' / '%s'", textA, textB) 49 | } 50 | } 51 | 52 | return nil 53 | } 54 | 55 | var errorUnequal = "files are not equal" 56 | 57 | // Equal ensures that all files and folders in 'a' exist in 'b'. 58 | // This function will return true even if dirB contains more items than 'a', as long as 59 | // all items in 'a' exist in 'b'. 60 | func Equal(a fs.FS, b fs.FS) (bool, error) { 61 | if err := fs.WalkDir(a, ".", func(path string, d fs.DirEntry, err error) error { 62 | if path == "." { 63 | return nil 64 | } 65 | if err != nil { 66 | return nil 67 | } 68 | if d.IsDir() { 69 | return nil 70 | } 71 | if err := equalFiles(a, b, path); err != nil { 72 | return fmt.Errorf("files not equal '%s' for equality: %w", path, err) 73 | } 74 | 75 | return nil 76 | }); err != nil { 77 | return false, err 78 | } 79 | 80 | return true, nil 81 | } 82 | -------------------------------------------------------------------------------- /swfs/extract.go: -------------------------------------------------------------------------------- 1 | package swfs 2 | 3 | import ( 4 | "io" 5 | "io/fs" 6 | "os" 7 | "path/filepath" 8 | ) 9 | 10 | // CopyFS copies the filesystem, returned from "state.GetDirectory(arg)", to the destination (dst). 11 | func CopyFS(dir fs.FS, dst string) error { 12 | return fs.WalkDir(dir, ".", func(path string, d fs.DirEntry, err error) error { 13 | if err != nil { 14 | return nil 15 | } 16 | 17 | if path == "." { 18 | return nil 19 | } 20 | 21 | dst := filepath.Join(dst, path) 22 | info, err := d.Info() 23 | if err != nil { 24 | return err 25 | } 26 | 27 | if d.IsDir() { 28 | os.Mkdir(dst, info.Mode()) 29 | return nil 30 | } 31 | 32 | r, err := dir.Open(path) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | w, err := os.Create(dst) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | if _, err := io.Copy(w, r); err != nil { 43 | return err 44 | } 45 | 46 | return nil 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /swfs/extract_test.go: -------------------------------------------------------------------------------- 1 | package swfs_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/grafana/scribe/swfs" 8 | ) 9 | 10 | func TestExtract(t *testing.T) { 11 | tmp := t.TempDir() 12 | fs := os.DirFS("testdata") 13 | 14 | if err := swfs.CopyFS(fs, tmp); err != nil { 15 | t.Fatal(err) 16 | } 17 | 18 | copied := os.DirFS(tmp) 19 | equal, err := swfs.Equal(fs, copied) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | if !equal { 25 | t.Fatal("Expected copied filesystem to equal the original one") 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /swfs/hash.go: -------------------------------------------------------------------------------- 1 | package swfs 2 | 3 | import ( 4 | "crypto/sha256" 5 | "io" 6 | "io/fs" 7 | "os" 8 | ) 9 | 10 | func HashDirectory(dir string) ([]byte, error) { 11 | fs := os.DirFS(dir) 12 | return HashFS(fs) 13 | } 14 | 15 | func HashFS(dir fs.FS) ([]byte, error) { 16 | hashes := map[string][]byte{} 17 | 18 | // Hash each file, add the hash to the map above 19 | if err := fs.WalkDir(dir, ".", func(path string, d fs.DirEntry, err error) error { 20 | if err != nil { 21 | return nil 22 | } 23 | 24 | if path == "." { 25 | return nil 26 | } 27 | 28 | if d.IsDir() { 29 | return nil 30 | } 31 | 32 | f, err := dir.Open(path) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | defer f.Close() 38 | 39 | b, err := HashFile(f) 40 | if err != nil { 41 | return err 42 | } 43 | hashes[path] = b 44 | return nil 45 | }); err != nil { 46 | return nil, err 47 | } 48 | 49 | // For each hashed file, add it to one big hash 50 | hash := sha256.New() 51 | for k, b := range hashes { 52 | hash.Sum([]byte(k)) 53 | hash.Sum(b) 54 | } 55 | 56 | return hash.Sum(nil), nil 57 | } 58 | 59 | func HashFile(r io.Reader) ([]byte, error) { 60 | hash := sha256.New() 61 | 62 | _, err := io.Copy(hash, r) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | return hash.Sum(nil), nil 68 | } 69 | -------------------------------------------------------------------------------- /swfs/hash_test.go: -------------------------------------------------------------------------------- 1 | package swfs_test 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/grafana/scribe/swfs" 11 | ) 12 | 13 | func TestHashFile(t *testing.T) { 14 | file := `{ 15 | "example": "json", 16 | "file": [] 17 | }` 18 | 19 | expect := "c443cfbc0ed9c6097346eed4fe6581999c82032f2d81065e095fd407a1d20fd7" 20 | 21 | buf := bytes.NewBufferString(file) 22 | b, err := swfs.HashFile(buf) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | 27 | hash := hex.EncodeToString(b) 28 | if hash != expect { 29 | t.Fatalf("Unexpected result from hashfile:\nExpected: '%s'\nReceived: '%s'", expect, hash) 30 | } 31 | } 32 | 33 | func TestEncodeDir(t *testing.T) { 34 | dir := filepath.Clean("testdata") 35 | 36 | b, err := swfs.HashDirectory(dir) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | hash := sha256.New() 42 | hash.Sum([]byte("a.json")) 43 | hash.Sum([]byte("e8f1fae1d192acff9666ffb429757fb60cb92dfa39e3ac074777fd01e1bfabbf")) 44 | hash.Sum([]byte("b.json")) 45 | hash.Sum([]byte("497ec934da3f4dc5708e4be58a11f72224b23127b8b402256c114a892ae2aba2")) 46 | hash.Sum([]byte("c/c.json")) 47 | hash.Sum([]byte("bbd82e48900b9f9bbe1a00eca6a9ec646eb7126a442dc60b6dd0255de6abd48c")) 48 | 49 | expect := hash.Sum(nil) 50 | 51 | if !bytes.Equal(b, expect) { 52 | t.Fatalf("Unexpected result from HashDirectory:\nExpected: '%x'\nReceived: '%x'", expect, b) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /swfs/testdata/a.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "a", 3 | "data": [] 4 | } 5 | -------------------------------------------------------------------------------- /swfs/testdata/b.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "b", 3 | "data": ["some", "sample", "data", {}] 4 | } 5 | -------------------------------------------------------------------------------- /swfs/testdata/c/c.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "c", 3 | "data": [{}, {}, {}, {}] 4 | } 5 | -------------------------------------------------------------------------------- /swhttp/default_client.go: -------------------------------------------------------------------------------- 1 | package swhttp 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | ) 7 | 8 | var DefaultClient = http.Client{ 9 | Timeout: time.Minute * 5, 10 | } 11 | -------------------------------------------------------------------------------- /swhttp/download.go: -------------------------------------------------------------------------------- 1 | package swhttp 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "net/http" 8 | ) 9 | 10 | // Download downloads the file at the provided URL using the default client. It returns the response in a bytes.Buffer. 11 | func Download(ctx context.Context, url string) (*bytes.Buffer, error) { 12 | return DownloadWithClient(ctx, DefaultClient, url) 13 | } 14 | 15 | // DownloadWithClient downloads the file at the provided URL using the provided client. It returns the response in a bytes.Buffer if successful. 16 | func DownloadWithClient(ctx context.Context, client http.Client, url string) (*bytes.Buffer, error) { 17 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | res, err := client.Do(req) 23 | if err := HandleResponse(res, err); err != nil { 24 | return nil, err 25 | } 26 | 27 | buf := bytes.NewBuffer(nil) 28 | if _, err := io.Copy(buf, res.Body); err != nil { 29 | return nil, err 30 | } 31 | 32 | return buf, nil 33 | } 34 | -------------------------------------------------------------------------------- /swhttp/download_test.go: -------------------------------------------------------------------------------- 1 | package swhttp_test 2 | 3 | import "testing" 4 | 5 | func TestDownload(t *testing.T) { 6 | } 7 | -------------------------------------------------------------------------------- /swhttp/request.go: -------------------------------------------------------------------------------- 1 | package swhttp 2 | -------------------------------------------------------------------------------- /swhttp/response.go: -------------------------------------------------------------------------------- 1 | package swhttp 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | // HandleResponse is a utility function for standardizing failed responses. It checks the HTTP status code for a success response (200-299), and will attach the body of the response in the event of a non-200 response. 10 | // This should be called immediately after an HTTP request, rather than checking immediately for the error. 11 | func HandleResponse(res *http.Response, err error) error { 12 | if err != nil { 13 | return err 14 | } 15 | 16 | if res.StatusCode != http.StatusOK { 17 | b, err := io.ReadAll(res.Body) 18 | if err != nil { 19 | return fmt.Errorf("non-200 response: %s. Error reading response body: %w", res.Status, err) 20 | } 21 | 22 | return fmt.Errorf("non-200 response: %s. body: '%s'", res.Status, string(b)) 23 | } 24 | 25 | return nil 26 | } 27 | -------------------------------------------------------------------------------- /swhttp/response_test.go: -------------------------------------------------------------------------------- 1 | package swhttp_test 2 | 3 | import "testing" 4 | 5 | // type Response struct { 6 | // Status string // e.g. "200 OK" 7 | // StatusCode int // e.g. 200 8 | // Proto string // e.g. "HTTP/1.0" 9 | // ProtoMajor int // e.g. 1 10 | // ProtoMinor int // e.g. 0 11 | // Header Header 12 | // Body io.ReadCloser 13 | // ContentLength int64 14 | // TransferEncoding []string 15 | // Close bool 16 | // Uncompressed bool 17 | // Trailer Header 18 | // Request *Request 19 | // TLS *tls.ConnectionState 20 | // } 21 | func TestHandleResponse(t *testing.T) { 22 | 23 | } 24 | -------------------------------------------------------------------------------- /syncutil/docs.go: -------------------------------------------------------------------------------- 1 | // Package syncutil provides utilities for working with asynchronous tasks and provides wrappers around the "sync" package. 2 | package syncutil 3 | -------------------------------------------------------------------------------- /syncutil/pipelinewaitgroup.go: -------------------------------------------------------------------------------- 1 | package syncutil 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | ) 8 | 9 | // PipelineWaitGroup is a wrapper around a WaitGroup that runs the actions of a list of steps, handles errors, and watches for context cancellation. 10 | type PipelineWaitGroup struct { 11 | wg *WaitGroup 12 | } 13 | 14 | // Add adds a new Action to the waitgroup. The provided function will be run in parallel with all other added functions. 15 | func (w *PipelineWaitGroup) Add(f pipeline.Pipeline, walker *pipeline.Collection, wf pipeline.StepWalkFunc) { 16 | w.wg.Add(func(ctx context.Context) error { 17 | return walker.WalkSteps(ctx, f.ID, wf) 18 | }) 19 | } 20 | 21 | // Wait runs all provided functions (via Add(...)) and runs them in parallel and waits for them to finish. 22 | // If they are not all finished before the provided timeout (via NewPipelineWaitGroup), then an error is returned. 23 | // If any functions return an error, the first error encountered is returned. 24 | func (w *PipelineWaitGroup) Wait(ctx context.Context) error { 25 | return w.wg.Wait(ctx) 26 | } 27 | 28 | func NewPipelineWaitGroup() *PipelineWaitGroup { 29 | return &PipelineWaitGroup{ 30 | wg: NewWaitGroup(), 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /syncutil/stepwaitgroup.go: -------------------------------------------------------------------------------- 1 | package syncutil 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | ) 8 | 9 | // StepWaitGroup is a wrapper around a WaitGroup that runs the actions of a list of steps, handles errors, and watches for context cancellation. 10 | type StepWaitGroup struct { 11 | wg *WaitGroup 12 | } 13 | 14 | // Add adds a new Action to the waitgroup. The provided function will be run in parallel with all other added functions. 15 | func (w *StepWaitGroup) Add(f pipeline.Step, opts pipeline.ActionOpts) { 16 | w.wg.Add(func(ctx context.Context) error { 17 | return f.Action(ctx, opts) 18 | }) 19 | } 20 | 21 | // Wait runs all provided functions (via Add(...)) and runs them in parallel and waits for them to finish. 22 | // If they are not all finished before the provided timeout (via NewStepWaitGroup), then an error is returned. 23 | // If any functions return an error, the first error encountered is returned. 24 | func (w *StepWaitGroup) Wait(ctx context.Context) error { 25 | return w.wg.Wait(ctx) 26 | } 27 | 28 | func NewStepWaitGroup() *StepWaitGroup { 29 | return &StepWaitGroup{ 30 | wg: NewWaitGroup(), 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /syncutil/waitgroup.go: -------------------------------------------------------------------------------- 1 | package syncutil 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | ) 8 | 9 | type WaitGroupFunc func(context.Context) error 10 | 11 | type WaitGroup struct { 12 | funcs []WaitGroupFunc 13 | 14 | wg *sync.WaitGroup 15 | } 16 | 17 | func (w *WaitGroup) Add(f WaitGroupFunc) { 18 | w.funcs = append(w.funcs, f) 19 | } 20 | 21 | func (w *WaitGroup) Wait(ctx context.Context) error { 22 | var ( 23 | doneChan = make(chan bool) 24 | errChan = make(chan error) 25 | ) 26 | 27 | w.wg.Add(len(w.funcs)) 28 | 29 | for _, v := range w.funcs { 30 | go func(f WaitGroupFunc) { 31 | if err := f(ctx); err != nil { 32 | errChan <- err 33 | } 34 | 35 | w.wg.Done() 36 | }(v) 37 | } 38 | 39 | go func() { 40 | w.wg.Wait() 41 | doneChan <- true 42 | }() 43 | 44 | select { 45 | case err := <-ctx.Done(): 46 | return fmt.Errorf("%w: %s", context.Canceled, err) 47 | case <-doneChan: 48 | return nil 49 | case err := <-errChan: 50 | return err 51 | } 52 | 53 | } 54 | 55 | func NewWaitGroup() *WaitGroup { 56 | return &WaitGroup{ 57 | funcs: []WaitGroupFunc{}, 58 | wg: &sync.WaitGroup{}, 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /tarfs/testdir/a.txt: -------------------------------------------------------------------------------- 1 | test file A 2 | -------------------------------------------------------------------------------- /tarfs/testdir/folder-1/folder-4/b.txt: -------------------------------------------------------------------------------- 1 | test file b 2 | -------------------------------------------------------------------------------- /tarfs/testdir/folder-3/c.txt: -------------------------------------------------------------------------------- 1 | test file c 2 | -------------------------------------------------------------------------------- /tarfs/untar.go: -------------------------------------------------------------------------------- 1 | package tarfs 2 | 3 | import ( 4 | "archive/tar" 5 | "compress/gzip" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "io/fs" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | ) 14 | 15 | // validate validates the name of the file being extracted to ensure that it does not have illegal directory traversal names before extracting. 16 | // Courtesy of https://snyk.io/research/zip-slip-vulnerability. 17 | func validate(name, dst string) error { 18 | dstPath := filepath.Join(dst, name) 19 | if !strings.HasPrefix(dstPath, filepath.Clean(dst)+string(os.PathSeparator)) { 20 | return fmt.Errorf("%s: illegal file path", name) 21 | } 22 | 23 | return nil 24 | } 25 | 26 | // Untar unarchives and uncompresses a gzipped archive into a path. 27 | func Untar(path string, r io.Reader) error { 28 | gr, err := gzip.NewReader(r) 29 | if err != nil { 30 | return fmt.Errorf("error creating gzip reader: %w", err) 31 | } 32 | 33 | defer gr.Close() 34 | 35 | tr := tar.NewReader(gr) 36 | 37 | for { 38 | header, err := tr.Next() 39 | if err != nil { 40 | if err == io.EOF { 41 | // Then there's no more to read 42 | return nil 43 | } 44 | } 45 | if header == nil { 46 | continue 47 | } 48 | 49 | if err := validate(header.Name, path); err != nil { 50 | return err 51 | } 52 | 53 | // use header.Name because we manually set it before and includes the directory name. 54 | path := filepath.Join(path, header.Name) 55 | if header.Typeflag == tar.TypeDir { 56 | if _, err := os.Stat(path); err != nil { 57 | if err := os.MkdirAll(path, header.FileInfo().Mode()); err != nil { 58 | return err 59 | } 60 | } 61 | } 62 | 63 | if header.Typeflag == tar.TypeReg { 64 | dir := filepath.Dir(path) 65 | if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) { 66 | if err := os.MkdirAll(dir, 0755); err != nil { 67 | return err 68 | } 69 | } 70 | f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, header.FileInfo().Mode()) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | if _, err := io.Copy(f, tr); err != nil { 76 | return err 77 | } 78 | 79 | // manually close here after each file operation; defering would cause each file close 80 | // to wait until all operations have completed. 81 | f.Close() 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /tarfs/untar_test.go: -------------------------------------------------------------------------------- 1 | package tarfs_test 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io/fs" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | 11 | "github.com/grafana/scribe/tarfs" 12 | ) 13 | 14 | func ensureEqualFS(t *testing.T, a fs.FS, b fs.FS) { 15 | t.Helper() 16 | 17 | fs.WalkDir(a, ".", func(path string, d fs.DirEntry, err error) error { 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | info, err := fs.Stat(b, path) 22 | if errors.Is(err, fs.ErrNotExist) { 23 | t.Fatal("Expected file or folder at", path, "but was not found", err) 24 | } 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | if info.IsDir() { 29 | // Nothing left to do with directories 30 | return nil 31 | } 32 | if _, err := b.Open(path); err != nil { 33 | t.Fatal(err) 34 | } 35 | return nil 36 | }) 37 | } 38 | 39 | func TestUntar(t *testing.T) { 40 | dir := os.DirFS("testdir") 41 | buf := bytes.NewBuffer(nil) 42 | if err := tarfs.Write(buf, dir); err != nil { 43 | t.Fatal(err) 44 | } 45 | 46 | tmp := t.TempDir() 47 | out := filepath.Join(tmp, "testdir") 48 | if err := tarfs.Untar(out, buf); err != nil { 49 | t.Fatal(err) 50 | } 51 | 52 | ensureEqualFS(t, dir, os.DirFS(out)) 53 | } 54 | -------------------------------------------------------------------------------- /tarfs/write.go: -------------------------------------------------------------------------------- 1 | package tarfs 2 | 3 | import ( 4 | "archive/tar" 5 | "compress/gzip" 6 | "io" 7 | "io/fs" 8 | "os" 9 | ) 10 | 11 | // WriteFile writes the filesystem (dir) into a gzipped tar archive at the name provided. 12 | // This function closes the File. 13 | func WriteFile(name string, dir fs.FS) (*os.File, error) { 14 | file, err := os.Create(name) 15 | if err != nil { 16 | return nil, err 17 | } 18 | defer file.Close() 19 | if err := Write(file, dir); err != nil { 20 | return nil, err 21 | } 22 | 23 | return file, nil 24 | } 25 | 26 | // Write writes the filesystem (dir) into a gzipped tar archive in the writer provided. 27 | func Write(writer io.Writer, dir fs.FS) error { 28 | gzw := gzip.NewWriter(writer) 29 | defer gzw.Close() 30 | 31 | tw := tar.NewWriter(gzw) 32 | defer tw.Close() 33 | 34 | return fs.WalkDir(dir, ".", func(path string, d fs.DirEntry, err error) error { 35 | // If there's something that we can't package, like maybe a symbolic link, we should probably return an error. 36 | // In the future, should we try allow the user to define what to do? 37 | if err != nil { 38 | return err 39 | } 40 | if path == "." { 41 | return nil 42 | } 43 | 44 | info, err := fs.Stat(dir, path) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | h, err := tar.FileInfoHeader(info, path) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | h.Name = path 55 | 56 | if err := tw.WriteHeader(h); err != nil { 57 | return err 58 | } 59 | 60 | if !info.IsDir() { 61 | file, err := dir.Open(path) 62 | if err != nil { 63 | return err 64 | } 65 | if _, err := io.Copy(tw, file); err != nil { 66 | return err 67 | } 68 | file.Close() 69 | } 70 | 71 | return nil 72 | }) 73 | } 74 | -------------------------------------------------------------------------------- /tarfs/write_test.go: -------------------------------------------------------------------------------- 1 | package tarfs_test 2 | 3 | import ( 4 | "archive/tar" 5 | "compress/gzip" 6 | "io" 7 | "io/fs" 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | 12 | tarfs "github.com/grafana/scribe/tarfs" 13 | ) 14 | 15 | func TestWrite(t *testing.T) { 16 | tmp := t.TempDir() 17 | dir := os.DirFS("testdir") 18 | 19 | path := filepath.Join(tmp, "test.tar.gz") 20 | _, err := tarfs.WriteFile(path, dir) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | f, err := os.Open(path) 26 | if err != nil { 27 | t.Fatal("expected file to be openable, but enountered an error", err) 28 | } 29 | defer f.Close() 30 | gz, err := gzip.NewReader(f) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | defer gz.Close() 35 | tr := tar.NewReader(gz) 36 | 37 | fs.WalkDir(dir, ".", func(path string, d fs.DirEntry, err error) error { 38 | if err != nil { 39 | t.Fatalf("did not expect un-walkable path (%s): %s", path, err) 40 | } 41 | 42 | if path == "." { 43 | return nil 44 | } 45 | 46 | if _, err := fs.Stat(dir, path); err != nil { 47 | t.Fatalf("did not expect error from fs.Stat (%s): %s", path, err) 48 | } 49 | 50 | // if info.IsDir() { 51 | // return nil 52 | // } 53 | 54 | h, err := tr.Next() 55 | if err != nil { 56 | t.Fatalf("did not expect error from getting next file header (%s): %s / %t", path, err, err == io.EOF) 57 | } 58 | 59 | if h.Name != path { 60 | t.Fatalf("Expected file '%s' in archive, but got '%s'", path, h.Name) 61 | } 62 | return nil 63 | }) 64 | } 65 | -------------------------------------------------------------------------------- /testutil/errors.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | ) 7 | 8 | func EnsureError(t *testing.T, err, expect error) { 9 | t.Helper() 10 | if err == nil && expect != nil { 11 | t.Fatal("Expected error but none was received") 12 | } 13 | 14 | if expect == nil && err != nil { 15 | t.Fatalf("Expected no error but received '%s'", err.Error()) 16 | } 17 | 18 | if !errors.Is(err, expect) { 19 | t.Fatalf("Expected error '%s' but received '%s'", expect.Error(), err.Error()) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /testutil/io.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | "testing" 8 | ) 9 | 10 | func ReadersEqual(t *testing.T, value io.Reader, expected io.Reader) { 11 | vScanner := bufio.NewScanner(value) 12 | eScanner := bufio.NewScanner(expected) 13 | i := 1 14 | for eScanner.Scan() { 15 | if !vScanner.Scan() { 16 | t.Fatal("expected has more lines than provided reader") 17 | } 18 | 19 | if !bytes.Equal(eScanner.Bytes(), vScanner.Bytes()) { 20 | t.Fatalf("[%d] Lines not equal: \n%s\n%s\n", i, string(eScanner.Bytes()), string(vScanner.Bytes())) 21 | } 22 | i++ 23 | } 24 | 25 | if vScanner.Scan() { 26 | t.Fatal("provided reader has more lines than expected") 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /testutil/pipeline.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "testing" 8 | 9 | "github.com/grafana/scribe/args" 10 | "github.com/grafana/scribe/cmd/commands" 11 | "github.com/grafana/scribe/pipeline" 12 | ) 13 | 14 | func RunPipeline(ctx context.Context, t *testing.T, path string, stdout io.Writer, stderr io.Writer, args *args.PipelineArgs) { 15 | stderrBuf := bytes.NewBuffer(nil) 16 | stdoutBuf := bytes.NewBuffer(nil) 17 | t.Log("Running pipeline with args", args) 18 | cmd := commands.Run(ctx, &commands.RunOpts{ 19 | Path: path, 20 | Stdout: io.MultiWriter(stdout, stdoutBuf), 21 | Stderr: io.MultiWriter(stderr, stderrBuf), 22 | Args: args, 23 | }) 24 | 25 | if err := cmd.Run(); err != nil { 26 | t.Fatalf("Error running pipeline. Error: '%s'\nStdout: '%s'\nStderr: '%s'\n", err, stdoutBuf.String(), stderrBuf.String()) 27 | } 28 | } 29 | 30 | // NewTestStep creates a new TestStep that emits data into the channel 'b' when the action is ran 31 | func NewTestStep(b chan bool) pipeline.Step { 32 | return pipeline.Step{ 33 | Name: "test", 34 | Action: func(context.Context, pipeline.ActionOpts) error { 35 | b <- true 36 | return nil 37 | }, 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /testutil/scribe.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe" 7 | "github.com/grafana/scribe/pipeline/clients" 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | func NewScribe(initializer scribe.InitializerFunc) *scribe.Scribe { 12 | log := logrus.New() 13 | 14 | opts := clients.CommonOpts{ 15 | Log: log, 16 | } 17 | client, _ := initializer(context.Background(), opts) 18 | 19 | return &scribe.Scribe{ 20 | Opts: opts, 21 | Client: client, 22 | Collection: scribe.NewDefaultCollection(opts), 23 | } 24 | } 25 | 26 | func NewScribeMulti(initializer scribe.InitializerFunc) *scribe.Scribe { 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /testutil/slices.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | func Int64SlicesEqual(a []int64, b []int64) bool { 4 | // sort.Slice(a, int64SortFunc(a)) 5 | // sort.Slice(b, int64SortFunc(b)) 6 | 7 | for i := range a { 8 | if a[i] != b[i] { 9 | return false 10 | } 11 | } 12 | 13 | return true 14 | } 15 | 16 | func int64SortFunc(int64s []int64) func(i, j int) bool { 17 | return func(i, j int) bool { 18 | return int64s[i] < int64s[j] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /testutil/timeout.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | // WithTimeout adds a timeout to the test function (f). If the `-timeout` flag is provided, then `f` will be called without a timeout and the `go test` command will handle the deadline. 10 | func WithTimeout(d time.Duration, f func(t *testing.T)) func(t *testing.T) { 11 | return func(t *testing.T) { 12 | if _, ok := t.Deadline(); ok { 13 | f(t) 14 | return 15 | } 16 | 17 | go func() { 18 | <-time.After(d) 19 | panic(fmt.Sprintf("timeout '%s' exceeded", d)) 20 | }() 21 | 22 | f(t) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /wrappers/log_wrapper.go: -------------------------------------------------------------------------------- 1 | package wrappers 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | "github.com/grafana/scribe/pipeline/clients" 8 | "github.com/grafana/scribe/plog" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | type LogWrapper struct { 13 | Opts clients.CommonOpts 14 | Log logrus.FieldLogger 15 | } 16 | 17 | func (l *LogWrapper) Fields(ctx context.Context, step pipeline.Step) logrus.Fields { 18 | fields := plog.DefaultFields(ctx, step, l.Opts) 19 | 20 | return fields 21 | } 22 | 23 | func (l *LogWrapper) WrapStep(step pipeline.Step) pipeline.Step { 24 | action := step.Action 25 | 26 | // Steps that provide a nil action should continue to provide a nil action. 27 | // There is nothing for us to log in the execution of this action anyways, though there is an implication that 28 | // this step may execute something that is not defined in the pipeline. 29 | if step.Action == nil { 30 | return step 31 | } 32 | 33 | step.Action = func(ctx context.Context, opts pipeline.ActionOpts) error { 34 | l.Log.WithFields(l.Fields(ctx, step)).Infoln("starting step") 35 | 36 | stdoutFields := l.Fields(ctx, step) 37 | stdoutFields["stream"] = "stdout" 38 | 39 | stderrFields := l.Fields(ctx, step) 40 | stderrFields["stream"] = "stderr" 41 | 42 | opts.Stdout = l.Log.WithFields(stdoutFields).Writer() 43 | opts.Stderr = l.Log.WithFields(stderrFields).Writer() 44 | 45 | if err := action(ctx, opts); err != nil { 46 | l.Log.WithFields(l.Fields(ctx, step)).Infoln("encountered error", err.Error()) 47 | return err 48 | } 49 | 50 | l.Log.WithFields(l.Fields(ctx, step)).Infoln("done running step without error") 51 | return nil 52 | } 53 | 54 | return step 55 | } 56 | 57 | func (l *LogWrapper) Wrap(wf pipeline.StepWalkFunc) pipeline.StepWalkFunc { 58 | return func(ctx context.Context, step pipeline.Step) error { 59 | steps := l.WrapStep(step) 60 | 61 | if err := wf(ctx, steps); err != nil { 62 | return err 63 | } 64 | return nil 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /wrappers/trace_wrapper.go: -------------------------------------------------------------------------------- 1 | package wrappers 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/scribe/pipeline" 7 | "github.com/grafana/scribe/pipeline/clients" 8 | "github.com/grafana/scribe/plog" 9 | "github.com/opentracing/opentracing-go" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | type TraceWrapper struct { 14 | Opts clients.CommonOpts 15 | Tracer opentracing.Tracer 16 | } 17 | 18 | func (l *TraceWrapper) Fields(ctx context.Context, step pipeline.Step) logrus.Fields { 19 | fields := plog.DefaultFields(ctx, step, l.Opts) 20 | 21 | return fields 22 | } 23 | 24 | func TagSpan(span opentracing.Span, opts clients.CommonOpts, step pipeline.Step) { 25 | span.SetTag("job", "scribe") 26 | span.SetTag("build_id", opts.Args.BuildID) 27 | } 28 | 29 | func (l *TraceWrapper) WrapStep(step pipeline.Step) pipeline.Step { 30 | // Steps that provide a nil action should continue to provide a nil action. 31 | // There is nothing for us to trace in the execution of this action anyways, though there is an implication that 32 | // this step may execute something that is not defined in the pipeline. 33 | if step.Action == nil { 34 | return step 35 | } 36 | 37 | action := step.Action 38 | step.Action = func(ctx context.Context, opts pipeline.ActionOpts) error { 39 | parent := opentracing.SpanFromContext(ctx) 40 | 41 | span, ctx := opentracing.StartSpanFromContextWithTracer(ctx, l.Tracer, step.Name, opentracing.ChildOf(parent.Context())) 42 | TagSpan(span, l.Opts, step) 43 | defer span.Finish() 44 | 45 | if err := action(ctx, opts); err != nil { 46 | span.SetTag("error", err) 47 | return err 48 | } 49 | 50 | return nil 51 | } 52 | 53 | return step 54 | } 55 | 56 | func (l *TraceWrapper) Wrap(wf pipeline.StepWalkFunc) pipeline.StepWalkFunc { 57 | return func(ctx context.Context, step pipeline.Step) error { 58 | steps := l.WrapStep(step) 59 | 60 | if err := wf(ctx, steps); err != nil { 61 | return err 62 | } 63 | return nil 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /yarn/run.go: -------------------------------------------------------------------------------- 1 | package yarn 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/grafana/scribe/exec" 9 | "github.com/grafana/scribe/pipeline" 10 | "github.com/grafana/scribe/state" 11 | ) 12 | 13 | var ( 14 | ArgumentYarnCache = state.NewDirectoryArgument("yarn-cache-dir") 15 | ) 16 | 17 | func InstallAction() pipeline.Action { 18 | return func(ctx context.Context, opts pipeline.ActionOpts) error { 19 | if err := exec.Run(ctx, opts, "yarn", "install"); err != nil { 20 | return err 21 | } 22 | 23 | return opts.State.SetDirectory(ctx, ArgumentYarnCache, filepath.Join(".yarn", "cache")) 24 | } 25 | } 26 | 27 | func InstallStep() pipeline.Step { 28 | return pipeline. 29 | NewStep(InstallAction()). 30 | WithName("yarn install"). 31 | Requires(pipeline.ArgumentSourceFS). 32 | Provides(ArgumentYarnCache) 33 | } 34 | 35 | func RunAction(script ...string) pipeline.Action { 36 | return func(ctx context.Context, opts pipeline.ActionOpts) error { 37 | // For now, just run the yarn command. 38 | // In the future, we can verify that the script exists in the "scripts" object. 39 | return exec.Run(ctx, opts, "yarn", script...) 40 | } 41 | } 42 | 43 | func RunStep(script ...string) pipeline.Step { 44 | n := append([]string{"yarn"}, script...) 45 | name := strings.Join(n, " ") 46 | 47 | action := RunAction(script...) 48 | 49 | return pipeline.NewStep(action). 50 | WithName(name). 51 | Requires(pipeline.ArgumentSourceFS) 52 | } 53 | --------------------------------------------------------------------------------