├── pkg ├── enhancers │ ├── azure │ │ └── testdata │ │ │ ├── file │ │ │ └── azure-org │ │ │ └── proj │ │ │ └── _apis │ │ │ └── git │ │ │ └── repositories │ │ │ └── repo │ │ │ └── items │ ├── gitlab │ │ └── testdata │ │ │ ├── pipeline.yaml │ │ │ └── group │ │ │ └── subgroup │ │ │ └── project │ │ │ └── - │ │ │ └── raw │ │ │ └── master │ │ │ └── pipeline.yaml │ ├── github │ │ ├── testdata │ │ │ └── org │ │ │ │ └── repo │ │ │ │ └── version │ │ │ │ └── file │ │ ├── github.go │ │ ├── github_test.go │ │ └── reusable_workflows.go │ ├── general │ │ ├── config │ │ │ ├── config.go │ │ │ └── common.go │ │ ├── jobs.go │ │ ├── enhancers.go │ │ └── steps.go │ ├── enhancer.go │ └── bitbucket │ │ └── bitbucket.go ├── loaders │ ├── bitbucket │ │ ├── models │ │ │ ├── caches.go │ │ │ ├── trigger.go │ │ │ ├── definitions.go │ │ │ ├── pipeline.go │ │ │ ├── services.go │ │ │ ├── global_settings.go │ │ │ ├── clone.go │ │ │ ├── common.go │ │ │ ├── artifacts.go │ │ │ ├── image.go │ │ │ ├── variable.go │ │ │ ├── build_pipelines.go │ │ │ └── step.go │ │ └── bitbucket.go │ ├── loader.go │ ├── gitlab │ │ ├── models │ │ │ ├── common │ │ │ │ ├── common.go │ │ │ │ ├── script.go │ │ │ │ ├── retry.go │ │ │ │ ├── image.go │ │ │ │ ├── rules.go │ │ │ │ ├── environment_variables.go │ │ │ │ └── include.go │ │ │ ├── job │ │ │ │ ├── needs.go │ │ │ │ ├── inherit.go │ │ │ │ ├── trigger.go │ │ │ │ ├── allow_failure.go │ │ │ │ ├── controls.go │ │ │ │ └── parallel.go │ │ │ ├── artifacts.go │ │ │ └── pipeline.go │ │ └── gitlab.go │ ├── azure │ │ ├── azure.go │ │ └── models │ │ │ ├── pool.go │ │ │ ├── strategy.go │ │ │ ├── common.go │ │ │ ├── variables.go │ │ │ ├── pipeline.go │ │ │ └── stage.go │ ├── github │ │ ├── github.go │ │ └── models │ │ │ ├── common.go │ │ │ ├── workflow.go │ │ │ ├── runner.go │ │ │ ├── step.go │ │ │ └── permissions.go │ ├── common │ │ └── models │ │ │ ├── map.go │ │ │ └── map_test.go │ └── utils │ │ └── yaml_test.go ├── models │ ├── credentials.go │ ├── metadata.go │ ├── permission.go │ ├── import.go │ ├── runner.go │ ├── trigger.go │ ├── pipeline.go │ ├── common.go │ ├── step.go │ └── job.go ├── utils │ ├── array.go │ ├── pointer.go │ ├── regexp.go │ ├── file_reference.go │ ├── http.go │ ├── map.go │ └── slice.go ├── consts │ ├── bool.go │ ├── yaml.go │ ├── outputs.go │ ├── platforms.go │ ├── os.go │ └── errors.go ├── parsers │ ├── parsers.go │ ├── azure │ │ ├── common.go │ │ ├── parameters.go │ │ ├── variables.go │ │ ├── runner.go │ │ ├── resouces.go │ │ ├── variables_test.go │ │ ├── common_test.go │ │ ├── stages.go │ │ ├── parameters_test.go │ │ ├── azure.go │ │ ├── extends.go │ │ └── extends_test.go │ ├── github │ │ ├── common.go │ │ ├── runner.go │ │ ├── github.go │ │ ├── permissions.go │ │ ├── common_test.go │ │ ├── runner_test.go │ │ └── permissions_test.go │ ├── gitlab │ │ ├── common │ │ │ ├── envs.go │ │ │ ├── runner.go │ │ │ ├── envs_test.go │ │ │ ├── script.go │ │ │ └── runner_test.go │ │ ├── triggers │ │ │ ├── expressions.go │ │ │ ├── controls.go │ │ │ └── expressions_test.go │ │ ├── job │ │ │ ├── dependencies.go │ │ │ └── job.go │ │ └── gitlab.go │ ├── bitbucket │ │ ├── bitbucket.go │ │ └── defaults.go │ └── utils │ │ ├── map.go │ │ ├── ref.go │ │ ├── ref_test.go │ │ ├── image.go │ │ ├── map_test.go │ │ └── image_test.go ├── testutils │ ├── file.go │ ├── diff.go │ ├── reference.go │ └── sort.go └── handler │ ├── azure.go │ ├── github.go │ ├── gitlab.go │ └── bitbucket.go ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── FEATURE_REQUEST.md │ └── BUG_REPORT.md ├── workflows │ └── test.yml └── pull_request_template.md ├── test ├── fixtures │ ├── azure │ │ ├── no-trigger.yaml │ │ ├── testdata │ │ │ ├── imported.yaml │ │ │ ├── azure-org │ │ │ │ └── proj │ │ │ │ │ └── _apis │ │ │ │ │ └── git │ │ │ │ │ └── repositories │ │ │ │ │ └── repo │ │ │ │ │ └── items │ │ │ │ │ └── remote_import.yml │ │ │ └── imported-stage.yaml │ │ ├── branch-list-trigger.yaml │ │ ├── pool.yaml │ │ ├── default-job.yaml │ │ ├── local-import.yaml │ │ ├── parameter-templates.yaml │ │ ├── extends.yaml │ │ ├── stages.yaml │ │ ├── variables.yaml │ │ ├── parameters.yaml │ │ ├── all-triggers.yaml │ │ ├── jobs.yaml │ │ ├── steps.yaml │ │ └── resources.yaml │ ├── gitlab │ │ ├── invalid-import.yaml │ │ ├── include-local.yaml │ │ ├── include-remote.yaml │ │ ├── trigger-include.yaml │ │ ├── trivy.yaml │ │ ├── include-multiple.yaml │ │ ├── terraform.yaml │ │ ├── build-job.yaml │ │ ├── gradle.yaml │ │ └── testdata │ │ │ └── gitlab-org │ │ │ └── gitlab │ │ │ └── - │ │ │ └── raw │ │ │ └── master │ │ │ ├── imported.yaml │ │ │ └── lib │ │ │ └── gitlab │ │ │ └── ci │ │ │ └── templates │ │ │ └── Android.gitlab-ci.yml │ ├── github │ │ ├── concurrent-jobs.yaml │ │ ├── dependant-jobs.yaml │ │ ├── token-permissions.yaml │ │ ├── runners.yaml │ │ ├── continue-on-error-jobs.yaml │ │ ├── testdata │ │ │ └── org │ │ │ │ └── repo │ │ │ │ └── main │ │ │ │ └── path.yaml │ │ ├── workflow-call.yaml │ │ ├── environment-variables.yaml │ │ ├── all-triggers.yaml │ │ ├── matrix.yaml │ │ └── steps.yaml │ └── bitbucket │ │ ├── image.yml │ │ ├── config-options.yml │ │ ├── sync-steps.yml │ │ ├── parallel-steps.yml │ │ ├── alias-nodes.yml │ │ ├── multiple-pipelines-types.yml │ │ ├── image-step.yml │ │ ├── merge-step-pipeline.yml │ │ ├── alias-pipeline.yml │ │ ├── variables-pipeline.yml │ │ ├── definitions.yml │ │ └── simple-pipeline.yml └── blackbox │ └── models.go ├── .gitignore ├── .githooks └── pre-commit ├── Makefile ├── scripts └── tag.sh └── go.mod /pkg/enhancers/azure/testdata/file: -------------------------------------------------------------------------------- 1 | file content -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @argonsecurity/supply-chain 2 | -------------------------------------------------------------------------------- /pkg/enhancers/gitlab/testdata/pipeline.yaml: -------------------------------------------------------------------------------- 1 | test data 2 | -------------------------------------------------------------------------------- /pkg/enhancers/github/testdata/org/repo/version/file: -------------------------------------------------------------------------------- 1 | file data -------------------------------------------------------------------------------- /test/fixtures/azure/no-trigger.yaml: -------------------------------------------------------------------------------- 1 | name: no-trigger 2 | trigger: none 3 | pr: none -------------------------------------------------------------------------------- /test/fixtures/gitlab/invalid-import.yaml: -------------------------------------------------------------------------------- 1 | include: https://gitlab.com/invalid.yaml 2 | -------------------------------------------------------------------------------- /pkg/enhancers/azure/testdata/azure-org/proj/_apis/git/repositories/repo/items: -------------------------------------------------------------------------------- 1 | file content -------------------------------------------------------------------------------- /pkg/enhancers/gitlab/testdata/group/subgroup/project/-/raw/master/pipeline.yaml: -------------------------------------------------------------------------------- 1 | test data 2 | -------------------------------------------------------------------------------- /test/fixtures/gitlab/include-local.yaml: -------------------------------------------------------------------------------- 1 | include: /../../test/fixtures/gitlab/gradle.yaml 2 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/caches.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Caches map[string]*string 4 | -------------------------------------------------------------------------------- /test/fixtures/gitlab/include-remote.yaml: -------------------------------------------------------------------------------- 1 | include: https://gitlab.com/gitlab-org/gitlab/-/raw/master/imported.yaml 2 | -------------------------------------------------------------------------------- /pkg/loaders/loader.go: -------------------------------------------------------------------------------- 1 | package loaders 2 | 3 | type Loader[T any] interface { 4 | Load(data []byte) (*T, error) 5 | } 6 | -------------------------------------------------------------------------------- /test/fixtures/azure/testdata/imported.yaml: -------------------------------------------------------------------------------- 1 | jobs: 2 | - job: PostBuild 3 | steps: 4 | - script: npm test 5 | -------------------------------------------------------------------------------- /pkg/models/credentials.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Credentials struct { 4 | Username string 5 | Token string 6 | } 7 | -------------------------------------------------------------------------------- /pkg/utils/array.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func IsArray(input any) bool { 4 | _, ok := input.([]any) 5 | return ok 6 | } 7 | -------------------------------------------------------------------------------- /test/fixtures/azure/branch-list-trigger.yaml: -------------------------------------------------------------------------------- 1 | name: branch-list-trigger 2 | trigger: 3 | - main 4 | - development 5 | pr: 6 | - main 7 | - develop -------------------------------------------------------------------------------- /test/fixtures/gitlab/trigger-include.yaml: -------------------------------------------------------------------------------- 1 | trivy-parent: 2 | stage: aqua 3 | trigger: 4 | include: "/../../test/fixtures/gitlab/trivy.yaml" 5 | -------------------------------------------------------------------------------- /pkg/consts/bool.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | var TrueValues = []string{"true", "on", "y", "yes"} 4 | var FalseValues = []string{"false", "off", "n", "no"} 5 | -------------------------------------------------------------------------------- /test/fixtures/azure/testdata/azure-org/proj/_apis/git/repositories/repo/items/remote_import.yml: -------------------------------------------------------------------------------- 1 | currently not supported in blackbox test due to fileserver limitation 2 | -------------------------------------------------------------------------------- /pkg/parsers/parsers.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import "github.com/argonsecurity/pipeline-parser/pkg/models" 4 | 5 | type Parser[T any] interface { 6 | Parse(*T) (*models.Pipeline, error) 7 | } 8 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/trigger.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type StepTriggerType string 4 | 5 | const ( 6 | AUTOMATIC StepTriggerType = "automatic" 7 | MANUAL StepTriggerType = "manual" 8 | ) 9 | -------------------------------------------------------------------------------- /pkg/models/metadata.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Metadata struct { 4 | Build bool `json:"build,omitempty"` 5 | Test bool `json:"test,omitempty"` 6 | Deploy bool `json:"deploy,omitempty"` 7 | } 8 | -------------------------------------------------------------------------------- /test/fixtures/github/concurrent-jobs.yaml: -------------------------------------------------------------------------------- 1 | name: concurrent-jobs 2 | jobs: 3 | job1: 4 | name: Job 1 5 | concurrency: ci 6 | 7 | job2: 8 | name: Job 2 9 | concurrency: ci 10 | -------------------------------------------------------------------------------- /test/fixtures/github/dependant-jobs.yaml: -------------------------------------------------------------------------------- 1 | name: dependable jobs 2 | 3 | jobs: 4 | dependable-job: 5 | name: Dependable Job 6 | 7 | dependant-job: 8 | name: Dependant Job 9 | needs: [dependable-job] 10 | -------------------------------------------------------------------------------- /pkg/consts/yaml.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | const ( 4 | // YAML node tags 5 | StringTag = "!!str" 6 | IntTag = "!!int" 7 | SequenceTag = "!!seq" 8 | BooleanTag = "!!bool" 9 | MapTag = "!!map" 10 | ) 11 | -------------------------------------------------------------------------------- /test/fixtures/azure/testdata/imported-stage.yaml: -------------------------------------------------------------------------------- 1 | stages: 2 | - stage: build_stage 3 | jobs: 4 | - job: build 5 | steps: 6 | - script: echo this builds the project 7 | displayName: "Build" 8 | -------------------------------------------------------------------------------- /test/fixtures/github/token-permissions.yaml: -------------------------------------------------------------------------------- 1 | name: permissions 2 | permissions: 3 | actions: read 4 | statuses: write 5 | pull-requests: read 6 | 7 | jobs: 8 | job1: 9 | name: Job 1 10 | permissions: read-all 11 | -------------------------------------------------------------------------------- /test/fixtures/gitlab/trivy.yaml: -------------------------------------------------------------------------------- 1 | trivy: 2 | image: docker.com/dev-sec-ops/aqua/aqua-scanner:latest 3 | script: 4 | - export TRIVY_RUN_AS_PLUGIN=aqua 5 | - trivy fs --skip-db-update --sast --reachability --scanners config,vuln,secret . 6 | -------------------------------------------------------------------------------- /pkg/consts/outputs.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | type OutputTarget string 4 | 5 | const ( 6 | Stdout OutputTarget = "stdout" 7 | File OutputTarget = "file" 8 | ) 9 | 10 | var OutputTargets = []OutputTarget{ 11 | Stdout, 12 | File, 13 | } 14 | -------------------------------------------------------------------------------- /pkg/testutils/file.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | func ReadFile(filepath string) []byte { 8 | data, err := os.ReadFile(filepath) 9 | if err != nil { 10 | panic(err) 11 | } 12 | 13 | return data 14 | } 15 | -------------------------------------------------------------------------------- /pkg/utils/pointer.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func GetPtr[T any](v T) *T { 4 | return &v 5 | } 6 | 7 | func GetPtrOrNil[T comparable](v T) *T { 8 | var zeroValue T 9 | if zeroValue == v { 10 | return nil 11 | } 12 | 13 | return &v 14 | } 15 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | labels: kind/feature 4 | about: I have a suggestion (and might want to implement myself)! 5 | --- 6 | 7 | -------------------------------------------------------------------------------- /test/blackbox/models.go: -------------------------------------------------------------------------------- 1 | package blackbox 2 | 3 | import "github.com/argonsecurity/pipeline-parser/pkg/models" 4 | 5 | type TestCase struct { 6 | Filename string 7 | TestdataDir string 8 | Expected *models.Pipeline 9 | ShouldFail bool 10 | } 11 | -------------------------------------------------------------------------------- /test/fixtures/azure/pool.yaml: -------------------------------------------------------------------------------- 1 | name: pool 2 | pool: 3 | name: MyPool 4 | demands: [demand1, demand2] 5 | vmImage: ubuntu-latest 6 | 7 | 8 | jobs: 9 | - job: jobId 10 | pool: 11 | name: jobPool 12 | demands: demand 13 | vmImage: ubuntu-latest -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/definitions.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Definitions struct { 4 | Caches *Caches `yaml:"caches,omitempty"` 5 | Services map[string]*Service `yaml:"services,omitempty"` 6 | Steps []*Step `yaml:"steps,omitempty"` 7 | } 8 | -------------------------------------------------------------------------------- /test/fixtures/azure/default-job.yaml: -------------------------------------------------------------------------------- 1 | name: stages 2 | 3 | stages: 4 | - stage: BuildWin 5 | displayName: Build for Windows 6 | - stage: BuildMac 7 | displayName: Build for Mac 8 | dependsOn: [] # by specifying an empty array, this stage doesn't depend on the stage before it 9 | -------------------------------------------------------------------------------- /test/fixtures/github/runners.yaml: -------------------------------------------------------------------------------- 1 | name: runners 2 | 3 | jobs: 4 | job1: 5 | name: Job 1 6 | runs-on: ubuntu-latest 7 | job2: 8 | name: Job 2 9 | runs-on: [self-hosted, windows-latest] 10 | job3: 11 | name: Job 3 12 | runs-on: [self-hosted, linux, x64] 13 | -------------------------------------------------------------------------------- /test/fixtures/gitlab/include-multiple.yaml: -------------------------------------------------------------------------------- 1 | include: 2 | - file: /imported.yaml 3 | project: gitlab-org/gitlab 4 | ref: master 5 | - https://gitlab.com/gitlab-org/gitlab/-/raw/master/imported.yaml 6 | - /../../test/fixtures/gitlab/gradle.yaml 7 | - template: Android.gitlab-ci.yml 8 | -------------------------------------------------------------------------------- /pkg/utils/regexp.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "regexp" 4 | 5 | func AnyMatch(regexes []*regexp.Regexp, s *string) bool { 6 | if s == nil { 7 | return false 8 | } 9 | 10 | for _, regexp := range regexes { 11 | if regexp.MatchString(*s) { 12 | return true 13 | } 14 | } 15 | return false 16 | } 17 | -------------------------------------------------------------------------------- /pkg/utils/file_reference.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | ) 8 | 9 | func CompareFileReferences(a, b *models.FileReference) bool { 10 | if a == nil || b == nil { 11 | return false 12 | } 13 | return reflect.DeepEqual(a, b) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type Cache struct { 4 | When string `yaml:"when,omitempty"` 5 | Key any `yaml:"key,omitempty"` 6 | Paths []string `yaml:"paths,omitempty"` 7 | Policy string `yaml:"policy,omitempty"` 8 | Untracked bool `yaml:"untracked,omitempty"` 9 | } 10 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/image.yml: -------------------------------------------------------------------------------- 1 | image: node:16 2 | 3 | pipelines: 4 | default: 5 | - step: 6 | image: 7 | name: node:16 8 | email: test@test.com 9 | username: test 10 | password: test 11 | aws: 12 | access-key: "123456" 13 | secret-key: "7891011" 14 | -------------------------------------------------------------------------------- /test/fixtures/github/continue-on-error-jobs.yaml: -------------------------------------------------------------------------------- 1 | name: continue-on-error-jobs 2 | jobs: 3 | job1: 4 | name: Job 1 5 | continue-on-error: true 6 | job2: 7 | name: Job 2 8 | continue-on-error: false 9 | job3: 10 | name: Job 3 11 | continue-on-error: "${{ inputs.continue-on-error || github.event_name == 'schedule' }}" 12 | -------------------------------------------------------------------------------- /pkg/models/permission.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | const ( 4 | PullRequestPermission = "pull-request" 5 | PushPermission = "push" 6 | RunPipelinePermission = "run-pipeline" 7 | ) 8 | 9 | type Permission struct { 10 | Read bool `json:"read,omitempty"` 11 | Write bool `json:"write,omitempty"` 12 | Admin bool `json:"admin,omitempty"` 13 | } 14 | -------------------------------------------------------------------------------- /test/fixtures/github/testdata/org/repo/main/path.yaml: -------------------------------------------------------------------------------- 1 | name: continue-on-error-jobs 2 | jobs: 3 | job1: 4 | name: Job 1 5 | continue-on-error: true 6 | job2: 7 | name: Job 2 8 | continue-on-error: false 9 | job3: 10 | name: Job 3 11 | continue-on-error: "${{ inputs.continue-on-error || github.event_name == 'schedule' }}" 12 | -------------------------------------------------------------------------------- /test/fixtures/azure/local-import.yaml: -------------------------------------------------------------------------------- 1 | trigger: none 2 | 3 | pool: 4 | vmImage: "windows-2019" 5 | 6 | stages: 7 | - template: /../../test/fixtures/azure/testdata/imported-stage.yaml 8 | parameters: 9 | name: test 10 | 11 | extends: 12 | template: ../../test/fixtures/azure/testdata/imported.yaml@self 13 | parameters: 14 | runMode: local 15 | -------------------------------------------------------------------------------- /test/fixtures/github/workflow-call.yaml: -------------------------------------------------------------------------------- 1 | name: workflow-call 2 | 3 | jobs: 4 | call-local-workflow: 5 | uses: ./../fixtures/github/dependant-jobs.yaml 6 | with: 7 | param: value 8 | secrets: 9 | test: ${{ secrets.test }} 10 | call-remote-workflow: 11 | uses: org/repo/path.yaml@main 12 | with: 13 | param: value 14 | secrets: inherit -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/pipeline.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Pipeline struct { 4 | Image *Image `yaml:"image"` 5 | Clone *Clone `yaml:"clone,omitempty"` 6 | Options *GlobalSettings `yaml:"options,omitempty"` 7 | Definitions *Definitions `yaml:"definitions,omitempty"` 8 | Pipelines *BuildPipelines `yaml:"pipelines"` 9 | } 10 | -------------------------------------------------------------------------------- /pkg/testutils/diff.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/go-test/deep" 7 | ) 8 | 9 | func DeepCompare(t *testing.T, struct1 any, struct2 any) bool { 10 | if diffs := deep.Equal(struct1, struct2); diffs != nil { 11 | for _, diff := range diffs { 12 | t.Errorf(diff) 13 | } 14 | 15 | return false 16 | } 17 | 18 | return true 19 | } 20 | -------------------------------------------------------------------------------- /test/fixtures/azure/parameter-templates.yaml: -------------------------------------------------------------------------------- 1 | name: "parameter templates" 2 | 3 | extends: 4 | template: parameters.yml 5 | parameters: 6 | foo: bar 7 | 8 | testSteps: 9 | - template: test-steps.yml 10 | parameters: 11 | foo: bar2 12 | 13 | testSteps2: 14 | - template: test-steps2.yml 15 | parameters: 16 | bar: foo 17 | -------------------------------------------------------------------------------- /test/fixtures/github/environment-variables.yaml: -------------------------------------------------------------------------------- 1 | name: environment-variables 2 | 3 | env: 4 | STRING: string 5 | NUMBER: 1 6 | 7 | jobs: 8 | job1: 9 | name: Job 1 10 | env: 11 | STRING: string 12 | NUMBER: 1 13 | steps: 14 | - name: Step 1 15 | run: command line 16 | env: 17 | STRING: string 18 | NUMBER: 1 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | __debug_bin 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | vendor/ 17 | .vscode/ 18 | pipeline-parser -------------------------------------------------------------------------------- /pkg/loaders/azure/azure.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 5 | "gopkg.in/yaml.v3" 6 | ) 7 | 8 | type AzureLoader struct{} 9 | 10 | func (g *AzureLoader) Load(data []byte) (*models.Pipeline, error) { 11 | pipeline := &models.Pipeline{} 12 | err := yaml.Unmarshal(data, pipeline) 13 | return pipeline, err 14 | } 15 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/services.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Service struct { 4 | Image *Image `yaml:"image"` 5 | Memory *int64 `yaml:"memory,omitempty"` // Memory limit for the service container, in megabytes 6 | Variables *EnvironmentVariablesRef `yaml:"variables,omitempty"` // Environment variables passed to the service container 7 | } 8 | -------------------------------------------------------------------------------- /pkg/enhancers/general/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "regexp" 4 | 5 | type ObjectiveConfiguration struct { 6 | Tasks []string 7 | Names []*regexp.Regexp 8 | ShellRegexes []*regexp.Regexp 9 | } 10 | 11 | type EnhancementConfiguration struct { 12 | Build ObjectiveConfiguration 13 | Test ObjectiveConfiguration 14 | Deploy ObjectiveConfiguration 15 | } 16 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/global_settings.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type GlobalSettings struct { 4 | Docker *bool `yaml:"docker,omitempty"` // A flag to add Docker to all build steps in all pipelines 5 | MaxTime *int64 `yaml:"max-time,omitempty"` 6 | Size *Size `yaml:"size,omitempty"` 7 | } 8 | 9 | type Size string 10 | 11 | const ( 12 | X1 Size = "1x" 13 | X2 Size = "2x" 14 | ) 15 | -------------------------------------------------------------------------------- /pkg/loaders/github/github.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/github/models" 5 | "gopkg.in/yaml.v3" 6 | ) 7 | 8 | type GitHubLoader struct{} 9 | 10 | func (g *GitHubLoader) Load(data []byte) (*models.Workflow, error) { 11 | workflow := &models.Workflow{} 12 | err := yaml.Unmarshal(data, workflow) 13 | return workflow, err 14 | } 15 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/config-options.yml: -------------------------------------------------------------------------------- 1 | clone: 2 | lfs: true 3 | enabled: true 4 | depth: 1 5 | options: 6 | max-time: 30 7 | docker: true 8 | size: 1x 9 | definitions: 10 | caches: 11 | custom-npm: node_modules 12 | services: 13 | service: 14 | memory: 128 15 | image: node:16 16 | variables: 17 | TEST: development 18 | TEST2: production 19 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/bitbucket.go: -------------------------------------------------------------------------------- 1 | package bitbucket 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/bitbucket/models" 5 | "gopkg.in/yaml.v3" 6 | ) 7 | 8 | type BitbucketLoader struct{} 9 | 10 | func (b *BitbucketLoader) Load(data []byte) (*models.Pipeline, error) { 11 | pipeline := &models.Pipeline{} 12 | err := yaml.Unmarshal(data, pipeline) 13 | return pipeline, err 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test Pipeline Parser 2 | on: [pull_request] 3 | 4 | jobs: 5 | test: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - name: Checkout 9 | uses: actions/checkout@v2 10 | 11 | - name: Install Go 12 | uses: actions/setup-go@v2 13 | with: 14 | go-version: 1.21 15 | 16 | - name: Execute Tests 17 | run: make test-coverage 18 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/clone.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Clone struct { 4 | Depth any `yaml:"depth"` // Depth of Git clones for all pipelines (supported only for Git repositories) 5 | Enabled *bool `yaml:"enabled,omitempty"` // Enables cloning of the repository 6 | LFS *bool `yaml:"lfs,omitempty"` // Enables the download of LFS files in the clone (supported only for Git repositories) 7 | } 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/BUG_REPORT.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | labels: kind/bug 4 | about: If something isn't working as expected. 5 | --- 6 | 7 | ## Description 8 | 9 | 12 | 13 | ## What did you expect to happen? 14 | 15 | 16 | ## What happened instead? 17 | 18 | 19 | ``` 20 | (paste your output here) 21 | ``` 22 | 23 | ## Additional details: -------------------------------------------------------------------------------- /pkg/loaders/gitlab/gitlab.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models" 5 | "gopkg.in/yaml.v3" 6 | ) 7 | 8 | type GitLabLoader struct{} 9 | 10 | func (g *GitLabLoader) Load(data []byte) (*models.GitlabCIConfiguration, error) { 11 | gitlabCIConfig := &models.GitlabCIConfiguration{} 12 | err := yaml.Unmarshal(data, gitlabCIConfig) 13 | return gitlabCIConfig, err 14 | } 15 | -------------------------------------------------------------------------------- /pkg/consts/platforms.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | import "github.com/argonsecurity/pipeline-parser/pkg/models" 4 | 5 | const ( 6 | GitHubPlatform models.Platform = "github" 7 | GitLabPlatform models.Platform = "gitlab" 8 | AzurePlatform models.Platform = "azure" 9 | BitbucketPlatform models.Platform = "bitbucket" 10 | ) 11 | 12 | var Platforms = []models.Platform{ 13 | GitHubPlatform, 14 | GitLabPlatform, 15 | AzurePlatform, 16 | BitbucketPlatform, 17 | } 18 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/sync-steps.yml: -------------------------------------------------------------------------------- 1 | image: node:16 2 | 3 | pipelines: 4 | pull-requests: 5 | "**": 6 | - step: 7 | name: Build and Test 8 | caches: 9 | - node 10 | script: 11 | - npm install 12 | - npm test 13 | - step: 14 | name: Code linting 15 | script: 16 | - npm install eslint 17 | - npx eslint . 18 | caches: 19 | - node 20 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/parallel-steps.yml: -------------------------------------------------------------------------------- 1 | image: node:16 2 | 3 | pipelines: 4 | default: 5 | - parallel: 6 | - step: 7 | name: Build and Test 8 | caches: 9 | - node 10 | script: 11 | - npm install 12 | - npm test 13 | - step: 14 | name: Code linting 15 | script: 16 | - npm install eslint 17 | - npx eslint . 18 | caches: 19 | - node 20 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/alias-nodes.yml: -------------------------------------------------------------------------------- 1 | definitions: 2 | caches: 3 | cypress: /root/.cache/Cypress 4 | services: 5 | docker: 6 | memory: 2048 7 | steps: 8 | - step: &install-build 9 | name: Install and build 10 | script: 11 | - yarn build 12 | artifacts: 13 | - dist/** 14 | 15 | pipelines: 16 | pull-requests: 17 | "*": 18 | - step: *install-build 19 | "**": 20 | - step: 21 | <<: *install-build 22 | name: merge test 23 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/multiple-pipelines-types.yml: -------------------------------------------------------------------------------- 1 | image: node:16 2 | 3 | pipelines: 4 | custom: 5 | notify: 6 | - step: 7 | name: Notify Teams 8 | caches: 9 | - node 10 | script: 11 | - npx notify -s "deployment" 12 | branches: 13 | master: 14 | - step: 15 | name: step 1 16 | - step: 17 | name: step 2 18 | - parallel: 19 | - step: 20 | name: step 3 21 | - step: 22 | name: step 4 23 | -------------------------------------------------------------------------------- /pkg/parsers/azure/common.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | func parseEnvironmentVariablesRef(envRef *azureModels.EnvironmentVariablesRef) *models.EnvironmentVariablesRef { 9 | if envRef == nil { 10 | return nil 11 | } 12 | 13 | return &models.EnvironmentVariablesRef{ 14 | EnvironmentVariables: envRef.EnvironmentVariables, 15 | FileReference: envRef.FileReference, 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /pkg/parsers/github/common.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | githubModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/github/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | func parseEnvironmentVariablesRef(envRef *githubModels.EnvironmentVariablesRef) *models.EnvironmentVariablesRef { 9 | if envRef == nil { 10 | return nil 11 | } 12 | 13 | return &models.EnvironmentVariablesRef{ 14 | EnvironmentVariables: envRef.EnvironmentVariables, 15 | FileReference: envRef.FileReference, 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /test/fixtures/azure/extends.yaml: -------------------------------------------------------------------------------- 1 | trigger: none 2 | 3 | pool: 4 | vmImage: "windows-2019" 5 | 6 | resources: 7 | repositories: 8 | - repository: CeTemplates 9 | type: git 10 | name: ORG/Templates 11 | 12 | extends: 13 | template: blueprints/template.yml@CeTemplates 14 | parameters: 15 | runMode: ${{parameters.runMode}} 16 | 17 | preBuildSteps: 18 | - template: /pipelines/steps/pre-build-steps.yml@self 19 | 20 | testSteps2: 21 | - template: test-steps2.yml 22 | parameters: 23 | bar: foo 24 | -------------------------------------------------------------------------------- /test/fixtures/azure/stages.yaml: -------------------------------------------------------------------------------- 1 | name: stages 2 | 3 | stages: 4 | - stage: BuildWin 5 | displayName: Build for Windows 6 | - stage: BuildMac 7 | displayName: Build for Mac 8 | dependsOn: [] # by specifying an empty array, this stage doesn't depend on the stage before it 9 | 10 | - template: stages/build.yml # Template reference 11 | parameters: 12 | param: value 13 | 14 | - template: stages/test.yml # Template reference 15 | parameters: 16 | name: Full 17 | testFile: tests/fullSuite.js 18 | 19 | - ${{ parameters.stages }} # Parameter reference -------------------------------------------------------------------------------- /test/fixtures/bitbucket/image-step.yml: -------------------------------------------------------------------------------- 1 | pipelines: 2 | pull-requests: 3 | 'master': 4 | - step: 5 | name: Run Aqua scanner 6 | image: aquasec/aqua-scanner 7 | script: 8 | - trivy fs --security-checks config,vuln,secret --sast . 9 | # To customize which severities to scan for, add the following flag: --severity UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL 10 | # To enable SAST scanning, add: --sast 11 | # To enable npm/dotnet non-lock file scanning, add: --package-json / --dotnet-proj -------------------------------------------------------------------------------- /test/fixtures/azure/variables.yaml: -------------------------------------------------------------------------------- 1 | name: variables 2 | variables: # pipeline-level 3 | - name: var1 4 | value: value1 5 | - name: var2 6 | value: value2 7 | readonly: true 8 | - group: my-group 9 | - template: variables/var.yml # Template reference 10 | parameters: 11 | param: value 12 | 13 | stages: 14 | - stage: Build 15 | variables: # stage-level 16 | STAGE_VAR: 'that happened' 17 | 18 | jobs: 19 | - job: FirstJob 20 | variables: # job-level 21 | JOB_VAR: 'a job var' 22 | steps: 23 | - script: echo $(MY_VAR) $(STAGE_VAR) $(JOB_VAR) -------------------------------------------------------------------------------- /pkg/parsers/github/runner.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/models" 5 | 6 | githubModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/github/models" 7 | ) 8 | 9 | func parseRunsOnToRunner(runsOn *githubModels.RunsOn) *models.Runner { 10 | if runsOn == nil { 11 | return nil 12 | } 13 | 14 | runner := &models.Runner{ 15 | OS: runsOn.OS, 16 | Arch: runsOn.Arch, 17 | Labels: &runsOn.Tags, 18 | SelfHosted: &runsOn.SelfHosted, 19 | FileReference: runsOn.FileReference, 20 | } 21 | return runner 22 | } 23 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/merge-step-pipeline.yml: -------------------------------------------------------------------------------- 1 | image: atlassian/default-image:3 2 | 3 | definitions: 4 | steps: 5 | - step: &test-run 6 | name: Test Run 7 | script: 8 | - echo testing... 9 | - npm run test 10 | - step: &send-result 11 | name: Send Result 12 | script: 13 | - echo sending result... 14 | - npm run send-result 15 | pipelines: 16 | branches: 17 | main: 18 | - parallel: 19 | - step: 20 | <<: *test-run 21 | name: Test 22 | - step: 23 | <<: *send-result 24 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/common/envs.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | gitlabModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models/common" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | func ParseEnvironmentVariables(environmentVariables *gitlabModels.EnvironmentVariablesRef) *models.EnvironmentVariablesRef { 9 | if environmentVariables == nil { 10 | return nil 11 | } 12 | 13 | return &models.EnvironmentVariablesRef{ 14 | FileReference: environmentVariables.FileReference, 15 | EnvironmentVariables: (map[string]any)(*environmentVariables.Variables), 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /pkg/parsers/bitbucket/bitbucket.go: -------------------------------------------------------------------------------- 1 | package bitbucket 2 | 3 | import ( 4 | bitbucketModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/bitbucket/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | type BitbucketParser struct{} 9 | 10 | func (g *BitbucketParser) Parse(bitbucketPipeline *bitbucketModels.Pipeline) (*models.Pipeline, error) { 11 | if bitbucketPipeline == nil { 12 | return nil, nil 13 | } 14 | 15 | var pipeline models.Pipeline 16 | 17 | pipeline.Defaults = parsePipelineDefaults(bitbucketPipeline) 18 | pipeline.Jobs = parseJobs(bitbucketPipeline) 19 | 20 | return &pipeline, nil 21 | } 22 | -------------------------------------------------------------------------------- /.githooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # get all the staged files in th commit 4 | STAGED_GO_FILES=$(git diff --cached --name-only -- '*.go') 5 | 6 | # check to see if this is empty 7 | if [[ $STAGED_GO_FILES == "" ]]; then 8 | echo "No Go Files to Update" 9 | # format all the staged go files 10 | else 11 | for file in $STAGED_GO_FILES; do 12 | # format the file 13 | go fmt $file 14 | # add any potential changes from the formatting to the commit 15 | git add $file 16 | done 17 | fi 18 | 19 | # execute go mod tidy 20 | go mod tidy 21 | # add potential changes from the tidy command to the commit 22 | git add go.mod go.sum 23 | -------------------------------------------------------------------------------- /pkg/enhancers/enhancer.go: -------------------------------------------------------------------------------- 1 | package enhancers 2 | 3 | import "github.com/argonsecurity/pipeline-parser/pkg/models" 4 | 5 | type ImportedPipeline struct { 6 | JobName string 7 | OriginFileReference *models.FileReference 8 | Data []byte 9 | Pipeline *models.Pipeline 10 | } 11 | 12 | type Enhancer interface { 13 | InheritParentPipelineData(parent, child *models.Pipeline) *models.Pipeline 14 | LoadImportedPipelines(data *models.Pipeline, credentials *models.Credentials, organization, baseUrl *string) ([]*ImportedPipeline, error) 15 | Enhance(data *models.Pipeline, importedPipelines []*ImportedPipeline) (*models.Pipeline, error) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/parsers/utils/map.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | commonLoaderModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/common/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | func ParseMapToParameters(mapNode commonLoaderModels.Map) []*models.Parameter { 9 | parameters := make([]*models.Parameter, 0) 10 | for _, entry := range mapNode.Values { 11 | var key = entry.Key // define key here so the pointer won't change in the loop 12 | parameters = append(parameters, &models.Parameter{ 13 | Name: &key, 14 | Value: entry.Value, 15 | FileReference: entry.FileReference, 16 | }) 17 | } 18 | 19 | return parameters 20 | } 21 | -------------------------------------------------------------------------------- /pkg/utils/http.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "github.com/imroc/req/v3" 8 | ) 9 | 10 | // ToDo: change to SetCommonBearerAuthToken and test 11 | func GetHttpClient(credentials *models.Credentials) *req.Client { 12 | client := req.C() 13 | if credentials == nil { 14 | return client 15 | } 16 | 17 | return client.SetCommonHeader("Authorization", fmt.Sprintf("token %s", credentials.Token)) 18 | } 19 | 20 | func GetHttpClientWithBasicAuth(credentials *models.Credentials) *req.Client { 21 | client := req.C() 22 | if credentials == nil { 23 | return client 24 | } 25 | 26 | return client.SetCommonBasicAuth("", credentials.Token) 27 | } 28 | -------------------------------------------------------------------------------- /test/fixtures/gitlab/terraform.yaml: -------------------------------------------------------------------------------- 1 | # To contribute improvements to CI/CD templates, please follow the Development guide at: 2 | # https://docs.gitlab.com/ee/development/cicd/templates.html 3 | # This specific template is located at: 4 | # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml 5 | 6 | stages: 7 | - validate 8 | - test 9 | - build 10 | - deploy 11 | 12 | fmt: 13 | extends: .terraform:fmt 14 | needs: [] 15 | 16 | validate: 17 | extends: .terraform:validate 18 | needs: [] 19 | 20 | build: 21 | extends: .terraform:build 22 | 23 | deploy: 24 | extends: .terraform:deploy 25 | dependencies: 26 | - build 27 | environment: 28 | name: $TF_STATE_NAME 29 | -------------------------------------------------------------------------------- /pkg/enhancers/bitbucket/bitbucket.go: -------------------------------------------------------------------------------- 1 | package bitbucket 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | type BitbucketEnhancer struct{} 9 | 10 | func (b *BitbucketEnhancer) LoadImportedPipelines(data *models.Pipeline, credentials *models.Credentials, _, _ *string) ([]*enhancers.ImportedPipeline, error) { 11 | return nil, nil 12 | } 13 | 14 | func (b *BitbucketEnhancer) Enhance(data *models.Pipeline, importedPipelines []*enhancers.ImportedPipeline) (*models.Pipeline, error) { 15 | return data, nil 16 | } 17 | 18 | func (b *BitbucketEnhancer) InheritParentPipelineData(parent, child *models.Pipeline) *models.Pipeline { 19 | return child 20 | } 21 | -------------------------------------------------------------------------------- /test/fixtures/github/all-triggers.yaml: -------------------------------------------------------------------------------- 1 | name: all-triggers 2 | on: 3 | schedule: 4 | - cron: 30 2 * * * 5 | push: 6 | branches: 7 | - master 8 | pull_request: 9 | paths-ignore: 10 | - "*/test/*" 11 | pull_request_target: 12 | paths: 13 | - "*/test/*" 14 | workflow_dispatch: 15 | inputs: 16 | workflow-input: 17 | description: "The workflow input" 18 | default: "default-value" 19 | required: true 20 | workflow_call: 21 | inputs: 22 | workflow-input: 23 | description: "The workflow input" 24 | default: "default-value" 25 | required: true 26 | workflow_run: 27 | branches-ignore: 28 | - master 29 | label: 30 | types: [created] 31 | -------------------------------------------------------------------------------- /pkg/loaders/github/models/common.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "gopkg.in/yaml.v3" 7 | ) 8 | 9 | type EnvironmentVariablesRef struct { 10 | models.EnvironmentVariables 11 | FileReference *models.FileReference 12 | } 13 | 14 | func (e *EnvironmentVariablesRef) UnmarshalYAML(node *yaml.Node) error { 15 | var env models.EnvironmentVariables 16 | if err := node.Decode(&env); err != nil { 17 | return err 18 | } 19 | 20 | e.EnvironmentVariables = env 21 | e.FileReference = loadersUtils.GetFileReference(node) 22 | e.FileReference.StartRef.Line-- // The "env" node is not accessible, this is a patch 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/parsers/azure/parameters.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | func parseParameters(parameters *azureModels.Parameters) []*models.Parameter { 9 | if parameters == nil || len(*parameters) == 0 { 10 | return nil 11 | } 12 | 13 | var parsedParameters []*models.Parameter 14 | for _, param := range *parameters { 15 | name := param.Name 16 | 17 | parsedParameters = append(parsedParameters, &models.Parameter{ 18 | Name: &name, 19 | Default: param.Default, 20 | Options: param.Values, 21 | FileReference: param.FileReference, 22 | }) 23 | } 24 | return parsedParameters 25 | } 26 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/common.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "gopkg.in/yaml.v3" 7 | ) 8 | 9 | type EnvironmentVariablesRef struct { 10 | models.EnvironmentVariables 11 | FileReference *models.FileReference 12 | } 13 | 14 | func (e *EnvironmentVariablesRef) UnmarshalYAML(node *yaml.Node) error { 15 | var env models.EnvironmentVariables 16 | if err := node.Decode(&env); err != nil { 17 | return err 18 | } 19 | 20 | e.EnvironmentVariables = env 21 | e.FileReference = loadersUtils.GetFileReference(node) 22 | e.FileReference.StartRef.Line-- // The "env" node is not accessible, this is a patch 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/testutils/reference.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import "github.com/argonsecurity/pipeline-parser/pkg/models" 4 | 5 | func CreateFileReference(l1, c1, l2, c2 int) *models.FileReference { 6 | return &models.FileReference{ 7 | StartRef: &models.FileLocation{ 8 | Line: l1, 9 | Column: c1, 10 | }, 11 | EndRef: &models.FileLocation{ 12 | Line: l2, 13 | Column: c2, 14 | }, 15 | IsAlias: false, 16 | } 17 | } 18 | 19 | func CreateAliasFileReference(l1, c1, l2, c2 int, isAlias bool) *models.FileReference { 20 | return &models.FileReference{ 21 | StartRef: &models.FileLocation{ 22 | Line: l1, 23 | Column: c1, 24 | }, 25 | EndRef: &models.FileLocation{ 26 | Line: l2, 27 | Column: c2, 28 | }, 29 | IsAlias: isAlias, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /pkg/utils/map.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func GetMapKeys[T comparable, U any](m map[T]U) []T { 4 | keys := make([]T, len(m)) 5 | i := 0 6 | for k := range m { 7 | keys[i] = k 8 | i += 1 9 | } 10 | return keys 11 | } 12 | 13 | func MapToSlice[T any, U any, K comparable](m map[K]T, cb func(k K, v T) U) []U { 14 | result := make([]U, len(m)) 15 | var i int 16 | for k, v := range m { 17 | item := cb(k, v) 18 | result[i] = item 19 | i += 1 20 | } 21 | return result 22 | } 23 | 24 | func MapToSliceErr[T any, U any, K comparable](m map[K]T, cb func(k K, v T) (U, error)) ([]U, error) { 25 | result := make([]U, len(m)) 26 | var i int 27 | for k, v := range m { 28 | item, err := cb(k, v) 29 | if err != nil { 30 | return nil, err 31 | } 32 | result[i] = item 33 | i += 1 34 | } 35 | return result, nil 36 | } 37 | -------------------------------------------------------------------------------- /test/fixtures/gitlab/build-job.yaml: -------------------------------------------------------------------------------- 1 | stages: 2 | - build 3 | 4 | python-build: 5 | only: 6 | refs: 7 | - merge_requests 8 | - /^feature-.*/ 9 | - main 10 | - api 11 | stage: build 12 | rules: 13 | - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/ 14 | script: 15 | - cd requests 16 | - python3 setup.py sdist 17 | 18 | before_script: 19 | - echo "before_script" 20 | 21 | workflow: 22 | rules: 23 | - if: $CI_PIPELINE_SOURCE == "merge_request_event" 24 | - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 25 | when: never 26 | 27 | default: 28 | artifacts: 29 | reports: 30 | secret_detection: secrets.json 31 | sast: sast.json 32 | terraform: terraform.json 33 | license_scanning: license.json 34 | dependency_scanning: dependency.json 35 | -------------------------------------------------------------------------------- /pkg/parsers/utils/ref.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | ) 8 | 9 | var ( 10 | sha1Regex = regexp.MustCompile(`[0-9a-fA-F]{40}`) 11 | semverRegex = regexp.MustCompile(`v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?`) 12 | ) 13 | 14 | func DetectVersionType(version string) models.VersionType { 15 | versionType := models.None 16 | 17 | if version != "" { 18 | if sha1Regex.MatchString(version) { 19 | versionType = models.CommitSHA 20 | } else if semverRegex.MatchString(version) { 21 | versionType = models.TagVersion 22 | } else if version == "latest" { 23 | versionType = models.Latest 24 | } else { 25 | versionType = models.BranchVersion 26 | } 27 | } 28 | 29 | return versionType 30 | } 31 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION := $(shell git describe --tags --always) 2 | LDFLAGS=-ldflags "-s -w -X=main.version=$(VERSION)" 3 | 4 | # If the first argument is "run"... 5 | ifeq (run,$(firstword $(MAKECMDGOALS))) 6 | # use the rest as arguments for "run" 7 | RUN_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) 8 | # ...and turn them into do-nothing targets 9 | $(eval $(RUN_ARGS):;@:) 10 | endif 11 | 12 | .PHONY: build 13 | build: 14 | go build $(LDFLAGS) ./cmd/pipeline-parser 15 | 16 | .PHONY: run 17 | run: 18 | go run $(LDFLAGS) ./cmd/pipeline-parser $(RUN_ARGS) 19 | 20 | .PHONY: tag 21 | tag: 22 | @LEVEL=$(LEVEL) ./scripts/tag.sh 23 | 24 | .PHONY: test 25 | test: 26 | go clean -testcache 27 | go test ./... 28 | 29 | .PHONY: test-coverage 30 | test-coverage: 31 | go clean -testcache 32 | go test -coverprofile=coverage.out -covermode=atomic -v ./... -------------------------------------------------------------------------------- /test/fixtures/github/matrix.yaml: -------------------------------------------------------------------------------- 1 | name: matrix 2 | 3 | jobs: 4 | matrix-job: 5 | strategy: 6 | matrix: 7 | artifact: 8 | [ 9 | docker/image, 10 | docker/tar, 11 | go, 12 | java, 13 | node, 14 | php, 15 | python/tar, 16 | python/wheel, 17 | ruby/gemspec, 18 | ] 19 | os: [ubuntu-latest, macos-latest, windows-latest] 20 | include: 21 | - os: ubuntu-latest 22 | artifact: docker/image 23 | exclude: 24 | - os: ubuntu-latest 25 | artifact: docker/tar 26 | 27 | steps: 28 | - name: task without params 29 | uses: actions/checkout@v1 30 | 31 | - name: task with params 32 | uses: actions/checkout@v1 33 | with: 34 | repo: ${{ matrix.artifact }} 35 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/common/runner.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | gitlabModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models/common" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | parsersUtils "github.com/argonsecurity/pipeline-parser/pkg/parsers/utils" 7 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 8 | ) 9 | 10 | func ParseRunner(image *gitlabModels.Image) *models.Runner { 11 | if image == nil { 12 | return nil 13 | } 14 | registry, namespace, imageName, tag := parsersUtils.ParseImageName(image.Name) 15 | if namespace != "" { 16 | imageName = namespace + "/" + imageName 17 | } 18 | 19 | return &models.Runner{ 20 | DockerMetadata: &models.DockerMetadata{ 21 | Image: utils.GetPtrOrNil(imageName), 22 | Label: utils.GetPtrOrNil(tag), 23 | RegistryURL: utils.GetPtrOrNil(registry), 24 | }, 25 | FileReference: image.FileReference, 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | ## Related issues 4 | - Close #XXX 5 | 6 | ## Related PRs 7 | - [ ] #XXX 8 | - [ ] #YYY 9 | 10 | Remove this section if you don't have related PRs. 11 | 12 | ## Checklist 13 | - [ ] I've read the [guidelines for contributing](https://github.com/argonsecurity/pipeline-parser/blob/main/CONTRIBUTING.md) to this repository. 14 | - [ ] I've followed the [conventions](https://github.com/argonsecurity/pipeline-parser/blob/main/CONTRIBUTING.md#pull-requests) in the PR title. 15 | - [ ] I've added tests that prove my fix is effective or that my feature works. 16 | - [ ] I've updated the [readme](https://github.com/argonsecurity/pipeline-parser/blob/main/README.md) with the relevant information (if needed). 17 | - [ ] I've added usage information (if the PR introduces new options) 18 | - [ ] I've included a "before" and "after" example to the description (if the PR is a user interface change). -------------------------------------------------------------------------------- /test/fixtures/azure/parameters.yaml: -------------------------------------------------------------------------------- 1 | name: parameters 2 | parameters: 3 | - name: myString 4 | type: string 5 | default: a string 6 | - name: myMultiString 7 | type: string 8 | default: default 9 | values: 10 | - default 11 | - ubuntu 12 | - name: myNumber 13 | type: number 14 | default: 2 15 | values: 16 | - 1 17 | - 2 18 | - 4 19 | - 8 20 | - 16 21 | - name: myBoolean 22 | type: boolean 23 | default: true 24 | - name: myObject 25 | type: object 26 | default: 27 | foo: FOO 28 | bar: BAR 29 | things: 30 | - one 31 | - two 32 | - three 33 | nested: 34 | one: apple 35 | two: pear 36 | count: 3 37 | - name: myStep 38 | type: step 39 | default: 40 | script: echo my step 41 | 42 | extends: 43 | template: parameters.yml 44 | parameters: 45 | foo: bar 46 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/job/needs.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "gopkg.in/yaml.v3" 6 | ) 7 | 8 | type NeedsItem struct { 9 | Artifacts bool `yaml:"artifacts"` 10 | Project string `yaml:"project"` 11 | Pipeline string `yaml:"pipeline"` 12 | Job string `yaml:"job"` 13 | Ref string `yaml:"ref"` 14 | } 15 | 16 | type Needs []*NeedsItem 17 | 18 | func (n *Needs) UnmarshalYAML(node *yaml.Node) error { 19 | needs := []*NeedsItem{} 20 | for _, item := range node.Content { 21 | if item.Tag == consts.StringTag { 22 | needs = append(needs, 23 | &NeedsItem{ 24 | Job: item.Value, 25 | }, 26 | ) 27 | } else if item.Tag == consts.MapTag { 28 | needsItem := &NeedsItem{} 29 | if err := item.Decode(&needsItem); err != nil { 30 | return err 31 | } 32 | needs = append(needs, needsItem) 33 | } 34 | } 35 | 36 | *n = needs 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/common/script.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type Script struct { 11 | Commands []string 12 | FileReference *models.FileReference 13 | } 14 | 15 | func (s *Script) UnmarshalYAML(node *yaml.Node) error { 16 | s.FileReference = utils.GetFileReference(node) 17 | 18 | if node.Tag == consts.StringTag { 19 | s.Commands = []string{node.Value} 20 | s.FileReference.EndRef.Column += len("script: ") 21 | return nil 22 | } 23 | 24 | if node.Tag == consts.SequenceTag { 25 | commands, err := utils.ParseYamlStringSequenceToSlice(node, "Script") 26 | if err != nil { 27 | return err 28 | } 29 | s.Commands = commands 30 | return nil 31 | } 32 | 33 | return consts.NewErrInvalidYamlTag(node.Tag, "Script") 34 | } 35 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/alias-pipeline.yml: -------------------------------------------------------------------------------- 1 | image: node:14.17.6 2 | 3 | definitions: 4 | caches: 5 | cypress: /root/.cache/Cypress 6 | services: 7 | docker: 8 | memory: 2048 9 | steps: 10 | - step: &install-build 11 | name: build 12 | size: 2x 13 | caches: 14 | - node 15 | - cypress 16 | script: 17 | - yarn 18 | - yarn build 19 | artifacts: 20 | - demo/** 21 | - dist/** 22 | after-script: 23 | - npx notify -s "Install and build" --only-failure 24 | - step: &deploy-env 25 | name: deploy 26 | script: 27 | - echo deploy 28 | 29 | pipelines: 30 | pull-requests: 31 | "**": 32 | - step: *install-build 33 | custom: 34 | deploy-staging: 35 | - step: *install-build 36 | - step: *deploy-env 37 | branches: 38 | master: 39 | - step: *install-build 40 | - step: *deploy-env 41 | -------------------------------------------------------------------------------- /pkg/testutils/sort.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | ) 8 | 9 | func SortParameters(params []models.Parameter) { 10 | sort.Slice(params, func(i, j int) bool { 11 | return *params[i].Name < *params[j].Name 12 | }) 13 | } 14 | 15 | func SortTrigger(trigger *models.Trigger) { 16 | sort.Slice(trigger.Parameters, func(i, j int) bool { 17 | return *trigger.Parameters[i].Name < *trigger.Parameters[j].Name 18 | }) 19 | } 20 | 21 | func SortTriggers(triggers []*models.Trigger) { 22 | sort.Slice(triggers, func(i, j int) bool { 23 | return triggers[i].Event < triggers[j].Event 24 | }) 25 | 26 | SortMany(triggers, SortTrigger) 27 | } 28 | 29 | func SortJobs(jobs []*models.Job) { 30 | sort.Slice(jobs, func(i, j int) bool { 31 | return *jobs[i].ID < *jobs[j].ID 32 | }) 33 | } 34 | 35 | func SortMany[T any](s []T, sortFunc func(T)) { 36 | for _, item := range s { 37 | sortFunc(item) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /test/fixtures/azure/all-triggers.yaml: -------------------------------------------------------------------------------- 1 | name: all-triggers 2 | trigger: 3 | batch: true 4 | branches: 5 | include: 6 | - master 7 | - main 8 | exclude: 9 | - test/* 10 | paths: 11 | include: 12 | - path/to/file 13 | - another/path/to/file 14 | exclude: 15 | - all/* 16 | tags: 17 | include: 18 | - v1.0.* 19 | exclude: 20 | - v2.0.* 21 | pr: 22 | autoCancel: true 23 | branches: 24 | include: 25 | - features/* 26 | exclude: 27 | - features/experimental/* 28 | paths: 29 | include: 30 | - path/to/file 31 | exclude: 32 | - README.md 33 | drafts: true 34 | schedules: 35 | - cron: "0 0 * * *" 36 | displayName: Daily midnight build 37 | branches: 38 | include: 39 | - main 40 | - releases/* 41 | exclude: 42 | - releases/ancient/* 43 | - cron: "0 12 * * 0" 44 | displayName: Weekly Sunday build 45 | branches: 46 | include: 47 | - releases/* 48 | always: true -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/job/inherit.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "gopkg.in/yaml.v3" 7 | ) 8 | 9 | type Inherit struct { 10 | Default *InheritValues `yaml:"default,omitempty"` 11 | Variables *InheritValues `yaml:"variables,omitempty"` 12 | } 13 | 14 | type InheritValues struct { 15 | Enabled *bool 16 | Keys []string `yaml:"default_keys,omitempty"` 17 | } 18 | 19 | func (i *InheritValues) UnmarshalYAML(node *yaml.Node) error { 20 | if node.Tag == consts.BooleanTag { 21 | i.Enabled = utils.MustParseYamlBooleanValue(node) 22 | return nil 23 | } 24 | 25 | if node.Tag == consts.SequenceTag { 26 | keys, err := utils.ParseYamlStringSequenceToSlice(node, "InheritValues") 27 | if err != nil { 28 | return err 29 | } 30 | 31 | i.Keys = keys 32 | return nil 33 | } 34 | 35 | return consts.NewErrInvalidYamlTag(node.Tag, "InheritValues") 36 | } 37 | -------------------------------------------------------------------------------- /pkg/enhancers/general/jobs.go: -------------------------------------------------------------------------------- 1 | package general 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers/general/config" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 7 | ) 8 | 9 | func enhanceJob(job *models.Job, config *config.EnhancementConfiguration) *models.Job { 10 | if utils.AnyMatch(config.Build.Names, job.Name) { 11 | job.Metadata.Build = true 12 | } 13 | 14 | if utils.AnyMatch(config.Test.Names, job.Name) { 15 | job.Metadata.Test = true 16 | } 17 | 18 | if utils.AnyMatch(config.Deploy.Names, job.Name) { 19 | job.Metadata.Deploy = true 20 | } 21 | 22 | if job.Steps != nil { 23 | for _, step := range job.Steps { 24 | if step.Metadata.Build { 25 | job.Metadata.Build = true 26 | } 27 | 28 | if step.Metadata.Test { 29 | job.Metadata.Test = true 30 | } 31 | 32 | if step.Metadata.Deploy { 33 | job.Metadata.Deploy = true 34 | } 35 | } 36 | } 37 | 38 | return job 39 | } 40 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/artifacts.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 5 | "gopkg.in/yaml.v3" 6 | ) 7 | 8 | type Artifacts struct { 9 | SharedStepFiles *SharedStepFiles 10 | Paths []*string 11 | } 12 | 13 | type SharedStepFiles struct { 14 | Download *bool `yaml:"download,omitempty"` // Indicates whether to download artifact in the step 15 | Paths []*string `yaml:"paths"` 16 | } 17 | 18 | func (a *Artifacts) UnmarshalYAML(node *yaml.Node) error { 19 | if node.Kind == yaml.SequenceNode { 20 | var paths []*string 21 | if err := loadersUtils.ParseSequenceOrOne(node, &paths); err != nil { 22 | return err 23 | } 24 | a.Paths = paths 25 | return nil 26 | } 27 | if node.Kind == yaml.MappingNode { 28 | var sharedStepFiles SharedStepFiles 29 | if err := node.Decode(&sharedStepFiles); err != nil { 30 | return err 31 | } 32 | a.SharedStepFiles = &sharedStepFiles 33 | return nil 34 | } 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/variables-pipeline.yml: -------------------------------------------------------------------------------- 1 | image: atlassian/default-image:3 2 | 3 | pipelines: 4 | branches: 5 | master: 6 | - step: 7 | name: Deploy to Production 8 | deployment: Production 9 | trigger: manual 10 | script: 11 | - pipe: atlassian/aws-elasticbeanstalk-deploy:1.0.2 12 | variables: 13 | AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID 14 | AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY 15 | AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION 16 | APPLICATION_NAME: "pipes-templates-java-spring-boot-app" 17 | ENVIRONMENT_NAME: "Production" 18 | S3_BUCKET: "pipes-template-java-spring-boot-source" 19 | ZIP_FILE: "application.zip" 20 | VERSION_LABEL: "prod-0.1.$BITBUCKET_BUILD_NUMBER" 21 | - pipe: atlassian/aws-elasticbeanstalk-run:1.0.2 22 | variables: 23 | KEY: "value" 24 | FOO: "bar" 25 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/job/trigger.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models/common" 6 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type Trigger struct { 11 | Include *common.Include 12 | Strategy string 13 | Forward *TriggerForward 14 | } 15 | 16 | type TriggerForward struct { 17 | YAMLVariables bool 18 | PipelineVariables bool 19 | } 20 | 21 | func (t *Trigger) UnmarshalYAML(node *yaml.Node) error { 22 | if node.Tag == consts.StringTag { 23 | t.Include = &common.Include{ 24 | common.ParseIncludeString(node), 25 | } 26 | return nil 27 | } 28 | 29 | return utils.IterateOnMap(node, func(key string, value *yaml.Node) error { 30 | switch key { 31 | case "include": 32 | t.Include = &common.Include{} 33 | value.Decode(t.Include) 34 | case "strategy": 35 | t.Strategy = value.Value 36 | case "forward": 37 | value.Decode(&t.Forward) 38 | } 39 | return nil 40 | }, "Trigger") 41 | } 42 | -------------------------------------------------------------------------------- /pkg/models/import.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type SourceType string 4 | 5 | const ( 6 | SourceTypeLocal SourceType = "local" 7 | SourceTypeRemote SourceType = "remote" 8 | ) 9 | 10 | type ImportSource struct { 11 | SCM Platform `json:"scm,omitempty"` 12 | Organization *string `json:"organization,omitempty"` 13 | Repository *string `json:"repository,omitempty"` 14 | Path *string `json:"path,omitempty"` 15 | Type SourceType `json:"type,omitempty"` 16 | RepositoryAlias *string `json:"alias,omitempty"` 17 | Reference *string `json:"reference,omitempty"` 18 | } 19 | 20 | type Import struct { 21 | Source *ImportSource `json:"source,omitempty"` 22 | Version *string `json:"version,omitempty"` 23 | VersionType VersionType `json:"version_type,omitempty"` 24 | Pipeline *Pipeline `json:"pipeline,omitempty"` 25 | Parameters map[string]any `json:"parameters,omitempty"` 26 | Secrets *SecretsRef `json:"secrets,omitempty"` 27 | FileReference *FileReference `json:"file_reference,omitempty"` 28 | } 29 | -------------------------------------------------------------------------------- /test/fixtures/github/steps.yaml: -------------------------------------------------------------------------------- 1 | name: steps 2 | 3 | jobs: 4 | job1: 5 | name: Job 1 6 | steps: 7 | - name: task without params 8 | uses: actions/checkout@v1 9 | 10 | - name: task with params 11 | uses: actions/checkout@v1 12 | with: 13 | repo: repository 14 | 15 | - name: task with multiline params 16 | uses: actions/checkout@v1 17 | with: 18 | repos: | 19 | repository1 20 | repository2 21 | input: value 22 | 23 | - name: task with commit ID version 24 | uses: actions/checkout@c44948622e1b6bb0eb0cec5b813c1ac561158e1e 25 | 26 | - name: task with branch version 27 | uses: actions/checkout@master 28 | 29 | - name: task with tag version 30 | uses: actions/checkout@v1.1.1 31 | 32 | - name: shell 33 | run: command line 34 | 35 | - name: custom shell 36 | shell: cmd 37 | run: command line 38 | 39 | - name: shell with break rows 40 | run: | 41 | echo 1 42 | echo 2 43 | echo 3 44 | -------------------------------------------------------------------------------- /pkg/enhancers/general/config/common.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | var ( 8 | commonBuildShellRegexes = []*regexp.Regexp{ 9 | regexp.MustCompile(`docker build`), 10 | regexp.MustCompile(`docker-compose build`), 11 | regexp.MustCompile(`npm run build`), 12 | regexp.MustCompile(`yarn( run)? build`), 13 | regexp.MustCompile(`go build`), 14 | } 15 | 16 | commonTestShellRegexes = []*regexp.Regexp{ 17 | regexp.MustCompile(`go test`), 18 | regexp.MustCompile(`npm run test`), 19 | regexp.MustCompile(`yarn( run)? test`), 20 | } 21 | 22 | commonBuildNameRegexes = []*regexp.Regexp{ 23 | regexp.MustCompile(`(?i)build.*`), 24 | } 25 | 26 | commonTestNameRegexes = []*regexp.Regexp{ 27 | regexp.MustCompile(`(?i)tests?`), 28 | } 29 | 30 | CommonConfiguration = &EnhancementConfiguration{ 31 | Build: ObjectiveConfiguration{ 32 | Names: commonBuildNameRegexes, 33 | ShellRegexes: commonBuildShellRegexes, 34 | }, 35 | Test: ObjectiveConfiguration{ 36 | Names: commonTestNameRegexes, 37 | ShellRegexes: commonTestShellRegexes, 38 | }, 39 | } 40 | ) 41 | -------------------------------------------------------------------------------- /pkg/models/runner.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | const ( 4 | DockerRunnerType RunnerType = "docker" 5 | VmRunnerType RunnerType = "vm" 6 | ServerRunnerType RunnerType = "server" 7 | 8 | WindowsOS OS = "windows" 9 | LinuxOS OS = "linux" 10 | MacOS OS = "macos" 11 | ) 12 | 13 | type OS string 14 | type RunnerType string 15 | 16 | type DockerMetadata struct { 17 | Image *string `json:"image,omitempty"` 18 | Label *string `json:"label,omitempty"` 19 | RegistryURL *string `json:"registry_url,omitempty"` 20 | RegistryCredentialsID *string `json:"registry_credentials_id,omitempty"` 21 | } 22 | 23 | type Runner struct { 24 | Type *string `json:"type,omitempty"` 25 | Labels *[]string `json:"labels,omitempty"` 26 | OS *string `json:"os,omitempty"` 27 | Arch *string `json:"arch,omitempty"` 28 | SelfHosted *bool `json:"self_hosted,omitempty"` 29 | DockerMetadata *DockerMetadata `json:"docker_metadata,omitempty"` 30 | FileReference *FileReference `json:"file_reference,omitempty"` 31 | } 32 | -------------------------------------------------------------------------------- /pkg/parsers/bitbucket/defaults.go: -------------------------------------------------------------------------------- 1 | package bitbucket 2 | 3 | import ( 4 | bitbucketModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/bitbucket/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | func parsePipelineDefaults(pipeline *bitbucketModels.Pipeline) *models.Defaults { 9 | if pipeline == nil { 10 | return nil 11 | } 12 | 13 | var defaults models.Defaults 14 | defaults.Runner = parseRunner(pipeline) 15 | 16 | if pipeline.Options != nil { 17 | defaults.Settings = &map[string]any{ 18 | "docker": pipeline.Options.Docker, 19 | "max-time": pipeline.Options.MaxTime, 20 | "size": pipeline.Options.Size, 21 | } 22 | } 23 | 24 | return &defaults 25 | } 26 | 27 | func parseRunner(pipeline *bitbucketModels.Pipeline) *models.Runner { 28 | if pipeline == nil { 29 | return nil 30 | } 31 | 32 | if pipeline.Image != nil && pipeline.Image.ImageData != nil { 33 | runner := &models.Runner{ 34 | DockerMetadata: &models.DockerMetadata{ 35 | Image: pipeline.Image.ImageData.Name, 36 | }, 37 | } 38 | 39 | return runner 40 | } 41 | return nil 42 | } 43 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/common/retry.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 7 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 8 | "gopkg.in/yaml.v3" 9 | ) 10 | 11 | type Retry struct { 12 | When *[]string `yaml:"when,omitempty"` 13 | Max *int `yaml:"max,omitempty"` 14 | } 15 | 16 | func (r *Retry) UnmarshalYAML(node *yaml.Node) error { 17 | if node.Tag == consts.IntTag { // format: "retry: 3" 18 | parsedInt, _ := strconv.Atoi(node.Value) 19 | r.Max = &parsedInt 20 | return nil 21 | } 22 | 23 | return utils.IterateOnMap(node, func(key string, value *yaml.Node) error { 24 | switch key { 25 | case "when": 26 | if value.Tag == consts.SequenceTag { 27 | parsedStrings, _ := utils.ParseYamlStringSequenceToSlice(value, "Retry.when") 28 | r.When = &parsedStrings 29 | } 30 | if value.Tag == consts.StringTag { 31 | r.When = &[]string{value.Value} 32 | } 33 | case "max": 34 | parsedInt, _ := strconv.Atoi(value.Value) 35 | r.Max = &parsedInt 36 | } 37 | return nil 38 | }, "Retry") 39 | } 40 | -------------------------------------------------------------------------------- /pkg/consts/os.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | import "github.com/argonsecurity/pipeline-parser/pkg/models" 4 | 5 | const ( 6 | Macos1015 = "macos-10.15" 7 | Macos11 = "macos-11" 8 | MacosLatest = "macos-latest" 9 | SelfHosted = "self-hosted" 10 | Ubuntu1804 = "ubuntu-18.04" 11 | Ubuntu2004 = "ubuntu-20.04" 12 | UbuntuLatest = "ubuntu-latest" 13 | Windows2016 = "windows-2016" 14 | Windows2019 = "windows-2019" 15 | Windows2022 = "windows-2022" 16 | WindowsLatest = "windows-latest" 17 | 18 | ArmArch = "arm32" 19 | X64Arch = "x64" 20 | X32Arch = "x32" 21 | ) 22 | 23 | var ( 24 | WindowsKeywords = []string{"windows", Windows2016, Windows2019, Windows2022, WindowsLatest} 25 | LinuxKeywords = []string{"linux", "ubuntu", "debian", Ubuntu1804, Ubuntu2004, UbuntuLatest} 26 | MacKeywords = []string{"macos", "darwin", "osx", Macos1015, Macos11, MacosLatest} 27 | 28 | ArchKeywords = []string{ArmArch, X64Arch, X32Arch} 29 | 30 | OsToKeywords = map[models.OS][]string{ 31 | models.WindowsOS: WindowsKeywords, 32 | models.LinuxOS: LinuxKeywords, 33 | models.MacOS: MacKeywords, 34 | } 35 | ) 36 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/triggers/expressions.go: -------------------------------------------------------------------------------- 1 | package triggers 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 7 | ) 8 | 9 | // Variable Expressions are GitLab's way to filter according to variable values. 10 | // https://docs.gitlab.com/ee/ci/jobs/job_control.html#cicd-variable-expressions 11 | // TODO: support exists and parentheses 12 | 13 | var ( 14 | // Operators 15 | equals Operator = "==" 16 | match Operator = "=~" 17 | 18 | // Regexes 19 | comparisonRegex = regexp.MustCompile(`(\$\w+)\s*(==|!=|=~|!~)\s*(["/].*?["/]|\$\w+)`) 20 | ) 21 | 22 | type Operator string 23 | 24 | type Comparison struct { 25 | Variable string 26 | Value string 27 | Operator Operator 28 | } 29 | 30 | func (c *Comparison) IsPositive() bool { 31 | return c.Operator == equals || c.Operator == match 32 | } 33 | 34 | func getComparisons(expression string) []*Comparison { 35 | matches := comparisonRegex.FindAllStringSubmatch(expression, -1) 36 | return utils.Map(matches, func(match []string) *Comparison { 37 | return &Comparison{ 38 | Variable: match[1], 39 | Value: match[3], 40 | Operator: Operator(match[2]), 41 | } 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/handler/azure.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers" 6 | azureEnhancer "github.com/argonsecurity/pipeline-parser/pkg/enhancers/azure" 7 | "github.com/argonsecurity/pipeline-parser/pkg/loaders" 8 | azureLoader "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure" 9 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 10 | "github.com/argonsecurity/pipeline-parser/pkg/models" 11 | "github.com/argonsecurity/pipeline-parser/pkg/parsers" 12 | azureParser "github.com/argonsecurity/pipeline-parser/pkg/parsers/azure" 13 | ) 14 | 15 | type AzureHandler struct{} 16 | 17 | func (g *AzureHandler) GetPlatform() models.Platform { 18 | return consts.AzurePlatform 19 | } 20 | 21 | func (g *AzureHandler) GetLoader() loaders.Loader[azureModels.Pipeline] { 22 | return &azureLoader.AzureLoader{} 23 | } 24 | 25 | func (g *AzureHandler) GetParser() parsers.Parser[azureModels.Pipeline] { 26 | return &azureParser.AzureParser{} 27 | } 28 | 29 | func (g *AzureHandler) GetEnhancer() enhancers.Enhancer { 30 | return &azureEnhancer.AzureEnhancer{} 31 | } 32 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/image.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "gopkg.in/yaml.v3" 6 | ) 7 | 8 | type Image struct { 9 | ImageData *ImageData 10 | } 11 | 12 | type ImageData struct { 13 | Name *string `yaml:"name"` 14 | RunAsUser *int64 `yaml:"run-as-user,omitempty"` 15 | Email *string `yaml:"email,omitempty"` // Email to use to fetch the Docker image 16 | Password *string `yaml:"password,omitempty"` // Password to use to fetch the Docker image 17 | Username *string `yaml:"username,omitempty"` // Username to use to fetch the Docker image 18 | Aws *Aws `yaml:"aws,omitempty"` // AWS credentials 19 | } 20 | 21 | type Aws struct { 22 | AccessKey *string `yaml:"access-key"` // AWS Access Key 23 | SecretKey *string `yaml:"secret-key"` // AWS Secret Key 24 | } 25 | 26 | func (i *Image) UnmarshalYAML(node *yaml.Node) error { 27 | if node.Tag == consts.StringTag { 28 | *i = Image{&ImageData{Name: &node.Value}} 29 | return nil 30 | } 31 | var image ImageData 32 | if err := node.Decode(&image); err != nil { 33 | return err 34 | } 35 | *i = Image{ImageData: &image} 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /test/fixtures/gitlab/gradle.yaml: -------------------------------------------------------------------------------- 1 | # To contribute improvements to CI/CD templates, please follow the Development guide at: 2 | # https://docs.gitlab.com/ee/development/cicd/templates.html 3 | # This specific template is located at: 4 | # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml 5 | 6 | # This is the Gradle build system for JVM applications 7 | # https://gradle.org/ 8 | # https://github.com/gradle/gradle 9 | 10 | image: gradle:alpine 11 | 12 | # Disable the Gradle daemon for Continuous Integration servers as correctness 13 | # is usually a priority over speed in CI environments. Using a fresh 14 | # runtime for each build is more reliable since the runtime is completely 15 | # isolated from any previous builds. 16 | variables: 17 | GRADLE_OPTS: "-Dorg.gradle.daemon=false" 18 | 19 | before_script: 20 | - GRADLE_USER_HOME="$(pwd)/.gradle" 21 | - export GRADLE_USER_HOME 22 | 23 | build: 24 | stage: build 25 | script: gradle --build-cache assemble 26 | cache: 27 | key: "$CI_COMMIT_REF_NAME" 28 | policy: push 29 | paths: 30 | - build 31 | - .gradle 32 | 33 | test: 34 | stage: test 35 | script: gradle check 36 | -------------------------------------------------------------------------------- /pkg/loaders/azure/models/pool.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type Pool struct { 11 | Name string `yaml:"name"` 12 | Demands []string `yaml:"demands"` 13 | VmImage string `yaml:"vmImage"` 14 | FileReference *models.FileReference 15 | } 16 | 17 | func (p *Pool) UnmarshalYAML(node *yaml.Node) error { 18 | p.FileReference = loadersUtils.GetFileReference(node) 19 | if node.Tag == consts.StringTag { 20 | p.Name = node.Value 21 | return nil 22 | } 23 | 24 | p.FileReference.StartRef.Line-- 25 | return loadersUtils.IterateOnMap(node, func(key string, value *yaml.Node) error { 26 | switch key { 27 | case "name": 28 | p.Name = value.Value 29 | case "demands": 30 | var demands []string 31 | if err := loadersUtils.ParseSequenceOrOne(value, &demands); err != nil { 32 | return err 33 | } 34 | p.Demands = demands 35 | case "vmImage": 36 | p.VmImage = value.Value 37 | } 38 | 39 | return nil 40 | }, "Pool") 41 | } 42 | -------------------------------------------------------------------------------- /pkg/handler/github.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers" 6 | githubEnhancer "github.com/argonsecurity/pipeline-parser/pkg/enhancers/github" 7 | "github.com/argonsecurity/pipeline-parser/pkg/loaders" 8 | githubLoader "github.com/argonsecurity/pipeline-parser/pkg/loaders/github" 9 | githubModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/github/models" 10 | "github.com/argonsecurity/pipeline-parser/pkg/models" 11 | "github.com/argonsecurity/pipeline-parser/pkg/parsers" 12 | githubParser "github.com/argonsecurity/pipeline-parser/pkg/parsers/github" 13 | ) 14 | 15 | type GitHubHandler struct{} 16 | 17 | func (g *GitHubHandler) GetPlatform() models.Platform { 18 | return consts.GitHubPlatform 19 | } 20 | 21 | func (g *GitHubHandler) GetLoader() loaders.Loader[githubModels.Workflow] { 22 | return &githubLoader.GitHubLoader{} 23 | } 24 | 25 | func (g *GitHubHandler) GetParser() parsers.Parser[githubModels.Workflow] { 26 | return &githubParser.GitHubParser{} 27 | } 28 | 29 | func (g *GitHubHandler) GetEnhancer() enhancers.Enhancer { 30 | return &githubEnhancer.GitHubEnhancer{} 31 | } 32 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/common/image.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type Image struct { 11 | Name string `yaml:"name"` 12 | Entrypoint []string `yaml:"entrypoint"` 13 | FileReference *models.FileReference 14 | } 15 | 16 | func (im *Image) UnmarshalYAML(node *yaml.Node) error { 17 | 18 | im.FileReference = utils.GetFileReference(node) 19 | 20 | if node.Tag == consts.StringTag { // format - "image: image:tag" 21 | im.Name = node.Value 22 | im.FileReference.EndRef.Column += len("image: ") 23 | return nil 24 | } 25 | 26 | im.FileReference.StartRef.Line-- 27 | 28 | return utils.IterateOnMap(node, func(key string, value *yaml.Node) error { 29 | switch key { 30 | case "name": 31 | im.Name = value.Value 32 | case "entrypoint": 33 | entrypoints, err := utils.ParseYamlStringSequenceToSlice(value, "Image.entrypoint") 34 | if err != nil { 35 | return err 36 | } 37 | im.Entrypoint = entrypoints 38 | } 39 | return nil 40 | }, "Image") 41 | } 42 | -------------------------------------------------------------------------------- /pkg/enhancers/general/enhancers.go: -------------------------------------------------------------------------------- 1 | package general 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers/general/config" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | var ( 9 | platformToEnhancerMapping = map[models.Platform]*config.EnhancementConfiguration{} 10 | ) 11 | 12 | func Enhance(pipeline *models.Pipeline, platform models.Platform) (*models.Pipeline, error) { 13 | platformConfig := platformToEnhancerMapping[platform] 14 | 15 | if pipeline.Jobs != nil { 16 | jobs := make([]*models.Job, len(pipeline.Jobs)) 17 | for i, job := range pipeline.Jobs { 18 | if job.Steps != nil { 19 | steps := make([]*models.Step, len(job.Steps)) 20 | for i, step := range job.Steps { 21 | step = enhanceStep(step, config.CommonConfiguration) 22 | if platformConfig != nil { 23 | step = enhanceStep(step, platformConfig) 24 | } 25 | steps[i] = step 26 | } 27 | job.Steps = steps 28 | } 29 | job = enhanceJob(job, config.CommonConfiguration) 30 | if platformConfig != nil { 31 | job = enhanceJob(job, platformConfig) 32 | } 33 | jobs[i] = job 34 | } 35 | 36 | pipeline.Jobs = jobs 37 | } 38 | return pipeline, nil 39 | } 40 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/common/rules.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "gopkg.in/yaml.v3" 7 | ) 8 | 9 | type Rule struct { 10 | Changes []string `yaml:"changes,omitempty"` 11 | Exists []string `yaml:"exists,omitempty"` 12 | If string `yaml:"if,omitempty"` 13 | Variables *EnvironmentVariablesRef `yaml:"variables,omitempty"` 14 | When string `yaml:"when,omitempty"` 15 | FileReference *models.FileReference 16 | } 17 | 18 | type Rules struct { 19 | RulesList []*Rule 20 | FileReference *models.FileReference 21 | } 22 | 23 | func (r *Rules) UnmarshalYAML(node *yaml.Node) error { 24 | rules := make([]*Rule, len(node.Content)) 25 | for i, item := range node.Content { 26 | rule := &Rule{} 27 | if err := item.Decode(&rule); err != nil { 28 | return err 29 | } 30 | rule.FileReference = utils.GetFileReference(item) 31 | rules[i] = rule 32 | } 33 | 34 | *r = Rules{ 35 | RulesList: rules, 36 | FileReference: utils.GetFileReference(node), 37 | } 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /test/fixtures/gitlab/testdata/gitlab-org/gitlab/-/raw/master/imported.yaml: -------------------------------------------------------------------------------- 1 | # To contribute improvements to CI/CD templates, please follow the Development guide at: 2 | # https://docs.gitlab.com/ee/development/cicd/templates.html 3 | # This specific template is located at: 4 | # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml 5 | 6 | # This is the Gradle build system for JVM applications 7 | # https://gradle.org/ 8 | # https://github.com/gradle/gradle 9 | 10 | image: gradle:alpine 11 | 12 | # Disable the Gradle daemon for Continuous Integration servers as correctness 13 | # is usually a priority over speed in CI environments. Using a fresh 14 | # runtime for each build is more reliable since the runtime is completely 15 | # isolated from any previous builds. 16 | variables: 17 | GRADLE_OPTS: "-Dorg.gradle.daemon=false" 18 | 19 | before_script: 20 | - GRADLE_USER_HOME="$(pwd)/.gradle" 21 | - export GRADLE_USER_HOME 22 | 23 | build: 24 | stage: build 25 | script: gradle --build-cache assemble 26 | cache: 27 | key: "$CI_COMMIT_REF_NAME" 28 | policy: push 29 | paths: 30 | - build 31 | - .gradle 32 | 33 | test: 34 | stage: test 35 | script: gradle check 36 | -------------------------------------------------------------------------------- /pkg/handler/gitlab.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers" 6 | gitlabEnhancer "github.com/argonsecurity/pipeline-parser/pkg/enhancers/gitlab" 7 | "github.com/argonsecurity/pipeline-parser/pkg/loaders" 8 | gitlabLoader "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab" 9 | gitlabModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models" 10 | "github.com/argonsecurity/pipeline-parser/pkg/models" 11 | "github.com/argonsecurity/pipeline-parser/pkg/parsers" 12 | gitlabParser "github.com/argonsecurity/pipeline-parser/pkg/parsers/gitlab" 13 | ) 14 | 15 | type GitLabHandler struct{} 16 | 17 | func (g *GitLabHandler) GetPlatform() models.Platform { 18 | return consts.GitLabPlatform 19 | } 20 | 21 | func (g *GitLabHandler) GetLoader() loaders.Loader[gitlabModels.GitlabCIConfiguration] { 22 | return &gitlabLoader.GitLabLoader{} 23 | } 24 | 25 | func (g *GitLabHandler) GetParser() parsers.Parser[gitlabModels.GitlabCIConfiguration] { 26 | return &gitlabParser.GitLabParser{} 27 | } 28 | 29 | func (g *GitLabHandler) GetEnhancer() enhancers.Enhancer { 30 | return &gitlabEnhancer.GitLabEnhancer{} 31 | } 32 | -------------------------------------------------------------------------------- /test/fixtures/gitlab/testdata/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates/Android.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | # To contribute improvements to CI/CD templates, please follow the Development guide at: 2 | # https://docs.gitlab.com/ee/development/cicd/templates.html 3 | # This specific template is located at: 4 | # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml 5 | 6 | # This is the Gradle build system for JVM applications 7 | # https://gradle.org/ 8 | # https://github.com/gradle/gradle 9 | 10 | image: gradle:alpine 11 | 12 | # Disable the Gradle daemon for Continuous Integration servers as correctness 13 | # is usually a priority over speed in CI environments. Using a fresh 14 | # runtime for each build is more reliable since the runtime is completely 15 | # isolated from any previous builds. 16 | variables: 17 | GRADLE_OPTS: "-Dorg.gradle.daemon=false" 18 | 19 | before_script: 20 | - GRADLE_USER_HOME="$(pwd)/.gradle" 21 | - export GRADLE_USER_HOME 22 | 23 | build: 24 | stage: build 25 | script: gradle --build-cache assemble 26 | cache: 27 | key: "$CI_COMMIT_REF_NAME" 28 | policy: push 29 | paths: 30 | - build 31 | - .gradle 32 | 33 | test: 34 | stage: test 35 | script: gradle check 36 | -------------------------------------------------------------------------------- /pkg/handler/bitbucket.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers" 6 | bitbucketEnhancer "github.com/argonsecurity/pipeline-parser/pkg/enhancers/bitbucket" 7 | "github.com/argonsecurity/pipeline-parser/pkg/loaders" 8 | bitbucketLoader "github.com/argonsecurity/pipeline-parser/pkg/loaders/bitbucket" 9 | bitbucketModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/bitbucket/models" 10 | "github.com/argonsecurity/pipeline-parser/pkg/models" 11 | "github.com/argonsecurity/pipeline-parser/pkg/parsers" 12 | bitbucketParser "github.com/argonsecurity/pipeline-parser/pkg/parsers/bitbucket" 13 | ) 14 | 15 | type BitbucketHandler struct{} 16 | 17 | func (g *BitbucketHandler) GetPlatform() models.Platform { 18 | return consts.BitbucketPlatform 19 | } 20 | 21 | func (g *BitbucketHandler) GetLoader() loaders.Loader[bitbucketModels.Pipeline] { 22 | return &bitbucketLoader.BitbucketLoader{} 23 | } 24 | 25 | func (g *BitbucketHandler) GetParser() parsers.Parser[bitbucketModels.Pipeline] { 26 | return &bitbucketParser.BitbucketParser{} 27 | } 28 | 29 | func (g *BitbucketHandler) GetEnhancer() enhancers.Enhancer { 30 | return &bitbucketEnhancer.BitbucketEnhancer{} 31 | } 32 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/definitions.yml: -------------------------------------------------------------------------------- 1 | definitions: 2 | steps: 3 | - step: 4 | name: scripts step 5 | script: 6 | - echo "hello world" 7 | after-script: 8 | - echo "goodbye world" 9 | - pipe: notify 10 | variables: 11 | FOO: bar 12 | - step: 13 | name: artifacts step 14 | caches: 15 | - package.json 16 | - step: 17 | name: shared artifact step 18 | artifacts: 19 | download: false 20 | paths: 21 | - dist/* 22 | - package-lock.json 23 | - parallel: 24 | - step: 25 | name: parallel step 1 26 | trigger: manual 27 | - step: 28 | name: parallel step 2 29 | trigger: automatic 30 | pipelines: 31 | custom: 32 | test: 33 | - variables: #list variable names under here 34 | - name: Username 35 | - name: Role 36 | default: "admin" # optionally provide a default variable value 37 | - name: Region 38 | default: "ap-southeast-2" 39 | allowed-values: # optionally restrict variable values 40 | - "ap-southeast-2" 41 | - "us-east-1" 42 | - "us-west-2" 43 | -------------------------------------------------------------------------------- /pkg/parsers/github/github.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | githubModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/github/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | type GitHubParser struct{} 9 | 10 | func (g *GitHubParser) Parse(workflow *githubModels.Workflow) (*models.Pipeline, error) { 11 | var err error 12 | pipeline := &models.Pipeline{ 13 | Name: &workflow.Name, 14 | } 15 | 16 | pipeline.Triggers = parseWorkflowTriggers(workflow) 17 | 18 | if workflow.Jobs != nil { 19 | if pipeline.Jobs, err = parseWorkflowJobs(workflow); err != nil { 20 | return nil, err 21 | } 22 | } 23 | 24 | if pipeline.Defaults, err = parseWorkflowDefaults(workflow); err != nil { 25 | return nil, err 26 | } 27 | 28 | return pipeline, nil 29 | } 30 | 31 | func parseWorkflowDefaults(workflow *githubModels.Workflow) (*models.Defaults, error) { 32 | if workflow.Permissions == nil && workflow.Env == nil { 33 | return nil, nil 34 | } 35 | 36 | defaults := &models.Defaults{} 37 | permissions, err := parseTokenPermissions(workflow.Permissions) 38 | if err != nil { 39 | return nil, err 40 | } 41 | defaults.TokenPermissions = permissions 42 | defaults.EnvironmentVariables = parseEnvironmentVariablesRef(workflow.Env) 43 | 44 | return defaults, nil 45 | } 46 | -------------------------------------------------------------------------------- /pkg/models/trigger.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | const ( 4 | PushEvent EventType = "push" 5 | PullRequestEvent EventType = "pull_request" 6 | ForkEvent EventType = "fork" 7 | ManualEvent EventType = "manual" 8 | PipelineTriggerEvent EventType = "pipeline_trigger" 9 | PipelineRunEvent EventType = "pipeline_run" 10 | ScheduledEvent EventType = "scheduled" 11 | ) 12 | 13 | type EventType string 14 | 15 | type Trigger struct { 16 | Branches *Filter `json:"branches,omitempty"` 17 | Paths *Filter `json:"paths,omitempty"` 18 | Tags *Filter `json:"tags,omitempty"` 19 | Exists *Filter `json:"exists,omitempty"` 20 | Parameters []Parameter `json:"parameters,omitempty"` 21 | Pipelines []string `json:"pipelines,omitempty"` 22 | Filters map[string]any `json:"filters,omitempty"` 23 | Event EventType `json:"event,omitempty"` 24 | Disabled *bool `json:"disabled,omitempty"` 25 | Schedules *[]string `json:"schedules,omitempty"` 26 | FileReference *FileReference `json:"file_reference,omitempty"` 27 | } 28 | 29 | type Triggers struct { 30 | Triggers []*Trigger `json:"triggers,omitempty"` 31 | FileReference *FileReference `json:"file_reference,omitempty"` 32 | } 33 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/job/dependencies.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | gitlabModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 7 | ) 8 | 9 | func parseDependencies(job *gitlabModels.Job) []*models.JobDependency { 10 | dependencies := parseJobDependencies(job) 11 | dependencies = append(dependencies, parseJobNeeds(job)...) 12 | return dependencies 13 | } 14 | 15 | func parseJobDependencies(job *gitlabModels.Job) []*models.JobDependency { 16 | var dependencies []*models.JobDependency 17 | if job.Dependencies != nil { 18 | dependencies = append(dependencies, utils.Map(job.Dependencies, func(dependency string) *models.JobDependency { 19 | return &models.JobDependency{ 20 | JobID: &dependency, 21 | } 22 | })...) 23 | } 24 | return dependencies 25 | } 26 | 27 | func parseJobNeeds(job *gitlabModels.Job) []*models.JobDependency { 28 | var dependencies []*models.JobDependency 29 | if job.Needs != nil { 30 | for _, item := range *job.Needs { 31 | dependencies = append(dependencies, &models.JobDependency{ 32 | JobID: utils.GetPtrOrNil(item.Job), 33 | Pipeline: utils.GetPtrOrNil(item.Pipeline), 34 | }) 35 | } 36 | } 37 | return dependencies 38 | } 39 | -------------------------------------------------------------------------------- /pkg/parsers/utils/ref_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | ) 9 | 10 | func TestDetectVersionType(t *testing.T) { 11 | type args struct { 12 | version string 13 | } 14 | tests := []struct { 15 | name string 16 | args args 17 | want models.VersionType 18 | }{ 19 | { 20 | name: "Empty version", 21 | args: args{ 22 | version: "", 23 | }, 24 | want: models.None, 25 | }, 26 | { 27 | name: "Commit SHA", 28 | args: args{ 29 | version: "1234567890123456789012345678901234567890", 30 | }, 31 | want: models.CommitSHA, 32 | }, 33 | { 34 | name: "Tag version", 35 | args: args{ 36 | version: "v1.2.3", 37 | }, 38 | want: models.TagVersion, 39 | }, 40 | { 41 | name: "Branch version", 42 | args: args{ 43 | version: "master", 44 | }, 45 | want: models.BranchVersion, 46 | }, 47 | { 48 | name: "Latest version", 49 | args: args{ 50 | version: "latest", 51 | }, 52 | want: models.Latest, 53 | }, 54 | } 55 | for _, tt := range tests { 56 | t.Run(tt.name, func(t *testing.T) { 57 | if got := DetectVersionType(tt.args.version); !reflect.DeepEqual(got, tt.want) { 58 | t.Errorf("DetectVersionType() = %v, want %v", got, tt.want) 59 | } 60 | }) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pkg/enhancers/github/github.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | type GitHubEnhancer struct{} 9 | 10 | func (g *GitHubEnhancer) LoadImportedPipelines(data *models.Pipeline, credentials *models.Credentials, _, baseUrl *string) ([]*enhancers.ImportedPipeline, error) { 11 | importedPipelines, err := getReusableWorkflows(data, credentials, baseUrl) 12 | if err != nil { 13 | return importedPipelines, err 14 | } 15 | 16 | return importedPipelines, nil 17 | } 18 | 19 | func (g *GitHubEnhancer) Enhance(data *models.Pipeline, importedPipelines []*enhancers.ImportedPipeline) (*models.Pipeline, error) { 20 | for _, importedPipeline := range importedPipelines { 21 | data = mergePipelines(data, importedPipeline) 22 | } 23 | 24 | return data, nil 25 | } 26 | 27 | func mergePipelines(pipeline *models.Pipeline, importedPipeline *enhancers.ImportedPipeline) *models.Pipeline { 28 | if pipeline == nil || pipeline.Jobs == nil { 29 | return pipeline 30 | } 31 | 32 | for _, job := range pipeline.Jobs { 33 | if *job.Name == importedPipeline.JobName { 34 | job.Imports.Pipeline = importedPipeline.Pipeline 35 | } 36 | } 37 | 38 | return pipeline 39 | } 40 | 41 | func (g *GitHubEnhancer) InheritParentPipelineData(parent, child *models.Pipeline) *models.Pipeline { 42 | return child 43 | } 44 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/job/allow_failure.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 7 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 8 | commonUtils "github.com/argonsecurity/pipeline-parser/pkg/utils" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | type AllowFailure struct { 13 | Enabled *bool 14 | ExitCodes []int `yaml:"exit_codes,omitempty"` 15 | } 16 | 17 | func (a *AllowFailure) UnmarshalYAML(node *yaml.Node) error { 18 | if node.Tag == consts.BooleanTag { 19 | a.Enabled = utils.MustParseYamlBooleanValue(node) 20 | return nil 21 | } 22 | 23 | if node.Tag == consts.MapTag { 24 | a.Enabled = commonUtils.GetPtr(true) 25 | if len(node.Content) != 2 { 26 | return errors.New("AllowFailure node must have exactly one key and value") 27 | } 28 | 29 | exitCodesNode := node.Content[1] 30 | if exitCodesNode.Tag == consts.IntTag { 31 | var exitCode int 32 | if err := exitCodesNode.Decode(&exitCode); err != nil { 33 | return err 34 | } 35 | a.ExitCodes = []int{exitCode} 36 | return nil 37 | } 38 | 39 | if exitCodesNode.Tag == consts.SequenceTag { 40 | var exitCodes []int 41 | if err := exitCodesNode.Decode(&exitCodes); err != nil { 42 | return err 43 | } 44 | return nil 45 | } 46 | return consts.NewErrInvalidYamlTag(exitCodesNode.Tag, "ExitCode") 47 | } 48 | return consts.NewErrInvalidYamlTag(node.Tag, "AllowFailure") 49 | } 50 | -------------------------------------------------------------------------------- /pkg/loaders/azure/models/strategy.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Matrix map[string]any 4 | 5 | type JobStrategy struct { 6 | Matrix *Matrix `yaml:"matrix,omitempty"` 7 | MaxParallel string `yaml:"maxParallel,omitempty"` 8 | Parallel string `yaml:"parallel,omitempty"` 9 | } 10 | 11 | type DeploymentHook struct { 12 | Steps *Steps `yaml:"steps,omitempty"` 13 | Pool *Pool `yaml:"pool,omitempty"` 14 | } 15 | 16 | type DeploymentStrategy struct { 17 | RunOnce *BaseDeploymentStrategy `yaml:"runOnce,omitempty"` 18 | Rolling *RollingStrategy `yaml:"rolling,omitempty"` 19 | Canary *CanaryStrategy `yaml:"canary,omitempty"` 20 | } 21 | 22 | type BaseDeploymentStrategy struct { 23 | PreDeploy *DeploymentHook `yaml:"preDeploy,omitempty"` 24 | Deploy *DeploymentHook `yaml:"deploy,omitempty"` 25 | RouteTraffic *DeploymentHook `yaml:"routeTraffic,omitempty"` 26 | PostRouteTraffic *DeploymentHook `yaml:"postRouteTraffic,omitempty"` 27 | On struct { 28 | Failure *DeploymentHook `yaml:"failure,omitempty"` 29 | Success *DeploymentHook `yaml:"success,omitempty"` 30 | } `yaml:"on,omitempty"` 31 | } 32 | 33 | type RollingStrategy struct { 34 | MaxParallel string `yaml:"maxParallel,omitempty"` 35 | BaseDeploymentStrategy `yaml:",inline"` 36 | } 37 | 38 | type CanaryStrategy struct { 39 | Increments []string `yaml:"increments,omitempty"` 40 | BaseDeploymentStrategy `yaml:",inline"` 41 | } 42 | -------------------------------------------------------------------------------- /test/fixtures/azure/jobs.yaml: -------------------------------------------------------------------------------- 1 | name: Jobs 2 | 3 | jobs: 4 | - job: MyJob 5 | displayName: My First Job 6 | continueOnError: true 7 | dependsOn: job 8 | container: ubuntu:18.04 9 | workspace: 10 | clean: outputs 11 | steps: 12 | - script: echo My first job 13 | strategy: 14 | maxParallel: 2 15 | matrix: 16 | ${{ if in(parameters.artifactType,'*', 'docker/image') }}: 17 | docker: 18 | ArtifactType: docker/image 19 | ${{ if in(parameters.artifactType,'*', 'docker/tar') }}: 20 | tar: tar 21 | - deployment: DeployWeb 22 | displayName: deploy Web App 23 | dependsOn: [job1, job2] 24 | pool: 25 | vmImage: ubuntu-latest 26 | # creates an environment if it doesn't exist 27 | environment: smarthotel-dev 28 | strategy: 29 | # default deployment strategy, more coming... 30 | runOnce: 31 | deploy: 32 | steps: 33 | - script: echo my first deployment 34 | - template: jobs/build.yml # Template reference 35 | parameters: 36 | name: macOS 37 | pool: 38 | vmImage: macOS-latest 39 | 40 | - template: jobs/build.yml # Template reference 41 | parameters: 42 | name: Linux 43 | pool: 44 | vmImage: ubuntu-latest 45 | - template: jobs/build.yml # Template reference 46 | parameters: 47 | name: Windows 48 | pool: 49 | vmImage: windows-latest 50 | sign: true # Extra step on Windows only 51 | - ${{ parameters.jobs }} # Parameter reference -------------------------------------------------------------------------------- /pkg/loaders/azure/models/common.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "gopkg.in/yaml.v3" 7 | ) 8 | 9 | type EnvironmentVariablesRef struct { 10 | models.EnvironmentVariables 11 | FileReference *models.FileReference 12 | } 13 | 14 | type Template struct { 15 | Template string `yaml:"template,omitempty"` 16 | Parameters map[string]any `yaml:"parameters,omitempty"` 17 | } 18 | 19 | type Extends struct { 20 | Template `yaml:"inline"` 21 | FileReference *models.FileReference 22 | } 23 | 24 | type DependsOn []string 25 | 26 | func (e *EnvironmentVariablesRef) UnmarshalYAML(node *yaml.Node) error { 27 | var env models.EnvironmentVariables 28 | if err := node.Decode(&env); err != nil { 29 | return err 30 | } 31 | 32 | e.EnvironmentVariables = env 33 | e.FileReference = loadersUtils.GetFileReference(node) 34 | e.FileReference.StartRef.Line-- // The "env" node is not accessible, this is a patch 35 | return nil 36 | } 37 | 38 | func (e *Extends) UnmarshalYAML(node *yaml.Node) error { 39 | e.FileReference = loadersUtils.GetFileReference(node) 40 | return node.Decode(&e.Template) 41 | } 42 | 43 | func (n *DependsOn) UnmarshalYAML(node *yaml.Node) error { 44 | var tags []string 45 | if err := loadersUtils.ParseSequenceOrOne(node, &tags); err != nil { 46 | return err 47 | } 48 | 49 | *n = tags 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /pkg/loaders/common/models/map.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "gopkg.in/yaml.v3" 7 | ) 8 | 9 | /* 10 | Map is a struct that is used to describe a yaml tag that is a map of key value pairs. 11 | */ 12 | type Map struct { 13 | FileReference *models.FileReference 14 | Values []*MapEntry 15 | } 16 | 17 | type MapEntry struct { 18 | Key string 19 | Value any 20 | FileReference *models.FileReference 21 | } 22 | 23 | func (m *Map) UnmarshalYAML(node *yaml.Node) error { 24 | keyValues := []*MapEntry{} 25 | if err := loadersUtils.IterateOnMap(node, 26 | func(key string, value *yaml.Node) error { 27 | keyValues = append(keyValues, &MapEntry{ 28 | Key: key, 29 | Value: loadersUtils.GetNodeValue(value), 30 | FileReference: loadersUtils.GetFileReference(value), 31 | }) 32 | return nil 33 | }, 34 | "Map"); err != nil { 35 | return err 36 | } 37 | 38 | m.Values = keyValues 39 | 40 | m.FileReference = loadersUtils.GetFileReference(node) 41 | 42 | // node.Line is the line number of the first key-value pair of the map 43 | // but we want to set the file reference from the map declaration 44 | // e.g ` 45 | // map: 46 | // key: value` 47 | // set the file reference from "map:" 48 | m.FileReference.StartRef.Line-- 49 | m.FileReference.StartRef.Column -= 2 50 | 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/variable.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "gopkg.in/yaml.v3" 7 | ) 8 | 9 | type CustomStepVariable struct { 10 | Name *string `yaml:"name,omitempty"` // Name of a variable for the custom pipeline 11 | Default *string `yaml:"default,omitempty"` 12 | AllowedValues []*string `yaml:"allowed-values,omitempty"` 13 | FileReference *models.FileReference 14 | } 15 | 16 | func (v *CustomStepVariable) UnmarshalYAML(node *yaml.Node) error { 17 | v.FileReference = loadersUtils.GetFileReference(node) 18 | return loadersUtils.IterateOnMap(node, func(key string, value *yaml.Node) error { 19 | switch key { 20 | case "allowed-values": 21 | var items []*string 22 | if err := loadersUtils.ParseSequenceOrOne(value, &items); err != nil { 23 | return err 24 | } 25 | v.AllowedValues = items 26 | return nil 27 | case "default": 28 | str, err := decodeValue(value) 29 | if err != nil { 30 | return err 31 | } 32 | v.Default = str 33 | case "name": 34 | str, err := decodeValue(value) 35 | if err != nil { 36 | return err 37 | } 38 | v.Name = str 39 | } 40 | return nil 41 | }, "Variable") 42 | } 43 | 44 | func decodeValue(value *yaml.Node) (*string, error) { 45 | var def *string 46 | if err := value.Decode(&def); err != nil { 47 | return nil, err 48 | } 49 | return def, nil 50 | } 51 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/common/environment_variables.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type EnvironmentVariablesRef struct { 11 | Variables *Variables 12 | FileReference *models.FileReference 13 | } 14 | 15 | type Variables map[string]any 16 | 17 | type Variable struct { 18 | Value string `yaml:"value"` 19 | Description string `yaml:"description"` 20 | } 21 | 22 | func (v *Variables) UnmarshalYAML(node *yaml.Node) error { 23 | variables := map[string]any{} 24 | if err := utils.IterateOnMap(node, func(key string, value *yaml.Node) error { 25 | if value.Tag == consts.MapTag { 26 | variable := Variable{} 27 | if err := value.Decode(&variable); err != nil { 28 | return err 29 | } 30 | variables[key] = variable.Value 31 | } else { // format - "VARIABLE: VALUE" 32 | variables[key] = value.Value 33 | } 34 | return nil 35 | }, "Variables"); err != nil { 36 | return err 37 | } 38 | 39 | *v = variables 40 | return nil 41 | } 42 | 43 | func (e *EnvironmentVariablesRef) UnmarshalYAML(node *yaml.Node) error { 44 | e.FileReference = utils.GetFileReference(node) 45 | 46 | e.FileReference.StartRef.Line-- 47 | e.FileReference.StartRef.Column = 1 48 | e.FileReference.EndRef.Column += 2 // +2 for the ": " after the key 49 | return node.Decode(&e.Variables) 50 | } 51 | -------------------------------------------------------------------------------- /pkg/utils/slice.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func ToSlice[T any](v any) ([]T, bool) { 4 | anySlice, ok := v.([]any) 5 | if !ok { 6 | return []T{}, false 7 | } 8 | 9 | result := make([]T, len(anySlice)) 10 | for i, item := range anySlice { 11 | result[i], ok = item.(T) 12 | if !ok { 13 | return []T{}, false 14 | } 15 | } 16 | return result, true 17 | } 18 | 19 | func Map[T any, U any](s []T, cb func(v T) U) []U { 20 | result := make([]U, len(s)) 21 | for i, item := range s { 22 | result[i] = cb(item) 23 | } 24 | return result 25 | } 26 | 27 | func MapWithIndex[T any, U any](s []T, cb func(v T, i int) U) []U { 28 | result := make([]U, len(s)) 29 | for i, item := range s { 30 | result[i] = cb(item, i) 31 | } 32 | return result 33 | } 34 | 35 | func Filter[T any](s []T, cb func(v T) bool) []T { 36 | result := make([]T, 0) 37 | for _, item := range s { 38 | if cb(item) { 39 | result = append(result, item) 40 | } 41 | } 42 | return result 43 | } 44 | 45 | func SliceContains[T comparable](s []T, v T) bool { 46 | for _, item := range s { 47 | if item == v { 48 | return true 49 | } 50 | } 51 | return false 52 | } 53 | 54 | func SliceToMap[T comparable, U any](s []T, cb func(v T) U) map[T]U { 55 | result := make(map[T]U) 56 | for _, item := range s { 57 | result[item] = cb(item) 58 | } 59 | return result 60 | } 61 | 62 | func SliceContainsBy[T comparable](s []T, v T, cb func(y, u T) bool) bool { 63 | for _, item := range s { 64 | if cb(item, v) { 65 | return true 66 | } 67 | } 68 | return false 69 | } 70 | -------------------------------------------------------------------------------- /pkg/parsers/azure/variables.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | func parseVariables(variables *azureModels.Variables) *models.EnvironmentVariablesRef { 9 | if variables == nil || len(*variables) == 0 { 10 | return nil 11 | } 12 | 13 | env := make(models.EnvironmentVariables) 14 | var imports *models.Import 15 | 16 | for _, variable := range *variables { 17 | if variable.Name != "" { 18 | env[variable.Name] = variable.Value 19 | } 20 | 21 | path, alias := parseTemplateString(variable.Template.Template) 22 | if variable.Template.Template != "" { 23 | imports = &models.Import{ 24 | Source: &models.ImportSource{ 25 | Path: &path, 26 | Type: calculateSourceType(alias), 27 | RepositoryAlias: &alias, 28 | }, 29 | Parameters: variable.Parameters, 30 | FileReference: variable.FileReference, 31 | } 32 | } 33 | } 34 | 35 | return &models.EnvironmentVariablesRef{ 36 | EnvironmentVariables: env, 37 | FileReference: &models.FileReference{ 38 | StartRef: &models.FileLocation{ 39 | Line: (*variables)[0].FileReference.StartRef.Line, 40 | Column: (*variables)[0].FileReference.StartRef.Column, 41 | }, 42 | EndRef: &models.FileLocation{ 43 | Line: (*variables)[len(*variables)-1].FileReference.EndRef.Line, 44 | Column: (*variables)[len(*variables)-1].FileReference.EndRef.Column, 45 | }, 46 | }, 47 | Imports: imports, 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/common/envs_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "testing" 5 | 6 | gitlabModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models/common" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | ) 10 | 11 | func TestParseEnvironmentVariables(t *testing.T) { 12 | testCases := []struct { 13 | name string 14 | environmentVariables *gitlabModels.EnvironmentVariablesRef 15 | expectedEnvs *models.EnvironmentVariablesRef 16 | }{ 17 | { 18 | name: "EnvironmentVariables is nil", 19 | environmentVariables: nil, 20 | expectedEnvs: nil, 21 | }, 22 | { 23 | name: "EnvironmentVariables with data", 24 | environmentVariables: &gitlabModels.EnvironmentVariablesRef{ 25 | Variables: &gitlabModels.Variables{ 26 | "string": "string", 27 | "number": 1, 28 | "bool": true, 29 | }, 30 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 31 | }, 32 | expectedEnvs: &models.EnvironmentVariablesRef{ 33 | EnvironmentVariables: models.EnvironmentVariables{ 34 | "string": "string", 35 | "number": 1, 36 | "bool": true, 37 | }, 38 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 39 | }, 40 | }, 41 | } 42 | 43 | for _, testCase := range testCases { 44 | t.Run(testCase.name, func(t *testing.T) { 45 | got := ParseEnvironmentVariables(testCase.environmentVariables) 46 | 47 | testutils.DeepCompare(t, testCase.expectedEnvs, got) 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /test/fixtures/azure/steps.yaml: -------------------------------------------------------------------------------- 1 | name: steps 2 | steps: 3 | - bash: | 4 | which bash 5 | echo Hello $name 6 | displayName: Multiline Bash script 7 | env: 8 | name: Microsoft 9 | - checkout: self 10 | submodules: true 11 | persistCredentials: true 12 | - download: current # refers to artifacts published by current pipeline 13 | artifact: WebApp 14 | patterns: '**/.js' 15 | displayName: Download artifact WebApp 16 | - downloadBuild: current # refers to artifacts published by current pipeline 17 | artifact: WebApp 18 | path: build 19 | patterns: '**/.js' 20 | displayName: Download artifact WebApp 21 | - getPackage: packageID 22 | path: dist 23 | - powershell: Write-Host Hello $(name) 24 | displayName: Say hello 25 | name: firstStep 26 | workingDirectory: $(build.sourcesDirectory) 27 | failOnStderr: true 28 | env: 29 | name: Microsoft 30 | - publish: $(Build.SourcesDirectory)/build 31 | artifact: WebApp 32 | displayName: Publish artifact WebApp 33 | - pwsh: Write-Host Hello $(name) 34 | displayName: Say hello 35 | name: firstStep 36 | workingDirectory: $(build.sourcesDirectory) 37 | failOnStderr: true 38 | env: 39 | name: Microsoft 40 | - reviewApp: review 41 | - script: echo This runs in the default shell on any machine 42 | - task: VSBuild@1 43 | displayName: Build 44 | timeoutInMinutes: 120 45 | inputs: 46 | solution: '**\*.sln' 47 | - template: steps/build.yml # Template reference 48 | parameters: 49 | key: value 50 | - ${{ parameters.trivyStep }} 51 | 52 | pool: 53 | vmImage: ubuntu-latest 54 | container: ubuntu:18.04 -------------------------------------------------------------------------------- /pkg/parsers/azure/runner.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | parsersUtils "github.com/argonsecurity/pipeline-parser/pkg/parsers/utils" 7 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 8 | ) 9 | 10 | func parseRunner(job *azureModels.BaseJob) *models.Runner { 11 | if job == nil || job.Pool == nil && job.Container == nil { 12 | return nil 13 | } 14 | 15 | runner := &models.Runner{} 16 | 17 | if job.Pool != nil { 18 | runner = parsePool(job.Pool, runner) 19 | } 20 | 21 | if job.Container != nil { 22 | runner = parseContainer(job.Container, runner) 23 | } 24 | return runner 25 | } 26 | 27 | func parsePool(pool *azureModels.Pool, runner *models.Runner) *models.Runner { 28 | if runner == nil || pool == nil || pool.VmImage == "" { 29 | return runner 30 | } 31 | 32 | return parsersUtils.ParseRunnerTag(pool.VmImage, runner) 33 | } 34 | 35 | func parseContainer(container *azureModels.JobContainer, runner *models.Runner) *models.Runner { 36 | if runner == nil || container == nil || container.Image == "" { 37 | return runner 38 | } 39 | 40 | registry, namespace, imageName, tag := parsersUtils.ParseImageName(container.Image) 41 | if namespace != "" { 42 | imageName = namespace + "/" + imageName 43 | } 44 | 45 | runner.DockerMetadata = &models.DockerMetadata{ 46 | Image: utils.GetPtrOrNil(imageName), 47 | Label: utils.GetPtrOrNil(tag), 48 | RegistryURL: utils.GetPtrOrNil(registry), 49 | } 50 | return runner 51 | } 52 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/job/controls.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | // Control represents "job:except/only" 11 | type Controls struct { 12 | Refs []string 13 | Variables []string 14 | Changes []string 15 | Kubernetes string 16 | 17 | FileReference *models.FileReference 18 | } 19 | 20 | func (c *Controls) UnmarshalYAML(node *yaml.Node) error { 21 | c.FileReference = utils.GetFileReference(node) 22 | if node.Tag == consts.SequenceTag { 23 | refs, err := utils.ParseYamlStringSequenceToSlice(node, "Controls") 24 | if err != nil { 25 | return err 26 | } 27 | c.Refs = refs 28 | return nil 29 | } 30 | 31 | return utils.IterateOnMap(node, func(key string, value *yaml.Node) error { 32 | switch key { 33 | case "refs": 34 | refs, err := utils.ParseYamlStringSequenceToSlice(value, "Controls.refs") 35 | if err != nil { 36 | return err 37 | } 38 | c.Refs = refs 39 | case "variables": 40 | variables, err := utils.ParseYamlStringSequenceToSlice(value, "Controls.variables") 41 | if err != nil { 42 | return err 43 | } 44 | c.Variables = variables 45 | case "changes": 46 | changes, err := utils.ParseYamlStringSequenceToSlice(value, "Controls.changes") 47 | if err != nil { 48 | return err 49 | } 50 | c.Changes = changes 51 | case "kubernetes": 52 | c.Kubernetes = value.Value 53 | } 54 | return nil 55 | }, "Controls") 56 | } 57 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/artifacts.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Artifacts struct { 4 | Exclude []string `yaml:"exclude,omitempty"` 5 | ExpireIn string `yaml:"expire_in,omitempty"` 6 | ExposeAs string `yaml:"expose_as,omitempty"` 7 | Name string `yaml:"name,omitempty"` 8 | Paths []string `yaml:"paths,omitempty"` 9 | Reports *Reports `yaml:"reports,omitempty"` 10 | Untracked bool `yaml:"untracked,omitempty"` 11 | When string `yaml:"when,omitempty"` 12 | } 13 | 14 | type Reports struct { 15 | CoverageReport *CoverageReport `yaml:"coverage_report,omitempty"` 16 | 17 | Codequality any `yaml:"codequality,omitempty"` 18 | ContainerScanning any `yaml:"container_scanning,omitempty"` 19 | Dast any `yaml:"dast,omitempty"` 20 | DependencyScanning any `yaml:"dependency_scanning,omitempty"` 21 | Dotenv any `yaml:"dotenv,omitempty"` 22 | Junit any `yaml:"junit,omitempty"` 23 | LicenseManagement any `yaml:"license_management,omitempty"` 24 | LicenseScanning any `yaml:"license_scanning,omitempty"` 25 | Lsif any `yaml:"lsif,omitempty"` 26 | Metrics any `yaml:"metrics,omitempty"` 27 | Performance any `yaml:"performance,omitempty"` 28 | Requirements any `yaml:"requirements,omitempty"` 29 | Sast any `yaml:"sast,omitempty"` 30 | SecretDetection any `yaml:"secret_detection,omitempty"` 31 | Terraform any `yaml:"terraform,omitempty"` 32 | } 33 | 34 | type CoverageReport struct { 35 | CoverageFormat any `yaml:"coverage_format,omitempty"` 36 | Path string `yaml:"path,omitempty"` 37 | } 38 | -------------------------------------------------------------------------------- /scripts/tag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | function print_usage() { 6 | echo "Usage: make tag LEVEL=" 7 | } 8 | 9 | function validate_input() { 10 | if [ -z "${LEVEL}" ]; then 11 | echo "Parameter LEVEL is not allowed to be empty" 12 | print_usage 13 | exit 1 14 | fi 15 | 16 | case "${LEVEL}" in 17 | patch|minor|major) ;; 18 | *) 19 | echo "LEVEL value is not valid" 20 | print_usage 21 | exit 2 22 | ;; 23 | esac 24 | } 25 | 26 | # Usage: incr_semver 27 | # Example: incr_semver 1.2.3 patch 28 | # Output: 1.2.4 29 | function incr_semver() { 30 | IFS='.' read -ra ver <<< "$1" 31 | [[ "${#ver[@]}" -ne 3 ]] && echo "Invalid semver string" && return 1 32 | [[ "$#" -eq 1 ]] && level='patch' || level=$2 33 | 34 | patch=${ver[2]} 35 | minor=${ver[1]} 36 | major=${ver[0]} 37 | 38 | case $level in 39 | patch) 40 | patch=$((patch+1)) 41 | ;; 42 | minor) 43 | patch=0 44 | minor=$((minor+1)) 45 | ;; 46 | major) 47 | patch=0 48 | minor=0 49 | major=$((major+1)) 50 | ;; 51 | *) 52 | return 2 53 | esac 54 | echo "${major}.${minor}.${patch}" 55 | } 56 | 57 | validate_input 58 | 59 | git checkout main 60 | git fetch --tags --all 61 | git pull 62 | 63 | LATEST_TAG=$(git describe --tags --abbrev=0) 64 | TAG=$(incr_semver ${LATEST_TAG} ${LEVEL}) 65 | echo "Using tag: ${TAG}" 66 | 67 | git tag -a ${TAG} -m ${TAG} 68 | git push --tag -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/pipeline.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models/common" 5 | ) 6 | 7 | type GitlabCIConfiguration struct { 8 | AfterScript *common.Script `yaml:"after_script"` 9 | BeforeScript *common.Script `yaml:"before_script"` 10 | Cache *common.Cache `yaml:"cache"` 11 | Default *Default `yaml:"default"` 12 | Image *common.Image `yaml:"image"` 13 | Include *common.Include `yaml:"include"` 14 | 15 | Pages any `yaml:"pages"` 16 | Services []any `yaml:"services"` 17 | 18 | // Groups jobs into stages. All jobs in one stage must complete before next stage is executed. Defaults to ['build', 'test', 'deploy']. 19 | Stages []string `yaml:"stages"` 20 | Variables *common.EnvironmentVariablesRef `yaml:"variables"` 21 | Workflow *Workflow `yaml:"workflow"` 22 | Jobs map[string]*Job `yaml:",inline,omitempty"` 23 | } 24 | 25 | type Default struct { 26 | AfterScript []*common.Script `yaml:"after_script"` 27 | Artifacts *Artifacts `yaml:"artifacts"` 28 | BeforeScript []*common.Script `yaml:"before_script"` 29 | Cache *common.Cache `yaml:"cache"` 30 | Image *common.Image `yaml:"image"` 31 | Interruptible bool `yaml:"interruptible"` 32 | Retry *common.Retry `yaml:"retry"` 33 | Services []any `yaml:"services"` 34 | Tags []string `yaml:"tags"` 35 | Timeout string `yaml:"timeout"` 36 | } 37 | 38 | type Workflow struct { 39 | Rules *common.Rules `yaml:"rules"` 40 | } 41 | -------------------------------------------------------------------------------- /pkg/parsers/azure/resouces.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | ) 8 | 9 | func parseResources(resources *azureModels.Resources) *models.Resources { 10 | if resources == nil || len(resources.Resources) == 0 { 11 | return nil 12 | } 13 | 14 | parsedResources := &models.Resources{ 15 | Repositories: make([]*models.ImportSource, 0), 16 | } 17 | 18 | for _, resource := range resources.Resources { 19 | if resource.Repositories == nil || len(resource.Repositories) == 0 { 20 | return nil 21 | } 22 | for _, repo := range resource.Repositories { 23 | parsedResources.Repositories = append(parsedResources.Repositories, &models.ImportSource{ 24 | RepositoryAlias: &repo.Repository.Repository, 25 | Reference: &repo.Repository.Ref, 26 | Type: parseRepoType(repo.Repository.Type), 27 | SCM: parseRepoSCM(repo.Repository.Type), 28 | Repository: &repo.Repository.Name, 29 | }) 30 | parsedResources.FileReference = resource.FileReference 31 | } 32 | } 33 | 34 | return parsedResources 35 | } 36 | 37 | func parseRepoType(repoType string) models.SourceType { 38 | switch repoType { 39 | case "github", "git": 40 | return models.SourceTypeRemote 41 | default: 42 | return models.SourceTypeLocal 43 | } 44 | } 45 | 46 | func parseRepoSCM(repoType string) models.Platform { 47 | switch repoType { 48 | case "github": 49 | return consts.GitHubPlatform 50 | default: 51 | return consts.AzurePlatform 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pkg/parsers/github/permissions.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | githubModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/github/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "github.com/mitchellh/mapstructure" 7 | ) 8 | 9 | const ( 10 | readPermission = "read" 11 | writePermission = "write" 12 | ) 13 | 14 | var ( 15 | customPermissionsMap = map[string]string{ 16 | "actions": models.RunPipelinePermission, 17 | "pull-requests": models.PullRequestPermission, 18 | } 19 | ) 20 | 21 | func parseTokenPermissions(permissions *githubModels.PermissionsEvent) (*models.TokenPermissions, error) { 22 | if permissions == nil { 23 | return nil, nil 24 | } 25 | 26 | var permissionsMap map[string]any 27 | if err := mapstructure.Decode(permissions, &permissionsMap); err != nil { 28 | return nil, err 29 | } 30 | 31 | tokenPermissions := make(map[string]models.Permission) 32 | for permissionName, value := range permissionsMap { 33 | if val, ok := value.(string); ok { 34 | if customPermissionsMap[permissionName] != "" { 35 | permissionName = customPermissionsMap[permissionName] 36 | } 37 | tokenPermissions[permissionName] = parsePermissionValue(val) 38 | } 39 | } 40 | 41 | return &models.TokenPermissions{ 42 | Permissions: tokenPermissions, 43 | FileReference: permissions.FileReference, 44 | }, nil 45 | } 46 | 47 | func parsePermissionValue(permission string) models.Permission { 48 | if permission == readPermission { 49 | return models.Permission{ 50 | Read: true, 51 | } 52 | } 53 | if permission == writePermission { 54 | return models.Permission{ 55 | Write: true, 56 | } 57 | } 58 | return models.Permission{} 59 | } 60 | -------------------------------------------------------------------------------- /pkg/parsers/utils/image.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 9 | ) 10 | 11 | func ParseImageName(imageName string) (string, string, string, string) { 12 | var registry, namespace, tag string 13 | image := imageName 14 | 15 | if split := strings.Split(imageName, "/"); len(split) == 3 { // imageName contains registry/repository/image 16 | registry = split[0] 17 | namespace = split[1] 18 | image = split[2] 19 | } else if len(split) == 2 { // imageName contains repository/image 20 | namespace = split[0] 21 | image = split[1] 22 | } 23 | 24 | if split := strings.Split(image, ":"); len(split) == 2 { // image contains image:tag 25 | image = split[0] 26 | tag = split[1] 27 | } 28 | 29 | return registry, namespace, image, tag 30 | } 31 | 32 | func ParseRunnerTag(tag string, runner *models.Runner) *models.Runner { 33 | if runner == nil { 34 | return runner 35 | } 36 | 37 | if tag == consts.SelfHosted { 38 | runner.SelfHosted = utils.GetPtr(true) 39 | } 40 | 41 | for os, keywords := range consts.OsToKeywords { 42 | didFind := false 43 | for _, keyword := range keywords { 44 | if strings.Contains(strings.ToLower(tag), keyword) { 45 | runner.OS = utils.GetPtr(string(os)) 46 | didFind = true 47 | break 48 | } 49 | } 50 | if didFind { 51 | break 52 | } 53 | } 54 | 55 | for _, arch := range consts.ArchKeywords { 56 | if strings.Contains(strings.ToLower(tag), arch) { 57 | runner.Arch = utils.GetPtr(arch) 58 | break 59 | } 60 | } 61 | 62 | return runner 63 | } 64 | -------------------------------------------------------------------------------- /pkg/loaders/utils/yaml_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestCalcParameterFileReference(t *testing.T) { 12 | testCases := []struct { 13 | name string 14 | startLine int 15 | startColumn int 16 | key string 17 | val any 18 | expectedFileReference *models.FileReference 19 | }{ 20 | { 21 | name: "Start line is -1", 22 | startLine: -1, 23 | expectedFileReference: nil, 24 | }, 25 | { 26 | name: "Start column is -1", 27 | startColumn: -1, 28 | expectedFileReference: nil, 29 | }, 30 | { 31 | name: "Value without \\n", 32 | startLine: 2, 33 | startColumn: 4, 34 | key: "key", 35 | val: "value", 36 | expectedFileReference: testutils.CreateFileReference(2, 4, 2, 14), 37 | }, 38 | { 39 | name: "Value with \\n", 40 | 41 | startLine: 2, 42 | startColumn: 4, 43 | key: "key", 44 | val: "value\nvalue\n", 45 | expectedFileReference: testutils.CreateFileReference(2, 4, 4, 14), 46 | }, 47 | } 48 | 49 | for _, testCase := range testCases { 50 | t.Run(testCase.name, func(t *testing.T) { 51 | got := CalculateParameterFileReference(testCase.startLine, testCase.startColumn, testCase.key, testCase.val) 52 | assert.Equal(t, testCase.expectedFileReference, got, testCase.name) 53 | }) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/parsers/azure/variables_test.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "testing" 5 | 6 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | ) 10 | 11 | func TestParseVariables(t *testing.T) { 12 | testCases := []struct { 13 | name string 14 | variables *azureModels.Variables 15 | expected *models.EnvironmentVariablesRef 16 | }{ 17 | { 18 | name: "variables are nil", 19 | variables: nil, 20 | expected: nil, 21 | }, 22 | { 23 | name: "variables are empty", 24 | variables: &azureModels.Variables{}, 25 | expected: nil, 26 | }, 27 | { 28 | name: "variables with data", 29 | variables: &azureModels.Variables{ 30 | { 31 | Name: "var1", 32 | Value: "value1", 33 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 34 | }, 35 | { 36 | Group: "group1", 37 | FileReference: testutils.CreateFileReference(5, 6, 7, 8), 38 | }, 39 | { 40 | Name: "var2", 41 | Value: "value2", 42 | FileReference: testutils.CreateFileReference(9, 10, 11, 12), 43 | }, 44 | }, 45 | expected: &models.EnvironmentVariablesRef{ 46 | EnvironmentVariables: models.EnvironmentVariables{ 47 | "var1": "value1", 48 | "var2": "value2", 49 | }, 50 | FileReference: testutils.CreateFileReference(1, 2, 11, 12), 51 | }, 52 | }, 53 | } 54 | 55 | for _, testCase := range testCases { 56 | t.Run(testCase.name, func(t *testing.T) { 57 | got := parseVariables(testCase.variables) 58 | 59 | testutils.DeepCompare(t, testCase.expected, got) 60 | }) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pkg/parsers/azure/common_test.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "testing" 5 | 6 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | ) 10 | 11 | func TestParseEnvironmentVariablesRef(t *testing.T) { 12 | testCases := []struct { 13 | name string 14 | envRef *azureModels.EnvironmentVariablesRef 15 | expectedEnv *models.EnvironmentVariablesRef 16 | }{ 17 | { 18 | name: "Input is nil", 19 | envRef: nil, 20 | expectedEnv: nil, 21 | }, 22 | { 23 | name: "Input is not nil", 24 | envRef: &azureModels.EnvironmentVariablesRef{ 25 | EnvironmentVariables: map[string]any{ 26 | "key1": "value1", 27 | "key2": "value2", 28 | }, 29 | FileReference: &models.FileReference{ 30 | StartRef: &models.FileLocation{ 31 | Line: 1, 32 | Column: 2, 33 | }, 34 | EndRef: &models.FileLocation{ 35 | Line: 3, 36 | Column: 4, 37 | }, 38 | }, 39 | }, 40 | expectedEnv: &models.EnvironmentVariablesRef{ 41 | EnvironmentVariables: map[string]any{ 42 | "key1": "value1", 43 | "key2": "value2", 44 | }, 45 | FileReference: &models.FileReference{ 46 | StartRef: &models.FileLocation{ 47 | Line: 1, 48 | Column: 2, 49 | }, 50 | EndRef: &models.FileLocation{ 51 | Line: 3, 52 | Column: 4, 53 | }, 54 | }, 55 | }, 56 | }, 57 | } 58 | 59 | for _, testCase := range testCases { 60 | t.Run(testCase.name, func(t *testing.T) { 61 | got := parseEnvironmentVariablesRef(testCase.envRef) 62 | testutils.DeepCompare(t, testCase.expectedEnv, got) 63 | }) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /pkg/parsers/github/common_test.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "testing" 5 | 6 | githubModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/github/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | ) 10 | 11 | func TestParseEnvironmentVariablesRef(t *testing.T) { 12 | testCases := []struct { 13 | name string 14 | envRef *githubModels.EnvironmentVariablesRef 15 | expectedEnv *models.EnvironmentVariablesRef 16 | }{ 17 | { 18 | name: "Input is nil", 19 | envRef: nil, 20 | expectedEnv: nil, 21 | }, 22 | { 23 | name: "Input is not nil", 24 | envRef: &githubModels.EnvironmentVariablesRef{ 25 | EnvironmentVariables: map[string]any{ 26 | "key1": "value1", 27 | "key2": "value2", 28 | }, 29 | FileReference: &models.FileReference{ 30 | StartRef: &models.FileLocation{ 31 | Line: 1, 32 | Column: 2, 33 | }, 34 | EndRef: &models.FileLocation{ 35 | Line: 3, 36 | Column: 4, 37 | }, 38 | }, 39 | }, 40 | expectedEnv: &models.EnvironmentVariablesRef{ 41 | EnvironmentVariables: map[string]any{ 42 | "key1": "value1", 43 | "key2": "value2", 44 | }, 45 | FileReference: &models.FileReference{ 46 | StartRef: &models.FileLocation{ 47 | Line: 1, 48 | Column: 2, 49 | }, 50 | EndRef: &models.FileLocation{ 51 | Line: 3, 52 | Column: 4, 53 | }, 54 | }, 55 | }, 56 | }, 57 | } 58 | 59 | for _, testCase := range testCases { 60 | t.Run(testCase.name, func(t *testing.T) { 61 | got := parseEnvironmentVariablesRef(testCase.envRef) 62 | 63 | testutils.DeepCompare(t, testCase.expectedEnv, got) 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/build_pipelines.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type BuildPipelines struct { 4 | Default []*Step `yaml:"default"` // The default pipeline runs on every push to the repository, unless a branch-specific; pipeline is defined.; You can define a branch pipeline in the branches section.; ; Note: The default pipeline doesn't run on tags or bookmarks. 5 | Branches *StepMap `yaml:"branches,omitempty"` // Defines a section for all branch-specific build pipelines. The names or expressions in; this section are matched against:; ; * branches in your Git repository; * named branches in your Mercurial repository; ; You can use glob patterns for handling the branch names. 6 | Tags *StepMap `yaml:"tags,omitempty"` // Defines all tag-specific build pipelines.; ; The names or expressions in this section are matched against tags and annotated tags in; your Git repository.; ; You can use glob patterns for handling the tag names. 7 | Bookmarks *StepMap `yaml:"bookmarks,omitempty"` // Defines all bookmark-specific build pipelines.; ; The names or expressions in this section are matched against bookmarks in your Mercurial; repository.; ; You can use glob patterns for handling the tag names. 8 | PullRequests *StepMap `yaml:"pull-requests,omitempty"` // A special pipeline which only runs on pull requests. Pull-requests has the same level of; indentation as branches.; ; This type of pipeline runs a little differently to other pipelines. When it's triggered,; we'll merge the destination branch into your working branch before it runs. If the merge; fails we will stop the pipeline. 9 | Custom *StepMap `yaml:"custom,omitempty"` // Defines pipelines that can only be triggered manually or scheduled from the Bitbucket; Cloud interface. 10 | } 11 | -------------------------------------------------------------------------------- /pkg/enhancers/general/steps.go: -------------------------------------------------------------------------------- 1 | package general 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers/general/config" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 7 | ) 8 | 9 | func enhanceStep(step *models.Step, config *config.EnhancementConfiguration) *models.Step { 10 | if step.Type == models.ShellStepType { 11 | step = enhanceShellStep(step, config) 12 | } 13 | 14 | if step.Type == models.TaskStepType { 15 | step = enhanceTaskStep(step, config) 16 | } 17 | 18 | if utils.AnyMatch(config.Build.Names, step.Name) { 19 | step.Metadata.Build = true 20 | } 21 | 22 | if utils.AnyMatch(config.Test.Names, step.Name) { 23 | step.Metadata.Test = true 24 | } 25 | 26 | if utils.AnyMatch(config.Deploy.Names, step.Name) { 27 | step.Metadata.Deploy = true 28 | } 29 | 30 | return step 31 | } 32 | 33 | func enhanceShellStep(step *models.Step, config *config.EnhancementConfiguration) *models.Step { 34 | if utils.AnyMatch(config.Build.ShellRegexes, step.Shell.Script) { 35 | step.Metadata.Build = true 36 | } 37 | if utils.AnyMatch(config.Test.ShellRegexes, step.Shell.Script) { 38 | step.Metadata.Test = true 39 | } 40 | 41 | if utils.AnyMatch(config.Deploy.ShellRegexes, step.Shell.Script) { 42 | step.Metadata.Deploy = true 43 | } 44 | 45 | return step 46 | } 47 | 48 | func enhanceTaskStep(step *models.Step, config *config.EnhancementConfiguration) *models.Step { 49 | if utils.SliceContains(config.Build.Tasks, *step.Task.Name) { 50 | step.Metadata.Build = true 51 | } 52 | 53 | if utils.SliceContains(config.Test.Tasks, *step.Task.Name) { 54 | step.Metadata.Test = true 55 | } 56 | 57 | if utils.SliceContains(config.Deploy.Tasks, *step.Task.Name) { 58 | step.Metadata.Deploy = true 59 | } 60 | 61 | return step 62 | } 63 | -------------------------------------------------------------------------------- /pkg/loaders/azure/models/variables.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type Variables []Variable 11 | 12 | type Variable struct { 13 | Name string `yaml:"name,omitempty"` 14 | Value string `yaml:"value,omitempty"` 15 | Readonly bool `yaml:"readonly,omitempty"` 16 | Group string `yaml:"group,omitempty"` 17 | Template `yaml:",inline"` 18 | FileReference *models.FileReference 19 | } 20 | 21 | func (v *Variables) UnmarshalYAML(node *yaml.Node) error { 22 | if node.Tag == consts.MapTag { 23 | *v = parseVariablesMap(node) 24 | return nil 25 | } 26 | 27 | variables, err := parseVariablesList(node) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | *v = variables 33 | return nil 34 | } 35 | 36 | func parseVariablesMap(node *yaml.Node) []Variable { 37 | variables := []Variable{} 38 | 39 | loadersUtils.IterateOnMap(node, func(key string, value *yaml.Node) error { 40 | variable := Variable{ 41 | Name: key, 42 | Value: value.Value, 43 | } 44 | 45 | variable.FileReference = loadersUtils.GetFileReference(value) 46 | 47 | variables = append(variables, variable) 48 | return nil 49 | }, "Variables") 50 | 51 | return variables 52 | } 53 | 54 | func parseVariablesList(node *yaml.Node) ([]Variable, error) { 55 | variables := []Variable{} 56 | for _, variableNode := range node.Content { 57 | var variable Variable 58 | if err := variableNode.Decode(&variable); err != nil { 59 | return nil, err 60 | } 61 | variable.FileReference = loadersUtils.GetFileReference(variableNode) 62 | variables = append(variables, variable) 63 | } 64 | return variables, nil 65 | } 66 | -------------------------------------------------------------------------------- /pkg/loaders/github/models/workflow.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/models" 5 | ) 6 | 7 | type Container struct { 8 | Credentials *Credentials `yaml:"credentials,omitempty"` 9 | Env interface{} `yaml:"env,omitempty"` 10 | Image string `yaml:"image"` 11 | Options string `yaml:"options,omitempty"` 12 | Ports []interface{} `yaml:"ports,omitempty"` 13 | Volumes []string `yaml:"volumes,omitempty"` 14 | } 15 | 16 | type Credentials struct { 17 | Password string `yaml:"password,omitempty"` 18 | Username string `yaml:"username,omitempty"` 19 | } 20 | 21 | type Defaults struct { 22 | Run *Run `yaml:"run,omitempty"` 23 | } 24 | 25 | type Environment struct { 26 | Name string `yaml:"name"` 27 | Url string `yaml:"url,omitempty"` 28 | } 29 | 30 | type Ref struct { 31 | Branches []string `yaml:"branches,omitempty"` 32 | BranchesIgnore []string `yaml:"branches-ignore,omitempty"` 33 | Paths []string `yaml:"paths,omitempty"` 34 | PathsIgnore []string `yaml:"paths-ignore,omitempty"` 35 | Tags []string `yaml:"tags,omitempty"` 36 | TagsIgnore []string `yaml:"tags-ignore,omitempty"` 37 | FileReference *models.FileReference 38 | } 39 | 40 | type Run struct { 41 | Shell interface{} `yaml:"shell,omitempty"` 42 | WorkingDirectory string `yaml:"working-directory,omitempty"` 43 | } 44 | 45 | type Workflow struct { 46 | Concurrency *Concurrency `yaml:"concurrency,omitempty"` 47 | Defaults *Defaults `yaml:"defaults,omitempty"` 48 | Env *EnvironmentVariablesRef `yaml:"env,omitempty"` 49 | Jobs *Jobs `yaml:"jobs"` 50 | Name string `yaml:"name,omitempty"` 51 | On *On `yaml:"on"` 52 | Permissions *PermissionsEvent `yaml:"permissions,omitempty"` 53 | } 54 | -------------------------------------------------------------------------------- /pkg/parsers/azure/stages.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | ) 7 | 8 | func parseStages(stages *azureModels.Stages) []*models.Job { 9 | if stages == nil || (stages.Stages == nil && stages.TemplateStages == nil) { 10 | return nil 11 | } 12 | 13 | var jobs []*models.Job 14 | 15 | for _, stage := range stages.Stages { 16 | if stage.Jobs != nil { 17 | jobs = append(jobs, parseStage(stage)...) 18 | } 19 | } 20 | 21 | for _, stage := range stages.TemplateStages { 22 | if stage.Template.Template != "" { 23 | jobs = append(jobs, parseTemplateStage(stage)) 24 | } 25 | } 26 | 27 | return jobs 28 | } 29 | 30 | func parseStage(stage *azureModels.Stage) []*models.Job { 31 | if stage == nil || stage.Jobs == nil { 32 | return nil 33 | } 34 | 35 | parsedJobs := parseJobs(stage.Jobs) 36 | 37 | if stage.Variables == nil { 38 | return parsedJobs 39 | } 40 | envs := parseVariables(stage.Variables) 41 | 42 | for _, job := range parsedJobs { 43 | if job.EnvironmentVariables == nil { 44 | job.EnvironmentVariables = envs 45 | continue 46 | } 47 | 48 | for k, v := range envs.EnvironmentVariables { 49 | job.EnvironmentVariables.EnvironmentVariables[k] = v 50 | } 51 | } 52 | 53 | return parsedJobs 54 | } 55 | 56 | func parseTemplateStage(stage *azureModels.TemplateStage) *models.Job { 57 | path, alias := parseTemplateString(stage.Template.Template) 58 | return &models.Job{ 59 | ID: &stage.Template.Template, 60 | Imports: &models.Import{ 61 | Source: &models.ImportSource{ 62 | Path: &path, 63 | Type: calculateSourceType(alias), 64 | RepositoryAlias: &alias, 65 | }, 66 | Parameters: stage.Parameters, 67 | FileReference: stage.FileReference, 68 | }, 69 | FileReference: stage.FileReference, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/loaders/github/models/runner.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 7 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 8 | "github.com/argonsecurity/pipeline-parser/pkg/models" 9 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 10 | "gopkg.in/yaml.v3" 11 | ) 12 | 13 | type RunsOn struct { 14 | OS *string 15 | Arch *string 16 | SelfHosted bool 17 | Tags []string 18 | FileReference *models.FileReference 19 | } 20 | 21 | func (r *RunsOn) UnmarshalYAML(node *yaml.Node) error { 22 | var tags []string 23 | var err error 24 | if node.Tag == consts.StringTag { 25 | tags = []string{node.Value} 26 | } else if node.Tag == consts.SequenceTag { 27 | if tags, err = loadersUtils.ParseYamlStringSequenceToSlice(node, "RunsOn"); err != nil { 28 | return err 29 | } 30 | } else { 31 | return consts.NewErrInvalidYamlTag(node.Tag, "RunsOn") 32 | } 33 | 34 | *r = *generateRunsOnFromTags(tags) 35 | r.FileReference = loadersUtils.GetFileReference(node) 36 | return nil 37 | } 38 | 39 | func generateRunsOnFromTags(tags []string) *RunsOn { 40 | r := &RunsOn{} 41 | r.Tags = tags 42 | for _, tag := range tags { 43 | r = parseTag(r, tag) 44 | } 45 | return r 46 | } 47 | 48 | func parseTag(r *RunsOn, tag string) *RunsOn { 49 | if tag == consts.SelfHosted { 50 | r.SelfHosted = true 51 | } 52 | 53 | for os, keywords := range consts.OsToKeywords { 54 | didFind := false 55 | for _, keyword := range keywords { 56 | if strings.Contains(strings.ToLower(tag), keyword) { 57 | r.OS = utils.GetPtr(string(os)) 58 | didFind = true 59 | break 60 | } 61 | } 62 | if didFind { 63 | break 64 | } 65 | } 66 | 67 | for _, arch := range consts.ArchKeywords { 68 | if strings.Contains(strings.ToLower(tag), arch) { 69 | r.Arch = utils.GetPtr(arch) 70 | break 71 | } 72 | } 73 | 74 | return r 75 | } 76 | -------------------------------------------------------------------------------- /pkg/models/pipeline.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Pipeline struct { 4 | Id *string `json:"id,omitempty"` 5 | Name *string `json:"name,omitempty"` 6 | Triggers *Triggers `json:"triggers,omitempty"` 7 | Jobs []*Job `json:"jobs,omitempty"` 8 | Imports []*Import `json:"imports,omitempty"` 9 | Parameters []*Parameter `json:"parameters,omitempty"` 10 | Defaults *Defaults `json:"defaults,omitempty"` 11 | Platform Platform `json:"platform,omitempty"` 12 | } 13 | 14 | type Scans struct { 15 | Secrets *bool `json:"secrets,omitempty"` 16 | Iac *bool `json:"iac,omitempty"` 17 | Pipelines *bool `json:"pipelines,omitempty"` 18 | SAST *bool `json:"sast,omitempty"` 19 | Dependencies *bool `json:"dependencies,omitempty"` 20 | License *bool `json:"license,omitempty"` 21 | } 22 | 23 | type Resources struct { 24 | Repositories []*ImportSource `json:"repositories,omitempty"` 25 | FileReference *FileReference `json:"file_reference,omitempty"` 26 | } 27 | 28 | type Defaults struct { 29 | EnvironmentVariables *EnvironmentVariablesRef `json:"environment_variables,omitempty"` 30 | Scans *Scans `json:"scans,omitempty"` 31 | Runner *Runner `json:"runner,omitempty"` 32 | Conditions []*Condition `json:"conditions,omitempty"` 33 | ContinueOnError *bool `json:"continue_on_error,omitempty"` 34 | TokenPermissions *TokenPermissions `json:"token_permissions,omitempty"` 35 | Settings *map[string]any `json:"settings,omitempty"` 36 | FileReference *FileReference `json:"file_reference,omitempty"` 37 | PostSteps []*Step `json:"post_steps,omitempty"` 38 | PreSteps []*Step `json:"pre_steps,omitempty"` 39 | Resources *Resources `json:"resources,omitempty"` 40 | } 41 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/common/include.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 7 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 8 | "github.com/argonsecurity/pipeline-parser/pkg/models" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | type Include []IncludeItem 13 | 14 | func (i *Include) UnmarshalYAML(node *yaml.Node) error { 15 | if node.Tag == consts.StringTag { 16 | *i = Include{ParseIncludeString(node)} 17 | return nil 18 | } 19 | 20 | var items []IncludeItem 21 | if err := utils.ParseSequenceOrOne(node, &items); err != nil { 22 | return err 23 | } 24 | *i = items 25 | return nil 26 | } 27 | 28 | type IncludeItem struct { 29 | Project string `yaml:"project"` 30 | Ref string `yaml:"ref"` 31 | Template string `yaml:"template"` 32 | File string `yaml:"file"` 33 | 34 | Local string `yaml:"local"` 35 | Remote string `yaml:"remote"` 36 | 37 | FileReference *models.FileReference 38 | } 39 | 40 | func (it *IncludeItem) UnmarshalYAML(node *yaml.Node) error { 41 | it.FileReference = utils.GetFileReference(node) 42 | if node.Tag == consts.StringTag { 43 | *it = ParseIncludeString(node) 44 | return nil 45 | } 46 | 47 | return utils.IterateOnMap(node, func(key string, value *yaml.Node) error { 48 | switch key { 49 | case "project": 50 | it.Project = value.Value 51 | case "ref": 52 | it.Ref = value.Value 53 | case "file": 54 | it.File = value.Value 55 | case "template": 56 | it.Template = value.Value 57 | case "local": 58 | it.Local = value.Value 59 | case "remote": 60 | it.Remote = value.Value 61 | } 62 | return nil 63 | }, "IncludeItem") 64 | } 65 | 66 | func ParseIncludeString(node *yaml.Node) IncludeItem { 67 | it := IncludeItem{} 68 | it.FileReference = utils.GetFileReference(node) 69 | 70 | if strings.HasPrefix(node.Value, "https://") { 71 | it.Remote = node.Value 72 | return it 73 | } 74 | 75 | it.Local = node.Value 76 | return it 77 | } 78 | -------------------------------------------------------------------------------- /pkg/loaders/gitlab/models/job/parallel.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 7 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 8 | "gopkg.in/yaml.v3" 9 | ) 10 | 11 | type Parallel struct { 12 | Max *int `yaml:"max,omitempty"` 13 | Matrix *Matrix `yaml:"matrix,omitempty"` 14 | } 15 | 16 | type Matrix []MatrixItem 17 | 18 | type MatrixItem map[string][]string 19 | 20 | func (p *Parallel) UnmarshalYAML(node *yaml.Node) error { 21 | if node.Tag == consts.IntTag { 22 | intValue, err := strconv.Atoi(node.Value) 23 | if err != nil { 24 | return err 25 | } 26 | p.Max = &intValue 27 | return nil 28 | } 29 | 30 | return utils.IterateOnMap(node, func(key string, value *yaml.Node) error { 31 | switch key { 32 | case "max": 33 | v, _ := strconv.Atoi(value.Value) 34 | p.Max = &v 35 | case "matrix": 36 | p.Matrix = &Matrix{} 37 | if err := p.Matrix.UnmarshalYAML(value); err != nil { 38 | return err 39 | } 40 | } 41 | return nil 42 | }, "Parallel") 43 | } 44 | 45 | func (m *Matrix) UnmarshalYAML(node *yaml.Node) error { 46 | if node.Tag != consts.SequenceTag { 47 | return consts.NewErrInvalidYamlTag(node.Tag, "Matrix") 48 | } 49 | 50 | for _, matrixNode := range node.Content { 51 | var matrixItem MatrixItem 52 | if err := matrixNode.Decode(&matrixItem); err != nil { 53 | return err 54 | } 55 | 56 | *m = append(*m, matrixItem) 57 | } 58 | 59 | return nil 60 | } 61 | 62 | func (mi *MatrixItem) UnmarshalYAML(node *yaml.Node) error { 63 | *mi = make(MatrixItem, 0) 64 | return utils.IterateOnMap(node, func(key string, value *yaml.Node) error { 65 | if value.Tag == consts.StringTag { 66 | (*mi)[key] = []string{value.Value} 67 | } 68 | 69 | if value.Tag == consts.SequenceTag { 70 | parsedStrings, err := utils.ParseYamlStringSequenceToSlice(value, "Matrix") 71 | if err != nil { 72 | return err 73 | } 74 | (*mi)[key] = parsedStrings 75 | } 76 | return nil 77 | }, "Matrix") 78 | } 79 | -------------------------------------------------------------------------------- /pkg/consts/errors.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | ) 8 | 9 | type ErrInvalidPlatform struct { 10 | Platform models.Platform 11 | } 12 | 13 | func (e *ErrInvalidPlatform) Error() string { 14 | return fmt.Sprintf("invalid platform: %s. Supported platforms: %v", e.Platform, Platforms) 15 | } 16 | 17 | func NewErrInvalidPlatform(platform models.Platform) error { 18 | return &ErrInvalidPlatform{Platform: platform} 19 | } 20 | 21 | type ErrInvalidOutputTarget struct { 22 | OutputTarget OutputTarget 23 | } 24 | 25 | func (e *ErrInvalidOutputTarget) Error() string { 26 | return fmt.Sprintf("invalid output target: %s. Supported output targets: %v", e.OutputTarget, OutputTargets) 27 | } 28 | 29 | func NewErrInvalidOutputTarget(outputTarget OutputTarget) error { 30 | return &ErrInvalidOutputTarget{OutputTarget: outputTarget} 31 | } 32 | 33 | type ErrInvalidYaml struct { 34 | Message string 35 | } 36 | 37 | func (e *ErrInvalidYaml) Error() string { 38 | return fmt.Sprintf("invalid yaml: %s", e.Message) 39 | } 40 | 41 | func NewErrInvalidYaml(message string) error { 42 | return &ErrInvalidYaml{Message: message} 43 | } 44 | 45 | type ErrInvalidYamlTag struct { 46 | Tag string 47 | Type string 48 | } 49 | 50 | func (e *ErrInvalidYamlTag) Error() string { 51 | return fmt.Sprintf("invalid yaml tag '%s' for type '%s'", e.Tag, e.Type) 52 | } 53 | 54 | func NewErrInvalidYamlTag(tag string, structType string) error { 55 | return &ErrInvalidYamlTag{Tag: tag, Type: structType} 56 | } 57 | 58 | type ErrInvalidArgumentsCount struct { 59 | Count int 60 | } 61 | 62 | func (e *ErrInvalidArgumentsCount) Error() string { 63 | return fmt.Sprintf("invalid number of arguments: %d. Expected minimum 1 argument", e.Count) 64 | } 65 | 66 | func NewErrInvalidArgumentsCount(count int) error { 67 | return &ErrInvalidArgumentsCount{Count: count} 68 | } 69 | 70 | type ErrEmptyData struct { 71 | } 72 | 73 | func (e *ErrEmptyData) Error() string { 74 | return "empty data" 75 | } 76 | 77 | func NewErrEmptyData() error { 78 | return &ErrEmptyData{} 79 | } 80 | -------------------------------------------------------------------------------- /pkg/parsers/github/runner_test.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "testing" 5 | 6 | githubModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/github/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 10 | ) 11 | 12 | func TestParseRunsOnToRunner(t *testing.T) { 13 | testCases := []struct { 14 | name string 15 | runsOn *githubModels.RunsOn 16 | expectedRunner *models.Runner 17 | }{ 18 | { 19 | name: "runsOn nil", 20 | runsOn: nil, 21 | expectedRunner: nil, 22 | }, 23 | { 24 | name: "Full runsOn", 25 | runsOn: &githubModels.RunsOn{ 26 | OS: utils.GetPtr("linux"), 27 | Arch: utils.GetPtr("amd64"), 28 | SelfHosted: true, 29 | Tags: []string{"tag1", "tag2"}, 30 | FileReference: &models.FileReference{ 31 | StartRef: &models.FileLocation{ 32 | Line: 1, 33 | Column: 1, 34 | }, 35 | EndRef: &models.FileLocation{ 36 | Line: 2, 37 | Column: 2, 38 | }, 39 | }, 40 | }, 41 | expectedRunner: &models.Runner{ 42 | OS: utils.GetPtr("linux"), 43 | Arch: utils.GetPtr("amd64"), 44 | SelfHosted: utils.GetPtr(true), 45 | Labels: &[]string{"tag1", "tag2"}, 46 | FileReference: &models.FileReference{ 47 | StartRef: &models.FileLocation{ 48 | Line: 1, 49 | Column: 1, 50 | }, 51 | EndRef: &models.FileLocation{ 52 | Line: 2, 53 | Column: 2, 54 | }, 55 | }, 56 | }, 57 | }, 58 | { 59 | name: "Empty runsOn", 60 | runsOn: &githubModels.RunsOn{}, 61 | expectedRunner: &models.Runner{ 62 | SelfHosted: utils.GetPtr(false), 63 | Labels: utils.GetPtr[[]string](nil), 64 | }, 65 | }, 66 | } 67 | 68 | for _, testCase := range testCases { 69 | t.Run(testCase.name, func(t *testing.T) { 70 | got := parseRunsOnToRunner(testCase.runsOn) 71 | 72 | testutils.DeepCompare(t, testCase.expectedRunner, got) 73 | }) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/argonsecurity/pipeline-parser 2 | 3 | go 1.21 4 | 5 | toolchain go1.21.9 6 | 7 | require ( 8 | github.com/go-test/deep v1.0.8 9 | github.com/imroc/req/v3 v3.42.3 10 | github.com/mitchellh/mapstructure v1.4.3 11 | github.com/pkg/errors v0.9.1 12 | github.com/r3labs/diff/v3 v3.0.0 13 | github.com/spf13/cobra v1.4.0 14 | gopkg.in/yaml.v3 v3.0.1 15 | ) 16 | 17 | require ( 18 | github.com/andybalholm/brotli v1.0.6 // indirect 19 | github.com/cloudflare/circl v1.3.7 // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect 22 | github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42 // indirect 23 | github.com/hashicorp/errwrap v1.1.0 // indirect 24 | github.com/hashicorp/go-multierror v1.1.1 // indirect 25 | github.com/klauspost/compress v1.17.4 // indirect 26 | github.com/kr/text v0.2.0 // indirect 27 | github.com/onsi/ginkgo/v2 v2.13.2 // indirect 28 | github.com/pmezard/go-difflib v1.0.0 // indirect 29 | github.com/quic-go/qpack v0.4.0 // indirect 30 | github.com/quic-go/quic-go v0.40.1 // indirect 31 | github.com/refraction-networking/utls v1.6.0 // indirect 32 | go.uber.org/mock v0.4.0 // indirect 33 | golang.org/x/crypto v0.21.0 // indirect 34 | golang.org/x/mod v0.14.0 // indirect 35 | golang.org/x/sys v0.18.0 // indirect 36 | golang.org/x/text v0.14.0 // indirect 37 | golang.org/x/tools v0.16.1 // indirect 38 | google.golang.org/protobuf v1.33.0 // indirect 39 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 40 | ) 41 | 42 | require ( 43 | github.com/golang/protobuf v1.5.3 // indirect 44 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 45 | github.com/spf13/pflag v1.0.5 // indirect 46 | github.com/stretchr/testify v1.8.0 47 | github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect 48 | golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc 49 | golang.org/x/net v0.23.0 // indirect 50 | google.golang.org/appengine v1.6.6 // indirect 51 | ) 52 | 53 | // fix CVE-2024-22189 54 | replace github.com/quic-go/quic-go v0.40.1 => github.com/quic-go/quic-go v0.42.0 55 | -------------------------------------------------------------------------------- /pkg/loaders/github/models/step.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | commonModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/common/models" 5 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type Steps []Step 11 | 12 | type ShellCommand struct { 13 | Script string 14 | FileReference *models.FileReference 15 | } 16 | 17 | type With commonModels.Map 18 | 19 | type Step struct { 20 | ContinueOnError *string `yaml:"continue-on-error,omitempty"` 21 | Env *EnvironmentVariablesRef `yaml:"env,omitempty"` 22 | Id string `yaml:"id,omitempty"` 23 | If string `yaml:"if,omitempty"` 24 | Name string `yaml:"name,omitempty"` 25 | Run *ShellCommand `yaml:"run,omitempty"` 26 | Shell string `yaml:"shell,omitempty"` 27 | TimeoutMinutes int `yaml:"timeout-minutes,omitempty"` 28 | Uses string `yaml:"uses,omitempty"` 29 | With *With `yaml:"with,omitempty"` 30 | WorkingDirectory string `yaml:"working-directory,omitempty"` 31 | FileReference *models.FileReference 32 | } 33 | 34 | func (w *With) UnmarshalYAML(value *yaml.Node) error { 35 | var m commonModels.Map 36 | if err := value.Decode(&m); err != nil { 37 | return err 38 | } 39 | *w = With(m) 40 | return nil 41 | } 42 | 43 | func (s *Steps) UnmarshalYAML(node *yaml.Node) error { 44 | steps := []Step{} 45 | for _, stepNode := range node.Content { 46 | var step Step 47 | if err := stepNode.Decode(&step); err != nil { 48 | return err 49 | } 50 | step.FileReference = loadersUtils.GetFileReference(stepNode) 51 | steps = append(steps, step) 52 | } 53 | *s = steps 54 | return nil 55 | } 56 | 57 | func (s *ShellCommand) UnmarshalYAML(node *yaml.Node) error { 58 | s.FileReference = loadersUtils.GetFileReference(node) 59 | s.Script = node.Value 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /test/fixtures/bitbucket/simple-pipeline.yml: -------------------------------------------------------------------------------- 1 | # Template docker-push 2 | 3 | # This template allows you to build and push your docker image to a Docker Hub account. 4 | # The workflow allows running tests, code linting and security scans on feature branches (as well as master). 5 | # The docker image will be validated and pushed to the docker registry after the code is merged to master. 6 | 7 | # Prerequisites: $DOCKERHUB_USERNAME, $DOCKERHUB_PASSWORD setup as deployment variables 8 | 9 | image: atlassian/default-image:3 10 | 11 | pipelines: 12 | default: 13 | - parallel: 14 | - step: 15 | name: Build and Test 16 | script: 17 | - IMAGE_NAME=$BITBUCKET_REPO_SLUG 18 | - docker build . --file Dockerfile --tag ${IMAGE_NAME} 19 | services: 20 | - docker 21 | caches: 22 | - docker 23 | - step: 24 | name: Lint the Dockerfile 25 | image: hadolint/hadolint:latest-debian 26 | script: 27 | - hadolint Dockerfile 28 | branches: 29 | master: 30 | - step: 31 | name: Build and Test 32 | script: 33 | - IMAGE_NAME=$BITBUCKET_REPO_SLUG 34 | - docker build . --file Dockerfile --tag ${IMAGE_NAME} 35 | - docker save ${IMAGE_NAME} --output "${IMAGE_NAME}.tar" 36 | services: 37 | - docker 38 | caches: 39 | - docker 40 | artifacts: 41 | - "*.tar" 42 | - step: 43 | name: Deploy to Production 44 | deployment: Production 45 | script: 46 | - echo ${DOCKERHUB_PASSWORD} | docker login --username "$DOCKERHUB_USERNAME" --password-stdin 47 | - IMAGE_NAME=$BITBUCKET_REPO_SLUG 48 | - docker load --input "${IMAGE_NAME}.tar" 49 | - VERSION="prod-0.1.${BITBUCKET_BUILD_NUMBER}" 50 | - IMAGE=${DOCKERHUB_NAMESPACE}/${IMAGE_NAME} 51 | - docker tag "${IMAGE_NAME}" "${IMAGE}:${VERSION}" 52 | - docker push "${IMAGE}:${VERSION}" 53 | services: 54 | - docker 55 | -------------------------------------------------------------------------------- /pkg/models/common.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Platform string 4 | 5 | type EnvironmentVariables map[string]any 6 | type EnvironmentVariablesRef struct { 7 | EnvironmentVariables `json:"environment_variables,omitempty"` 8 | FileReference *FileReference `json:"file_reference,omitempty"` 9 | Imports *Import `json:"imports,omitempty"` 10 | } 11 | 12 | type Condition struct { 13 | Statement string `json:"statement,omitempty"` 14 | Allow *bool `json:"allow,omitempty"` 15 | Paths *Filter `json:"paths,omitempty"` 16 | Exists *Filter `json:"exists,omitempty"` 17 | Branches *Filter `json:"branches,omitempty"` 18 | Events []EventType `json:"events,omitempty"` 19 | Variables map[string]string `json:"variables,omitempty"` 20 | } 21 | 22 | type Filter struct { 23 | AllowList []string `json:"allow_list,omitempty"` 24 | DenyList []string `json:"deny_list,omitempty"` 25 | } 26 | 27 | type Variable struct { 28 | Context *string `json:"context,omitempty"` 29 | Name *string `json:"name,omitempty"` 30 | Value *string `json:"value,omitempty"` 31 | Parent *Entity `json:"parent,omitempty"` 32 | Masked *bool `json:"masked,omitempty"` 33 | Protected *bool `json:"protected,omitempty"` 34 | } 35 | 36 | type Parameter struct { 37 | Name *string `json:"name,omitempty"` 38 | Value any `json:"value,omitempty"` 39 | Description *string `json:"description,omitempty"` 40 | Default any `json:"default,omitempty"` 41 | Options []string `json:"options,omitempty"` 42 | FileReference *FileReference `json:"file_reference,omitempty"` 43 | } 44 | 45 | type Entity struct { 46 | ID *string `json:"id,omitempty"` 47 | Name *string `json:"name,omitempty"` 48 | } 49 | 50 | type FileLocation struct { 51 | Line int `json:"line,omitempty"` 52 | Column int `json:"column,omitempty"` 53 | } 54 | 55 | type FileReference struct { 56 | StartRef *FileLocation `json:"start_ref,omitempty"` 57 | EndRef *FileLocation `json:"end_ref,omitempty"` 58 | IsAlias bool `json:"is_alias,omitempty"` 59 | } 60 | -------------------------------------------------------------------------------- /pkg/loaders/common/models/map_test.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 7 | "github.com/r3labs/diff/v3" 8 | "github.com/stretchr/testify/assert" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | type testMap struct { 13 | Map *Map `yaml:"map"` 14 | } 15 | 16 | func TestMapLoader(t *testing.T) { 17 | testCases := []struct { 18 | name string 19 | yamlBuffer []byte 20 | expectedMap *testMap 21 | }{ 22 | { 23 | name: "Map is nil", 24 | yamlBuffer: nil, 25 | expectedMap: nil, 26 | }, 27 | { 28 | name: "Map is empty", 29 | yamlBuffer: []byte{}, 30 | expectedMap: nil, 31 | }, 32 | { 33 | name: "Map with data", 34 | yamlBuffer: []byte(`map: 35 | string: "string" 36 | int: 1 37 | bool: true 38 | list: [1, 2, 3]`, 39 | ), 40 | expectedMap: &testMap{ 41 | Map: &Map{ 42 | Values: []*MapEntry{ 43 | { 44 | Key: "string", 45 | Value: "string", 46 | FileReference: testutils.CreateFileReference(2, 3, 2, 9), 47 | }, 48 | { 49 | Key: "int", 50 | Value: 1, 51 | FileReference: testutils.CreateFileReference(3, 3, 3, 4), 52 | }, 53 | { 54 | Key: "bool", 55 | Value: true, 56 | FileReference: testutils.CreateFileReference(4, 3, 4, 7), 57 | }, 58 | { 59 | Key: "list", 60 | Value: []any{1, 2, 3}, 61 | FileReference: testutils.CreateFileReference(5, 3, 5, 18), 62 | }, 63 | }, 64 | FileReference: testutils.CreateFileReference(1, 1, 5, 17), 65 | }, 66 | }, 67 | }, 68 | } 69 | 70 | for _, testCase := range testCases { 71 | t.Run(testCase.name, func(t *testing.T) { 72 | var m *testMap 73 | err := yaml.Unmarshal(testCase.yamlBuffer, &m) 74 | assert.NoError(t, err) 75 | 76 | changelog, err := diff.Diff(testCase.expectedMap, m) 77 | assert.NoError(t, err) 78 | assert.Len(t, changelog, 0, "test case failed with %d modifications", len(changelog)) 79 | for _, change := range changelog { 80 | t.Log(change) 81 | } 82 | }) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /pkg/loaders/azure/models/pipeline.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "gopkg.in/yaml.v3" 7 | ) 8 | 9 | type Pipeline struct { 10 | Extends *Extends `yaml:"extends,omitempty"` 11 | Jobs *Jobs `yaml:"jobs,omitempty"` 12 | Stages *Stages `yaml:"stages,omitempty"` 13 | ContinueOnError *bool `yaml:"continueOnError,omitempty"` 14 | Pool *Pool `yaml:"pool,omitempty"` 15 | Container *JobContainer `yaml:"container,omitempty"` 16 | Name string `yaml:"name,omitempty"` 17 | Trigger *TriggerRef `yaml:"trigger,omitempty"` 18 | Parameters *Parameters `yaml:"parameters,omitempty"` 19 | PR *PRRef `yaml:"pr,omitempty"` 20 | Schedules *Schedules `yaml:"schedules,omitempty"` 21 | Resources *Resources `yaml:"resources,omitempty"` 22 | Steps *Steps `yaml:"steps,omitempty"` 23 | Variables *Variables `yaml:"variables,omitempty"` 24 | LockBehavior string `yaml:"lockBehavior,omitempty"` 25 | } 26 | 27 | type Parameter struct { 28 | Name string `yaml:"name,omitempty"` 29 | DisplayName string `yaml:"displayName,omitempty"` 30 | Type string `yaml:"type,omitempty"` 31 | Default any `yaml:"default,omitempty"` 32 | Values []string `yaml:"values,omitempty"` 33 | FileReference *models.FileReference 34 | } 35 | 36 | type Parameters []Parameter 37 | 38 | func (p *Parameters) UnmarshalYAML(node *yaml.Node) error { 39 | var parameters Parameters 40 | for _, parameterNode := range node.Content { 41 | parameter, err := parseParameter(parameterNode) 42 | if err != nil { 43 | return err 44 | } 45 | parameters = append(parameters, parameter) 46 | } 47 | *p = parameters 48 | return nil 49 | } 50 | 51 | func parseParameter(node *yaml.Node) (Parameter, error) { 52 | var parameter Parameter 53 | if err := node.Decode(¶meter); err != nil { 54 | return parameter, err 55 | } 56 | parameter.FileReference = loadersUtils.GetFileReference(node) 57 | return parameter, nil 58 | } 59 | -------------------------------------------------------------------------------- /pkg/parsers/azure/parameters_test.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "testing" 5 | 6 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 10 | ) 11 | 12 | func TestParseParameters(t *testing.T) { 13 | testCases := []struct { 14 | name string 15 | parameters *azureModels.Parameters 16 | expectedParameters []*models.Parameter 17 | }{ 18 | { 19 | name: "parameters are nil", 20 | parameters: nil, 21 | expectedParameters: nil, 22 | }, 23 | { 24 | name: "parameters are empty", 25 | parameters: &azureModels.Parameters{}, 26 | expectedParameters: nil, 27 | }, 28 | { 29 | name: "parameters with data", 30 | parameters: &azureModels.Parameters{ 31 | { 32 | Name: "param1", 33 | DisplayName: "Param 1", 34 | Type: "string", 35 | Default: "default1", 36 | Values: []string{"value1", "value2"}, 37 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 38 | }, 39 | { 40 | Name: "param2", 41 | DisplayName: "Param 2", 42 | Type: "number", 43 | Default: 2, 44 | Values: []string{"1", "2"}, 45 | FileReference: testutils.CreateFileReference(5, 6, 7, 8), 46 | }, 47 | }, 48 | expectedParameters: []*models.Parameter{ 49 | { 50 | Name: utils.GetPtr("param1"), 51 | Default: "default1", 52 | Options: []string{"value1", "value2"}, 53 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 54 | }, 55 | { 56 | Name: utils.GetPtr("param2"), 57 | Default: 2, 58 | Options: []string{"1", "2"}, 59 | FileReference: testutils.CreateFileReference(5, 6, 7, 8), 60 | }, 61 | }, 62 | }, 63 | } 64 | 65 | for _, testCase := range testCases { 66 | t.Run(testCase.name, func(t *testing.T) { 67 | got := parseParameters(testCase.parameters) 68 | testutils.DeepCompare(t, testCase.expectedParameters, got) 69 | }) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/parsers/azure/azure.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 7 | ) 8 | 9 | type AzureParser struct{} 10 | 11 | func (g *AzureParser) Parse(azurePipeline *azureModels.Pipeline) (*models.Pipeline, error) { 12 | if azurePipeline == nil { 13 | return nil, nil 14 | } 15 | 16 | pipeline := &models.Pipeline{ 17 | Name: &azurePipeline.Name, 18 | } 19 | 20 | pipeline.Defaults = parsePipelineDefaults(azurePipeline) 21 | pipeline.Triggers = parsePipelineTriggers(azurePipeline) 22 | pipeline.Parameters = parseParameters(azurePipeline.Parameters) 23 | pipeline.Imports = parseExtends(azurePipeline.Extends) 24 | 25 | var jobs []*models.Job 26 | 27 | if azurePipeline.Stages != nil { 28 | jobs = append(jobs, parseStages(azurePipeline.Stages)...) 29 | } 30 | 31 | if azurePipeline.Jobs != nil { 32 | jobs = append(pipeline.Jobs, parseJobs(azurePipeline.Jobs)...) 33 | } 34 | 35 | if len(jobs) == 0 { 36 | jobs = []*models.Job{generateDefaultJob()} 37 | if azurePipeline.Pool != nil { 38 | jobs[0].Runner = parsePool(azurePipeline.Pool, jobs[0].Runner) 39 | } 40 | 41 | if azurePipeline.Container != nil { 42 | jobs[0].Runner = parseContainer(azurePipeline.Container, jobs[0].Runner) 43 | } 44 | } 45 | 46 | pipeline.Jobs = jobs 47 | 48 | if azurePipeline.Steps != nil { 49 | pipeline.Jobs[0].Steps = parseSteps(azurePipeline.Steps) 50 | } 51 | 52 | return pipeline, nil 53 | } 54 | 55 | func parsePipelineDefaults(pipeline *azureModels.Pipeline) *models.Defaults { 56 | if pipeline == nil { 57 | return nil 58 | } 59 | 60 | defaults := &models.Defaults{ 61 | ContinueOnError: pipeline.ContinueOnError, 62 | } 63 | 64 | if pipeline.Variables != nil { 65 | defaults.EnvironmentVariables = parseVariables(pipeline.Variables) 66 | } 67 | 68 | if pipeline.Resources != nil { 69 | defaults.Resources = parseResources(pipeline.Resources) 70 | } 71 | 72 | return defaults 73 | } 74 | 75 | func generateDefaultJob() *models.Job { 76 | return &models.Job{ 77 | Name: utils.GetPtr("default"), 78 | Runner: &models.Runner{}, 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /test/fixtures/azure/resources.yaml: -------------------------------------------------------------------------------- 1 | name: resources 2 | 3 | resources: 4 | - repo: self 5 | builds: 6 | - build: Spaceworkz 7 | type: Jenkins 8 | connection: MyJenkinsServer 9 | source: SpaceworkzProj # name of the jenkins source project 10 | trigger: true 11 | containers: 12 | - container: linux 13 | image: ubuntu:16.04 14 | - container: windows 15 | image: myprivate.azurecr.io/windowsservercore:1803 16 | endpoint: my_acr_connection 17 | - container: my_service 18 | image: my_service:tag 19 | ports: 20 | - 8080:80 # bind container port 80 to 8080 on the host machine 21 | - 6379 # bind container port 6379 to a random available port on the host machine 22 | volumes: 23 | - /src/dir:/dst/dir # mount /src/dir on the host into /dst/dir in the container 24 | pipelines: 25 | - pipeline: SmartHotel 26 | project: DevOpsProject 27 | source: SmartHotel-CI 28 | trigger: 29 | branches: 30 | include: 31 | - releases/* 32 | - main 33 | exclude: 34 | - topic/* 35 | tags: 36 | - Verified 37 | - Signed 38 | stages: 39 | - Production 40 | - PreProduction 41 | repositories: 42 | - repository: common 43 | type: github 44 | name: Contoso/CommonTools 45 | endpoint: MyContosoServiceConnection 46 | webhooks: 47 | - webhook: MyWebhookTriggerAlias ### Webhook alias 48 | connection: IncomingWebhookConnection ### Incoming webhook service connection 49 | filters: ### List of JSON parameters to filter; Parameters are AND'ed 50 | - path: JSONParameterPath ### JSON path in the payload 51 | value: JSONParameterExpectedValue ### Expected value in the path provided 52 | packages: 53 | - package: myPackageAlias # alias for the package resource 54 | type: Npm # type of the package NuGet/npm 55 | connection: GitHubConnectionName # GitHub service connection with the PAT type 56 | name: nugetTest/nodeapp # / 57 | version: 1.0.1 # Version of the package to consume; Optional; Defaults to latest 58 | trigger: true # To enable automated triggers (true/false); Optional; Defaults to no triggers -------------------------------------------------------------------------------- /pkg/parsers/utils/map_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | 6 | loadersCommonModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/common/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 10 | ) 11 | 12 | func TestParseMapToParameters(t *testing.T) { 13 | testCases := []struct { 14 | name string 15 | with loadersCommonModels.Map 16 | expectedParameters []*models.Parameter 17 | }{ 18 | { 19 | name: "with nil", 20 | with: loadersCommonModels.Map{}, 21 | expectedParameters: []*models.Parameter{}, 22 | }, 23 | { 24 | name: "with values", 25 | with: loadersCommonModels.Map{ 26 | Values: []*loadersCommonModels.MapEntry{ 27 | { 28 | Key: "string", 29 | Value: "string", 30 | FileReference: testutils.CreateFileReference(112, 224, 112, 238), 31 | }, 32 | { 33 | Key: "int", 34 | Value: 1, 35 | FileReference: testutils.CreateFileReference(113, 224, 113, 230), 36 | }, 37 | { 38 | Key: "bool", 39 | Value: true, 40 | FileReference: testutils.CreateFileReference(114, 224, 114, 234), 41 | }, 42 | }, 43 | FileReference: testutils.CreateFileReference(111, 222, 333, 444), 44 | }, 45 | expectedParameters: []*models.Parameter{ 46 | { 47 | Name: utils.GetPtr("string"), 48 | Value: "string", 49 | FileReference: testutils.CreateFileReference(112, 224, 112, 238), 50 | }, 51 | { 52 | Name: utils.GetPtr("int"), 53 | Value: 1, 54 | FileReference: testutils.CreateFileReference(113, 224, 113, 230), 55 | }, 56 | { 57 | Name: utils.GetPtr("bool"), 58 | Value: true, 59 | FileReference: testutils.CreateFileReference(114, 224, 114, 234), 60 | }, 61 | }, 62 | }, 63 | } 64 | 65 | for _, testCase := range testCases { 66 | t.Run(testCase.name, func(t *testing.T) { 67 | got := ParseMapToParameters(testCase.with) 68 | testutils.DeepCompare(t, testCase.expectedParameters, got) 69 | }) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/common/script.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models/common" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 9 | ) 10 | 11 | func ParseScript(script *common.Script) []*models.Step { 12 | if script == nil { 13 | return nil 14 | } 15 | 16 | if len(script.Commands) == 1 { // format: script: command 17 | return []*models.Step{ 18 | { 19 | Type: models.ShellStepType, 20 | Shell: &models.Shell{ 21 | Script: &script.Commands[0], 22 | }, 23 | FileReference: script.FileReference, 24 | }, 25 | } 26 | } 27 | 28 | // format 29 | // script: 30 | // - command1 31 | // - command2 32 | return utils.MapWithIndex(script.Commands, func(command string, index int) *models.Step { 33 | return &models.Step{ 34 | Type: models.ShellStepType, 35 | Shell: &models.Shell{ 36 | Script: &command, 37 | }, 38 | FileReference: parseCommandFileReference(script, index), 39 | } 40 | }) 41 | } 42 | 43 | func parseCommandFileReference(script *common.Script, commandIndex int) *models.FileReference { 44 | command := script.Commands[commandIndex] 45 | 46 | firstLine := script.FileReference.StartRef.Line + countLines(script, commandIndex) 47 | endLine := firstLine 48 | 49 | // handle multiline command 50 | if strings.Contains(command, "\n") { 51 | splitValue := strings.Split(command, "\n") 52 | command = splitValue[len(splitValue)-1] 53 | endLine = firstLine + len(splitValue) - 1 54 | } 55 | 56 | return &models.FileReference{ 57 | StartRef: &models.FileLocation{ 58 | Line: firstLine, // +1 for the script header 59 | Column: script.FileReference.StartRef.Column + 2, // +2 for the "- " section 60 | }, 61 | EndRef: &models.FileLocation{ 62 | Line: endLine, 63 | Column: script.FileReference.EndRef.Column + len(command), // start column + the length of the last command 64 | }, 65 | } 66 | } 67 | 68 | func countLines(script *common.Script, commandIndex int) int { 69 | lines := 0 70 | for i := 0; i < commandIndex; i++ { 71 | splitCommand := strings.Split(script.Commands[i], "\n") 72 | lines += len(splitCommand) 73 | } 74 | 75 | return lines 76 | } 77 | -------------------------------------------------------------------------------- /pkg/models/step.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | const ( 4 | CommitSHA VersionType = "commit" 5 | TagVersion VersionType = "tag" 6 | BranchVersion VersionType = "branch" 7 | Latest VersionType = "latest" 8 | None VersionType = "none" 9 | 10 | ShellStepType StepType = "shell" 11 | TaskStepType StepType = "task" 12 | 13 | DockerTaskType TaskType = "docker" 14 | CITaskType TaskType = "ci" 15 | ) 16 | 17 | type VersionType string 18 | type StepType string 19 | type TaskType string 20 | 21 | type Shell struct { 22 | Type *string `json:"type,omitempty"` 23 | Script *string `json:"script,omitempty"` 24 | FileReference *FileReference `json:"file_reference,omitempty"` 25 | } 26 | 27 | type Step struct { 28 | ID *string `json:"id,omitempty"` 29 | Name *string `json:"name,omitempty"` 30 | Type StepType `json:"type,omitempty"` 31 | Runner *Runner `json:"runner,omitempty"` 32 | FailsPipeline *bool `json:"fails_pipeline,omitempty"` 33 | Disabled *bool `json:"disabled,omitempty"` 34 | EnvironmentVariables *EnvironmentVariablesRef `json:"environment_variables,omitempty"` 35 | WorkingDirectory *string `json:"working_directory,omitempty"` 36 | Timeout *int `json:"timeout,omitempty"` 37 | Conditions *[]Condition `json:"conditions,omitempty"` 38 | Shell *Shell `json:"shell,omitempty"` 39 | Task *Task `json:"task,omitempty"` 40 | Metadata Metadata `json:"metadata,omitempty"` 41 | AfterScript *Shell `json:"after_script,omitempty"` 42 | FileReference *FileReference `json:"file_reference,omitempty"` 43 | Imports *Import `json:"imports,omitempty"` 44 | } 45 | 46 | type Task struct { 47 | ID *string `json:"id,omitempty"` 48 | Name *string `json:"name,omitempty"` 49 | Inputs []*Parameter `json:"inputs,omitempty"` 50 | Version *string `json:"version,omitempty"` 51 | VersionType VersionType `json:"version_type,omitempty"` 52 | Type TaskType `json:"type,omitempty"` 53 | } 54 | -------------------------------------------------------------------------------- /pkg/models/job.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type TokenPermissions struct { 4 | Permissions map[string]Permission 5 | FileReference *FileReference 6 | } 7 | 8 | type ConcurrencyGroup string 9 | 10 | type SecretsRef struct { 11 | Secrets map[string]any `json:"secrets,omitempty"` 12 | Inherit bool `json:"inherit,omitempty"` 13 | FileReference *FileReference `json:"file_reference,omitempty"` 14 | } 15 | 16 | type Job struct { 17 | ID *string `json:"id,omitempty"` 18 | Name *string `json:"name,omitempty"` 19 | Steps []*Step `json:"steps,omitempty"` 20 | ContinueOnError *string `json:"continue_on_error,omitempty"` 21 | PreSteps []*Step `json:"pre_steps,omitempty"` 22 | PostSteps []*Step `json:"post_steps,omitempty"` 23 | EnvironmentVariables *EnvironmentVariablesRef `json:"environment_variables,omitempty"` 24 | Runner *Runner `json:"runner,omitempty"` 25 | Conditions []*Condition `json:"conditions,omitempty"` 26 | ConcurrencyGroup *ConcurrencyGroup `json:"concurrency_group,omitempty"` 27 | Inputs []*Parameter `json:"inputs,omitempty"` 28 | TimeoutMS *int `json:"timeout_ms,omitempty"` 29 | Tags []string `json:"tags,omitempty"` 30 | TokenPermissions *TokenPermissions `json:"token_permissions,omitempty"` 31 | Dependencies []*JobDependency `json:"dependencies,omitempty"` 32 | Metadata Metadata `json:"metadata,omitempty"` 33 | Matrix *Matrix `json:"matrix,omitempty"` 34 | FileReference *FileReference `json:"file_reference,omitempty"` 35 | Imports *Import `json:"imports,omitempty"` 36 | } 37 | 38 | type Matrix struct { 39 | Matrix map[string]any 40 | Include []map[string]any 41 | Exclude []map[string]any 42 | FileReference *FileReference 43 | } 44 | 45 | type JobDependency struct { 46 | JobID *string `json:"job_id,omitempty"` 47 | ConcurrencyGroup *ConcurrencyGroup `json:"concurrency_group,omitempty"` 48 | Pipeline *string `json:"pipeline,omitempty"` 49 | } 50 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/job/job.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "strconv" 5 | 6 | gitlabModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/parsers/gitlab/common" 9 | "github.com/argonsecurity/pipeline-parser/pkg/parsers/gitlab/triggers" 10 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 11 | ) 12 | 13 | func ParseJobs(gitlabCIConfiguration *gitlabModels.GitlabCIConfiguration) ([]*models.Job, error) { 14 | jobs, err := utils.MapToSliceErr(gitlabCIConfiguration.Jobs, parseJob) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | return jobs, nil 20 | } 21 | 22 | func parseJob(jobID string, job *gitlabModels.Job) (*models.Job, error) { 23 | parsedJob := &models.Job{ 24 | ID: &jobID, 25 | Name: &jobID, 26 | ContinueOnError: getJobContinueOnError(job), 27 | ConcurrencyGroup: getJobConcurrencyGroup(job), 28 | Dependencies: parseDependencies(job), 29 | PreSteps: common.ParseScript(job.BeforeScript), 30 | PostSteps: common.ParseScript(job.AfterScript), 31 | Steps: common.ParseScript(job.Script), 32 | EnvironmentVariables: common.ParseEnvironmentVariables(job.Variables), 33 | Tags: job.Tags, 34 | Runner: common.ParseRunner(job.Image), 35 | Conditions: getJobConditions(job), 36 | FileReference: job.FileReference, 37 | } 38 | return parsedJob, nil 39 | } 40 | 41 | func getJobContinueOnError(job *gitlabModels.Job) *string { 42 | if job.AllowFailure != nil { 43 | return utils.GetPtr(strconv.FormatBool(*job.AllowFailure.Enabled)) 44 | } 45 | return nil 46 | } 47 | 48 | func getJobConcurrencyGroup(job *gitlabModels.Job) *models.ConcurrencyGroup { 49 | if job.Stage == "" { 50 | return nil 51 | } 52 | 53 | return utils.GetPtr(models.ConcurrencyGroup(job.Stage)) 54 | } 55 | 56 | func getJobConditions(job *gitlabModels.Job) []*models.Condition { 57 | conditions := triggers.ParseConditionRules(job.Rules) 58 | if parsedExcept := triggers.ParseControls(job.Except, true); parsedExcept != nil { 59 | conditions = append(conditions, parsedExcept) 60 | } 61 | if parsedOnly := triggers.ParseControls(job.Only, false); parsedOnly != nil { 62 | conditions = append(conditions, parsedOnly) 63 | } 64 | return conditions 65 | } 66 | -------------------------------------------------------------------------------- /pkg/parsers/utils/image_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestParseImageName(t *testing.T) { 10 | testCases := []struct { 11 | name string 12 | imageName string 13 | expectedRegistry string 14 | expectedNamespace string 15 | expectedImageName string 16 | expectedTag string 17 | }{ 18 | { 19 | name: "Empty image name", 20 | imageName: "", 21 | expectedRegistry: "", 22 | expectedNamespace: "", 23 | expectedImageName: "", 24 | expectedTag: "", 25 | }, 26 | { 27 | name: "Image name with tag", 28 | imageName: "image:tag", 29 | expectedRegistry: "", 30 | expectedNamespace: "", 31 | expectedImageName: "image", 32 | expectedTag: "tag", 33 | }, 34 | { 35 | name: "Image name without registry and tag", 36 | imageName: "repository/image", 37 | expectedRegistry: "", 38 | expectedNamespace: "repository", 39 | expectedImageName: "image", 40 | expectedTag: "", 41 | }, 42 | { 43 | name: "Image name with registry and namespace without tag", 44 | imageName: "registry/namespace/image", 45 | expectedRegistry: "registry", 46 | expectedNamespace: "namespace", 47 | expectedImageName: "image", 48 | expectedTag: "", 49 | }, 50 | 51 | { 52 | name: "Image name with tag and namespace", 53 | imageName: "namespace/image:tag", 54 | expectedRegistry: "", 55 | expectedNamespace: "namespace", 56 | expectedImageName: "image", 57 | expectedTag: "tag", 58 | }, 59 | { 60 | name: "Image name with tag and registry and namespace", 61 | imageName: "registry/namespace/image:tag", 62 | expectedRegistry: "registry", 63 | expectedNamespace: "namespace", 64 | expectedImageName: "image", 65 | expectedTag: "tag", 66 | }, 67 | } 68 | 69 | for _, testCase := range testCases { 70 | t.Run(testCase.name, func(t *testing.T) { 71 | registry, namespace, imageName, tag := ParseImageName(testCase.imageName) 72 | assert.Equal(t, testCase.expectedRegistry, registry) 73 | assert.Equal(t, testCase.expectedNamespace, namespace) 74 | assert.Equal(t, testCase.expectedImageName, imageName) 75 | assert.Equal(t, testCase.expectedTag, tag) 76 | }) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/triggers/controls.go: -------------------------------------------------------------------------------- 1 | package triggers 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models/job" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 7 | ) 8 | 9 | var ( 10 | refKeywords = []string{ 11 | "api", 12 | "branches", 13 | "chat", 14 | "external", 15 | "external_pull_requests", 16 | "merge_requests", 17 | "pipelines", 18 | "pushes", 19 | "schedules", 20 | "tags", 21 | "triggers", 22 | "web", 23 | } 24 | 25 | refToEventMapping = map[string]models.EventType{ 26 | "pushes": models.PushEvent, 27 | "merge_requests": models.PullRequestEvent, 28 | "external_pull_requests": models.PullRequestEvent, 29 | "pipelines": models.PipelineRunEvent, 30 | "trigger": models.PipelineTriggerEvent, 31 | "schedules": models.ScheduledEvent, 32 | } 33 | ) 34 | 35 | func ParseControls(controls *job.Controls, isDeny bool) *models.Condition { 36 | if controls == nil { 37 | return nil 38 | } 39 | 40 | branches, events := getBranchesAndEvents(controls.Refs) 41 | 42 | return &models.Condition{ 43 | Allow: utils.GetPtr(!isDeny), 44 | Branches: generateFilter(branches, isDeny), 45 | Paths: generateFilter(controls.Changes, isDeny), 46 | Events: events, 47 | Variables: parseVariables(controls.Variables), 48 | } 49 | } 50 | 51 | func parseVariables(expressions []string) map[string]string { 52 | if expressions == nil { 53 | return nil 54 | } 55 | variables := make(map[string]string) 56 | for _, expression := range expressions { 57 | comparisons := getComparisons(expression) 58 | for _, comparison := range comparisons { 59 | if comparison.IsPositive() { 60 | variables[comparison.Variable] = comparison.Value 61 | } 62 | } 63 | } 64 | return variables 65 | } 66 | 67 | func getBranchesAndEvents(refs []string) ([]string, []models.EventType) { 68 | branches := []string{} 69 | events := []models.EventType{} 70 | 71 | for _, ref := range refs { 72 | if event, ok := refToEventMapping[ref]; ok { 73 | events = append(events, event) 74 | } else if utils.SliceContains(refKeywords, ref) { 75 | events = append(events, models.EventType(ref)) 76 | } else { 77 | branches = append(branches, ref) 78 | } 79 | } 80 | 81 | return branches, events 82 | } 83 | 84 | func generateFilter(list []string, isDeny bool) *models.Filter { 85 | if list == nil { 86 | return nil 87 | } 88 | 89 | if isDeny { 90 | return &models.Filter{ 91 | DenyList: list, 92 | } 93 | } 94 | 95 | return &models.Filter{ 96 | AllowList: list, 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /pkg/enhancers/github/github_test.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test_mergePipelines(t *testing.T) { 13 | type args struct { 14 | pipeline *models.Pipeline 15 | importedPipeline *enhancers.ImportedPipeline 16 | } 17 | tests := []struct { 18 | name string 19 | args args 20 | want *models.Pipeline 21 | }{ 22 | { 23 | name: "happy flow", 24 | args: args{ 25 | pipeline: &models.Pipeline{ 26 | Jobs: []*models.Job{ 27 | { 28 | Name: utils.GetPtr("job1"), 29 | Imports: &models.Import{}, 30 | }, 31 | }, 32 | }, 33 | importedPipeline: &enhancers.ImportedPipeline{ 34 | JobName: "job1", 35 | Pipeline: &models.Pipeline{ 36 | Name: utils.GetPtr("imported"), 37 | }, 38 | }, 39 | }, 40 | want: &models.Pipeline{ 41 | Jobs: []*models.Job{ 42 | { 43 | Name: utils.GetPtr("job1"), 44 | Imports: &models.Import{ 45 | Pipeline: &models.Pipeline{ 46 | Name: utils.GetPtr("imported"), 47 | }, 48 | }, 49 | }, 50 | }, 51 | }, 52 | }, 53 | { 54 | name: "jobs names don't match", 55 | args: args{ 56 | pipeline: &models.Pipeline{ 57 | Jobs: []*models.Job{ 58 | { 59 | Name: utils.GetPtr("job1"), 60 | Imports: &models.Import{}, 61 | }, 62 | }, 63 | }, 64 | importedPipeline: &enhancers.ImportedPipeline{ 65 | JobName: "job2", 66 | Pipeline: &models.Pipeline{ 67 | Name: utils.GetPtr("imported"), 68 | }, 69 | }, 70 | }, 71 | want: &models.Pipeline{ 72 | Jobs: []*models.Job{ 73 | { 74 | Name: utils.GetPtr("job1"), 75 | Imports: &models.Import{}, 76 | }, 77 | }, 78 | }, 79 | }, 80 | { 81 | name: "pipeline is nil", 82 | args: args{ 83 | pipeline: nil, 84 | }, 85 | want: nil, 86 | }, 87 | { 88 | name: "pipeline jobs are nil", 89 | args: args{ 90 | pipeline: &models.Pipeline{ 91 | Name: utils.GetPtr("pipeline"), 92 | }, 93 | }, 94 | want: &models.Pipeline{ 95 | Name: utils.GetPtr("pipeline"), 96 | }, 97 | }, 98 | } 99 | for _, tt := range tests { 100 | t.Run(tt.name, func(t *testing.T) { 101 | got := mergePipelines(tt.args.pipeline, tt.args.importedPipeline) 102 | assert.Equal(t, tt.want, got) 103 | }) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/common/runner_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "testing" 5 | 6 | gitlabModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models/common" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 10 | ) 11 | 12 | func TestParseRunner(t *testing.T) { 13 | testCases := []struct { 14 | name string 15 | image *gitlabModels.Image 16 | expectedRunner *models.Runner 17 | }{ 18 | { 19 | name: "Image is nil", 20 | image: nil, 21 | expectedRunner: nil, 22 | }, 23 | { 24 | name: "Image is empty", 25 | image: &gitlabModels.Image{}, 26 | expectedRunner: &models.Runner{ 27 | DockerMetadata: &models.DockerMetadata{ 28 | Image: utils.GetPtrOrNil(""), 29 | Label: utils.GetPtrOrNil(""), 30 | RegistryURL: utils.GetPtrOrNil(""), 31 | }, 32 | }, 33 | }, 34 | { 35 | name: "Image with full data", 36 | image: &gitlabModels.Image{ 37 | Name: "registry/namespace/image:tag", 38 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 39 | }, 40 | expectedRunner: &models.Runner{ 41 | DockerMetadata: &models.DockerMetadata{ 42 | Image: utils.GetPtr("namespace/image"), 43 | Label: utils.GetPtr("tag"), 44 | RegistryURL: utils.GetPtr("registry"), 45 | }, 46 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 47 | }, 48 | }, 49 | { 50 | name: "Image without registry", 51 | image: &gitlabModels.Image{ 52 | Name: "namespace/image:tag", 53 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 54 | }, 55 | expectedRunner: &models.Runner{ 56 | DockerMetadata: &models.DockerMetadata{ 57 | Image: utils.GetPtr("namespace/image"), 58 | Label: utils.GetPtr("tag"), 59 | }, 60 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 61 | }, 62 | }, 63 | { 64 | name: "Image without namespace", 65 | image: &gitlabModels.Image{ 66 | Name: "image:tag", 67 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 68 | }, 69 | expectedRunner: &models.Runner{ 70 | DockerMetadata: &models.DockerMetadata{ 71 | Image: utils.GetPtr("image"), 72 | Label: utils.GetPtr("tag"), 73 | }, 74 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 75 | }, 76 | }, 77 | } 78 | 79 | for _, testCase := range testCases { 80 | t.Run(testCase.name, func(t *testing.T) { 81 | got := ParseRunner(testCase.image) 82 | 83 | testutils.DeepCompare(t, testCase.expectedRunner, got) 84 | }) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/triggers/expressions_test.go: -------------------------------------------------------------------------------- 1 | package triggers 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 7 | ) 8 | 9 | func TestIsPositive(t *testing.T) { 10 | testCases := []struct { 11 | name string 12 | comparison *Comparison 13 | expected bool 14 | }{ 15 | { 16 | name: "Comparison is empty", 17 | comparison: &Comparison{}, 18 | expected: false, 19 | }, 20 | { 21 | name: "Comparison is equals", 22 | comparison: &Comparison{Operator: equals}, 23 | expected: true, 24 | }, 25 | { 26 | name: "Comparison is match", 27 | comparison: &Comparison{Operator: match}, 28 | expected: true, 29 | }, 30 | } 31 | 32 | for _, testCase := range testCases { 33 | t.Run(testCase.name, func(t *testing.T) { 34 | got := testCase.comparison.IsPositive() 35 | 36 | testutils.DeepCompare(t, testCase.expected, got) 37 | 38 | }) 39 | } 40 | } 41 | 42 | func TestGetComparisons(t *testing.T) { 43 | testCases := []struct { 44 | name string 45 | expression string 46 | expectedComparisons []*Comparison 47 | }{ 48 | { 49 | name: "Expressions is empty", 50 | expression: "", 51 | expectedComparisons: []*Comparison{}, 52 | }, 53 | { 54 | name: "Expressions with == operator and parentheses", 55 | expression: `$var == "value"`, 56 | expectedComparisons: []*Comparison{ 57 | { 58 | Variable: "$var", 59 | Value: `"value"`, 60 | Operator: equals, 61 | }, 62 | }, 63 | }, 64 | { 65 | name: "Expressions with == operator and regex", 66 | expression: `$var == /value/`, 67 | expectedComparisons: []*Comparison{ 68 | { 69 | Variable: "$var", 70 | Value: `/value/`, 71 | Operator: equals, 72 | }, 73 | }, 74 | }, 75 | { 76 | name: "Expressions with =~ operator and parentheses", 77 | expression: `$var =~ "value"`, 78 | expectedComparisons: []*Comparison{ 79 | { 80 | Variable: "$var", 81 | Value: `"value"`, 82 | Operator: match, 83 | }, 84 | }, 85 | }, 86 | { 87 | name: "Expressions with =~ operator and regex", 88 | expression: `$var =~ /value/`, 89 | expectedComparisons: []*Comparison{ 90 | { 91 | Variable: "$var", 92 | Value: `/value/`, 93 | Operator: match, 94 | }, 95 | }, 96 | }, 97 | } 98 | 99 | for _, testCase := range testCases { 100 | t.Run(testCase.name, func(t *testing.T) { 101 | got := getComparisons(testCase.expression) 102 | 103 | testutils.DeepCompare(t, testCase.expectedComparisons, got) 104 | }) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /pkg/parsers/azure/extends.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "strings" 5 | 6 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 9 | "github.com/mitchellh/mapstructure" 10 | ) 11 | 12 | func parseExtends(extends *azureModels.Extends) []*models.Import { 13 | if extends == nil { 14 | return nil 15 | } 16 | 17 | path, alias := parseTemplateString(extends.Template.Template) 18 | parameters, paramImports := parseExtendParameters(extends.Parameters, extends.FileReference) 19 | imports := []*models.Import{{ 20 | FileReference: extends.FileReference, 21 | Parameters: parameters, 22 | Source: &models.ImportSource{ 23 | Path: &path, 24 | RepositoryAlias: &alias, 25 | Type: calculateSourceType(alias), 26 | }, 27 | }} 28 | 29 | imports = append(imports, paramImports...) 30 | 31 | return imports 32 | } 33 | 34 | func parseTemplateString(template string) (path string, alias string) { 35 | // common.yml@templates # Template reference 36 | parts := strings.Split(template, "@") 37 | path = parts[0] 38 | alias = "" 39 | if len(parts) == 2 { 40 | alias = parts[1] 41 | } 42 | return path, alias 43 | } 44 | 45 | func parseExtendParameters(params map[string]any, rootFileRef *models.FileReference) (parameters map[string]any, imports []*models.Import) { 46 | if params == nil { 47 | return nil, nil 48 | } 49 | 50 | for key, param := range params { 51 | var items []any 52 | if utils.IsArray(param) { 53 | items = append(items, param.([]any)...) 54 | } else { 55 | items = append(items, param) 56 | } 57 | for _, item := range items { 58 | value, ok := tryToParseTemplate(item) 59 | if ok { 60 | path, alias := parseTemplateString(value.Template) 61 | parameters, paramImports := parseExtendParameters(value.Parameters, rootFileRef) 62 | 63 | imports = append(imports, &models.Import{ 64 | Parameters: parameters, 65 | Source: &models.ImportSource{ 66 | Path: &path, 67 | RepositoryAlias: &alias, 68 | Type: calculateSourceType(alias), 69 | }, 70 | FileReference: rootFileRef, 71 | }) 72 | imports = append(imports, paramImports...) 73 | continue 74 | } 75 | if parameters == nil { 76 | parameters = make(map[string]any) 77 | } 78 | parameters[key] = item 79 | } 80 | } 81 | 82 | return parameters, imports 83 | } 84 | 85 | func tryToParseTemplate(input any) (azureModels.Template, bool) { 86 | var azureTemplate azureModels.Template 87 | return azureTemplate, mapstructure.Decode(input, &azureTemplate) == nil 88 | } 89 | 90 | func calculateSourceType(alias string) models.SourceType { 91 | if alias == "self" || alias == "" { 92 | return models.SourceTypeLocal 93 | } 94 | return models.SourceTypeRemote 95 | } 96 | -------------------------------------------------------------------------------- /pkg/loaders/github/models/permissions.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "github.com/mitchellh/mapstructure" 8 | "gopkg.in/yaml.v3" 9 | ) 10 | 11 | const ( 12 | readAll = "read-all" 13 | writeAll = "write-all" 14 | ) 15 | 16 | type PermissionsEvent struct { 17 | Actions string `mapstructure:"actions,omitempty" yaml:"actions,omitempty"` 18 | Checks string `mapstructure:"checks,omitempty" yaml:"checks,omitempty"` 19 | Contents string `mapstructure:"contents,omitempty" yaml:"contents,omitempty"` 20 | Deployments string `mapstructure:"deployments,omitempty" yaml:"deployments,omitempty"` 21 | Discussions string `mapstructure:"discussions,omitempty" yaml:"discussions,omitempty"` 22 | IdToken string `mapstructure:"id-token,omitempty" yaml:"id-token,omitempty"` 23 | Issues string `mapstructure:"issues,omitempty" yaml:"issues,omitempty"` 24 | Packages string `mapstructure:"packages,omitempty" yaml:"packages,omitempty"` 25 | Pages string `mapstructure:"pages,omitempty" yaml:"pages,omitempty"` 26 | PullRequests string `mapstructure:"pull-requests,omitempty" yaml:"pull-requests,omitempty"` 27 | RepositoryProjects string `mapstructure:"repository-projects,omitempty" yaml:"repository-projects,omitempty"` 28 | SecurityEvents string `mapstructure:"security-events,omitempty" yaml:"security-events,omitempty"` 29 | Statuses string `mapstructure:"statuses,omitempty" yaml:"statuses,omitempty"` 30 | 31 | FileReference *models.FileReference 32 | } 33 | 34 | func createFullPermissions(permission string) *PermissionsEvent { 35 | return &PermissionsEvent{ 36 | Actions: permission, 37 | Checks: permission, 38 | Contents: permission, 39 | Deployments: permission, 40 | Discussions: permission, 41 | IdToken: permission, 42 | Issues: permission, 43 | Packages: permission, 44 | Pages: permission, 45 | PullRequests: permission, 46 | RepositoryProjects: permission, 47 | SecurityEvents: permission, 48 | Statuses: permission, 49 | } 50 | } 51 | 52 | func (p *PermissionsEvent) UnmarshalYAML(node *yaml.Node) error { 53 | if node.Tag == consts.StringTag { 54 | switch node.Value { 55 | case readAll: 56 | *p = *createFullPermissions("read") 57 | case writeAll: 58 | *p = *createFullPermissions("write") 59 | } 60 | return nil 61 | } 62 | 63 | var tmpInterface any 64 | if err := node.Decode(&tmpInterface); err != nil { 65 | return err 66 | } 67 | 68 | if err := mapstructure.Decode(tmpInterface, &p); err != nil { 69 | return err 70 | } 71 | 72 | p.FileReference = loadersUtils.GetFileReference(node) 73 | p.FileReference.StartRef.Line-- // The "permissions" node is not accessible, this is a patch 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /pkg/parsers/github/permissions_test.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "testing" 5 | 6 | githubModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/github/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestParseTokenPermissions(t *testing.T) { 13 | testCases := []struct { 14 | name string 15 | permissions *githubModels.PermissionsEvent 16 | expectedPermissions *models.TokenPermissions 17 | }{ 18 | { 19 | name: "Permissions is nil", 20 | permissions: nil, 21 | expectedPermissions: nil, 22 | }, 23 | { 24 | name: "All permissions keys", 25 | permissions: &githubModels.PermissionsEvent{ 26 | Actions: "read", 27 | Checks: "write", 28 | Contents: "read", 29 | Deployments: "read", 30 | Issues: "write", 31 | Pages: "read", 32 | Statuses: "read", 33 | Packages: "nothing", 34 | FileReference: testutils.CreateFileReference(6, 7, 8, 9), 35 | }, 36 | expectedPermissions: &models.TokenPermissions{ 37 | Permissions: map[string]models.Permission{ 38 | "checks": { 39 | Write: true, 40 | }, 41 | "contents": { 42 | Read: true, 43 | }, 44 | "deployments": { 45 | Read: true, 46 | }, 47 | "issues": { 48 | Write: true, 49 | }, 50 | "pages": { 51 | Read: true, 52 | }, 53 | "run-pipeline": { 54 | Read: true, 55 | }, 56 | "statuses": { 57 | Read: true, 58 | }, 59 | "packages": {}, 60 | }, 61 | FileReference: testutils.CreateFileReference(6, 7, 8, 9), 62 | }, 63 | }, 64 | } 65 | 66 | for _, testCase := range testCases { 67 | t.Run(testCase.name, func(t *testing.T) { 68 | got, err := parseTokenPermissions(testCase.permissions) 69 | assert.NoError(t, err) 70 | 71 | testutils.DeepCompare(t, testCase.expectedPermissions, got) 72 | }) 73 | } 74 | } 75 | 76 | func TestParsePermissionValue(t *testing.T) { 77 | testCases := []struct { 78 | name string 79 | permission string 80 | expectedVal models.Permission 81 | }{ 82 | { 83 | name: "Read permission", 84 | permission: readPermission, 85 | expectedVal: models.Permission{ 86 | Read: true, 87 | }, 88 | }, 89 | { 90 | name: "Write permission", 91 | permission: writePermission, 92 | expectedVal: models.Permission{ 93 | Write: true, 94 | }, 95 | }, 96 | { 97 | name: "Mo read no write permissions", 98 | permission: "no_permission", 99 | expectedVal: models.Permission{}, 100 | }, 101 | } 102 | 103 | for _, testCase := range testCases { 104 | t.Run(testCase.name, func(t *testing.T) { 105 | got := parsePermissionValue(testCase.permission) 106 | 107 | testutils.DeepCompare(t, testCase.expectedVal, got) 108 | }) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /pkg/loaders/bitbucket/models/step.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 5 | "gopkg.in/yaml.v3" 6 | ) 7 | 8 | type StepMap map[string][]*Step 9 | 10 | type Step struct { 11 | Step *ExecutionUnitRef `yaml:"step,omitempty"` 12 | Parallel []*ParallelSteps `yaml:"parallel"` 13 | Variables []*CustomStepVariable `yaml:"variables"` // List of variables for the custom pipeline 14 | } 15 | 16 | type ParallelSteps struct { 17 | Step *ExecutionUnitRef `yaml:"step,omitempty"` 18 | } 19 | 20 | func (ps *ParallelSteps) UnmarshalYAML(node *yaml.Node) error { 21 | return loadersUtils.IterateOnMap(node, func(key string, value *yaml.Node) error { 22 | switch key { 23 | case "step": 24 | var step *ExecutionUnitRef 25 | if err := value.Decode(&step); err != nil { 26 | return err 27 | } 28 | if isAliasStep(value) { 29 | step.FileReference.IsAlias = true 30 | } 31 | ps.Step = step 32 | return nil 33 | } 34 | return nil 35 | }, "ParallelSteps") 36 | } 37 | 38 | func (s *Step) UnmarshalYAML(node *yaml.Node) error { 39 | return loadersUtils.IterateOnMap(node, func(key string, value *yaml.Node) error { 40 | switch key { 41 | case "parallel": 42 | var parallel []*ParallelSteps 43 | if err := loadersUtils.ParseSequenceOrOne(value, ¶llel); err != nil { 44 | return err 45 | } 46 | s.Parallel = parallel 47 | return nil 48 | case "step": 49 | var step *ExecutionUnitRef 50 | if err := value.Decode(&step); err != nil { 51 | return err 52 | } 53 | if isAliasStep(value) { 54 | step.FileReference.IsAlias = true 55 | } 56 | s.Step = step 57 | return nil 58 | case "variables": 59 | vars, err := parseVariables(value) 60 | if err != nil { 61 | return err 62 | } 63 | s.Variables = vars 64 | return nil 65 | } 66 | return nil 67 | }, "Step") 68 | } 69 | 70 | func (sm *StepMap) UnmarshalYAML(node *yaml.Node) error { 71 | var stepMap = make(map[string][]*Step) 72 | if err := loadersUtils.IterateOnMap(node, func(key string, value *yaml.Node) error { 73 | var steps []*Step 74 | if err := loadersUtils.ParseSequenceOrOne(value, &steps); err != nil { 75 | return err 76 | } 77 | stepMap[key] = steps 78 | return nil 79 | }, "StepMap"); err != nil { 80 | return err 81 | } 82 | *sm = stepMap 83 | return nil 84 | } 85 | 86 | func parseVariables(node *yaml.Node) ([]*CustomStepVariable, error) { 87 | var vars []*CustomStepVariable 88 | if err := loadersUtils.ParseSequenceOrOne(node, &vars); err != nil { 89 | return nil, err 90 | } 91 | return vars, nil 92 | } 93 | 94 | func isAliasStep(node *yaml.Node) bool { 95 | if node.Kind == yaml.AliasNode { 96 | return true 97 | } 98 | 99 | var res = false 100 | loadersUtils.IterateOnMap(node, func(key string, value *yaml.Node) error { 101 | if value.Kind == yaml.AliasNode { 102 | res = true 103 | } 104 | return nil 105 | }, "Step") 106 | return res 107 | } 108 | -------------------------------------------------------------------------------- /pkg/loaders/azure/models/stage.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "github.com/argonsecurity/pipeline-parser/pkg/consts" 5 | loadersUtils "github.com/argonsecurity/pipeline-parser/pkg/loaders/utils" 6 | "github.com/argonsecurity/pipeline-parser/pkg/models" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type Stage struct { 11 | Stage string `yaml:"stage,omitempty"` 12 | DisplayName string `yaml:"displayName,omitempty"` 13 | Pool *Pool `yaml:"pool,omitempty"` 14 | DependsOn *DependsOn `yaml:"dependsOn,omitempty"` 15 | Condition string `yaml:"condition,omitempty"` 16 | Variables *Variables `yaml:"variables,omitempty"` 17 | Jobs *Jobs `yaml:"jobs,omitempty"` 18 | LockBehavior string `yaml:"lockBehavior,omitempty"` 19 | TemplateContext map[string]any `yaml:"templateContext,omitempty"` 20 | FileReference *models.FileReference 21 | } 22 | 23 | type TemplateStage struct { 24 | Template `yaml:",inline"` 25 | FileReference *models.FileReference 26 | } 27 | 28 | type Stages struct { 29 | Stages []*Stage `yaml:"stages,omitempty"` 30 | TemplateStages []*TemplateStage `yaml:"templateStages,omitempty"` 31 | FileReference *models.FileReference 32 | } 33 | 34 | func (s *Stages) UnmarshalYAML(node *yaml.Node) error { 35 | var stages []*Stage 36 | var templateStages []*TemplateStage 37 | 38 | for _, stageNode := range node.Content { 39 | if stageNode.Tag == consts.StringTag { 40 | templateStages = append(templateStages, &TemplateStage{ 41 | Template: Template{ 42 | Template: stageNode.Value, 43 | }, 44 | FileReference: loadersUtils.GetFileReference(stageNode), 45 | }) 46 | continue 47 | } 48 | 49 | if isTemplateStage(stageNode) { 50 | stage, err := parseTemplateStage(stageNode) 51 | if err != nil { 52 | return err 53 | } 54 | templateStages = append(templateStages, &stage) 55 | continue 56 | } 57 | 58 | stage, err := parseStage(stageNode) 59 | if err != nil { 60 | return err 61 | } 62 | stages = append(stages, &stage) 63 | } 64 | 65 | *s = Stages{ 66 | Stages: stages, 67 | TemplateStages: templateStages, 68 | FileReference: loadersUtils.GetFileReference(node), 69 | } 70 | return nil 71 | } 72 | 73 | func parseStage(node *yaml.Node) (Stage, error) { 74 | var stage Stage 75 | if err := node.Decode(&stage); err != nil { 76 | return stage, err 77 | } 78 | stage.FileReference = loadersUtils.GetFileReference(node) 79 | return stage, nil 80 | } 81 | 82 | func parseTemplateStage(node *yaml.Node) (TemplateStage, error) { 83 | var templateStage TemplateStage 84 | if err := node.Decode(&templateStage); err != nil { 85 | return templateStage, err 86 | } 87 | templateStage.FileReference = loadersUtils.GetFileReference(node) 88 | return templateStage, nil 89 | } 90 | 91 | func isTemplateStage(job *yaml.Node) bool { 92 | for _, node := range job.Content { 93 | if node.Tag == consts.StringTag && node.Value == "template" { 94 | return true 95 | } 96 | } 97 | return false 98 | } 99 | -------------------------------------------------------------------------------- /pkg/parsers/gitlab/gitlab.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | gitlabModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/gitlab/models" 5 | "github.com/argonsecurity/pipeline-parser/pkg/models" 6 | "github.com/argonsecurity/pipeline-parser/pkg/parsers/gitlab/common" 7 | "github.com/argonsecurity/pipeline-parser/pkg/parsers/gitlab/job" 8 | "github.com/argonsecurity/pipeline-parser/pkg/parsers/gitlab/triggers" 9 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 10 | ) 11 | 12 | type GitLabParser struct{} 13 | 14 | func (g *GitLabParser) Parse(gitlabCIConfiguration *gitlabModels.GitlabCIConfiguration) (*models.Pipeline, error) { 15 | var err error 16 | pipeline := &models.Pipeline{ 17 | Imports: ParseImports(gitlabCIConfiguration.Include), 18 | } 19 | 20 | pipeline.Defaults = parseDefaults(gitlabCIConfiguration) 21 | 22 | if gitlabCIConfiguration.Workflow != nil { 23 | pipeline.Triggers, pipeline.Defaults.Conditions = triggers.ParseRules(gitlabCIConfiguration.Workflow.Rules) 24 | } 25 | pipeline.Jobs, err = job.ParseJobs(gitlabCIConfiguration) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | if len(gitlabCIConfiguration.Jobs) > 0 { 31 | _, err := utils.MapToSliceErr(gitlabCIConfiguration.Jobs, func(jobID string, job *gitlabModels.Job) ([]*models.Import, error) { 32 | return appendJobTriggerIncludes(job, &pipeline.Imports) 33 | }) 34 | if err != nil { 35 | return nil, err 36 | } 37 | } 38 | return pipeline, nil 39 | } 40 | 41 | func parseScans(gitlabCIConfiguration *gitlabModels.GitlabCIConfiguration) *models.Scans { 42 | if gitlabCIConfiguration.Default == nil || gitlabCIConfiguration.Default.Artifacts == nil { 43 | return nil 44 | } 45 | reports := gitlabCIConfiguration.Default.Artifacts.Reports 46 | 47 | return &models.Scans{ 48 | Secrets: utils.GetPtr(reports.SecretDetection != nil), 49 | SAST: utils.GetPtr(reports.Sast != nil), 50 | Dependencies: utils.GetPtr(reports.DependencyScanning != nil), 51 | Iac: utils.GetPtr(reports.Terraform != nil), 52 | License: utils.GetPtr(reports.LicenseScanning != nil), 53 | } 54 | } 55 | 56 | func parseDefaults(gitlabCIConfiguration *gitlabModels.GitlabCIConfiguration) *models.Defaults { 57 | defaults := &models.Defaults{ 58 | EnvironmentVariables: common.ParseEnvironmentVariables(gitlabCIConfiguration.Variables), 59 | Runner: common.ParseRunner(gitlabCIConfiguration.Image), 60 | PostSteps: common.ParseScript(gitlabCIConfiguration.AfterScript), 61 | PreSteps: common.ParseScript(gitlabCIConfiguration.BeforeScript), 62 | Scans: parseScans(gitlabCIConfiguration), 63 | } 64 | return defaults 65 | } 66 | 67 | func appendJobTriggerIncludes(job *gitlabModels.Job, imports *[]*models.Import) ([]*models.Import, error) { 68 | if job.Trigger != nil && job.Trigger.Include != nil { 69 | if jobImport := ParseImports(job.Trigger.Include); jobImport != nil { 70 | if imports == nil { 71 | *imports = []*models.Import{} 72 | } 73 | *imports = append(*imports, jobImport...) 74 | } 75 | } 76 | return *imports, nil 77 | } 78 | -------------------------------------------------------------------------------- /pkg/enhancers/github/reusable_workflows.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/pkg/errors" 8 | 9 | "github.com/argonsecurity/pipeline-parser/pkg/enhancers" 10 | "github.com/argonsecurity/pipeline-parser/pkg/models" 11 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 12 | ) 13 | 14 | var ( 15 | GITHUB_BASE_URL = "https://raw.githubusercontent.com" 16 | ) 17 | 18 | func getReusableWorkflows(pipeline *models.Pipeline, credentials *models.Credentials, baseUrl *string) ([]*enhancers.ImportedPipeline, error) { 19 | var errs error 20 | importedPipelines := []*enhancers.ImportedPipeline{} 21 | for _, job := range pipeline.Jobs { 22 | if job.Imports != nil { 23 | importedPipelineBuf, err := handleImport(job.Imports, credentials, baseUrl) 24 | if err != nil { 25 | if errs == nil { 26 | errs = errors.New("got error(s) importing pipeline(s):") 27 | } 28 | errs = errors.Wrap(errs, fmt.Sprintf("error importing pipeline for job %s: %s", *job.Name, err.Error())) 29 | } 30 | importedPipelines = append(importedPipelines, &enhancers.ImportedPipeline{ 31 | JobName: *job.Name, 32 | Data: importedPipelineBuf, 33 | }) 34 | } 35 | 36 | } 37 | return importedPipelines, errs 38 | } 39 | 40 | func handleImport(jobImport *models.Import, credentials *models.Credentials, baseUrl *string) ([]byte, error) { 41 | if jobImport == nil || jobImport.Source == nil { 42 | return nil, nil 43 | } 44 | 45 | if jobImport.Source.Type == models.SourceTypeRemote && jobImport.Source.Organization != nil && jobImport.Source.Repository != nil && jobImport.Source.Path != nil && jobImport.Version != nil { 46 | return loadRemoteFile(*jobImport.Source.Organization, *jobImport.Source.Repository, *jobImport.Version, *jobImport.Source.Path, credentials, baseUrl) 47 | } 48 | 49 | if jobImport.Source.Type == models.SourceTypeLocal && jobImport.Source.Path != nil { 50 | return loadLocalFile(*jobImport.Source.Path) 51 | } 52 | 53 | return nil, nil 54 | } 55 | 56 | func loadRemoteFile(org, repo, version, path string, credentials *models.Credentials, baseUrl *string) ([]byte, error) { 57 | if org == "" || repo == "" || path == "" { 58 | return nil, nil 59 | } 60 | 61 | if version == "" { 62 | version = "main" 63 | } 64 | 65 | url := fmt.Sprintf("%s/%s/%s/%s/%s", GITHUB_BASE_URL, org, repo, version, path) 66 | if baseUrl != nil && *baseUrl != "" { 67 | url = fmt.Sprintf("%s/raw/%s/%s/%s/%s", *baseUrl, org, repo, version, path) 68 | } 69 | 70 | client := utils.GetHttpClient(credentials) 71 | resp, err := client.R().Get(url) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | if resp.IsErrorState() { 77 | return nil, errors.New(resp.Response.Status) 78 | } 79 | 80 | buf := resp.Bytes() 81 | return buf, nil 82 | } 83 | 84 | func loadLocalFile(path string) ([]byte, error) { 85 | if path == "" { 86 | return nil, nil 87 | } 88 | 89 | if _, err := os.Stat(path); os.IsNotExist(err) { 90 | return nil, err 91 | } 92 | 93 | buf, err := os.ReadFile(path) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | return buf, nil 99 | } 100 | -------------------------------------------------------------------------------- /pkg/parsers/azure/extends_test.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "testing" 5 | 6 | azureModels "github.com/argonsecurity/pipeline-parser/pkg/loaders/azure/models" 7 | "github.com/argonsecurity/pipeline-parser/pkg/models" 8 | "github.com/argonsecurity/pipeline-parser/pkg/testutils" 9 | "github.com/argonsecurity/pipeline-parser/pkg/utils" 10 | ) 11 | 12 | func TestParseExtends(t *testing.T) { 13 | testCases := []struct { 14 | name string 15 | extends *azureModels.Extends 16 | expectedImports []*models.Import 17 | }{ 18 | { 19 | name: "Extends is nil", 20 | extends: nil, 21 | expectedImports: nil, 22 | }, 23 | { 24 | name: "Extends with relative path", 25 | extends: &azureModels.Extends{ 26 | Template: azureModels.Template{ 27 | Template: "template1", 28 | }, 29 | }, 30 | expectedImports: []*models.Import{{ 31 | Source: &models.ImportSource{ 32 | Path: utils.GetPtr("template1"), 33 | RepositoryAlias: utils.GetPtr(""), 34 | Type: models.SourceTypeLocal, 35 | }, 36 | }}, 37 | }, 38 | { 39 | name: "Extends with repository alias", 40 | extends: &azureModels.Extends{ 41 | Template: azureModels.Template{ 42 | Template: "template1@repo1", 43 | }, 44 | }, 45 | expectedImports: []*models.Import{{ 46 | Source: &models.ImportSource{ 47 | Path: utils.GetPtr("template1"), 48 | RepositoryAlias: utils.GetPtr("repo1"), 49 | Type: models.SourceTypeRemote, 50 | }, 51 | }}, 52 | }, 53 | { 54 | name: "Extends with parameter template", 55 | extends: &azureModels.Extends{ 56 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 57 | Template: azureModels.Template{ 58 | Template: "template1@repo1", 59 | Parameters: map[string]any{ 60 | "foo": "bar", 61 | "testSteps": map[string]any{ 62 | "template": "template2@repo2", 63 | "parameters": map[string]any{ 64 | "foo2": "bar2", 65 | }, 66 | }, 67 | }, 68 | }, 69 | }, 70 | expectedImports: []*models.Import{ 71 | { 72 | FileReference: testutils.CreateAliasFileReference(1, 2, 3, 4, false), 73 | Parameters: map[string]any{ 74 | "foo": "bar", 75 | }, 76 | Source: &models.ImportSource{ 77 | Path: utils.GetPtr("template1"), 78 | RepositoryAlias: utils.GetPtr("repo1"), 79 | Type: models.SourceTypeRemote, 80 | }, 81 | }, 82 | { 83 | Parameters: map[string]any{ 84 | "foo2": "bar2", 85 | }, 86 | Source: &models.ImportSource{ 87 | Path: utils.GetPtr("template2"), 88 | RepositoryAlias: utils.GetPtr("repo2"), 89 | Type: models.SourceTypeRemote, 90 | }, 91 | FileReference: testutils.CreateFileReference(1, 2, 3, 4), 92 | }, 93 | }, 94 | }, 95 | } 96 | 97 | for _, testCase := range testCases { 98 | t.Run(testCase.name, func(t *testing.T) { 99 | got := parseExtends(testCase.extends) 100 | testutils.DeepCompare(t, testCase.expectedImports, got) 101 | }) 102 | } 103 | } 104 | --------------------------------------------------------------------------------