├── sem-context ├── .gitignore ├── sem-context ├── Dockerfile.dev ├── pkg │ ├── flags │ │ └── flags.go │ ├── store │ │ └── store_interface.go │ ├── utils │ │ ├── error_utils.go │ │ └── pipeline_context_utils.go │ └── validators │ │ └── validators.go ├── main.go ├── go.mod ├── docker-compose.yml ├── cmd │ ├── root.go │ ├── delete.go │ ├── put.go │ └── get.go ├── go.sum └── Makefile ├── tests ├── test_helper ├── test-results │ ├── junit-summary.json │ ├── staticcheck.json │ ├── golang.xml │ ├── rspec2.xml │ ├── revive.json │ └── junit-sample.xml ├── sem_service │ ├── rabbitmq │ ├── cassandra │ ├── memcached │ ├── rethinkdb │ ├── valkey │ ├── redis │ ├── opensearch │ ├── mysql │ └── elasticsearch ├── compiler │ └── test-repo │ │ └── .semaphore │ │ └── semaphore.yml ├── sftp_server │ ├── id_rsa.pub │ ├── start_on_linux.sh │ ├── start_on_docker.sh │ ├── Dockerfile │ ├── sshd_config │ ├── start_on_mac.sh │ └── id_rsa ├── install_package.bats ├── xcode15_sem_version.bats ├── sem_version_container.bats ├── sem_version_jammy │ ├── kubectl.bats │ ├── scala.bats │ ├── java.bats │ ├── gcc.bats │ ├── python.bats │ ├── node.bats │ ├── php.bats │ ├── go.bats │ └── elixir.bats ├── sem_version_noble │ ├── kubectl.bats │ ├── scala.bats │ ├── java.bats │ ├── gcc.bats │ ├── python.bats │ ├── node.bats │ ├── php.bats │ ├── go.bats │ └── elixir.bats ├── macOS_sem_version.bats ├── sem_version_focal │ ├── scala.bats │ ├── java.bats │ ├── kubectl.bats │ ├── go.bats │ ├── python.bats │ ├── gcc.bats │ ├── firefox.bats │ ├── node.bats │ └── php.bats ├── shm │ ├── shm_docker.bats │ └── shm_ubuntu.bats ├── compiler.bats ├── system_metrics_collector.bats ├── artifacts.bats ├── enetwork.bats └── toolbox_metrics.bats ├── cache-cli ├── test │ ├── autocache │ │ ├── go │ │ │ └── go.sum │ │ ├── maven │ │ │ └── pom.xml │ │ ├── nvm │ │ │ └── .nvmrc │ │ ├── elixir │ │ │ └── mix.lock │ │ ├── gems │ │ │ └── Gemfile.lock │ │ ├── yarn │ │ │ └── yarn.lock │ │ ├── cocoapods │ │ │ └── Podfile.lock │ │ ├── composer │ │ │ └── composer.lock │ │ ├── npm │ │ │ └── package-lock.json │ │ ├── pip │ │ │ └── requirements.txt │ │ └── multiple-files │ │ │ ├── package-lock.json │ │ │ └── requirements.txt │ └── gcs │ │ └── data │ │ └── semaphore-cache │ │ └── .gitkeep ├── pkg │ ├── files │ │ ├── testdata │ │ │ └── test.txt │ │ ├── size.go │ │ ├── checksum.go │ │ ├── checksum_test.go │ │ ├── download.go │ │ ├── size_test.go │ │ └── download_test.go │ ├── storage │ │ ├── s3_is_not_empty.go │ │ ├── sftp_is_not_empty.go │ │ ├── sftp_delete.go │ │ ├── s3_usage.go │ │ ├── gcs_usage.go │ │ ├── sftp_clear.go │ │ ├── sftp_has_key.go │ │ ├── gcs_delete.go │ │ ├── s3_delete.go │ │ ├── sftp_usage.go │ │ ├── gcs_is_not_empty.go │ │ ├── gcs_has_key.go │ │ ├── gcs_clear.go │ │ ├── s3_has_key.go │ │ ├── gcs_restore.go │ │ ├── s3_store.go │ │ ├── sftp_restore.go │ │ ├── s3_restore.go │ │ ├── gcs_store.go │ │ ├── has_key_test.go │ │ ├── is_not_empty_test.go │ │ ├── delete_test.go │ │ ├── gcs.go │ │ ├── restore_test.go │ │ ├── usage_test.go │ │ ├── gcs_list.go │ │ ├── clear_test.go │ │ ├── s3_clear.go │ │ └── sftp_list.go │ ├── logging │ │ └── formatter.go │ ├── metrics │ │ ├── no_op.go │ │ ├── context.go │ │ ├── metrics.go │ │ ├── local_test.go │ │ └── local.go │ ├── utils │ │ └── check.go │ └── archive │ │ └── archiver.go ├── .gitignore ├── .htpasswd ├── wrapper.sh ├── Dockerfile.dev ├── default.conf ├── id_rsa.pub ├── nginx.conf ├── cmd │ ├── root.go │ ├── clear.go │ ├── is_not_empty_test.go │ ├── is_not_empty.go │ ├── usage.go │ ├── usage_test.go │ ├── delete.go │ ├── has_key.go │ ├── clear_test.go │ ├── list_test.go │ ├── delete_test.go │ ├── has_key_test.go │ └── list.go ├── lint.toml ├── sshd_config ├── Dockerfile.sftp_server ├── main.go ├── docker-compose.yml ├── Makefile └── id_rsa ├── .github ├── CODEOWNERS └── workflows │ └── test.yml ├── .gitignore ├── self-hosted-toolbox ├── test-results ├── priv │ ├── workflow │ │ └── summary-out.json │ ├── merging │ │ └── pipeline │ │ │ ├── job2 │ │ │ ├── junit.xml │ │ │ └── junit.json │ │ │ ├── job3 │ │ │ ├── junit.xml │ │ │ └── junit.json │ │ │ └── job1 │ │ │ ├── junit.xml │ │ │ └── junit.json │ └── parsers │ │ ├── junit_mocha │ │ └── in.xml │ │ ├── junit_generic │ │ └── in.xml │ │ ├── go_staticcheck │ │ └── in.json │ │ ├── go_revive │ │ └── in.json │ │ ├── junit_exunit │ │ └── in.xml │ │ ├── junit_rspec │ │ └── in.xml │ │ └── junit_phpunit │ │ └── in.xml ├── main.go ├── pkg │ ├── parser │ │ ├── parser.go │ │ ├── time.go │ │ ├── mermaid.go │ │ ├── xmlelement_test.go │ │ ├── mermaid_test.go │ │ └── xmlelement.go │ ├── fileloader │ │ ├── fileloader_test.go │ │ └── fileloader.go │ └── parsers │ │ ├── junit_mocha_test.go │ │ ├── junit_rspec_test.go │ │ ├── junit_golang_test.go │ │ ├── junit_exunit_test.go │ │ ├── junit_generic_test.go │ │ ├── go_staticcheck_test.go │ │ ├── junit_phpunit_test.go │ │ ├── junit_embedded_test.go │ │ └── helpers.go ├── Dockerfile.dev ├── docker-compose.yml ├── .gitignore ├── lint.toml ├── docs │ └── id-generation.md ├── go.mod └── cmd │ └── combine.go ├── toolbox ├── docker-compose.yml ├── scripts ├── run-linter.ps1 └── run-tests.ps1 ├── sem-dockerize ├── libchecksum ├── README.md ├── .gitmodules ├── Dockerfile.dev ├── Makefile ├── release └── install_in_tests.sh ├── .semaphore ├── sem-version_focal.txt ├── sem-version_bionic.txt └── release.yml └── retry /sem-context/.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | -------------------------------------------------------------------------------- /tests/test_helper: -------------------------------------------------------------------------------- 1 | load "../ 2 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/go/go.sum: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/maven/pom.xml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/nvm/.nvmrc: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | @bogyo210 @lucaspin 2 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/elixir/mix.lock: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/gems/Gemfile.lock: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/yarn/yarn.lock: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/pkg/files/testdata/test.txt: -------------------------------------------------------------------------------- 1 | Test 123 -------------------------------------------------------------------------------- /cache-cli/test/autocache/cocoapods/Podfile.lock: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/composer/composer.lock: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/npm/package-lock.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/pip/requirements.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/test/gcs/data/semaphore-cache/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tmp* 2 | report.xml 3 | **/.vscode/ 4 | -------------------------------------------------------------------------------- /cache-cli/.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | junit-report.xml 3 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/multiple-files/package-lock.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/test/autocache/multiple-files/requirements.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cache-cli/.htpasswd: -------------------------------------------------------------------------------- 1 | test:$apr1$kCL/ipwt$qOeC2mnzu9dJrmVYw2agO. 2 | -------------------------------------------------------------------------------- /cache-cli/wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | /usr/sbin/nginx 3 | /usr/sbin/sshd -D -------------------------------------------------------------------------------- /self-hosted-toolbox: -------------------------------------------------------------------------------- 1 | source ~/.toolbox/libcheckout 2 | source ~/.toolbox/libchecksum 3 | -------------------------------------------------------------------------------- /sem-context/sem-context: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/semaphoreci/toolbox/HEAD/sem-context/sem-context -------------------------------------------------------------------------------- /sem-context/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM golang:1.20 2 | 3 | RUN go get gotest.tools/gotestsum 4 | 5 | WORKDIR /app 6 | -------------------------------------------------------------------------------- /sem-context/pkg/flags/flags.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var IgnoreFailure bool 4 | var Fallback string 5 | var Force bool 6 | -------------------------------------------------------------------------------- /tests/test-results/junit-summary.json: -------------------------------------------------------------------------------- 1 | {"total":6,"passed":2,"skipped":2,"error":0,"failed":2,"disabled":0,"duration":13640000} -------------------------------------------------------------------------------- /test-results/priv/workflow/summary-out.json: -------------------------------------------------------------------------------- 1 | {"total":104,"passed":99,"skipped":0,"error":1,"failed":4,"disabled":0,"duration":680829000} -------------------------------------------------------------------------------- /toolbox: -------------------------------------------------------------------------------- 1 | source ~/.toolbox/sem-install 2 | source ~/.toolbox/sem-version 3 | source ~/.toolbox/libcheckout 4 | source ~/.toolbox/libchecksum 5 | -------------------------------------------------------------------------------- /tests/sem_service/rabbitmq: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | sem-service start rabbitmq 6 | sem-service status rabbitmq 7 | -------------------------------------------------------------------------------- /sem-context/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/semaphoreci/toolbox/sem-context/cmd" 4 | 5 | func main() { 6 | cmd.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /test-results/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/semaphoreci/toolbox/test-results/cmd" 4 | 5 | func main() { 6 | cmd.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /tests/sem_service/cassandra: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | sem-service start cassandra 6 | docker ps -a | grep cassandra 7 | -------------------------------------------------------------------------------- /tests/sem_service/memcached: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | sem-service start memcached 6 | sem-service status memcached 7 | -------------------------------------------------------------------------------- /tests/sem_service/rethinkdb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | sem-service start rethinkdb 6 | docker ps -a | grep rethinkdb 7 | -------------------------------------------------------------------------------- /cache-cli/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM golang:1.23 2 | 3 | RUN go install gotest.tools/gotestsum@v1.12.3 4 | RUN mkdir /root/.ssh 5 | COPY id_rsa /root/.ssh/semaphore_cache_key 6 | 7 | WORKDIR /app 8 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/s3_is_not_empty.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | func (s *S3Storage) IsNotEmpty() (bool, error) { 4 | keys, err := s.List() 5 | if err != nil { 6 | return false, err 7 | } 8 | 9 | return len(keys) != 0, nil 10 | } 11 | -------------------------------------------------------------------------------- /test-results/pkg/parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | type Parser interface { 4 | Parse(string) TestResults 5 | IsApplicable(string) bool 6 | GetName() string 7 | GetDescription() string 8 | GetSupportedExtensions() []string 9 | } 10 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/sftp_is_not_empty.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | func (s *SFTPStorage) IsNotEmpty() (bool, error) { 4 | keys, err := s.List() 5 | if err != nil { 6 | return false, err 7 | } 8 | 9 | return len(keys) != 0, nil 10 | } 11 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | services: 3 | toolbox: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile.dev 7 | tty: true 8 | command: "sleep 0" 9 | container_name: 'toolbox' 10 | volumes: 11 | - .:/app 12 | -------------------------------------------------------------------------------- /scripts/run-linter.ps1: -------------------------------------------------------------------------------- 1 | # Don't display progress bar when installing PSScriptAnalyzer 2 | $ProgressPreference = 'SilentlyContinue' 3 | 4 | Install-Module -Name PSScriptAnalyzer -Force 5 | if ($?) { 6 | Invoke-ScriptAnalyzer * -EnableExit -ReportSummary 7 | } 8 | -------------------------------------------------------------------------------- /sem-dockerize: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | run::dockerize(){ 4 | 5 | local port=$1 6 | local service=$(echo "${@: -1}") 7 | 8 | dockerize -wait tcp://0.0.0.0:${port} -timeout 60s 2>/dev/null 9 | 10 | echo $? 11 | } 12 | 13 | run::dockerize $@ 14 | -------------------------------------------------------------------------------- /test-results/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM golang:1.24 2 | 3 | WORKDIR /app 4 | 5 | COPY go.* ./ 6 | 7 | RUN go install gotest.tools/gotestsum@v1.13.0 8 | RUN go install honnef.co/go/tools/cmd/staticcheck@v0.6.1 9 | RUN go install github.com/mgechev/revive@v1.9.0 10 | -------------------------------------------------------------------------------- /cache-cli/default.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80 default_server; 3 | server_name _; 4 | root /home/tester; 5 | auth_basic "Auth required"; 6 | auth_basic_user_file /etc/nginx/.htpasswd; 7 | location / { 8 | autoindex on; 9 | } 10 | } -------------------------------------------------------------------------------- /sem-context/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/semaphoreci/toolbox/sem-context 2 | 3 | go 1.20 4 | 5 | require github.com/spf13/cobra v1.5.0 6 | 7 | require ( 8 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 9 | github.com/spf13/pflag v1.0.5 // indirect 10 | ) 11 | -------------------------------------------------------------------------------- /test-results/priv/merging/pipeline/job2/junit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /sem-context/pkg/store/store_interface.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | type Store interface { 4 | Get(key, contextId string) (string, error) 5 | Put(key, value, contextId string) error 6 | Delete(key, contextId string) error 7 | CheckIfKeyDeleted(key, contextId string) (bool, error) 8 | } 9 | -------------------------------------------------------------------------------- /test-results/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | cli: 3 | build: 4 | context: . 5 | dockerfile: Dockerfile.dev 6 | tty: true 7 | volumes: 8 | - go-pkg-cache:/go/pkg 9 | - .:/app 10 | 11 | volumes: 12 | go-pkg-cache: 13 | driver: local 14 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/sftp_delete.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "strings" 4 | 5 | func (s *SFTPStorage) Delete(key string) error { 6 | err := s.SFTPClient.Remove(key) 7 | if err != nil && strings.Contains(err.Error(), "file does not exist") { 8 | return nil 9 | } 10 | 11 | return err 12 | } 13 | -------------------------------------------------------------------------------- /test-results/priv/merging/pipeline/job3/junit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /libchecksum: -------------------------------------------------------------------------------- 1 | function checksum() { 2 | DIST=$(uname) 3 | 4 | case $DIST in 5 | Darwin) 6 | md5 $1 | tr -d " " | awk -F= {'print $2'} 7 | ;; 8 | Linux) 9 | md5sum $1 | awk '{ print $1 }' 10 | ;; 11 | *) 12 | echo "Unsupported distro $DIST" 13 | exit 1 14 | ;; 15 | esac 16 | } 17 | 18 | export -f checksum 19 | -------------------------------------------------------------------------------- /sem-context/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | services: 3 | cli: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile.dev 7 | tty: true 8 | command: "sleep 0" 9 | container_name: 'sem-context' 10 | volumes: 11 | - go-pkg-cache:/go 12 | - .:/app 13 | 14 | volumes: 15 | go-pkg-cache: 16 | driver: local 17 | -------------------------------------------------------------------------------- /test-results/priv/merging/pipeline/job1/junit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/compiler/test-repo/.semaphore/semaphore.yml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | name: Test 3 | agent: 4 | machine: 5 | type: e1-standard-2 6 | 7 | blocks: 8 | - name: Test 9 | run: 10 | when: "branch = 'master' and change_in('/lib')" 11 | task: 12 | jobs: 13 | - name: Hello 14 | commands: 15 | - echo "Hello World" 16 | -------------------------------------------------------------------------------- /cache-cli/pkg/logging/formatter.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "fmt" 5 | 6 | log "github.com/sirupsen/logrus" 7 | ) 8 | 9 | type CustomFormatter struct { 10 | } 11 | 12 | // We just care about the actual message here 13 | func (f *CustomFormatter) Format(entry *log.Entry) ([]byte, error) { 14 | return []byte(fmt.Sprintf("%s\n", entry.Message)), nil 15 | } 16 | -------------------------------------------------------------------------------- /cache-cli/pkg/metrics/no_op.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | type NoOpMetricsManager struct { 4 | } 5 | 6 | func NewNoOpMetricsManager() *NoOpMetricsManager { 7 | return &NoOpMetricsManager{} 8 | } 9 | 10 | func (b *NoOpMetricsManager) Enabled() bool { 11 | return false 12 | } 13 | 14 | func (b *NoOpMetricsManager) LogEvent(event CacheEvent) error { 15 | return nil 16 | } 17 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/s3_usage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | func (s *S3Storage) Usage() (*UsageSummary, error) { 4 | keys, err := s.List() 5 | if err != nil { 6 | return nil, err 7 | } 8 | 9 | var total int64 10 | for _, key := range keys { 11 | total = total + key.Size 12 | } 13 | 14 | return &UsageSummary{ 15 | Used: total, 16 | Free: -1, 17 | }, nil 18 | } 19 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/gcs_usage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | func (s *GCSStorage) Usage() (*UsageSummary, error) { 4 | keys, err := s.List() 5 | if err != nil { 6 | return nil, err 7 | } 8 | 9 | var total int64 10 | for _, key := range keys { 11 | total = total + key.Size 12 | } 13 | 14 | return &UsageSummary{ 15 | Used: total, 16 | Free: -1, 17 | }, nil 18 | } 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Toolbox 2 | 3 | Toolset used in Semaphore 2.0 jobs. 4 | 5 | ## Installation 6 | 7 | ``` bash 8 | # Install binaries 9 | bash ~/.toolbox/install-toolbox 10 | 11 | # Source functions into current session 12 | source ~/.toolbox/toolbox 13 | 14 | # Add toolbox to your bash_profile to activate it in every SSH session 15 | echo 'source ~/.toolbox/toolbox' >> ~/.bash_profile 16 | ``` 17 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/sftp_clear.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | func (s *SFTPStorage) Clear() error { 4 | keys, err := s.List() 5 | if err != nil { 6 | return err 7 | } 8 | 9 | if len(keys) == 0 { 10 | return nil 11 | } 12 | 13 | for _, key := range keys { 14 | err := s.SFTPClient.Remove(key.Name) 15 | if err != nil { 16 | return err 17 | } 18 | } 19 | 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/sftp_has_key.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "strings" 4 | 5 | func (s *SFTPStorage) HasKey(key string) (bool, error) { 6 | file, err := s.SFTPClient.Stat(key) 7 | if file == nil { 8 | if err != nil && strings.Contains(err.Error(), "file does not exist") { 9 | return false, nil 10 | } 11 | 12 | return false, err 13 | } 14 | 15 | return true, nil 16 | } 17 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tests/support/bats-assert"] 2 | path = tests/support/bats-assert 3 | url = git@github.com:ztombol/bats-assert.git 4 | [submodule "tests/support/bats-support"] 5 | path = tests/support/bats-support 6 | url = https://github.com/ztombol/bats-support 7 | [submodule "tests/support/bats-core"] 8 | path = tests/support/bats-core 9 | url = https://github.com/bats-core/bats-core.git 10 | -------------------------------------------------------------------------------- /cache-cli/id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC03+iJDsMojaLJ7b0TkBbANC9LIs1R/BCxGGOwOPmYaNuu4i9oTGTNEHYEszq+19G0cROfwwimO0mdrGvTi79EvJENR5UYHPA5n0I4GdznUlJYSACgoRj6mnJuWFB0K9t5FoPzHrPaQ7Bez7boxAo8AFbjZfgfhK0O1ncHH29Pr04wh4Fx1KSRPENSPrNy1LWqFc1mKolQMNZ8XboNjG0Lc5sS++L4S/kJ1LZyUSIbMv1suh0dWoQ004mklGqZJCL5AE31Bcj6Keu7eTGVjV66zr4iCI58gzeGkitfFg+iwuHGZzom/97ca8fdX2EZUzgqkYlsPdeuHxeGYukUSWpB vagrant@boxbox 2 | -------------------------------------------------------------------------------- /tests/sftp_server/id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC03+iJDsMojaLJ7b0TkBbANC9LIs1R/BCxGGOwOPmYaNuu4i9oTGTNEHYEszq+19G0cROfwwimO0mdrGvTi79EvJENR5UYHPA5n0I4GdznUlJYSACgoRj6mnJuWFB0K9t5FoPzHrPaQ7Bez7boxAo8AFbjZfgfhK0O1ncHH29Pr04wh4Fx1KSRPENSPrNy1LWqFc1mKolQMNZ8XboNjG0Lc5sS++L4S/kJ1LZyUSIbMv1suh0dWoQ004mklGqZJCL5AE31Bcj6Keu7eTGVjV66zr4iCI58gzeGkitfFg+iwuHGZzom/97ca8fdX2EZUzgqkYlsPdeuHxeGYukUSWpB vagrant@boxbox 2 | -------------------------------------------------------------------------------- /tests/sftp_server/start_on_linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker build -t sftp_server tests/sftp_server 4 | docker run -p 9000:22 -d sftp_server 5 | 6 | sleep 2 7 | 8 | cp tests/sftp_server/id_rsa ~/.ssh/semaphore_cache_key 9 | chmod 0600 ~/.ssh/semaphore_cache_key 10 | 11 | ssh-keyscan -p 9000 -H 127.0.0.1 >> ~/.ssh/known_hosts 12 | 13 | export SEMAPHORE_CACHE_URL="127.0.0.1:9000" 14 | export SEMAPHORE_CACHE_USERNAME=tester 15 | -------------------------------------------------------------------------------- /cache-cli/pkg/files/size.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func HumanReadableSize(b int64) string { 8 | const unit = 1024 9 | 10 | if b < unit { 11 | return fmt.Sprintf("%.1f", float64(b)) 12 | } 13 | 14 | div, exp := int64(unit), 0 15 | for n := b / unit; n >= unit; n /= unit { 16 | div *= unit 17 | exp++ 18 | } 19 | 20 | return fmt.Sprintf("%.1f%c", float64(b)/float64(div), "KMGTPE"[exp]) 21 | } 22 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/gcs_delete.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | gcs "cloud.google.com/go/storage" 9 | ) 10 | 11 | func (s *GCSStorage) Delete(key string) error { 12 | bucketKey := fmt.Sprintf("%s/%s", s.Project, key) 13 | err := s.Bucket.Object(bucketKey).Delete(context.TODO()) 14 | if errors.Is(err, gcs.ErrObjectNotExist) { 15 | return nil 16 | } 17 | 18 | return err 19 | } 20 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/s3_delete.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aws/aws-sdk-go-v2/service/s3" 8 | ) 9 | 10 | func (s *S3Storage) Delete(key string) error { 11 | bucketKey := fmt.Sprintf("%s/%s", s.Project, key) 12 | 13 | _, err := s.Client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ 14 | Bucket: &s.Bucket, 15 | Key: &bucketKey, 16 | }) 17 | 18 | return err 19 | } 20 | -------------------------------------------------------------------------------- /Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | ARG DEBIAN_FRONTEND=noninteractive 4 | 5 | RUN apt-get update -qy 6 | RUN apt-get install -y --fix-missing wget apt-transport-https software-properties-common git 7 | 8 | RUN wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb && \ 9 | dpkg -i packages-microsoft-prod.deb && \ 10 | apt-get update && \ 11 | apt-get install -y powershell 12 | 13 | WORKDIR /app 14 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/sftp_usage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | func (s *SFTPStorage) Usage() (*UsageSummary, error) { 4 | files, err := s.SFTPClient.ReadDir(".") 5 | if err != nil { 6 | return nil, err 7 | } 8 | 9 | var totalUsed int64 10 | for _, file := range files { 11 | totalUsed = totalUsed + file.Size() 12 | } 13 | 14 | return &UsageSummary{ 15 | Used: totalUsed, 16 | Free: s.Config().MaxSpace - totalUsed, 17 | }, nil 18 | } 19 | -------------------------------------------------------------------------------- /tests/sftp_server/start_on_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker build -t sftp-server tests/sftp_server 4 | docker run --net tmp_default --name sftp_server -d sftp-server 5 | 6 | sleep 2 7 | 8 | cp tests/sftp_server/id_rsa ~/.ssh/semaphore_cache_key 9 | chmod 0600 ~/.ssh/semaphore_cache_key 10 | 11 | ssh-keyscan -H sftp_server >> ~/.ssh/known_hosts 12 | 13 | export SEMAPHORE_CACHE_URL=sftp_server:22 14 | export SEMAPHORE_CACHE_USERNAME=tester 15 | -------------------------------------------------------------------------------- /cache-cli/nginx.conf: -------------------------------------------------------------------------------- 1 | user root; 2 | worker_processes auto; 3 | pid /run/nginx.pid; 4 | events { 5 | worker_connections 10; 6 | } 7 | http { 8 | sendfile off; 9 | tcp_nopush on; 10 | tcp_nodelay on; 11 | keepalive_timeout 65; 12 | types_hash_max_size 2048; 13 | server_tokens off; 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | gzip off; 17 | include /etc/nginx/sites-enabled/*; 18 | } -------------------------------------------------------------------------------- /tests/install_package.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "support/bats-support/load" 4 | load "support/bats-assert/load" 5 | 6 | @test "test install-package" { 7 | run install-package mtr 8 | assert_success 9 | } 10 | @test "test install_package files" { 11 | run ls -lah ~/.deb-cache 12 | assert_success 13 | assert_output --partial "mtr" 14 | } 15 | @test "test caching of packages" { 16 | cache has_key install_package_cache 17 | assert_success 18 | } -------------------------------------------------------------------------------- /tests/sem_service/valkey: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | sem-service start valkey 6 | sem-service status valkey 7 | sem-service stop valkey 8 | 9 | sem-service stop valkey 10 | sem-service start valkey 8.1.2 11 | sem-service status valkey 12 | 13 | sem-service stop valkey 14 | sem-service start valkey 7.2.6 15 | sem-service status valkey 16 | 17 | sem-service stop valkey 18 | sem-service start valkey 7 19 | sem-service status valkey 20 | 21 | 22 | -------------------------------------------------------------------------------- /cache-cli/pkg/metrics/context.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | func CacheUsername() string { 9 | return os.Getenv("SEMAPHORE_CACHE_USERNAME") 10 | } 11 | 12 | func CacheServerIP() string { 13 | cacheURL := os.Getenv("SEMAPHORE_CACHE_URL") 14 | if cacheURL == "" { 15 | return "" 16 | } 17 | 18 | ipAndPort := strings.Split(cacheURL, ":") 19 | if len(ipAndPort) != 2 { 20 | return "" 21 | } 22 | 23 | return ipAndPort[0] 24 | } 25 | -------------------------------------------------------------------------------- /test-results/.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | *.lcov 14 | COVERAGE.md 15 | 16 | # Dependency directories (remove the comment below to include it) 17 | # vendor/ 18 | .vscode 19 | .devcontainer 20 | bin 21 | samples 22 | dist/ 23 | test-results 24 | junit-report.xml 25 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/gcs_is_not_empty.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | 6 | "cloud.google.com/go/storage" 7 | "google.golang.org/api/iterator" 8 | ) 9 | 10 | func (s *GCSStorage) IsNotEmpty() (bool, error) { 11 | it := s.Bucket.Objects(context.TODO(), &storage.Query{Prefix: s.Project}) 12 | 13 | _, err := it.Next() 14 | if err == iterator.Done { 15 | return false, nil 16 | } else if err != nil { 17 | return false, err 18 | } 19 | 20 | return true, nil 21 | } 22 | -------------------------------------------------------------------------------- /test-results/priv/parsers/junit_mocha/in.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /test-results/priv/parsers/junit_generic/in.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/gcs_has_key.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "cloud.google.com/go/storage" 9 | ) 10 | 11 | func (s *GCSStorage) HasKey(key string) (bool, error) { 12 | gcsKey := fmt.Sprintf("%s/%s", s.Project, key) 13 | _, err := s.Bucket.Object(gcsKey).Attrs(context.TODO()) 14 | if err != nil { 15 | if errors.Is(err, storage.ErrObjectNotExist) { 16 | return false, nil 17 | } 18 | 19 | return false, err 20 | } 21 | 22 | return true, nil 23 | } 24 | -------------------------------------------------------------------------------- /cache-cli/pkg/files/checksum.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "crypto/md5" // #nosec 5 | "encoding/hex" 6 | "io" 7 | "os" 8 | ) 9 | 10 | func GenerateChecksum(filePath string) (string, error) { 11 | // #nosec 12 | file, err := os.Open(filePath) 13 | if err != nil { 14 | return "", err 15 | } 16 | 17 | defer file.Close() 18 | 19 | // #nosec 20 | hash := md5.New() 21 | if _, err := io.Copy(hash, file); err != nil { 22 | return "", err 23 | } 24 | 25 | return hex.EncodeToString(hash.Sum(nil)), nil 26 | } 27 | -------------------------------------------------------------------------------- /tests/sftp_server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | 3 | RUN apt-get update && apt-get install -y openssh-server 4 | RUN mkdir -p /var/run/sshd 5 | 6 | COPY sshd_config /etc/ssh/sshd_config 7 | 8 | RUN addgroup ftpaccess 9 | RUN adduser tester --ingroup ftpaccess --shell /bin/bash --disabled-password --gecos '' 10 | RUN chown tester:ftpaccess /home/tester 11 | RUN mkdir /etc/ssh/authorized_keys 12 | 13 | COPY id_rsa.pub /tmp/id_rsa.pub 14 | RUN cat /tmp/id_rsa.pub >> /etc/ssh/authorized_keys/tester 15 | 16 | EXPOSE 22 17 | CMD ["/usr/sbin/sshd", "-D"] 18 | -------------------------------------------------------------------------------- /cache-cli/pkg/utils/check.go: -------------------------------------------------------------------------------- 1 | //revive:disable-next-line:var-naming 2 | package utils 3 | 4 | import ( 5 | "os" 6 | 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | func Check(err error) { 11 | if err != nil { 12 | log.Errorf("error: %s", err.Error()) 13 | 14 | failOnError := os.Getenv("CACHE_FAIL_ON_ERROR") 15 | if failOnError == "true" { 16 | os.Exit(1) 17 | } else { 18 | os.Exit(0) 19 | } 20 | } 21 | } 22 | 23 | func CheckWithMessage(err error, message string) { 24 | if err != nil { 25 | log.Errorf("error: %+v", message) 26 | 27 | os.Exit(1) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/gcs_clear.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | 6 | "cloud.google.com/go/storage" 7 | "google.golang.org/api/iterator" 8 | ) 9 | 10 | func (s *GCSStorage) Clear() error { 11 | it := s.Bucket.Objects(context.TODO(), &storage.Query{Prefix: s.Project}) 12 | for { 13 | attrs, err := it.Next() 14 | if err == iterator.Done { 15 | break 16 | } 17 | if err != nil { 18 | return err 19 | } 20 | 21 | err = s.Bucket.Object(attrs.Name).Delete(context.TODO()) 22 | if err != nil { 23 | return err 24 | } 25 | } 26 | 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /test-results/pkg/fileloader/fileloader_test.go: -------------------------------------------------------------------------------- 1 | package fileloader 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestLoad(t *testing.T) { 11 | 12 | reader := bytes.NewReader([]byte(`Some data`)) 13 | 14 | _, found1 := Load("foo", reader) 15 | _, found2 := Load("foo", reader) 16 | _, found3 := Load("bar", reader) 17 | 18 | assert.Equal(t, false, found1, "Decoders should be the same") 19 | assert.Equal(t, true, found2, "Decoders should be the same") 20 | assert.Equal(t, false, found3, "Decoders should be the same") 21 | } 22 | -------------------------------------------------------------------------------- /tests/sem_service/redis: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | sem-service start redis 6 | sem-service status redis 7 | sem-service stop redis 8 | sem-service start redis 5 9 | sem-service status redis 10 | 11 | sem-service stop redis 12 | sem-service start redis 6.2.7 13 | sem-service status redis 14 | 15 | sem-service stop redis 16 | sem-service start redis 7.0.5 17 | sem-service status redis 18 | 19 | sem-service stop redis 20 | sem-service start redis 7.2.4 21 | sem-service status redis 22 | 23 | sem-service stop redis 24 | sem-service start redis 8.0.2 25 | sem-service status redis 26 | 27 | -------------------------------------------------------------------------------- /test-results/lint.toml: -------------------------------------------------------------------------------- 1 | ignoreGeneratedHeader = false 2 | severity = "warning" 3 | confidence = 0.8 4 | errorCode = 1 5 | warningCode = 1 6 | 7 | [rule.blank-imports] 8 | [rule.context-as-argument] 9 | [rule.context-keys-type] 10 | [rule.dot-imports] 11 | [rule.error-return] 12 | [rule.error-strings] 13 | [rule.error-naming] 14 | [rule.if-return] 15 | [rule.increment-decrement] 16 | [rule.var-naming] 17 | [rule.var-declaration] 18 | [rule.range] 19 | [rule.receiver-naming] 20 | [rule.time-naming] 21 | [rule.unexported-return] 22 | [rule.indent-error-flow] 23 | [rule.errorf] 24 | 25 | [rule.package-comments] 26 | Disabled = true 27 | -------------------------------------------------------------------------------- /test-results/pkg/parser/time.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | var layouts = []string{ 9 | time.RFC1123, 10 | time.RFC1123Z, 11 | "Mon 02 Jan 2006 03:04:05 PM MST", 12 | "Mon Jan 2 15:04:05 MST 2006", 13 | "Mon Jan 2 03:04:05 PM MST 2006", 14 | "Mon Jan 2 15:04:05 UTC 2006", 15 | } 16 | 17 | func ParseTimeAuto(input string) (time.Time, string, error) { 18 | for _, layout := range layouts { 19 | t, err := time.Parse(layout, input) 20 | if err == nil { 21 | return t, layout, nil 22 | } 23 | } 24 | return time.Time{}, "", fmt.Errorf("no matching layout found for: %s", input) 25 | } 26 | -------------------------------------------------------------------------------- /cache-cli/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | // RootCmd represents the base command when called without any subcommands 11 | var RootCmd = &cobra.Command{ 12 | Use: "cache", 13 | Short: "Utility for managing dependency caches", 14 | } 15 | 16 | // Execute adds all child commands to the root command and sets flags appropriately. 17 | // This is called by main.main(). It only needs to happen once to the RootCmd. 18 | func Execute() { 19 | if err := RootCmd.Execute(); err != nil { 20 | log.Error(err) 21 | os.Exit(1) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /cache-cli/lint.toml: -------------------------------------------------------------------------------- 1 | ignoreGeneratedHeader = false 2 | severity = "warning" 3 | confidence = 0.8 4 | errorCode = 1 5 | warningCode = 1 6 | 7 | [rule.blank-imports] 8 | [rule.context-as-argument] 9 | [rule.context-keys-type] 10 | [rule.dot-imports] 11 | [rule.error-return] 12 | [rule.error-strings] 13 | [rule.error-naming] 14 | # [rule.exported] 15 | [rule.if-return] 16 | [rule.increment-decrement] 17 | [rule.var-naming] 18 | [rule.var-declaration] 19 | [rule.range] 20 | [rule.receiver-naming] 21 | [rule.time-naming] 22 | [rule.unexported-return] 23 | [rule.indent-error-flow] 24 | [rule.errorf] 25 | 26 | [rule.package-comments] 27 | Disabled = true -------------------------------------------------------------------------------- /cache-cli/pkg/archive/archiver.go: -------------------------------------------------------------------------------- 1 | package archive 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/semaphoreci/toolbox/cache-cli/pkg/metrics" 7 | ) 8 | 9 | type Archiver interface { 10 | Compress(dst, src string) error 11 | Decompress(src string) (string, error) 12 | } 13 | 14 | func NewArchiver(metricsManager metrics.MetricsManager) Archiver { 15 | method := os.Getenv("SEMAPHORE_CACHE_ARCHIVE_METHOD") 16 | switch method { 17 | case "native": 18 | return NewNativeArchiver(metricsManager, false) 19 | case "native-parallel": 20 | return NewNativeArchiver(metricsManager, true) 21 | default: 22 | return NewShellOutArchiver(metricsManager) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /tests/test-results/staticcheck.json: -------------------------------------------------------------------------------- 1 | {"code":"ST1003","severity":"error","location":{"file":"main.go","line":10,"column":6},"end":{"file":"main.go","line":10,"column":14},"message":"should not use underscores in package names"} 2 | {"code":"SA1019","severity":"error","location":{"file":"utils/helper.go","line":23,"column":10},"end":{"file":"utils/helper.go","line":23,"column":25},"message":"io/ioutil.ReadFile is deprecated: As of Go 1.16, this function simply calls os.ReadFile."} 3 | {"code":"SA4006","severity":"error","location":{"file":"server/handler.go","line":45,"column":2},"end":{"file":"server/handler.go","line":45,"column":5},"message":"this value of err is never used"} -------------------------------------------------------------------------------- /tests/test-results/golang.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | sample_test.go:15: 10 | expected: 5 11 | got: 3 12 | 13 | 14 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/s3_has_key.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/aws/aws-sdk-go-v2/service/s3" 9 | "github.com/aws/smithy-go" 10 | ) 11 | 12 | func (s *S3Storage) HasKey(key string) (bool, error) { 13 | s3Key := fmt.Sprintf("%s/%s", s.Project, key) 14 | input := s3.HeadObjectInput{ 15 | Bucket: &s.Bucket, 16 | Key: &s3Key, 17 | } 18 | 19 | _, err := s.Client.HeadObject(context.TODO(), &input) 20 | if err != nil { 21 | var apiErr *smithy.GenericAPIError 22 | if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NotFound" { 23 | return false, nil 24 | } 25 | 26 | return false, err 27 | } 28 | 29 | return true, nil 30 | } 31 | -------------------------------------------------------------------------------- /cache-cli/sshd_config: -------------------------------------------------------------------------------- 1 | Port 22 2 | 3 | # Secure defaults 4 | Protocol 2 5 | HostKey /etc/ssh/ssh_host_ed25519_key 6 | HostKey /etc/ssh/ssh_host_rsa_key 7 | 8 | # Faster connection 9 | # See: https://github.com/atmoz/sftp/issues/11 10 | UseDNS no 11 | 12 | RSAAuthentication yes 13 | PubkeyAuthentication yes 14 | ChallengeResponseAuthentication no 15 | TCPKeepAlive yes 16 | 17 | # Limited access 18 | PermitRootLogin no 19 | X11Forwarding no 20 | AllowTcpForwarding no 21 | 22 | # Force sftp and chroot jail 23 | Subsystem sftp internal-sftp 24 | # ForceCommand internal-sftp 25 | # ChrootDirectory %h 26 | AuthorizedKeysFile /etc/ssh/authorized_keys/%u 27 | 28 | # Enable this for more logs 29 | #LogLevel VERBOSE 30 | -------------------------------------------------------------------------------- /tests/sftp_server/sshd_config: -------------------------------------------------------------------------------- 1 | Port 22 2 | 3 | # Secure defaults 4 | Protocol 2 5 | HostKey /etc/ssh/ssh_host_ed25519_key 6 | HostKey /etc/ssh/ssh_host_rsa_key 7 | 8 | # Faster connection 9 | # See: https://github.com/atmoz/sftp/issues/11 10 | UseDNS no 11 | 12 | RSAAuthentication yes 13 | PubkeyAuthentication yes 14 | ChallengeResponseAuthentication no 15 | TCPKeepAlive yes 16 | 17 | # Limited access 18 | PermitRootLogin no 19 | X11Forwarding no 20 | AllowTcpForwarding no 21 | 22 | # Force sftp and chroot jail 23 | Subsystem sftp internal-sftp 24 | # ForceCommand internal-sftp 25 | # ChrootDirectory %h 26 | AuthorizedKeysFile /etc/ssh/authorized_keys/%u 27 | 28 | # Enable this for more logs 29 | #LogLevel VERBOSE 30 | -------------------------------------------------------------------------------- /scripts/run-tests.ps1: -------------------------------------------------------------------------------- 1 | $ProgressPreference = 'SilentlyContinue' 2 | $ErrorActionPreference = 'Stop' 3 | 4 | $ToolboxPath = "$HOME\.toolbox" 5 | if (Test-Path $ToolboxPath) { 6 | Remove-Item -Path $ToolboxPath -Force -Recurse 7 | } 8 | 9 | New-Item -Path $ToolboxPath -ItemType Directory > $null 10 | 11 | # Copy toolbox to right place and install it 12 | Copy-Item -Path Checkout.psm1 -Destination "$ToolboxPath\Checkout.psm1" 13 | Copy-Item -Path .\install-self-hosted-toolbox.ps1 -Destination "$ToolboxPath\install-self-hosted-toolbox.ps1" 14 | & "$ToolboxPath\install-self-hosted-toolbox.ps1" 15 | 16 | # Import and run tests using Pester 17 | Install-Module -Name Pester -Force 18 | if ($?) { 19 | Invoke-Pester -Output Detailed -CI 20 | } 21 | -------------------------------------------------------------------------------- /cache-cli/pkg/files/checksum_test.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | assert "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test__GeneratesChecksum(t *testing.T) { 12 | t.Run("file is present", func(t *testing.T) { 13 | tempFile, _ := ioutil.TempFile(os.TempDir(), "*") 14 | tempFile.WriteString("hello, hello\n") 15 | 16 | checksum, err := GenerateChecksum(tempFile.Name()) 17 | assert.Nil(t, err) 18 | assert.Equal(t, "db243d472932e6e19fcb85468f962c46", checksum) 19 | 20 | os.Remove(tempFile.Name()) 21 | }) 22 | 23 | t.Run("file is not present", func(t *testing.T) { 24 | _, err := GenerateChecksum("/tmp/this-file-does-not-exist") 25 | assert.NotNil(t, err) 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /cache-cli/Dockerfile.sftp_server: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | RUN apt-get update && apt-get install -y openssh-server nginx --no-install-recommends 4 | RUN mkdir -p /var/run/sshd 5 | 6 | COPY sshd_config /etc/ssh/sshd_config 7 | COPY nginx.conf /etc/nginx/nginx.conf 8 | COPY default.conf /etc/nginx/sites-enabled/default 9 | COPY wrapper.sh /wrapper.sh 10 | COPY .htpasswd /etc/nginx/.htpasswd 11 | 12 | RUN addgroup ftpaccess 13 | RUN adduser tester --ingroup ftpaccess --shell /bin/bash --disabled-password --gecos '' 14 | RUN chown tester:ftpaccess /home/tester 15 | RUN mkdir /etc/ssh/authorized_keys 16 | 17 | COPY id_rsa.pub /tmp/id_rsa.pub 18 | RUN cat /tmp/id_rsa.pub >> /etc/ssh/authorized_keys/tester 19 | 20 | EXPOSE 80 21 | EXPOSE 22 22 | CMD ["/wrapper.sh"] -------------------------------------------------------------------------------- /tests/xcode15_sem_version.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "support/bats-support/load" 4 | load "support/bats-assert/load" 5 | 6 | PROJECT_ROOT=$(pwd) 7 | 8 | setup() { 9 | eval "$(rbenv init -)" 10 | source ~/toolbox/sem-version 11 | source ~/toolbox/sem-install 12 | source ~/.nvm/nvm.sh 13 | export NVM_DIR=~/.nvm 14 | } 15 | 16 | @test "[macOS] sem-version ruby - 3.3.3 " { 17 | 18 | run sem-version ruby 3.3.3 19 | assert_success 20 | run ruby --version 21 | assert_success 22 | assert_output --partial "3.3.3" 23 | } 24 | 25 | @test "[macOS] sem-version node - 14.16.1 " { 26 | run sem-version node 14.16.1 27 | assert_success 28 | assert_output --partial "14.16.1" 29 | run node --version 30 | assert_success 31 | } 32 | 33 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/gcs_restore.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | ) 10 | 11 | func (s *GCSStorage) Restore(key string) (*os.File, error) { 12 | tempFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-*", key)) 13 | if err != nil { 14 | return nil, err 15 | } 16 | 17 | bucketKey := fmt.Sprintf("%s/%s", s.Project, key) 18 | reader, err := s.Bucket.Object(bucketKey).NewReader(context.TODO()) 19 | if err != nil { 20 | _ = tempFile.Close() 21 | return nil, err 22 | } 23 | 24 | defer reader.Close() 25 | 26 | _, err = io.Copy(tempFile, reader) 27 | if err != nil { 28 | _ = tempFile.Close() 29 | return nil, err 30 | } 31 | 32 | return tempFile, tempFile.Close() 33 | } 34 | -------------------------------------------------------------------------------- /sem-context/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/semaphoreci/toolbox/sem-context/pkg/flags" 8 | "github.com/semaphoreci/toolbox/sem-context/pkg/store" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | var IgnoreFailure bool 13 | 14 | var Store store.Store 15 | 16 | var RootCmd = &cobra.Command{ 17 | Use: "sem-context", 18 | Short: "Share variables between your Semaphore jobs", 19 | } 20 | 21 | func Execute() { 22 | if err := RootCmd.Execute(); err != nil { 23 | fmt.Println(err) 24 | os.Exit(1) 25 | } 26 | } 27 | 28 | func init() { 29 | Store = &store.ArtifactStore{} 30 | RootCmd.PersistentFlags().BoolVar(&flags.IgnoreFailure, "ignore-failure", false, "Ignore if failure occurs, and always return 0.") 31 | } 32 | -------------------------------------------------------------------------------- /cache-cli/cmd/clear.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 5 | "github.com/semaphoreci/toolbox/cache-cli/pkg/utils" 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var clearCmd = &cobra.Command{ 11 | Use: "clear", 12 | Short: "Remove all keys in the cache.", 13 | Long: ``, 14 | Args: cobra.ArbitraryArgs, 15 | Run: func(cmd *cobra.Command, args []string) { 16 | RunClear(cmd, args) 17 | }, 18 | } 19 | 20 | func RunClear(cmd *cobra.Command, args []string) { 21 | storage, err := storage.InitStorage() 22 | utils.Check(err) 23 | 24 | err = storage.Clear() 25 | utils.Check(err) 26 | log.Infof("Deleted all caches.") 27 | } 28 | 29 | func init() { 30 | RootCmd.AddCommand(clearCmd) 31 | } 32 | -------------------------------------------------------------------------------- /tests/sem_version_container.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "support/bats-support/load" 4 | load "support/bats-assert/load" 5 | 6 | setup() { 7 | source ~/.toolbox/toolbox 8 | } 9 | 10 | @test "sem-version flutter 3.10.6" { 11 | 12 | run sem-version flutter 3.10.6 13 | assert_success 14 | assert_line --partial "3.10.6" 15 | } 16 | 17 | @test "sem-version flutter 3.10" { 18 | 19 | run sem-version flutter 3.10 20 | assert_success 21 | assert_line --partial "3.10.6" 22 | } 23 | 24 | @test "sem-version flutter 3.16.1" { 25 | 26 | run sem-version flutter 3.16.1 27 | assert_success 28 | assert_line --partial "3.16.1" 29 | } 30 | 31 | @test "sem-version flutter 3.16" { 32 | 33 | run sem-version flutter 3.16 34 | assert_success 35 | assert_line --partial "3.16.1" 36 | } 37 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/s3_store.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/aws/aws-sdk-go-v2/feature/s3/manager" 9 | "github.com/aws/aws-sdk-go-v2/service/s3" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | func (s *S3Storage) Store(key, path string) error { 14 | // #nosec 15 | file, err := os.Open(path) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | destination := fmt.Sprintf("%s/%s", s.Project, key) 21 | uploader := manager.NewUploader(s.Client) 22 | _, err = uploader.Upload(context.TODO(), &s3.PutObjectInput{ 23 | Bucket: &s.Bucket, 24 | Key: &destination, 25 | Body: file, 26 | }) 27 | 28 | if err != nil { 29 | log.Errorf("Error uploading: %v", err) 30 | return err 31 | } 32 | 33 | return file.Close() 34 | } 35 | -------------------------------------------------------------------------------- /cache-cli/pkg/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | const ( 9 | LocalBackend = "local" 10 | 11 | MeasurementName = "usercache" 12 | CommandStore = "store" 13 | CommandRestore = "restore" 14 | ) 15 | 16 | type CacheEvent struct { 17 | Command string 18 | Server string 19 | User string 20 | SizeBytes int64 21 | Duration time.Duration 22 | Corrupt bool 23 | } 24 | 25 | type MetricsManager interface { 26 | Enabled() bool 27 | LogEvent(event CacheEvent) error 28 | } 29 | 30 | func InitMetricsManager(backend string) (MetricsManager, error) { 31 | switch backend { 32 | case LocalBackend: 33 | return NewLocalMetricsBackend() 34 | default: 35 | return nil, fmt.Errorf("metrics backend '%s' is not available", backend) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/sftp_restore.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | ) 8 | 9 | func (s *SFTPStorage) Restore(key string) (*os.File, error) { 10 | localFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-*", key)) 11 | if err != nil { 12 | return nil, err 13 | } 14 | 15 | remoteFile, err := s.SFTPClient.Open(key) 16 | if err != nil { 17 | _ = localFile.Close() 18 | _ = os.Remove(localFile.Name()) 19 | return nil, err 20 | } 21 | 22 | _, err = localFile.ReadFrom(remoteFile) 23 | if err != nil { 24 | _ = localFile.Close() 25 | _ = remoteFile.Close() 26 | return nil, err 27 | } 28 | 29 | err = remoteFile.Close() 30 | if err != nil { 31 | _ = localFile.Close() 32 | return nil, err 33 | } 34 | return localFile, localFile.Close() 35 | } 36 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/s3_restore.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | 9 | "github.com/aws/aws-sdk-go-v2/feature/s3/manager" 10 | "github.com/aws/aws-sdk-go-v2/service/s3" 11 | ) 12 | 13 | func (s *S3Storage) Restore(key string) (*os.File, error) { 14 | tempFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-*", key)) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | bucketKey := fmt.Sprintf("%s/%s", s.Project, key) 20 | downloader := manager.NewDownloader(s.Client) 21 | _, err = downloader.Download(context.TODO(), tempFile, &s3.GetObjectInput{ 22 | Bucket: &s.Bucket, 23 | Key: &bucketKey, 24 | }) 25 | 26 | if err != nil { 27 | _ = tempFile.Close() 28 | return nil, err 29 | } 30 | 31 | return tempFile, tempFile.Close() 32 | } 33 | -------------------------------------------------------------------------------- /cache-cli/cmd/is_not_empty_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 10 | assert "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func Test__IsNotEmpty(t *testing.T) { 14 | runTestForAllBackends(t, func(backend string, storage storage.Storage) { 15 | t.Run(fmt.Sprintf("%s cache is empty", backend), func(*testing.T) { 16 | storage.Clear() 17 | assert.False(t, RunIsNotEmpty(isNotEmptyCmd, []string{})) 18 | }) 19 | 20 | t.Run(fmt.Sprintf("%s cache is not empty", backend), func(*testing.T) { 21 | storage.Clear() 22 | tempFile, _ := ioutil.TempFile(os.TempDir(), "*") 23 | storage.Store("abc001", tempFile.Name()) 24 | 25 | assert.True(t, RunIsNotEmpty(isNotEmptyCmd, []string{})) 26 | }) 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /sem-context/pkg/utils/error_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/semaphoreci/toolbox/sem-context/pkg/flags" 8 | ) 9 | 10 | type Error struct { 11 | ErrorMessage string 12 | ExitCode int 13 | } 14 | 15 | func (err *Error) Error() string { 16 | return err.ErrorMessage 17 | } 18 | 19 | // If error is present, exit and log error message, if it isn't, dont do anything. 20 | // If flag --ignore-failure is set, then exit with code 0, else exit with given error code 21 | // Error passed to this function must be of type Error defined inside this module 22 | func CheckError(err error) { 23 | if err != nil { 24 | castedError := err.(*Error) 25 | fmt.Fprintln(os.Stderr, err.Error()) 26 | if !flags.IgnoreFailure { 27 | os.Exit(castedError.ExitCode) 28 | } else { 29 | os.Exit(0) 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /cache-cli/cmd/is_not_empty.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 7 | "github.com/semaphoreci/toolbox/cache-cli/pkg/utils" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var isNotEmptyCmd = &cobra.Command{ 12 | Use: "is_not_empty", 13 | Short: "Check if the cache is not empty.", 14 | Long: ``, 15 | Args: cobra.ArbitraryArgs, 16 | Run: func(cmd *cobra.Command, args []string) { 17 | if RunIsNotEmpty(cmd, args) { 18 | os.Exit(0) 19 | } else { 20 | os.Exit(1) 21 | } 22 | }, 23 | } 24 | 25 | func RunIsNotEmpty(cmd *cobra.Command, args []string) bool { 26 | storage, err := storage.InitStorage() 27 | utils.Check(err) 28 | 29 | isNotEmpty, err := storage.IsNotEmpty() 30 | utils.Check(err) 31 | 32 | return isNotEmpty 33 | } 34 | 35 | func init() { 36 | RootCmd.AddCommand(isNotEmptyCmd) 37 | } 38 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | pwsh.lint: 2 | docker-compose run --rm toolbox pwsh .\\scripts\\run-linter.ps1 3 | 4 | pwsh.test: 5 | docker-compose run --rm toolbox pwsh .\\scripts\\run-tests.ps1 6 | 7 | release.major: 8 | git fetch --tags 9 | latest=$$(git tag | sort --version-sort | tail -n 1); new=$$(echo $$latest | cut -c 2- | awk -F '.' '{ print "v" $$1+1 ".0.0" }'); echo $$new; git tag $$new; git push origin $$new 10 | 11 | release.minor: 12 | git fetch --tags 13 | latest=$$(git tag | sort --version-sort | tail -n 1); new=$$(echo $$latest | cut -c 2- | awk -F '.' '{ print "v" $$1 "." $$2 + 1 ".0" }'); echo $$new; git tag $$new; git push origin $$new 14 | 15 | release.patch: 16 | git fetch --tags 17 | latest=$$(git tag | sort --version-sort | tail -n 1); new=$$(echo $$latest | cut -c 2- | awk -F '.' '{ print "v" $$1 "." $$2 "." $$3+1 }'); echo $$new; git tag $$new; git push origin $$new 18 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/gcs_store.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func (s *GCSStorage) Store(key, path string) error { 13 | // #nosec 14 | file, err := os.Open(path) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | ctx, cancel := context.WithCancel(context.TODO()) 20 | defer cancel() 21 | 22 | destination := fmt.Sprintf("%s/%s", s.Project, key) 23 | writer := s.Bucket.Object(destination).NewWriter(ctx) 24 | 25 | _, err = io.Copy(writer, file) 26 | if err != nil { 27 | log.Errorf("Error uploading: %v", err) 28 | _ = file.Close() 29 | 30 | // canceled context will abort the save, closing writer would save a partial object 31 | return err 32 | } 33 | 34 | err = writer.Close() 35 | if err != nil { 36 | return err 37 | } 38 | 39 | return file.Close() 40 | } 41 | -------------------------------------------------------------------------------- /tests/test-results/rspec2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Failure/Error: expect(user).to be_valid 9 | 10 | expected: true 11 | got: false 12 | 13 | ./spec/models/user_spec.rb:15:in `block (3 levels) in <top (required)>' 14 | 15 | -------------------------------------------------------------------------------- /sem-context/go.sum: -------------------------------------------------------------------------------- 1 | github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 2 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 3 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 4 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 5 | github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= 6 | github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= 7 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 8 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 10 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 11 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/has_key_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | assert "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test__HasKey(t *testing.T) { 13 | runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { 14 | t.Run(fmt.Sprintf("%s non-existing key", storageType), func(t *testing.T) { 15 | _ = storage.Clear() 16 | exists, err := storage.HasKey("this-key-does-not-exist") 17 | assert.Nil(t, err) 18 | assert.False(t, exists) 19 | }) 20 | 21 | t.Run(fmt.Sprintf("%s existing key", storageType), func(t *testing.T) { 22 | _ = storage.Clear() 23 | 24 | file, _ := ioutil.TempFile(os.TempDir(), "*") 25 | _ = storage.Store("abc001", file.Name()) 26 | 27 | exists, err := storage.HasKey("abc001") 28 | assert.Nil(t, err) 29 | assert.True(t, exists) 30 | 31 | os.Remove(file.Name()) 32 | }) 33 | }) 34 | } 35 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/is_not_empty_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | assert "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test__IsNotEmpty(t *testing.T) { 13 | runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { 14 | t.Run(fmt.Sprintf("%s empty cache", storageType), func(t *testing.T) { 15 | _ = storage.Clear() 16 | isNotEmpty, err := storage.IsNotEmpty() 17 | assert.Nil(t, err) 18 | assert.False(t, isNotEmpty) 19 | }) 20 | 21 | t.Run(fmt.Sprintf("%s non-empty cache", storageType), func(t *testing.T) { 22 | _ = storage.Clear() 23 | 24 | file, _ := ioutil.TempFile(os.TempDir(), "*") 25 | _ = storage.Store("abc001", file.Name()) 26 | 27 | isNotEmpty, err := storage.IsNotEmpty() 28 | assert.Nil(t, err) 29 | assert.True(t, isNotEmpty) 30 | 31 | os.Remove(file.Name()) 32 | }) 33 | }) 34 | } 35 | -------------------------------------------------------------------------------- /release/install_in_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | prefix_cmd() { 4 | local cmd=$@ 5 | if [ `whoami` == 'root' ]; then 6 | `$@` 7 | else 8 | `sudo $@` 9 | fi 10 | } 11 | 12 | # Before running this, you need to run release/create.sh 13 | 14 | # Remove installed toolbox 15 | prefix_cmd rm -rf ~/.toolbox 16 | prefix_cmd rm -f $(which artifact) 17 | prefix_cmd rm -f $(which spc) 18 | prefix_cmd rm -f $(which when) 19 | prefix_cmd rm -f $(which enetwork) 20 | cd ~ 21 | arch="" 22 | case $(uname) in 23 | Darwin) 24 | [[ "$(uname -m)" =~ "arm64" ]] && arch="-arm" 25 | tar -xvf /tmp/Darwin${arch}/darwin${arch}.tar -C /tmp 26 | mv /tmp/toolbox ~/.toolbox 27 | ;; 28 | Linux) 29 | [[ "$(uname -m)" =~ "aarch" ]] && arch="-arm" 30 | tar -xvf /tmp/"Linux${arch}"/"linux${arch}".tar -C /tmp 31 | mv /tmp/toolbox ~/.toolbox 32 | ;; 33 | esac 34 | 35 | cd - 36 | 37 | bash ~/.toolbox/install-toolbox 38 | source ~/.toolbox/toolbox 39 | -------------------------------------------------------------------------------- /tests/sem_version_jammy/kubectl.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # kubectl 27 | @test "change kubectl to 1.27.1" { 28 | sem-version kubectl 1.27.1 29 | run kubectl version 30 | assert_line --partial "1.27.1" 31 | } 32 | -------------------------------------------------------------------------------- /tests/sem_version_noble/kubectl.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # kubectl 27 | @test "change kubectl to 1.27.1" { 28 | sem-version kubectl 1.27.1 29 | run kubectl version 30 | assert_line --partial "1.27.1" 31 | } 32 | -------------------------------------------------------------------------------- /cache-cli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/semaphoreci/toolbox/cache-cli/cmd" 9 | "github.com/semaphoreci/toolbox/cache-cli/pkg/logging" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | func main() { 14 | logfile := OpenLogfile() 15 | log.SetOutput(logfile) 16 | log.SetFormatter(new(logging.CustomFormatter)) 17 | log.SetLevel(log.InfoLevel) 18 | cmd.Execute() 19 | } 20 | 21 | func OpenLogfile() io.Writer { 22 | // #nosec 23 | filePath := filepath.Join(os.TempDir(), "cache_log") 24 | 25 | // #nosec 26 | f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) 27 | 28 | /* 29 | * We shouldn't fail if we can't create 30 | * the log file for whatever reason. Just proceed logging only to stdout. 31 | */ 32 | if err != nil { 33 | log.Errorf("Error creating file '%s': %v - proceeding", filePath, err) 34 | return os.Stdout 35 | } 36 | 37 | return io.MultiWriter(f, os.Stdout) 38 | } 39 | -------------------------------------------------------------------------------- /sem-context/cmd/delete.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/semaphoreci/toolbox/sem-context/pkg/utils" 7 | "github.com/semaphoreci/toolbox/sem-context/pkg/validators" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var deleteCmd = &cobra.Command{ 12 | Use: "delete [key]", 13 | Short: "Delete a variable", 14 | Run: RunDeleteCmd, 15 | } 16 | 17 | func RunDeleteCmd(cmd *cobra.Command, args []string) { 18 | utils.CheckError(validators.ValidateGetAndDeleteArguments(args)) 19 | key := args[0] 20 | 21 | value, err := SearchForKeyInAllContexts(key) 22 | utils.CheckError(err) 23 | if value == "" { 24 | utils.CheckError(&utils.Error{ErrorMessage: fmt.Sprintf("Key %s does not exist", key), ExitCode: 1}) 25 | } 26 | 27 | contextId := utils.GetPipelineContextHierarchy()[0] 28 | err = Store.Delete(key, contextId) 29 | utils.CheckError(err) 30 | fmt.Println("Key successfully deleted") 31 | } 32 | 33 | func init() { 34 | RootCmd.AddCommand(deleteCmd) 35 | } 36 | -------------------------------------------------------------------------------- /test-results/pkg/parser/mermaid.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "unicode" 7 | ) 8 | 9 | // EscapeGanttLabel sanitizes text for use in Mermaid gantt chart labels. 10 | // It escapes special characters that have meaning in Mermaid syntax using 11 | // entity codes as documented at: 12 | // https://mermaid.js.org/syntax/sequenceDiagram.html#entity-codes-to-escape-characters 13 | func EscapeGanttLabel(text string) string { 14 | var result strings.Builder 15 | result.Grow(len(text)) 16 | 17 | for _, r := range text { 18 | result.WriteString(escapeChar(r)) 19 | } 20 | 21 | return result.String() 22 | } 23 | 24 | func escapeChar(r rune) string { 25 | switch r { 26 | case '\r', '\n', '\t': 27 | return " " 28 | case '[', ']', '{', '}', '<', '>', ',', ':': 29 | // Use Mermaid's entity code format: #[decimal]; 30 | return fmt.Sprintf("#%d;", r) 31 | default: 32 | if unicode.IsControl(r) { 33 | return "" 34 | } 35 | return string(r) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /tests/sem_version_jammy/scala.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | # scala 26 | @test "change scala to 3.2" { 27 | 28 | run scala -version 29 | assert_line --partial "3.2" 30 | 31 | sem-version scala 3.2 32 | run scala -version 33 | assert_line --partial "3.2" 34 | } 35 | -------------------------------------------------------------------------------- /tests/sem_version_noble/scala.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | # scala 26 | @test "change scala to 3.2" { 27 | 28 | run scala -version 29 | assert_line --partial "3.2" 30 | 31 | sem-version scala 3.2 32 | run scala -version 33 | assert_line --partial "3.2" 34 | } 35 | -------------------------------------------------------------------------------- /tests/macOS_sem_version.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "support/bats-support/load" 4 | load "support/bats-assert/load" 5 | 6 | PROJECT_ROOT=$(pwd) 7 | 8 | setup() { 9 | eval "$(rbenv init -)" 10 | source ~/toolbox/sem-version 11 | source ~/toolbox/sem-install 12 | source ~/.nvm/nvm.sh 13 | export NVM_DIR=~/.nvm 14 | } 15 | 16 | @test "[macOS] sem-version ruby - 2.7.2 " { 17 | 18 | run sem-version ruby 2.7.2 19 | assert_success 20 | run ruby --version 21 | assert_success 22 | assert_output --partial "2.7.2" 23 | } 24 | @test "[macOS] sem-version ruby - 3.0.1 " { 25 | 26 | run sem-version ruby 3.0.1 27 | assert_success 28 | run ruby --version 29 | assert_success 30 | assert_output --partial "3.0.1" 31 | } 32 | @test "[macOS] sem-version php - 8.0.5 " { 33 | 34 | run sem-version php 8.0.5 35 | assert_failure 36 | } 37 | 38 | @test "[macOS] sem-version node - 14.16.1 " { 39 | 40 | run sem-version node 14.16.1 41 | assert_success 42 | assert_output --partial "14.16.1" 43 | node --version 44 | } 45 | 46 | -------------------------------------------------------------------------------- /tests/sem_version_focal/scala.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source /opt/change-firefox-version.sh 14 | source ~/.phpbrew/bashrc 15 | . /home/semaphore/.nvm/nvm.sh 16 | export PATH="$PATH:/home/semaphore/.yarn/bin" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # scala 27 | @test "change scala to 2.12" { 28 | 29 | run scala -version 30 | assert_line --partial "2.12" 31 | 32 | sem-version scala 2.12 33 | run scala -version 34 | assert_line --partial "2.12" 35 | } 36 | -------------------------------------------------------------------------------- /cache-cli/cmd/usage.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/semaphoreci/toolbox/cache-cli/pkg/files" 5 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 6 | "github.com/semaphoreci/toolbox/cache-cli/pkg/utils" 7 | log "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var usageCmd = &cobra.Command{ 12 | Use: "usage", 13 | Short: "Get a summary of cache usage.", 14 | Long: ``, 15 | Args: cobra.ArbitraryArgs, 16 | Run: func(cmd *cobra.Command, args []string) { 17 | RunUsage(cmd, args) 18 | }, 19 | } 20 | 21 | func RunUsage(cmd *cobra.Command, args []string) { 22 | storage, err := storage.InitStorage() 23 | utils.Check(err) 24 | 25 | summary, err := storage.Usage() 26 | utils.Check(err) 27 | 28 | if summary.Free == -1 { 29 | log.Info("FREE SPACE: (unlimited)") 30 | } else { 31 | log.Infof("FREE SPACE: %s", files.HumanReadableSize(summary.Free)) 32 | } 33 | 34 | log.Infof("USED SPACE: %s", files.HumanReadableSize(summary.Used)) 35 | } 36 | 37 | func init() { 38 | RootCmd.AddCommand(usageCmd) 39 | } 40 | -------------------------------------------------------------------------------- /cache-cli/cmd/usage_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/semaphoreci/toolbox/cache-cli/pkg/logging" 8 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 9 | log "github.com/sirupsen/logrus" 10 | assert "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func Test__Usage(t *testing.T) { 14 | log.SetFormatter(new(logging.CustomFormatter)) 15 | log.SetLevel(log.InfoLevel) 16 | log.SetOutput(openLogfileForTests(t)) 17 | 18 | runTestForAllBackends(t, func(backend string, storage storage.Storage) { 19 | t.Run(fmt.Sprintf("%s empty cache", backend), func(t *testing.T) { 20 | storage.Clear() 21 | 22 | RunUsage(usageCmd, []string{}) 23 | output := readOutputFromFile(t) 24 | 25 | switch backend { 26 | case "s3": 27 | assert.Contains(t, output, "FREE SPACE: (unlimited)") 28 | assert.Contains(t, output, "USED SPACE: 0.0") 29 | case "sftp": 30 | assert.Contains(t, output, "FREE SPACE: 9.0G") 31 | assert.Contains(t, output, "USED SPACE: 0.0") 32 | } 33 | }) 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/delete_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | assert "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test__Delete(t *testing.T) { 13 | runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { 14 | t.Run(fmt.Sprintf("%s non-existing key", storageType), func(t *testing.T) { 15 | _ = storage.Clear() 16 | err := storage.Delete("this-key-does-not-exist") 17 | assert.Nil(t, err) 18 | }) 19 | 20 | t.Run(fmt.Sprintf("%s existing key", storageType), func(t *testing.T) { 21 | _ = storage.Clear() 22 | 23 | file, _ := ioutil.TempFile(os.TempDir(), "*") 24 | _ = storage.Store("abc001", file.Name()) 25 | 26 | keys, err := storage.List() 27 | assert.Nil(t, err) 28 | assert.Len(t, keys, 1) 29 | 30 | err = storage.Delete("abc001") 31 | assert.Nil(t, err) 32 | 33 | keys, err = storage.List() 34 | assert.Nil(t, err) 35 | assert.Len(t, keys, 0) 36 | 37 | os.Remove(file.Name()) 38 | }) 39 | }) 40 | } 41 | -------------------------------------------------------------------------------- /cache-cli/pkg/files/download.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | ) 8 | 9 | func DownloadFromHTTP(URL, username, password, key string) (*os.File, error) { 10 | client := &http.Client{} 11 | downloadURL := fmt.Sprintf("%s/%s", URL, key) 12 | req, err := http.NewRequest("GET", downloadURL, nil) 13 | if err != nil { 14 | return nil, err 15 | } 16 | 17 | req.SetBasicAuth(username, password) 18 | resp, err := client.Do(req) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | if resp.StatusCode != http.StatusOK { 24 | return nil, fmt.Errorf("failed to download file: %s", resp.Status) 25 | } 26 | 27 | localFile, err := os.CreateTemp(os.TempDir(), fmt.Sprintf("%s-*", key)) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | _, err = localFile.ReadFrom(resp.Body) 33 | if err != nil { 34 | _ = localFile.Close() 35 | return nil, err 36 | } 37 | 38 | err = resp.Body.Close() 39 | if err != nil { 40 | _ = localFile.Close() 41 | return nil, err 42 | } 43 | 44 | return localFile, localFile.Close() 45 | } 46 | -------------------------------------------------------------------------------- /tests/sem_version_jammy/java.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | @test "change java to 11" { 27 | sem-version java 11 28 | run java --version 29 | assert_line --partial "openjdk 11." 30 | } 31 | 32 | @test "change java to 17" { 33 | sem-version java 17 34 | run java --version 35 | assert_line --partial "openjdk 17." 36 | } 37 | 38 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/gcs.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | 6 | "cloud.google.com/go/storage" 7 | ) 8 | 9 | type GCSStorage struct { 10 | Client *storage.Client 11 | Bucket *storage.BucketHandle 12 | Project string 13 | StorageConfig StorageConfig 14 | } 15 | 16 | type GCSStorageOptions struct { 17 | Bucket string 18 | Project string 19 | Config StorageConfig 20 | } 21 | 22 | func NewGCSStorage(options GCSStorageOptions) (*GCSStorage, error) { 23 | return createDefaultGCSStorage(options.Bucket, options.Project, options.Config) 24 | } 25 | 26 | func createDefaultGCSStorage(gcsBucket string, project string, storageConfig StorageConfig) (*GCSStorage, error) { 27 | client, err := storage.NewClient(context.TODO()) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | return &GCSStorage{ 33 | Client: client, 34 | Bucket: client.Bucket(gcsBucket), 35 | Project: project, 36 | StorageConfig: storageConfig, 37 | }, nil 38 | } 39 | 40 | func (s *GCSStorage) Config() StorageConfig { 41 | return s.StorageConfig 42 | } 43 | -------------------------------------------------------------------------------- /cache-cli/pkg/files/size_test.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "testing" 5 | 6 | assert "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test__HumanReadableSize(t *testing.T) { 10 | t.Run("bytes", func(t *testing.T) { 11 | assert.Equal(t, "800.0", HumanReadableSize(800)) 12 | }) 13 | 14 | t.Run("kilobytes", func(t *testing.T) { 15 | assert.Equal(t, "100.0K", HumanReadableSize(1024*100)) 16 | }) 17 | 18 | t.Run("kilobytes with remainder", func(t *testing.T) { 19 | assert.Equal(t, "100.1K", HumanReadableSize(1024*100+128)) 20 | }) 21 | 22 | t.Run("megabytes", func(t *testing.T) { 23 | assert.Equal(t, "5.0M", HumanReadableSize(1024*1024*5)) 24 | }) 25 | 26 | t.Run("megabytes with remainder", func(t *testing.T) { 27 | assert.Equal(t, "5.1M", HumanReadableSize(1024*1024*5+1024*128)) 28 | }) 29 | 30 | t.Run("gigabytes", func(t *testing.T) { 31 | assert.Equal(t, "5.0G", HumanReadableSize(1024*1024*1024*5)) 32 | }) 33 | 34 | t.Run("gigabytes with remainder", func(t *testing.T) { 35 | assert.Equal(t, "5.1G", HumanReadableSize(1024*1024*1024*5+1024*1024*128)) 36 | }) 37 | } 38 | -------------------------------------------------------------------------------- /tests/sem_version_focal/java.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source /opt/change-firefox-version.sh 14 | source ~/.phpbrew/bashrc 15 | . /home/semaphore/.nvm/nvm.sh 16 | export PATH="$PATH:/home/semaphore/.yarn/bin" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # Java 27 | @test "change java to 11" { 28 | sem-version java 11 29 | run java --version 30 | assert_line --partial "openjdk 11." 31 | } 32 | 33 | @test "change java to 17" { 34 | sem-version java 17 35 | run java --version 36 | assert_line --partial "openjdk 17." 37 | } 38 | -------------------------------------------------------------------------------- /cache-cli/cmd/delete.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 5 | "github.com/semaphoreci/toolbox/cache-cli/pkg/utils" 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var deleteCmd = &cobra.Command{ 11 | Use: "delete [key]", 12 | Short: "Delete a key from the cache.", 13 | Long: ``, 14 | Args: cobra.ArbitraryArgs, 15 | Run: func(cmd *cobra.Command, args []string) { 16 | RunDelete(cmd, args) 17 | }, 18 | } 19 | 20 | func RunDelete(cmd *cobra.Command, args []string) { 21 | if len(args) != 1 { 22 | log.Errorf("Incorrect number of arguments!") 23 | _ = cmd.Help() 24 | return 25 | } 26 | 27 | storage, err := storage.InitStorage() 28 | utils.Check(err) 29 | 30 | rawKey := args[0] 31 | key := NormalizeKey(rawKey) 32 | 33 | if ok, _ := storage.HasKey(key); ok { 34 | err := storage.Delete(key) 35 | utils.Check(err) 36 | log.Infof("Key '%s' is deleted.", key) 37 | } else { 38 | log.Infof("Key '%s' doesn't exist in the cache store.", key) 39 | } 40 | } 41 | 42 | func init() { 43 | RootCmd.AddCommand(deleteCmd) 44 | } 45 | -------------------------------------------------------------------------------- /tests/sem_version_focal/kubectl.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source /opt/change-firefox-version.sh 14 | source ~/.phpbrew/bashrc 15 | . /home/semaphore/.nvm/nvm.sh 16 | export PATH="$PATH:/home/semaphore/.yarn/bin" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # kubectl 27 | @test "change kubectl to 1.15.3" { 28 | sem-version kubectl 1.15.3 29 | run kubectl version 30 | assert_line --partial "1.15.3" 31 | } 32 | 33 | @test "change kubectl to 1.28.2" { 34 | sem-version kubectl 1.28.2 35 | run kubectl version 36 | assert_line --partial "1.28.2" 37 | } 38 | -------------------------------------------------------------------------------- /tests/shm/shm_docker.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | curl http://packages.semaphoreci.com/tools/shm_512mb -o ~/toolbox/tests/shm/shm_512mb 8 | curl http://packages.semaphoreci.com/tools/shm_1024mb -o ~/toolbox/tests/shm/shm_1024mb 9 | chmod +x ~/toolbox/tests/shm/shm_*mb 10 | } 11 | 12 | @test "shm: write 512MB to shared memory" { 13 | run ~/toolbox/tests/shm/shm_512mb 14 | assert_line --partial "Writing Process: Shared Memory Write: Wrote 536870911 bytes" 15 | assert_line --partial "Writing Process: Complete" 16 | [ $(cat /tmp/system-metrics2 | awk -F "," '{print $5}' | grep -o '[0-9]\+' | sort -n | tail -n 1) -ge 510 ] 17 | run sleep 2 18 | } 19 | 20 | @test "shm: write 1024MB to shared memory" { 21 | run ~/toolbox/tests/shm/shm_1024mb 22 | assert_line --partial "Writing Process: Shared Memory Write: Wrote 1073741823 bytes" 23 | assert_line --partial "Writing Process: Complete" 24 | [ $(cat /tmp/system-metrics2 | awk -F "," '{print $5}' | grep -o '[0-9]\+' | sort -n | tail -n 1) -ge 1020 ] 25 | run sleep 2 26 | } 27 | -------------------------------------------------------------------------------- /tests/shm/shm_ubuntu.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | curl http://packages.semaphoreci.com/tools/shm_512mb -o ~/toolbox/tests/shm/shm_512mb 8 | curl http://packages.semaphoreci.com/tools/shm_1024mb -o ~/toolbox/tests/shm/shm_1024mb 9 | chmod +x ~/toolbox/tests/shm/shm_*mb 10 | } 11 | 12 | @test "shm: write 512MB to shared memory" { 13 | run ~/toolbox/tests/shm/shm_512mb 14 | assert_line --partial "Writing Process: Shared Memory Write: Wrote 536870911 bytes" 15 | assert_line --partial "Writing Process: Complete" 16 | [ $(cat /tmp/system-metrics2 | awk -F "," '{print $5}' | grep -o '[0-9]\+' | sort -n | tail -n 1) -ge 510 ] 17 | run sleep 2 18 | } 19 | 20 | @test "shm: write 1024MB to shared memory" { 21 | run ~/toolbox/tests/shm/shm_1024mb 22 | assert_line --partial "Writing Process: Shared Memory Write: Wrote 1073741823 bytes" 23 | assert_line --partial "Writing Process: Complete" 24 | [ $(cat /tmp/system-metrics2 | awk -F "," '{print $5}' | grep -o '[0-9]\+' | sort -n | tail -n 1) -ge 1020 ] 25 | run sleep 2 26 | } 27 | -------------------------------------------------------------------------------- /tests/test-results/revive.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Severity": "warning", 4 | "Failure": "receiver name ms should be consistent with previous receiver name s for Parser", 5 | "RuleName": "receiver-naming", 6 | "Category": "naming", 7 | "Position": { 8 | "Start": { 9 | "Filename": "pkg/parsers/parser.go", 10 | "Offset": 1046, 11 | "Line": 46, 12 | "Column": 1 13 | }, 14 | "End": { 15 | "Filename": "pkg/parsers/parser.go", 16 | "Offset": 1141, 17 | "Line": 48, 18 | "Column": 2 19 | } 20 | } 21 | }, 22 | { 23 | "Severity": "error", 24 | "Failure": "exported function Parse should have comment or be unexported", 25 | "RuleName": "exported", 26 | "Category": "comments", 27 | "Position": { 28 | "Start": { 29 | "Filename": "pkg/cli/cli.go", 30 | "Offset": 2500, 31 | "Line": 85, 32 | "Column": 1 33 | }, 34 | "End": { 35 | "Filename": "pkg/cli/cli.go", 36 | "Offset": 2600, 37 | "Line": 87, 38 | "Column": 2 39 | } 40 | } 41 | } 42 | ] -------------------------------------------------------------------------------- /tests/sem_version_focal/go.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source /opt/change-firefox-version.sh 14 | source ~/.phpbrew/bashrc 15 | . /home/semaphore/.nvm/nvm.sh 16 | export PATH="$PATH:/home/semaphore/.yarn/bin" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | @test "sem-version go 1.20.8 path check" { 27 | 28 | sem-version go 1.20.8 29 | run echo ${PATH} 30 | assert_line --partial "$(go env GOPATH)/bin" 31 | } 32 | 33 | @test "sem-version go 1.21.3 path check" { 34 | 35 | sem-version go 1.21.3 36 | run echo ${PATH} 37 | assert_line --partial "$(go env GOPATH)/bin" 38 | } 39 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/restore_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | assert "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test__Restore(t *testing.T) { 13 | runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { 14 | t.Run(fmt.Sprintf("%s key exists", storageType), func(t *testing.T) { 15 | _ = storage.Clear() 16 | 17 | file, _ := ioutil.TempFile(os.TempDir(), "*") 18 | file.WriteString("restore - key exists") 19 | 20 | err := storage.Store("abc001", file.Name()) 21 | assert.Nil(t, err) 22 | 23 | restoredFile, err := storage.Restore("abc001") 24 | assert.Nil(t, err) 25 | 26 | content, err := ioutil.ReadFile(restoredFile.Name()) 27 | assert.Nil(t, err) 28 | assert.Equal(t, "restore - key exists", string(content)) 29 | 30 | os.Remove(file.Name()) 31 | os.Remove(restoredFile.Name()) 32 | }) 33 | 34 | t.Run(fmt.Sprintf("%s key does not exist", storageType), func(t *testing.T) { 35 | _ = storage.Clear() 36 | 37 | _, err := storage.Restore("abc002") 38 | assert.NotNil(t, err) 39 | }) 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /tests/compiler.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "support/bats-support/load" 4 | load "support/bats-assert/load" 5 | 6 | setup() { 7 | unset SEMAPHORE_GIT_REF_TYPE 8 | unset SEMAPHORE_GIT_BRANCH 9 | unset SEMAPHORE_GIT_COMMIT_RANGE 10 | unset SEMAPHORE_GIT_SHA 11 | unset SEMAPHORE_MERGE_BASE 12 | unset SEMAPHORE_MERGE_BASE 13 | 14 | git config --global user.email "you@example.com" 15 | git config --global user.name "Your Name" 16 | 17 | rm -rf /tmp/test-repo-origin 18 | rm -rf /tmp/test-repo-clone 19 | cp -R tests/compiler/test-repo /tmp/test-repo-origin 20 | 21 | cd /tmp/test-repo-origin 22 | git init 23 | git add . 24 | git commit -m "Bootstrap" 25 | git clone /tmp/test-repo-origin /tmp/test-repo-clone 26 | cd - 27 | } 28 | 29 | @test "compiler can evaluate change_in expressions" { 30 | cd /tmp/test-repo-clone 31 | 32 | run spc evaluate change-in --input .semaphore/semaphore.yml --output .semaphore/semaphore.yml.compiler --logs .semaphore/semaphore.yml.logs 33 | assert_success 34 | 35 | run cat .semaphore/semaphore.yml.compiler 36 | assert_output --partial "(branch = 'master') and false" 37 | 38 | cd - 39 | } 40 | -------------------------------------------------------------------------------- /test-results/priv/merging/pipeline/job2/junit.json: -------------------------------------------------------------------------------- 1 | {"testResults":[{"id":"4ae71336-e44b-39bf-b9d2-752e234818a5","name":"","framework":"","isDisabled":false,"suites":[{"id":"5adf0dab-e505-39fe-99c7-298ef43a8f09","name":"foo","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"b6941c19-0674-36aa-965c-badc07c512bd","file":"","classname":"","package":"","name":"foo.3","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"7c947655-ee72-38d1-965d-35a6bdb0da6d","file":"","classname":"","package":"","name":"foo.4","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":2,"passed":2,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"systemOut":"","systemErr":""},{"id":"c348fb08-df86-3e06-a356-b951c48ea5a4","name":"bar","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[],"properties":null,"summary":{"total":0,"passed":0,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"systemOut":"","systemErr":""}],"summary":{"total":2,"passed":2,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"status":"success","statusMessage":""}]} -------------------------------------------------------------------------------- /tests/sem_version_focal/python.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | source "/home/semaphore/.kiex/scripts/kiex" 17 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 18 | export NVM_DIR=/home/semaphore/.nvm 19 | export PHPBREW_HOME=/home/semaphore/.phpbrew 20 | eval "$(rbenv init -)" 21 | 22 | source ~/.toolbox/toolbox 23 | } 24 | 25 | # Python 26 | @test "change python to 3.10" { 27 | sem-version python 3.10 28 | run python --version 29 | assert_line --partial "3.10" 30 | } 31 | 32 | @test "change python to 3.11" { 33 | sem-version python 3.11 34 | run python --version 35 | assert_line --partial "3.11" 36 | } 37 | 38 | @test "change python to 3.12" { 39 | sem-version python 3.12 40 | run python --version 41 | assert_line --partial "3.12" 42 | } 43 | -------------------------------------------------------------------------------- /cache-cli/cmd/has_key.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 7 | "github.com/semaphoreci/toolbox/cache-cli/pkg/utils" 8 | log "github.com/sirupsen/logrus" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | var hasKeyCmd = &cobra.Command{ 13 | Use: "has_key [key]", 14 | Short: "Check if a key is present in the cache.", 15 | Long: ``, 16 | Args: cobra.ArbitraryArgs, 17 | Run: func(cmd *cobra.Command, args []string) { 18 | if !RunHasKey(cmd, args) { 19 | os.Exit(1) 20 | } 21 | }, 22 | } 23 | 24 | func RunHasKey(cmd *cobra.Command, args []string) bool { 25 | if len(args) != 1 { 26 | log.Error("Incorrect number of arguments!") 27 | _ = cmd.Help() 28 | return true 29 | } 30 | 31 | storage, err := storage.InitStorage() 32 | utils.Check(err) 33 | 34 | rawKey := args[0] 35 | key := NormalizeKey(rawKey) 36 | exists, err := storage.HasKey(key) 37 | utils.Check(err) 38 | 39 | if exists { 40 | log.Infof("Key '%s' exists in the cache store.", key) 41 | return true 42 | } 43 | 44 | log.Infof("Key '%s' doesn't exist in the cache store.", key) 45 | return false 46 | } 47 | 48 | func init() { 49 | RootCmd.AddCommand(hasKeyCmd) 50 | } 51 | -------------------------------------------------------------------------------- /tests/sem_version_noble/java.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | @test "change java to 11" { 27 | sem-version java 11 28 | run java --version 29 | assert_line --partial "openjdk 11." 30 | } 31 | 32 | @test "change java to 17" { 33 | sem-version java 17 34 | run java --version 35 | assert_line --partial "openjdk 17." 36 | } 37 | 38 | @test "change java to 21" { 39 | sem-version java 21 40 | run java --version 41 | assert_line --partial "openjdk 21." 42 | } 43 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | on: [push] 2 | name: Unit tests 3 | jobs: 4 | unit-testing: 5 | runs-on: windows-latest 6 | steps: 7 | - name: Install Go 8 | uses: actions/setup-go@v2 9 | with: 10 | go-version: 1.22.x 11 | - name: Check out repository code 12 | uses: actions/checkout@v2 13 | - name: Install gotestsum 14 | run: go install gotest.tools/gotestsum@latest 15 | - name: Run tests 16 | env: 17 | SEMAPHORE_CACHE_S3_URL: "http://127.0.0.1:9000" 18 | SEMAPHORE_CACHE_S3_KEY: minioadmin 19 | SEMAPHORE_CACHE_S3_SECRET: minioadmin 20 | SEMAPHORE_TOOLBOX_METRICS_ENABLED: "true" 21 | run: | 22 | New-Item C:\minio -ItemType Directory > $null 23 | Invoke-WebRequest "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" -OutFile C:\minio\minio.exe 24 | New-Item C:\minio\data\semaphore-cache -ItemType Directory > $null 25 | Start-Process C:\minio\minio.exe -ArgumentList 'server','C:\minio\data' -RedirectStandardOutput C:\minio\logs -RedirectStandardError C:\minio\errors 26 | cd cache-cli 27 | gotestsum --format short-verbose --packages="./..." -- -p 1 28 | -------------------------------------------------------------------------------- /tests/system_metrics_collector.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "support/bats-support/load" 4 | load "support/bats-assert/load" 5 | 6 | setup() { 7 | kill -TERM `ps ax|grep system-metrics-collector|awk '{print $1}'` || true 8 | rm -f /tmp/system-metrics 9 | sh ~/toolbox/system-metrics-collector & 10 | PID=$! 11 | echo '[A-Z][a-z][a-z] [A-Z][a-z][a-z] [0-9]{1,2} [0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2} [A-Z]{3,4} [0-9]{4} \| cpu: [0-9]+(\.[0-9]{1,2})?%, mem: [0-9]{1,2}(.[0-9]{1,2})?%, system_disk: [0-9]{1,2}(.[0-9]{1,2})?%, docker_disk: [0-9]{1,2}(.[0-9]{1,2})?%, shared_memory: [0-9]+ M' > /tmp/pattern.txt 12 | sleep 5 13 | kill -TERM $PID 14 | cat /tmp/system-metrics 15 | } 16 | teardown() { 17 | rm -f /tmp/system-metrics 18 | } 19 | 20 | @test "Test if /tmp/system-metrics format is not empty" { 21 | 22 | result="$(wc -l /tmp/system-metrics | awk '{print $1}')" 23 | [ "$result" -gt 0 ] 24 | } 25 | 26 | 27 | @test "Test if /tmp/system-metrics format is correct" { 28 | 29 | run egrep -q -f /tmp/pattern.txt /tmp/system-metrics 30 | assert_success 31 | } 32 | 33 | @test "Test if /tmp/system-metrics has strange lines" { 34 | 35 | run egrep -q -v -f /tmp/pattern.txt /tmp/system-metrics 36 | assert_failure 37 | } 38 | -------------------------------------------------------------------------------- /tests/sem_version_focal/gcc.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source /opt/change-firefox-version.sh 14 | source ~/.phpbrew/bashrc 15 | . /home/semaphore/.nvm/nvm.sh 16 | export PATH="$PATH:/home/semaphore/.yarn/bin" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # C 27 | @test "change gcc to 9" { 28 | 29 | run sem-version c 9 30 | assert_success 31 | run gcc -v 32 | assert_line --partial "gcc version 9." 33 | } 34 | 35 | @test "change gcc to 10" { 36 | 37 | run sem-version c 10 38 | assert_success 39 | run gcc -v 40 | assert_line --partial "gcc version 10." 41 | } 42 | 43 | @test "change gcc to 16" { 44 | 45 | run sem-version c 16 46 | assert_failure 47 | } 48 | -------------------------------------------------------------------------------- /tests/sem_version_jammy/gcc.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # C 27 | @test "change gcc to 11" { 28 | 29 | run sem-version c 11 30 | assert_success 31 | run gcc -v 32 | assert_line --partial "gcc version 11." 33 | } 34 | 35 | @test "change gcc to 12" { 36 | 37 | run sem-version c 12 38 | assert_success 39 | run gcc -v 40 | assert_line --partial "gcc version 12." 41 | } 42 | 43 | @test "change gcc to 16" { 44 | 45 | run sem-version c 16 46 | assert_failure 47 | } 48 | 49 | -------------------------------------------------------------------------------- /tests/sem_version_jammy/python.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # Python 27 | @test "change python to 3.10" { 28 | sem-version python 3.10 29 | run python --version 30 | assert_line --partial "3.10" 31 | } 32 | 33 | @test "change python to 3.11" { 34 | sem-version python 3.11 35 | run python --version 36 | assert_line --partial "3.11" 37 | } 38 | 39 | @test "change python to 3.12" { 40 | sem-version python 3.12 41 | run python --version 42 | assert_line --partial "3.12" 43 | } 44 | -------------------------------------------------------------------------------- /tests/sem_version_noble/gcc.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # C 27 | @test "change gcc to 11" { 28 | 29 | run sem-version c 11 30 | assert_success 31 | run gcc -v 32 | assert_line --partial "gcc version 11." 33 | } 34 | 35 | @test "change gcc to 12" { 36 | 37 | run sem-version c 12 38 | assert_success 39 | run gcc -v 40 | assert_line --partial "gcc version 12." 41 | } 42 | 43 | @test "change gcc to 16" { 44 | 45 | run sem-version c 16 46 | assert_failure 47 | } 48 | 49 | -------------------------------------------------------------------------------- /tests/sem_version_noble/python.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # Python 27 | @test "change python to 3.10" { 28 | sem-version python 3.10 29 | run python --version 30 | assert_line --partial "3.10" 31 | } 32 | 33 | @test "change python to 3.11" { 34 | sem-version python 3.11 35 | run python --version 36 | assert_line --partial "3.11" 37 | } 38 | 39 | @test "change python to 3.12" { 40 | sem-version python 3.12 41 | run python --version 42 | assert_line --partial "3.12" 43 | } 44 | -------------------------------------------------------------------------------- /test-results/docs/id-generation.md: -------------------------------------------------------------------------------- 1 | ## ID generation 2 | 3 | This PR introduces `id` generator for tests results, test suites, and tests. 4 | 5 | `id`'s are being generated in form of UUID strings. 6 | 7 | To generate consistent `id`'s between builds following method is implemented for all parsers: 8 | 9 | - ID generation for `Test results`(top-level element) 10 | 11 | 1. If the element has an ID, generate UUID based on that ID 12 | 2. If the element doesn't have an ID - generate UUID based on the `name` attribute 13 | 3. If the element has a framework name - generate UUID based on the `name` attribute and `framework` 14 | 4. Otherwise, generate uuid based on an empty string `""` 15 | 16 | - ID generation for `Suites` 17 | 18 | The same rules apply as for `Test results` however every `Suite ID` is namespaced by `Test results` ID 19 | 20 | - ID generation for `Tests` 21 | 22 | The same rules apply as for `Test results` however every `Test ID` is namespaced by `Suite` ID and `Test classname` if present. 23 | If a test is failed/errored the state is also added as namespace, as failed/errored cases can happen simultaneously in the same suite. 24 | 25 | For generating IDs we're using [UUID v3 generator](https://pkg.go.dev/github.com/google/uuid#NewMD5). -------------------------------------------------------------------------------- /test-results/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/semaphoreci/toolbox/test-results 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.7 6 | 7 | require ( 8 | github.com/google/go-cmp v0.7.0 9 | github.com/google/uuid v1.6.0 10 | github.com/mitchellh/go-homedir v1.1.0 11 | github.com/sirupsen/logrus v1.9.3 12 | github.com/spf13/cobra v1.10.1 13 | github.com/spf13/viper v1.21.0 14 | github.com/stretchr/testify v1.11.1 15 | golang.org/x/text v0.29.0 16 | ) 17 | 18 | require ( 19 | github.com/davecgh/go-spew v1.1.1 // indirect 20 | github.com/fsnotify/fsnotify v1.9.0 // indirect 21 | github.com/go-viper/mapstructure/v2 v2.4.0 // indirect 22 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 23 | github.com/pelletier/go-toml/v2 v2.2.4 // indirect 24 | github.com/pmezard/go-difflib v1.0.0 // indirect 25 | github.com/sagikazarmark/locafero v0.11.0 // indirect 26 | github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect 27 | github.com/spf13/afero v1.15.0 // indirect 28 | github.com/spf13/cast v1.10.0 // indirect 29 | github.com/spf13/pflag v1.0.10 // indirect 30 | github.com/subosito/gotenv v1.6.0 // indirect 31 | go.yaml.in/yaml/v3 v3.0.4 // indirect 32 | golang.org/x/sys v0.36.0 // indirect 33 | gopkg.in/yaml.v3 v3.0.1 // indirect 34 | ) 35 | -------------------------------------------------------------------------------- /cache-cli/cmd/clear_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/semaphoreci/toolbox/cache-cli/pkg/logging" 10 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 11 | log "github.com/sirupsen/logrus" 12 | assert "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func Test__Clear(t *testing.T) { 16 | log.SetFormatter(new(logging.CustomFormatter)) 17 | log.SetLevel(log.InfoLevel) 18 | log.SetOutput(openLogfileForTests(t)) 19 | 20 | runTestForAllBackends(t, func(backend string, storage storage.Storage) { 21 | t.Run(fmt.Sprintf("%s no keys", backend), func(*testing.T) { 22 | err := storage.Clear() 23 | assert.Nil(t, err) 24 | 25 | RunClear(clearCmd, []string{}) 26 | output := readOutputFromFile(t) 27 | 28 | assert.Contains(t, output, "Deleted all caches.") 29 | }) 30 | 31 | t.Run(fmt.Sprintf("%s with keys", backend), func(*testing.T) { 32 | err := storage.Clear() 33 | assert.Nil(t, err) 34 | 35 | tempFile, _ := ioutil.TempFile(os.TempDir(), "*") 36 | storage.Store("abc001", tempFile.Name()) 37 | 38 | RunClear(hasKeyCmd, []string{}) 39 | output := readOutputFromFile(t) 40 | 41 | assert.Contains(t, output, "Deleted all caches.") 42 | }) 43 | }) 44 | } 45 | -------------------------------------------------------------------------------- /test-results/priv/parsers/go_staticcheck/in.json: -------------------------------------------------------------------------------- 1 | {"code":"ST1000","severity":"error","location":{"file":"main.go","line":5,"column":2},"end":{"file":"main.go","line":5,"column":8},"message":"at least one file in a package should have a package comment"} 2 | {"code":"ST1003","severity":"error","location":{"file":"main.go","line":10,"column":6},"end":{"file":"main.go","line":10,"column":14},"message":"should not use underscores in package names"} 3 | {"code":"SA1019","severity":"error","location":{"file":"utils/helper.go","line":23,"column":10},"end":{"file":"utils/helper.go","line":23,"column":25},"message":"io/ioutil.ReadFile is deprecated:As of Go 1.16, this function simply calls os.ReadFile."} 4 | {"code":"SA4006","severity":"error","location":{"file":"server/handler.go","line":45,"column":2},"end":{"file":"server/handler.go","line":45,"column":5},"message":"this value of err is never used"} 5 | {"code":"ST1020","severity":"error","location":{"file":"config/config.go","line":15,"column":1},"end":{"file":"config/config.go","line":15,"column":50},"message":"comment on exported function Config should be of the form \"Config ...\""} 6 | {"code":"SA5001","severity":"error","location":{"file":"main.go","line":75,"column":15},"end":{"file":"main.go","line":75,"column":30},"message":"must not defer Close on a writable file"} -------------------------------------------------------------------------------- /tests/sem_version_focal/firefox.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source /opt/change-firefox-version.sh 14 | source ~/.phpbrew/bashrc 15 | . /home/semaphore/.nvm/nvm.sh 16 | export PATH="$PATH:/home/semaphore/.yarn/bin" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # Firefox 27 | 28 | @test "change firefox to 68" { 29 | 30 | run sem-version firefox 68 31 | assert_success 32 | assert_line --partial "Mozilla Firefox 68" 33 | } 34 | 35 | @test "change firefox to 78" { 36 | 37 | run sem-version firefox 78 38 | assert_success 39 | assert_line --partial "Mozilla Firefox 78" 40 | } 41 | 42 | @test "change firefox to 102" { 43 | 44 | run sem-version firefox 102 45 | assert_success 46 | assert_line --partial "Mozilla Firefox 102" 47 | } 48 | -------------------------------------------------------------------------------- /sem-context/cmd/put.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/semaphoreci/toolbox/sem-context/pkg/flags" 8 | "github.com/semaphoreci/toolbox/sem-context/pkg/utils" 9 | "github.com/semaphoreci/toolbox/sem-context/pkg/validators" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | var putCmd = &cobra.Command{ 14 | Use: "put key=value", 15 | Short: "Stores a variable", 16 | Run: RunPutCmd, 17 | } 18 | 19 | func RunPutCmd(cmd *cobra.Command, args []string) { 20 | utils.CheckError(validators.ValidatePutArguments(args)) 21 | key_value := strings.Split(args[0], "=") 22 | key, value := key_value[0], key_value[1] 23 | 24 | existing_value, err := SearchForKeyInAllContexts(key) 25 | if err != nil && err.(*utils.Error).ExitCode == 2 { 26 | utils.CheckError(err) 27 | } 28 | if existing_value != "" && !flags.Force { 29 | utils.CheckError(&utils.Error{ErrorMessage: fmt.Sprintf("Key %s already exists", key), ExitCode: 1}) 30 | } 31 | contextId := utils.GetPipelineContextHierarchy()[0] 32 | err = Store.Put(key, value, contextId) 33 | utils.CheckError(err) 34 | fmt.Println("Key value pair successfully stored") 35 | } 36 | 37 | func init() { 38 | putCmd.Flags().BoolVarP(&flags.Force, "force", "f", false, "If same key already exists, overwrite it.") 39 | RootCmd.AddCommand(putCmd) 40 | } 41 | -------------------------------------------------------------------------------- /test-results/priv/parsers/go_revive/in.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Severity": "warning", 4 | "Failure": "receiver name ms should be consistent with previous receiver name s for GoStaticcheck", 5 | "RuleName": "receiver-naming", 6 | "Category": "naming", 7 | "Position": { 8 | "Start": { 9 | "Filename": "pkg/parsers/go_staticcheck.go", 10 | "Offset": 1046, 11 | "Line": 46, 12 | "Column": 1 13 | }, 14 | "End": { 15 | "Filename": "pkg/parsers/go_staticcheck.go", 16 | "Offset": 1141, 17 | "Line": 48, 18 | "Column": 2 19 | } 20 | }, 21 | "Confidence": 1, 22 | "ReplacementLine": "" 23 | }, 24 | { 25 | "Severity": "warning", 26 | "Failure": "receiver name ms should be consistent with previous receiver name s for GoStaticcheck", 27 | "RuleName": "receiver-naming", 28 | "Category": "naming", 29 | "Position": { 30 | "Start": { 31 | "Filename": "pkg/parsers/go_staticcheck.go", 32 | "Offset": 1173, 33 | "Line": 51, 34 | "Column": 1 35 | }, 36 | "End": { 37 | "Filename": "pkg/parsers/go_staticcheck.go", 38 | "Offset": 1260, 39 | "Line": 53, 40 | "Column": 2 41 | } 42 | }, 43 | "Confidence": 1, 44 | "ReplacementLine": "" 45 | } 46 | ] 47 | -------------------------------------------------------------------------------- /tests/sem_version_focal/node.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source /opt/change-firefox-version.sh 14 | source ~/.phpbrew/bashrc 15 | . /home/semaphore/.nvm/nvm.sh 16 | export PATH="$PATH:/home/semaphore/.yarn/bin" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # Node 27 | @test "change node to 12.16.1" { 28 | sem-version node 12.16.1 29 | run node --version 30 | assert_line --partial "v12.16.1" 31 | } 32 | 33 | @test "change node to 16.15.1" { 34 | sem-version node 16.15.1 35 | run node --version 36 | assert_line --partial "v16.15.1" 37 | } 38 | 39 | @test "change node to 18.18.0" { 40 | sem-version node 18.18.0 41 | run node --version 42 | assert_line --partial "v18.18.0" 43 | } 44 | 45 | @test "change node to 30.30.30" { 46 | run sem-version node 30.30.30 47 | assert_failure 48 | } 49 | -------------------------------------------------------------------------------- /tests/sem_service/opensearch: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | sem-service start opensearch 6 | sleep 5 7 | curl -XGET 'https://0.0.0.0:9200/_cluster/health?pretty' -ksu admin:admin | grep status 8 | 9 | sem-service status opensearch 10 | sleep 5 11 | curl -XGET 'https://0.0.0.0:9200' -ksu admin:admin| grep number | grep 2 12 | 13 | sem-service stop opensearch 14 | sem-service start opensearch 1 15 | sleep 5 16 | curl -XGET 'https://0.0.0.0:9200/_cluster/health?pretty' -ksu admin:admin | grep status 17 | 18 | sem-service stop opensearch 19 | sem-service start opensearch 1.3.9 20 | sleep 5 21 | curl -XGET 'https://0.0.0.0:9200' -ksu admin:admin | grep number | grep 1.3.9 22 | 23 | sem-service stop opensearch 24 | sem-service start opensearch 2.6.0 25 | sleep 5 26 | curl -XGET 'https://0.0.0.0:9200' -ksu admin:admin | grep number | grep 2.6.0 27 | 28 | sem-service stop opensearch 29 | sem-service start opensearch 2.7.0 30 | sleep 5 31 | curl -XGET 'https://0.0.0.0:9200' -ksu admin:admin | grep number | grep 2.7.0 32 | sem-service stop opensearch 33 | 34 | sem-service stop opensearch 35 | sem-service start opensearch 2.7.0 -e '"ES_JAVA_OPTS=-Xms256m -Xmx256m"' 36 | sleep 5 37 | curl -XGET 'https://0.0.0.0:9200' -ksu admin:admin | grep number | grep 2.7.0 38 | docker inspect opensearch | grep '"ES_JAVA_OPTS=-Xms256m -Xmx256m"' 39 | sem-service stop opensearch 40 | 41 | -------------------------------------------------------------------------------- /test-results/pkg/parser/xmlelement_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNewXMLElement(t *testing.T) { 11 | el := NewXMLElement() 12 | 13 | assert.Equal(t, el, XMLElement{}) 14 | } 15 | 16 | func TestXMLElement_Attr(t *testing.T) { 17 | 18 | reader := bytes.NewReader([]byte(``)) 19 | 20 | xmlelement := NewXMLElement() 21 | xmlelement.Parse(reader) 22 | 23 | assert.Equal(t, "1", xmlelement.Attr("foo")) 24 | assert.Equal(t, "2", xmlelement.Attr("bar")) 25 | assert.Equal(t, "", xmlelement.Attr("baz")) 26 | } 27 | 28 | func TestXMLElement_Tag(t *testing.T) { 29 | 30 | reader := bytes.NewReader([]byte(``)) 31 | 32 | xmlelement := NewXMLElement() 33 | xmlelement.Parse(reader) 34 | 35 | assert.Equal(t, "test", xmlelement.Tag()) 36 | } 37 | 38 | func TestXMLElement_Parse(t *testing.T) { 39 | malformedData := [...]string{ 40 | "test", 41 | "", 43 | "", 44 | "", 45 | "", 46 | } 47 | 48 | for _, data := range malformedData { 49 | reader := bytes.NewReader([]byte(data)) 50 | 51 | xmlelement := NewXMLElement() 52 | err := xmlelement.Parse(reader) 53 | assert.Error(t, err, "should error on malformed xml") 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /test-results/priv/merging/pipeline/job1/junit.json: -------------------------------------------------------------------------------- 1 | {"testResults":[{"id":"4ae71336-e44b-39bf-b9d2-752e234818a5","name":"","framework":"","isDisabled":false,"suites":[{"id":"5adf0dab-e505-39fe-99c7-298ef43a8f09","name":"foo","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"58f2ffeb-dc64-3100-b81f-cfc2e9646b8b","file":"","classname":"","package":"","name":"foo.1","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"03038e64-7367-39b9-8e6c-5f503139a200","file":"","classname":"","package":"","name":"foo.2","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":2,"passed":2,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"systemOut":"","systemErr":""},{"id":"c348fb08-df86-3e06-a356-b951c48ea5a4","name":"bar","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"0727aa44-1fc5-3024-804f-6e5936caa480","file":"","classname":"","package":"","name":"bar.1","duration":0,"state":"skipped","failure":null,"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":1,"passed":0,"skipped":1,"error":0,"failed":0,"disabled":0,"duration":0},"systemOut":"","systemErr":""}],"summary":{"total":3,"passed":2,"skipped":1,"error":0,"failed":0,"disabled":0,"duration":0},"status":"success","statusMessage":""}]} -------------------------------------------------------------------------------- /test-results/priv/merging/pipeline/job3/junit.json: -------------------------------------------------------------------------------- 1 | {"testResults":[{"id":"4ae71336-e44b-39bf-b9d2-752e234818a5","name":"","framework":"","isDisabled":false,"suites":[{"id":"5adf0dab-e505-39fe-99c7-298ef43a8f09","name":"foo","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"69fda129-a322-36c7-85b6-215eb8ce5c77","file":"","classname":"","package":"","name":"foo.5","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"08a2a4c8-158c-300f-b951-9bb99f6f5a49","file":"","classname":"","package":"","name":"foo.6","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":2,"passed":2,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"systemOut":"","systemErr":""},{"id":"c348fb08-df86-3e06-a356-b951c48ea5a4","name":"bar","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"0727aa44-1fc5-3024-804f-6e5936caa480","file":"","classname":"","package":"","name":"bar.1","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":1,"passed":1,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"systemOut":"","systemErr":""}],"summary":{"total":3,"passed":3,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"status":"success","statusMessage":""}]} -------------------------------------------------------------------------------- /tests/sftp_server/start_on_mac.sh: -------------------------------------------------------------------------------- 1 | sudo dscl . -create /Users/sftp 2 | sudo dscl . -create /Users/sftp UserShell /bin/bash 3 | sudo dscl . -create /Users/sftp RealName "SFTP" 4 | sudo dscl . -create /Users/sftp UniqueID "1010" 5 | sudo dscl . -create /Users/sftp PrimaryGroupID 80 6 | sudo dscl . -create /Users/sftp NFSHomeDirectory /Users/sftp 7 | 8 | sudo mkdir -p /Users/sftp 9 | 10 | sudo sh -c 'echo "Subsystem sftp internal-sftp" > /private/etc/ssh/sshd_config' 11 | sudo sh -c 'echo "AuthorizedKeysFile /etc/ssh/authorized_keys/%u" >> /private/etc/ssh/sshd_config' 12 | sudo sh -c 'echo "PubkeyAcceptedAlgorithms +ssh-rsa" >> /private/etc/ssh/sshd_config' 13 | sudo sh -c 'echo "PubkeyAcceptedKeyTypes=+ssh-rsa" >> /private/etc/ssh/sshd_config' 14 | 15 | sudo mkdir -p /private/etc/ssh/authorized_keys 16 | 17 | sudo sh -c 'cat tests/sftp_server/id_rsa.pub >> /private/etc/ssh/authorized_keys/sftp' 18 | sudo chown -R sftp /Users/sftp/ 19 | sudo chown -R sftp /private/etc/ssh/authorized_keys/sftp 20 | chmod 0600 /private/etc/ssh/authorized_keys/sftp 21 | 22 | sudo launchctl stop com.openssh.sshd 23 | sudo launchctl start com.openssh.sshd 24 | 25 | cp tests/sftp_server/id_rsa ~/.ssh/semaphore_cache_key 26 | chmod 0600 ~/.ssh/semaphore_cache_key 27 | 28 | ssh-keyscan -p 22 -H localhost >> ~/.ssh/known_hosts 29 | 30 | export SEMAPHORE_CACHE_URL=localhost:22 31 | export SEMAPHORE_CACHE_USERNAME=sftp 32 | -------------------------------------------------------------------------------- /cache-cli/cmd/list_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/semaphoreci/toolbox/cache-cli/pkg/logging" 10 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 11 | log "github.com/sirupsen/logrus" 12 | assert "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func Test__List(t *testing.T) { 16 | listCmd := NewListCommand() 17 | log.SetFormatter(new(logging.CustomFormatter)) 18 | log.SetLevel(log.InfoLevel) 19 | log.SetOutput(openLogfileForTests(t)) 20 | 21 | runTestForAllBackends(t, func(backend string, storage storage.Storage) { 22 | t.Run(fmt.Sprintf("%s no keys", backend), func(*testing.T) { 23 | storage.Clear() 24 | 25 | RunList(listCmd, []string{""}) 26 | output := readOutputFromFile(t) 27 | 28 | assert.Contains(t, output, "Cache is empty.") 29 | }) 30 | 31 | t.Run(fmt.Sprintf("%s with keys", backend), func(*testing.T) { 32 | storage.Clear() 33 | tempFile, _ := ioutil.TempFile(os.TempDir(), "*") 34 | storage.Store("abc001", tempFile.Name()) 35 | storage.Store("abc002", tempFile.Name()) 36 | storage.Store("abc003", tempFile.Name()) 37 | 38 | RunList(listCmd, []string{}) 39 | output := readOutputFromFile(t) 40 | 41 | assert.Contains(t, output, "abc001") 42 | assert.Contains(t, output, "abc002") 43 | assert.Contains(t, output, "abc003") 44 | }) 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /tests/sem_version_jammy/node.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # Node 27 | @test "change node to 14.21.3" { 28 | sem-version node 14.21.3 29 | run node --version 30 | assert_line --partial "v14.21.3" 31 | } 32 | 33 | @test "change node to 16.19.1" { 34 | sem-version node 16.19.1 35 | run node --version 36 | assert_line --partial "v16.19.1" 37 | } 38 | 39 | @test "change node to 18.14.2" { 40 | sem-version node 18.14.2 41 | run node --version 42 | assert_line --partial "v18.14.2" 43 | } 44 | 45 | @test "change node to 18.18.0" { 46 | sem-version node 18.18.0 47 | run node --version 48 | assert_line --partial "v18.18.0" 49 | } 50 | -------------------------------------------------------------------------------- /tests/sem_version_noble/node.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # Node 27 | @test "change node to 14.21.3" { 28 | sem-version node 14.21.3 29 | run node --version 30 | assert_line --partial "v14.21.3" 31 | } 32 | 33 | @test "change node to 16.19.1" { 34 | sem-version node 16.19.1 35 | run node --version 36 | assert_line --partial "v16.19.1" 37 | } 38 | 39 | @test "change node to 18.14.2" { 40 | sem-version node 18.14.2 41 | run node --version 42 | assert_line --partial "v18.14.2" 43 | } 44 | 45 | @test "change node to 18.18.0" { 46 | sem-version node 18.18.0 47 | run node --version 48 | assert_line --partial "v18.18.0" 49 | } 50 | -------------------------------------------------------------------------------- /cache-cli/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | services: 3 | cli: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile.dev 7 | depends_on: 8 | - s3 9 | - gcs 10 | - sftp-server 11 | tty: true 12 | command: "sleep 0" 13 | container_name: 'cache-cli' 14 | volumes: 15 | - go-pkg-cache:/go 16 | - .:/app 17 | environment: 18 | SEMAPHORE_CACHE_S3_URL: "http://s3:9000" 19 | SEMAPHORE_CACHE_S3_KEY: minioadmin 20 | SEMAPHORE_CACHE_S3_SECRET: minioadmin 21 | STORAGE_EMULATOR_HOST: "http://gcs:4443" 22 | SEMAPHORE_TOOLBOX_METRICS_ENABLED: "true" 23 | gcs: 24 | image: fsouza/fake-gcs-server 25 | container_name: 'gcs' 26 | volumes: 27 | - ./test/gcs/data:/data 28 | ports: 29 | - 4443:4443 30 | command: -backend memory -scheme http -port 4443 -public-host gcs:4443 -external-url http://gcs:4443 31 | s3: 32 | image: quay.io/minio/minio:RELEASE.2024-09-22T00-33-43Z 33 | container_name: 's3' 34 | ports: 35 | - 9000:9000 36 | entrypoint: sh 37 | command: -c 'mkdir -p /tmp/s3-data/semaphore-cache && minio server /tmp/s3-data' 38 | sftp-server: 39 | container_name: sftp-server 40 | ports: 41 | - "2222:22" 42 | - "8080:8080" 43 | build: 44 | context: . 45 | dockerfile: Dockerfile.sftp_server 46 | volumes: 47 | go-pkg-cache: 48 | driver: local -------------------------------------------------------------------------------- /test-results/pkg/fileloader/fileloader.go: -------------------------------------------------------------------------------- 1 | package fileloader 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | 8 | "github.com/semaphoreci/toolbox/test-results/pkg/logger" 9 | ) 10 | 11 | var readers = make(map[string]*bytes.Reader) 12 | 13 | // Load reader from internal buffer if path was already loaded or create new one if not 14 | func Load(path string, reader *bytes.Reader) (*bytes.Reader, bool) { 15 | return decode(path, reader) 16 | } 17 | 18 | // Ensure puts reader data into temporary created file. 19 | func Ensure(reader *bytes.Reader) (fileName string) { 20 | file, err := os.CreateTemp("", "") 21 | if err != nil { 22 | panic(err) 23 | } 24 | 25 | defer file.Close() 26 | 27 | fileName = file.Name() 28 | 29 | _, err = reader.WriteTo(file) 30 | if err != nil { 31 | panic(err) 32 | } 33 | 34 | if err = file.Sync(); err != nil { 35 | panic(err) 36 | } 37 | 38 | return 39 | } 40 | 41 | func decode(path string, reader *bytes.Reader) (*bytes.Reader, bool) { 42 | foundReader, exists := readers[path] 43 | if exists && foundReader != nil && foundReader.Size() == reader.Size() { 44 | logger.Debug("Path read from cache") 45 | _, err := foundReader.Seek(0, io.SeekStart) 46 | if err != nil { 47 | logger.Error("Cannot seek to start of reader: %v", err) 48 | } 49 | 50 | return foundReader, true 51 | } 52 | readers[path] = reader 53 | logger.Debug("No path in cache") 54 | return reader, false 55 | } 56 | -------------------------------------------------------------------------------- /test-results/pkg/parser/mermaid_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestEscapeGanttLabel(t *testing.T) { 10 | t.Parallel() 11 | 12 | testCases := []struct { 13 | name string 14 | input string 15 | expected string 16 | }{ 17 | { 18 | name: "no special characters", 19 | input: "echo hello world", 20 | expected: "echo hello world", 21 | }, 22 | { 23 | name: "brackets and braces", 24 | input: "if [ \"$VAR\" == 1 ] { echo ok }", 25 | expected: "if #91; \"$VAR\" == 1 #93; #123; echo ok #125;", 26 | }, 27 | { 28 | name: "angle brackets (heredoc)", 29 | input: "cat < ruby_test/.ruby-version 30 | cd ruby_test 31 | ruby --version| grep 2.7.2 32 | sem-version ruby 2.6.2 -f 33 | ruby --version| grep 2.6.2 34 | kubectl version --client 35 | sem-version kubectl 1.15.3 36 | kubectl version --client | grep -q 1.15.3 37 | echo "Erlang version" 38 | erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell 39 | sem-version erlang 23 40 | erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell 41 | sem-version erlang 23.2 42 | erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell 43 | echo "actual scala version" 44 | scala -version 45 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/usage_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | assert "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test__Usage(t *testing.T) { 13 | runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { 14 | t.Run(fmt.Sprintf("%s no usage", storageType), func(t *testing.T) { 15 | _ = storage.Clear() 16 | usage, err := storage.Usage() 17 | assert.Nil(t, err) 18 | assert.Equal(t, int64(0), usage.Used) 19 | 20 | switch storageType { 21 | case "s3": 22 | assert.Equal(t, int64(-1), usage.Free) 23 | case "sftp": 24 | assert.Equal(t, storage.Config().MaxSpace, usage.Free) 25 | } 26 | }) 27 | 28 | t.Run(fmt.Sprintf("%s some usage", storageType), func(t *testing.T) { 29 | _ = storage.Clear() 30 | 31 | fileContents := "usage - some usage" 32 | file, _ := ioutil.TempFile(os.TempDir(), "*") 33 | file.WriteString(fileContents) 34 | _ = storage.Store("abc001", file.Name()) 35 | 36 | usage, err := storage.Usage() 37 | assert.Nil(t, err) 38 | assert.Equal(t, int64(len(fileContents)), usage.Used) 39 | 40 | switch storageType { 41 | case "s3": 42 | assert.Equal(t, int64(-1), usage.Free) 43 | case "sftp": 44 | free := storage.Config().MaxSpace - int64(len(fileContents)) 45 | assert.Equal(t, free, usage.Free) 46 | } 47 | 48 | os.Remove(file.Name()) 49 | }) 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /tests/sem_version_jammy/php.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # PHP 27 | @test "change php to 8.1.29" { 28 | 29 | run sem-version php 8.1.29 30 | assert_success 31 | source ~/.phpbrew/bashrc 32 | run php -v 33 | assert_line --partial "PHP 8.1.29" 34 | run php -m 35 | assert_line --partial "gd" 36 | assert_line --partial "imap" 37 | } 38 | 39 | @test "php check composer 8.1.29" { 40 | 41 | run which composer 42 | assert_success 43 | source ~/.phpbrew/bashrc 44 | assert_line --partial "8.1.29" 45 | } 46 | 47 | @test "php check source 8.1.29" { 48 | 49 | run sem-version php 8.1.29 50 | assert_success 51 | source ~/.phpbrew/bashrc 52 | assert_line --partial "8.1.29" 53 | run phpbrew ext install iconv 54 | assert_success 55 | } 56 | -------------------------------------------------------------------------------- /tests/sem_version_noble/php.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # PHP 27 | @test "change php to 8.1.29" { 28 | 29 | run sem-version php 8.1.29 30 | assert_success 31 | source ~/.phpbrew/bashrc 32 | run php -v 33 | assert_line --partial "PHP 8.1.29" 34 | run php -m 35 | assert_line --partial "gd" 36 | assert_line --partial "imap" 37 | } 38 | 39 | @test "php check composer 8.1.29" { 40 | 41 | run which composer 42 | assert_success 43 | source ~/.phpbrew/bashrc 44 | assert_line --partial "8.1.29" 45 | } 46 | 47 | @test "php check source 8.1.29" { 48 | 49 | run sem-version php 8.1.29 50 | assert_success 51 | source ~/.phpbrew/bashrc 52 | assert_line --partial "8.1.29" 53 | run phpbrew ext install iconv 54 | assert_success 55 | } 56 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/gcs_list.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | "strings" 8 | 9 | "cloud.google.com/go/storage" 10 | "google.golang.org/api/iterator" 11 | ) 12 | 13 | func (s *GCSStorage) List() ([]CacheKey, error) { 14 | it := s.Bucket.Objects(context.TODO(), &storage.Query{Prefix: s.Project}) 15 | 16 | keys := make([]CacheKey, 0) 17 | for { 18 | attrs, err := it.Next() 19 | if err == iterator.Done { 20 | break 21 | } 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | keys = s.appendToListResult(keys, attrs) 27 | } 28 | 29 | return s.sortKeys(keys), nil 30 | } 31 | 32 | // S3 backend does not support sorting keys by ACCESS_TIME 33 | func (s *GCSStorage) sortKeys(keys []CacheKey) []CacheKey { 34 | switch s.Config().SortKeysBy { 35 | case SortBySize: 36 | sort.SliceStable(keys, func(i, j int) bool { 37 | return keys[i].Size > keys[j].Size 38 | }) 39 | default: 40 | sort.SliceStable(keys, func(i, j int) bool { 41 | return keys[i].StoredAt.After(*keys[j].StoredAt) 42 | }) 43 | } 44 | 45 | return keys 46 | } 47 | 48 | func (s *GCSStorage) appendToListResult(keys []CacheKey, object *storage.ObjectAttrs) []CacheKey { 49 | keyWithoutProject := strings.ReplaceAll(object.Name, fmt.Sprintf("%s/", s.Project), "") 50 | keys = append(keys, CacheKey{ 51 | Name: keyWithoutProject, 52 | StoredAt: &object.Updated, 53 | LastAccessedAt: &object.Updated, 54 | Size: object.Size, 55 | }) 56 | 57 | return keys 58 | } 59 | -------------------------------------------------------------------------------- /sem-context/pkg/validators/validators.go: -------------------------------------------------------------------------------- 1 | package validators 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | 7 | "github.com/semaphoreci/toolbox/sem-context/pkg/utils" 8 | ) 9 | 10 | var valueSizeLimit = 20000 11 | 12 | func ValidateGetAndDeleteArguments(args []string) error { 13 | if len(args) != 1 { 14 | return &utils.Error{ErrorMessage: "Exactly one argument expected", ExitCode: 3} 15 | } 16 | return isKeyValid(args[0]) 17 | } 18 | 19 | func ValidatePutArguments(args []string) error { 20 | if len(args) != 1 || len(strings.Split(args[0], "=")) != 2 { 21 | return &utils.Error{ErrorMessage: "Put command expects one argument in form of key=value", ExitCode: 3} 22 | } 23 | err := isKeyValid(strings.Split(args[0], "=")[0]) 24 | if err == nil { 25 | err = isValueValid(strings.Split(args[0], "=")[1]) 26 | } 27 | return err 28 | } 29 | 30 | func isKeyValid(key string) error { 31 | keyRegex := regexp.MustCompile(`[A-Za-z0-9-_]{3,256}`) 32 | if keyRegex.MatchString(key) { 33 | return nil 34 | } 35 | return &utils.Error{ 36 | ErrorMessage: "Key must be between 3 and 256 characters in length, and can contain letters, " + 37 | "digits, and characters _ and - (no spaces)", 38 | ExitCode: 3} 39 | } 40 | 41 | func isValueValid(value string) error { 42 | if value == "" { 43 | return &utils.Error{ErrorMessage: "Value cant be an empty string", ExitCode: 3} 44 | } 45 | if len([]byte(value)) > int(valueSizeLimit) { 46 | return &utils.Error{ErrorMessage: "Value cant be more than 20KB in size", ExitCode: 3} 47 | } 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /test-results/pkg/parser/xmlelement.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bytes" 5 | "encoding/xml" 6 | 7 | "github.com/semaphoreci/toolbox/test-results/pkg/logger" 8 | ) 9 | 10 | type XMLElement struct { 11 | XMLName xml.Name 12 | Attributes map[string]string `xml:"-"` 13 | Children []XMLElement `xml:",any"` 14 | Contents []byte `xml:",chardata"` 15 | } 16 | 17 | func NewXMLElement() XMLElement { 18 | return XMLElement{} 19 | } 20 | 21 | func (me *XMLElement) Attr(attr string) string { 22 | return me.Attributes[attr] 23 | } 24 | 25 | func (me *XMLElement) Tag() string { 26 | return me.XMLName.Local 27 | } 28 | 29 | func (me *XMLElement) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { 30 | logger.Trace("Decoding element: %s", start.Name.Local) 31 | type alias XMLElement 32 | if err := d.DecodeElement((*alias)(me), &start); err != nil { 33 | logger.Error("Decoding element failed: %v", err) 34 | return err 35 | } 36 | 37 | me.Attributes = parseAttributes(start.Attr) 38 | return nil 39 | } 40 | 41 | func (me *XMLElement) Parse(reader *bytes.Reader) error { 42 | decoder := xml.NewDecoder(reader) 43 | 44 | if err := decoder.Decode(&me); err != nil { 45 | logger.Error("Parsing element \"<%v>\" failed", me.Tag()) 46 | return err 47 | } 48 | return nil 49 | } 50 | 51 | func parseAttributes(attrs []xml.Attr) map[string]string { 52 | attributes := make(map[string]string) 53 | 54 | for _, attr := range attrs { 55 | attributes[attr.Name.Local] = attr.Value 56 | } 57 | 58 | return attributes 59 | } 60 | -------------------------------------------------------------------------------- /tests/sem_service/elasticsearch: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | sem-service start elasticsearch 6 | sleep 5 7 | curl -XGET '0.0.0.0:9200/_cluster/health?pretty' -s | grep green 8 | sem-service status elasticsearch 9 | curl -XGET '0.0.0.0:9200' -s | grep 6.5 10 | sem-service stop elasticsearch 11 | 12 | sem-service start elasticsearch 6.6 13 | sleep 5 14 | curl -XGET '0.0.0.0:9200/_cluster/health?pretty' -s| grep green 15 | sem-service stop elasticsearch 16 | 17 | sem-service start elasticsearch 7.9 18 | sleep 5 19 | curl -XGET '0.0.0.0:9200' -s | grep 7.9 20 | sem-service stop elasticsearch 21 | 22 | sem-service start elasticsearch 7.10.0 23 | sleep 5 24 | curl -XGET '0.0.0.0:9200' -s | grep 7.10 25 | sem-service stop elasticsearch 26 | sleep 5 27 | 28 | sem-service start elasticsearch 8.5.3 -e "xpack.security.enabled=false" -e "xpack.security.enrollment.enabled=false" 29 | sleep 5 30 | curl -XGET '0.0.0.0:9200' -s | grep 8.5.3 31 | sem-service stop elasticsearch 32 | sleep 5 33 | 34 | sem-service start elasticsearch 8.9.2 -e "xpack.security.enabled=false" -e "xpack.security.enrollment.enabled=false" 35 | sleep 10 36 | curl -XGET '0.0.0.0:9200' -s | grep 8.9.2 37 | sem-service stop elasticsearch 38 | sleep 5 39 | 40 | sem-service start elasticsearch 8.11.3 -e "xpack.security.enabled=false" -e "xpack.security.enrollment.enabled=false" -e '"ES_JAVA_OPTS=-Xms256m -Xmx256m"' 41 | sleep 10 42 | curl -XGET '0.0.0.0:9200' -s | grep 8.11.3 43 | docker inspect elasticsearch | grep '"ES_JAVA_OPTS=-Xms256m -Xmx256m"' 44 | sem-service stop elasticsearch 45 | 46 | -------------------------------------------------------------------------------- /tests/artifacts.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "support/bats-support/load" 4 | load "support/bats-assert/load" 5 | 6 | setup() { 7 | echo "hello" > /tmp/unique-file-$SEMAPHORE_JOB_ID 8 | } 9 | 10 | @test "artifacts - uploading to project level" { 11 | run artifact push project /tmp/unique-file-$SEMAPHORE_JOB_ID 12 | assert_success 13 | assert_output --regexp "Pushed [0-9]+ files?\. Total of .+" 14 | 15 | 16 | run artifact yank project unique-file-$SEMAPHORE_JOB_ID 17 | assert_success 18 | } 19 | 20 | @test "artifacts - uploading to workflow level" { 21 | run artifact push workflow /tmp/unique-file-$SEMAPHORE_JOB_ID 22 | assert_success 23 | assert_output --regexp "Pushed [0-9]+ files?\. Total of .+" 24 | 25 | run artifact yank workflow unique-file-$SEMAPHORE_JOB_ID 26 | assert_success 27 | } 28 | 29 | @test "artifacts - uploading to job level" { 30 | run artifact push job /tmp/unique-file-$SEMAPHORE_JOB_ID 31 | assert_success 32 | assert_output --regexp "Pushed [0-9]+ files?\. Total of .+" 33 | 34 | run artifact yank job unique-file-$SEMAPHORE_JOB_ID 35 | assert_success 36 | } 37 | 38 | @test "artifacts - pulling should display size summary" { 39 | run artifact push job /tmp/unique-file-$SEMAPHORE_JOB_ID 40 | assert_success 41 | assert_output --regexp "Pushed [0-9]+ files?\. Total of .+" 42 | 43 | run artifact pull job unique-file-$SEMAPHORE_JOB_ID 44 | assert_success 45 | assert_output --regexp "Pulled [0-9]+ files?\. Total of .+" 46 | 47 | run artifact yank job unique-file-$SEMAPHORE_JOB_ID 48 | assert_success 49 | } 50 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/clear_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | assert "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test__Clear(t *testing.T) { 13 | runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { 14 | setup := func(storage Storage) []string { 15 | _ = storage.Clear() 16 | 17 | file1, _ := ioutil.TempFile(os.TempDir(), "*") 18 | file1.WriteString("something, something") 19 | 20 | file2, _ := ioutil.TempFile(os.TempDir(), "*") 21 | file2.WriteString("else, else") 22 | 23 | _ = storage.Store("abc001", file1.Name()) 24 | _ = storage.Store("abc002", file2.Name()) 25 | 26 | return []string{file1.Name(), file2.Name()} 27 | } 28 | 29 | cleanup := func(files []string) { 30 | for _, file := range files { 31 | os.Remove(file) 32 | } 33 | } 34 | 35 | t.Run(fmt.Sprintf("%s no keys", storageType), func(t *testing.T) { 36 | err := storage.Clear() 37 | assert.Nil(t, err) 38 | 39 | keys, err := storage.List() 40 | assert.Nil(t, err) 41 | assert.Len(t, keys, 0) 42 | 43 | err = storage.Clear() 44 | assert.Nil(t, err) 45 | }) 46 | 47 | t.Run(fmt.Sprintf("%s with keys", storageType), func(t *testing.T) { 48 | filesToCleanup := setup(storage) 49 | 50 | keys, err := storage.List() 51 | assert.Nil(t, err) 52 | assert.Len(t, keys, 2) 53 | 54 | err = storage.Clear() 55 | assert.Nil(t, err) 56 | 57 | keys, err = storage.List() 58 | assert.Nil(t, err) 59 | assert.Len(t, keys, 0) 60 | 61 | cleanup(filesToCleanup) 62 | }) 63 | }) 64 | } 65 | -------------------------------------------------------------------------------- /test-results/pkg/parsers/junit_mocha_test.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestJUnitMocha(t *testing.T) { 8 | // Set up environment variables for consistent test output 9 | t.Setenv("SEMAPHORE_PIPELINE_ID", "ppl-id") 10 | t.Setenv("SEMAPHORE_WORKFLOW_ID", "wf-id") 11 | t.Setenv("SEMAPHORE_JOB_NAME", "job-name") 12 | t.Setenv("SEMAPHORE_JOB_ID", "job-id") 13 | t.Setenv("SEMAPHORE_PROJECT_ID", "project-id") 14 | t.Setenv("SEMAPHORE_AGENT_MACHINE_TYPE", "agent-machine-type") 15 | t.Setenv("SEMAPHORE_AGENT_MACHINE_OS_IMAGE", "agent-machine-os-image") 16 | t.Setenv("SEMAPHORE_JOB_CREATION_TIME", "job-creation-time") 17 | t.Setenv("SEMAPHORE_GIT_REF_TYPE", "git-ref-type") 18 | 19 | t.Run("Golden File Test", func(t *testing.T) { 20 | test := GoldenTest{ 21 | Name: "JUnitMocha", 22 | Parser: NewJUnitMocha(), 23 | InputFile: FixturePath("priv/parsers/junit_mocha/in.xml"), 24 | GoldenFile: FixturePath("priv/parsers/junit_mocha/out.json"), 25 | } 26 | RunGoldenTest(t, test) 27 | }) 28 | 29 | t.Run("Parser Identification", func(t *testing.T) { 30 | p := NewJUnitMocha() 31 | 32 | // Mocha parser currently returns false for all IsApplicable checks 33 | // because it doesn't have specific identifying characteristics 34 | AssertParserApplicable(t, p, FixturePath("priv/parsers/junit_generic/in.xml"), false) 35 | AssertParserApplicable(t, p, FixturePath("priv/parsers/junit_golang/in.xml"), false) 36 | }) 37 | 38 | t.Run("Extensions", func(t *testing.T) { 39 | p := NewJUnitMocha() 40 | AssertParserSupportsExtension(t, p, ".xml", true) 41 | AssertParserSupportsExtension(t, p, ".json", false) 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /tests/enetwork.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "support/bats-support/load" 4 | load "support/bats-assert/load" 5 | 6 | setup() { 7 | 8 | export SEMAPHORE_DOCKERGW_USER='user' 9 | export SEMAPHORE_DOCKERGW_PASSWORD='password' 10 | export SEMAPHORE_DOCKERGW_ENDPOINT='127.0.0.1' 11 | export SEMAPHORE_DOCKERGW_ENDPOINT_PORT='3128' 12 | 13 | mkdir -p /tmp/squid 14 | tee /tmp/squid/squid.conf > /dev/null < 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 1) test fails with 8531 + 6984 = 1547 0.09411569755491345 (Calculator.AdderTest) 10 | test/calculator/adder_test.exs:15 11 | Assertion with == failed 12 | code: assert Calculator.Adder.run(8531, 6984) == 1547 13 | left: 15515 14 | right: 1547 15 | stacktrace: 16 | test/calculator/adder_test.exs:16: (test) 17 | 18 | 19 | 20 | 21 | 3) test fails with 6981 + 3883 = 3098 0.5046993437439501 (Calculator.AdderTest) 22 | test/calculator/adder_test.exs:15 23 | Assertion with == failed 24 | code: assert Calculator.Adder.run(6981, 3883) == 3098 25 | left: 10864 26 | right: 3098 27 | stacktrace: 28 | test/calculator/adder_test.exs:16: (test) 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /cache-cli/cmd/delete_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/semaphoreci/toolbox/cache-cli/pkg/logging" 10 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 11 | log "github.com/sirupsen/logrus" 12 | assert "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func Test__Delete(t *testing.T) { 16 | log.SetFormatter(new(logging.CustomFormatter)) 17 | log.SetLevel(log.InfoLevel) 18 | log.SetOutput(openLogfileForTests(t)) 19 | 20 | runTestForAllBackends(t, func(backend string, storage storage.Storage) { 21 | t.Run(fmt.Sprintf("%s key is missing", backend), func(*testing.T) { 22 | RunDelete(deleteCmd, []string{"this-key-does-not-exist"}) 23 | output := readOutputFromFile(t) 24 | 25 | assert.Contains(t, output, "Key 'this-key-does-not-exist' doesn't exist in the cache store.") 26 | }) 27 | 28 | t.Run(fmt.Sprintf("%s key is present", backend), func(*testing.T) { 29 | storage.Clear() 30 | tempFile, _ := ioutil.TempFile(os.TempDir(), "*") 31 | storage.Store("abc001", tempFile.Name()) 32 | 33 | RunDelete(deleteCmd, []string{"abc001"}) 34 | output := readOutputFromFile(t) 35 | 36 | assert.Contains(t, output, "Key 'abc001' is deleted.") 37 | }) 38 | 39 | t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { 40 | storage.Clear() 41 | tempFile, _ := ioutil.TempFile(os.TempDir(), "*") 42 | RunStore(NewStoreCommand(), []string{"abc/00/33", tempFile.Name()}) 43 | 44 | RunDelete(deleteCmd, []string{"abc/00/33"}) 45 | output := readOutputFromFile(t) 46 | 47 | assert.Contains(t, output, "Key 'abc/00/33' is normalized to 'abc-00-33'") 48 | assert.Contains(t, output, "Key 'abc-00-33' is deleted.") 49 | }) 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /cache-cli/id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAtN/oiQ7DKI2iye29E5AWwDQvSyLNUfwQsRhjsDj5mGjbruIv 3 | aExkzRB2BLM6vtfRtHETn8MIpjtJnaxr04u/RLyRDUeVGBzwOZ9COBnc51JSWEgA 4 | oKEY+ppyblhQdCvbeRaD8x6z2kOwXs+26MQKPABW42X4H4StDtZ3Bx9vT69OMIeB 5 | cdSkkTxDUj6zctS1qhXNZiqJUDDWfF26DYxtC3ObEvvi+Ev5CdS2clEiGzL9bLod 6 | HVqENNOJpJRqmSQi+QBN9QXI+inru3kxlY1eus6+IgiOfIM3hpIrXxYPosLhxmc6 7 | Jv/e3GvH3V9hGVM4KpGJbD3Xrh8XhmLpFElqQQIDAQABAoIBAQCO5KGxva7iV9r2 8 | ilIopVnD6LZANrNchLWvIySGcMyG4rhDVj+7tzafZtTF686KPLhYREkLmyo3IXmM 9 | Dk51Jt8pb4tsoVOhZCNIco//ADerrDaPtfqA7CRuvxuOH0kYLhnKdMYziNrvoTAy 10 | BwlqpI2tT1tVdRZr2AZcRYRtZSEsIm+SVJHdoszp3kj37wsrG3u2rVWmjpWBP/+I 11 | YTUnXLPaKJX1w+0VxmluBRrbSPHZzgl8W2nmMa20J6s9t6UfZxogpYMwTD11RHb3 12 | mmxc2g1JEqlvTtzr2EiskEnUWIc2i0NNZYl2Jku8kVg12MvT5UeFuuV9m5fUB06e 13 | mher2gGtAoGBAN1Ck8FtKNWkihY3N0cbepjv+KlxbDZY9W8/uqQ48jagi9dLst8Y 14 | GrSVtZJRDv7CbjtU4I+Qs1AT8OER1T9zuOlXVldeF9BmcQCbTd02KYmqgialAH8a 15 | /FbJTLhxXmwkOok3hGi+SjGy0Z5BeWxU3VTMIitJMtSCkNqcphm9AOITAoGBANFG 16 | D9p9a/Um5XnT7rDB0rGk0BRJm/RrG7II8ua3fDBNTjk3wfZsXuYe74bHszt3lUUh 17 | LPM28qCJbKvrMasfqZE9J4x2XjyGwiD8RoErDPod7m0+NJ+bkaoGSpqk9m5AeOFg 18 | 2AjpigChfEY4wG8jwThVqpn1IhiRFbBDV7lLOGzbAoGAF7UAe+MkUu/UU9kKtTW+ 19 | hxB/Rjh+7bvOYGMlrl7jpPLCyQi1W64LfnnkAI9q2eUfRoArp3ZLoDLixlrQzIGr 20 | uGTBPY85Yfz38qaxSxh4juWUGGvph7TugKYE8GZFRbUECJvINijdbkHIr/o39mGJ 21 | bYKJzrWDGr36o1HRgkR8Cb0CgYB4hyKAUvYHdoR5uv7g3HCtkV+qH47roXiM/Cxx 22 | hsHGy+8+cweKFnY9kRmhBYkLud7Ftsq9tSreHOfyx/2OMikmFhOeCVpS/8X023b7 23 | 08CgmPaGAmqqdh3Zz8/N6y5m50j3fSBPWbn6jQcXJn5mRDB3cpdUHmOpxH26S77N 24 | YC46OwKBgB0JOaamfAfG63qq1mw9AkIG26cTyg5v/VHyMijLs3yXCBcicIa8aEUS 25 | VurhyU2C183EsYXfbM9dmjpH3KW4Zn8NUa06IAy6DbdbIjLrZja+Sa5T+VeWHrTm 26 | K5/cz+DRp+bOCcigzy+rm/9bHo9x4AZL8p0uqnITWuqYH4+49v03 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /test-results/pkg/parsers/junit_phpunit_test.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestJUnitPHPUnit(t *testing.T) { 8 | // Set up environment variables for consistent test output 9 | t.Setenv("SEMAPHORE_PIPELINE_ID", "ppl-id") 10 | t.Setenv("SEMAPHORE_WORKFLOW_ID", "wf-id") 11 | t.Setenv("SEMAPHORE_JOB_NAME", "job-name") 12 | t.Setenv("SEMAPHORE_JOB_ID", "job-id") 13 | t.Setenv("SEMAPHORE_PROJECT_ID", "project-id") 14 | t.Setenv("SEMAPHORE_AGENT_MACHINE_TYPE", "agent-machine-type") 15 | t.Setenv("SEMAPHORE_AGENT_MACHINE_OS_IMAGE", "agent-machine-os-image") 16 | t.Setenv("SEMAPHORE_JOB_CREATION_TIME", "job-creation-time") 17 | t.Setenv("SEMAPHORE_GIT_REF_TYPE", "git-ref-type") 18 | 19 | t.Run("Golden File Test", func(t *testing.T) { 20 | test := GoldenTest{ 21 | Name: "JUnitPHPUnit", 22 | Parser: NewJUnitPHPUnit(), 23 | InputFile: FixturePath("priv/parsers/junit_phpunit/in.xml"), 24 | GoldenFile: FixturePath("priv/parsers/junit_phpunit/out.json"), 25 | } 26 | RunGoldenTest(t, test) 27 | }) 28 | 29 | t.Run("Parser Identification", func(t *testing.T) { 30 | p := NewJUnitPHPUnit() 31 | 32 | // PHPUnit parser currently returns false for all IsApplicable checks 33 | // It needs to be explicitly selected with -p phpunit 34 | AssertParserApplicable(t, p, FixturePath("priv/parsers/junit_phpunit/in.xml"), false) 35 | AssertParserApplicable(t, p, FixturePath("priv/parsers/junit_golang/in.xml"), false) 36 | AssertParserApplicable(t, p, FixturePath("priv/parsers/junit_generic/in.xml"), false) 37 | }) 38 | 39 | t.Run("Extensions", func(t *testing.T) { 40 | p := NewJUnitPHPUnit() 41 | AssertParserSupportsExtension(t, p, ".xml", true) 42 | AssertParserSupportsExtension(t, p, ".json", false) 43 | }) 44 | } 45 | -------------------------------------------------------------------------------- /test-results/priv/parsers/junit_rspec/in.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Failure/Error: expect(result).to eq(-1) 14 | 15 | expected: -1 16 | got: 3 17 | 18 | (compared using ==) 19 | ./spec/calculator/adder_spec.rb:11:in `block (2 levels) in <top (required)>' 20 | 21 | 22 | 23 | Failure/Error: expect(result).to eq(3) 29 | 30 | expected: 3 31 | got: -1 32 | 33 | (compared using ==) 34 | ./spec/calculator/subtractor_spec.rb:11:in `block (2 levels) in <top (required)>' 35 | 36 | 37 | -------------------------------------------------------------------------------- /tests/sftp_server/id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAtN/oiQ7DKI2iye29E5AWwDQvSyLNUfwQsRhjsDj5mGjbruIv 3 | aExkzRB2BLM6vtfRtHETn8MIpjtJnaxr04u/RLyRDUeVGBzwOZ9COBnc51JSWEgA 4 | oKEY+ppyblhQdCvbeRaD8x6z2kOwXs+26MQKPABW42X4H4StDtZ3Bx9vT69OMIeB 5 | cdSkkTxDUj6zctS1qhXNZiqJUDDWfF26DYxtC3ObEvvi+Ev5CdS2clEiGzL9bLod 6 | HVqENNOJpJRqmSQi+QBN9QXI+inru3kxlY1eus6+IgiOfIM3hpIrXxYPosLhxmc6 7 | Jv/e3GvH3V9hGVM4KpGJbD3Xrh8XhmLpFElqQQIDAQABAoIBAQCO5KGxva7iV9r2 8 | ilIopVnD6LZANrNchLWvIySGcMyG4rhDVj+7tzafZtTF686KPLhYREkLmyo3IXmM 9 | Dk51Jt8pb4tsoVOhZCNIco//ADerrDaPtfqA7CRuvxuOH0kYLhnKdMYziNrvoTAy 10 | BwlqpI2tT1tVdRZr2AZcRYRtZSEsIm+SVJHdoszp3kj37wsrG3u2rVWmjpWBP/+I 11 | YTUnXLPaKJX1w+0VxmluBRrbSPHZzgl8W2nmMa20J6s9t6UfZxogpYMwTD11RHb3 12 | mmxc2g1JEqlvTtzr2EiskEnUWIc2i0NNZYl2Jku8kVg12MvT5UeFuuV9m5fUB06e 13 | mher2gGtAoGBAN1Ck8FtKNWkihY3N0cbepjv+KlxbDZY9W8/uqQ48jagi9dLst8Y 14 | GrSVtZJRDv7CbjtU4I+Qs1AT8OER1T9zuOlXVldeF9BmcQCbTd02KYmqgialAH8a 15 | /FbJTLhxXmwkOok3hGi+SjGy0Z5BeWxU3VTMIitJMtSCkNqcphm9AOITAoGBANFG 16 | D9p9a/Um5XnT7rDB0rGk0BRJm/RrG7II8ua3fDBNTjk3wfZsXuYe74bHszt3lUUh 17 | LPM28qCJbKvrMasfqZE9J4x2XjyGwiD8RoErDPod7m0+NJ+bkaoGSpqk9m5AeOFg 18 | 2AjpigChfEY4wG8jwThVqpn1IhiRFbBDV7lLOGzbAoGAF7UAe+MkUu/UU9kKtTW+ 19 | hxB/Rjh+7bvOYGMlrl7jpPLCyQi1W64LfnnkAI9q2eUfRoArp3ZLoDLixlrQzIGr 20 | uGTBPY85Yfz38qaxSxh4juWUGGvph7TugKYE8GZFRbUECJvINijdbkHIr/o39mGJ 21 | bYKJzrWDGr36o1HRgkR8Cb0CgYB4hyKAUvYHdoR5uv7g3HCtkV+qH47roXiM/Cxx 22 | hsHGy+8+cweKFnY9kRmhBYkLud7Ftsq9tSreHOfyx/2OMikmFhOeCVpS/8X023b7 23 | 08CgmPaGAmqqdh3Zz8/N6y5m50j3fSBPWbn6jQcXJn5mRDB3cpdUHmOpxH26S77N 24 | YC46OwKBgB0JOaamfAfG63qq1mw9AkIG26cTyg5v/VHyMijLs3yXCBcicIa8aEUS 25 | VurhyU2C183EsYXfbM9dmjpH3KW4Zn8NUa06IAy6DbdbIjLrZja+Sa5T+VeWHrTm 26 | K5/cz+DRp+bOCcigzy+rm/9bHo9x4AZL8p0uqnITWuqYH4+49v03 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /cache-cli/cmd/has_key_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/semaphoreci/toolbox/cache-cli/pkg/logging" 10 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 11 | log "github.com/sirupsen/logrus" 12 | assert "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func Test__HasKey(t *testing.T) { 16 | log.SetFormatter(new(logging.CustomFormatter)) 17 | log.SetLevel(log.InfoLevel) 18 | log.SetOutput(openLogfileForTests(t)) 19 | 20 | runTestForAllBackends(t, func(backend string, storage storage.Storage) { 21 | t.Run(fmt.Sprintf("%s key is missing", backend), func(*testing.T) { 22 | RunHasKey(hasKeyCmd, []string{"this-key-does-not-exist"}) 23 | output := readOutputFromFile(t) 24 | 25 | assert.Contains(t, output, "Key 'this-key-does-not-exist' doesn't exist in the cache store.") 26 | }) 27 | 28 | t.Run(fmt.Sprintf("%s key is present", backend), func(*testing.T) { 29 | storage.Clear() 30 | tempFile, _ := ioutil.TempFile(os.TempDir(), "*") 31 | storage.Store("abc001", tempFile.Name()) 32 | 33 | RunHasKey(hasKeyCmd, []string{"abc001"}) 34 | output := readOutputFromFile(t) 35 | 36 | assert.Contains(t, output, "Key 'abc001' exists in the cache store.") 37 | }) 38 | 39 | t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { 40 | storage.Clear() 41 | tempFile, _ := ioutil.TempFile(os.TempDir(), "*") 42 | RunStore(NewStoreCommand(), []string{"abc/00/33", tempFile.Name()}) 43 | 44 | RunHasKey(hasKeyCmd, []string{"abc/00/33"}) 45 | output := readOutputFromFile(t) 46 | 47 | assert.Contains(t, output, "Key 'abc/00/33' is normalized to 'abc-00-33'") 48 | assert.Contains(t, output, "Key 'abc-00-33' exists in the cache store.") 49 | }) 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /test-results/pkg/parsers/junit_embedded_test.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestJUnitEmbedded(t *testing.T) { 8 | // Set up environment variables for consistent test output 9 | t.Setenv("SEMAPHORE_PIPELINE_ID", "ppl-id") 10 | t.Setenv("SEMAPHORE_WORKFLOW_ID", "wf-id") 11 | t.Setenv("SEMAPHORE_JOB_NAME", "job-name") 12 | t.Setenv("SEMAPHORE_JOB_ID", "job-id") 13 | t.Setenv("SEMAPHORE_PROJECT_ID", "project-id") 14 | t.Setenv("SEMAPHORE_AGENT_MACHINE_TYPE", "agent-machine-type") 15 | t.Setenv("SEMAPHORE_AGENT_MACHINE_OS_IMAGE", "agent-machine-os-image") 16 | t.Setenv("SEMAPHORE_JOB_CREATION_TIME", "job-creation-time") 17 | t.Setenv("SEMAPHORE_GIT_REF_TYPE", "git-ref-type") 18 | 19 | t.Run("Golden File Test", func(t *testing.T) { 20 | test := GoldenTest{ 21 | Name: "JUnitEmbedded", 22 | Parser: NewJUnitEmbedded(), 23 | InputFile: FixturePath("priv/parsers/junit_embedded/in.xml"), 24 | GoldenFile: FixturePath("priv/parsers/junit_embedded/out.json"), 25 | } 26 | RunGoldenTest(t, test) 27 | }) 28 | 29 | t.Run("Parser Identification", func(t *testing.T) { 30 | p := NewJUnitEmbedded() 31 | 32 | // Embedded parser currently returns false for all IsApplicable checks 33 | // It needs to be explicitly selected with -p embedded 34 | AssertParserApplicable(t, p, FixturePath("priv/parsers/junit_embedded/in.xml"), false) 35 | AssertParserApplicable(t, p, FixturePath("priv/parsers/junit_generic/in.xml"), false) 36 | AssertParserApplicable(t, p, FixturePath("priv/parsers/junit_golang/in.xml"), false) 37 | }) 38 | 39 | t.Run("Extensions", func(t *testing.T) { 40 | p := NewJUnitEmbedded() 41 | AssertParserSupportsExtension(t, p, ".xml", true) 42 | AssertParserSupportsExtension(t, p, ".json", false) 43 | }) 44 | } 45 | -------------------------------------------------------------------------------- /tests/toolbox_metrics.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "support/bats-support/load" 4 | load "support/bats-assert/load" 5 | 6 | setup() { 7 | TMPFILE=$(mktemp /tmp/toolbox-XXXX) 8 | source /tmp/.env-* 9 | source /opt/change-erlang-version.sh 10 | source /opt/change-python-version.sh 11 | source /opt/change-go-version.sh 12 | source /opt/change-java-version.sh 13 | source /opt/change-scala-version.sh 14 | source ~/.phpbrew/bashrc 15 | . /home/semaphore/.nvm/nvm.sh 16 | export PATH="$PATH:/home/semaphore/.yarn/bin" 17 | export KIEX_HOME="$HOME/.kiex" 18 | source "/home/semaphore/.kiex/scripts/kiex" 19 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 20 | export NVM_DIR=/home/semaphore/.nvm 21 | export PHPBREW_HOME=/home/semaphore/.phpbrew 22 | eval "$(rbenv init -)" 23 | 24 | source ~/.toolbox/toolbox 25 | sem-service start postgres 26 | sem-service start redis 27 | sem-version go 1.20 28 | sed -E -i '/^semservice,service=[a-z]*,state=(success|fail),version=[0-9a-zA-Z.]+,location=(disk|local|remote) duration=[0-9]+$/d' /tmp/toolbox_metrics 29 | 30 | sed -E -i '/^semversion,software=[a-z]*,state=(success|fail),version=[0-9a-zA-Z.-]+,osversion=[0-9.]+ duration=[0-9]+$/d' /tmp/toolbox_metrics 31 | 32 | sed -E -i '/^libcheckout,provider=(github|bitbucket),reftype=[].*,status=(success|fail) size=[0-9]+$/d' /tmp/toolbox_metrics 33 | 34 | sed -E -i '/^usercache,server=[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3},user=[a-z,0-9,-]+,command=(store|restore),corrupt=[0,1] size=[0-9]+,duration=[0-9]+$/d' /tmp/toolbox_metrics 35 | 36 | } 37 | 38 | @test "metrics file should be empty" { 39 | if [[ $(wc -c /tmp/toolbox_metrics | awk '{print $1}') -eq 0 ]];then 40 | rm -f /tmp/toolbox_metrics 41 | fi 42 | 43 | run cat /tmp/toolbox_metrics 44 | 45 | assert_failure 46 | } 47 | -------------------------------------------------------------------------------- /.semaphore/sem-version_bionic.txt: -------------------------------------------------------------------------------- 1 | sem-version firefox 52 2 | firefox --version | grep -q 52 3 | sem-version firefox 78 4 | firefox --version | grep -q 78 5 | sem-version ruby 2.5.3 6 | ruby --version | grep 2.5.3 7 | sem-version ruby 2.5.2 8 | ruby --version | grep 2.5.2 9 | sem-version ruby 2.5.3 10 | ruby --version | grep 2.5.3 11 | sem-version ruby 2.3.7 12 | ruby --version | grep 2.3.7 13 | sem-version c 8 14 | gcc --version | grep " 8." 15 | sem-version cpp 7 16 | gcc --version | grep " 7." 17 | sem-version php 7.2.31 18 | php -v | grep 7.2.31 19 | sem-version php 7.0.33 20 | php -v | grep 7.0.33 21 | sem-version php 7.4 22 | php -v | grep 7.4 23 | phpbrew ext install xdebug 24 | php -m | grep xdebug 25 | sem-version php 7.3.19 26 | php -m | grep magick 27 | php -m | grep gd 28 | php -m | grep imap 29 | which composer | grep 7.3.19 30 | sem-version elixir 1.7.4 31 | time sem-version node 12.16.1 32 | node --version | grep 12.16.1 33 | nodejs --version | grep 12.16.1 34 | time sem-version node 14 35 | node --version | grep 14 36 | sem-version ruby 2.6.6 37 | ruby --version | grep 2.6.6 38 | bundler --version | grep 2 39 | mkdir ruby_test 40 | echo "2.7.2" > ruby_test/.ruby-version 41 | cd ruby_test 42 | ruby --version| grep 2.7.2 43 | sem-version ruby 2.3.7 -f 44 | ruby --version| grep 2.3.7 45 | kubectl version --client 46 | sem-version kubectl 1.15.3 47 | kubectl version --client | grep -q 1.15.3 48 | echo "Erlang version" 49 | erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell 50 | sem-version erlang 20 51 | erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell 52 | sem-version erlang 21 53 | erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell 54 | echo "actual scala version" 55 | scala -version 56 | sem-version scala 2.11 57 | sem-version scala 2.12 58 | -------------------------------------------------------------------------------- /retry: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copied from: https://raw.githubusercontent.com/renderedtext/scripts/8f67b78cef7f5a7556777f59a5f290db8b23abe1/utility/retry 4 | # 5 | # Retries a command several times. 6 | # 7 | # Options: 8 | # -t, --times - Number of times to retry the command (default: 3) 9 | # -s, --sleep - Number of seconds to sleep between retries (default: 0) 10 | # 11 | 12 | function retry_execution { 13 | local __n__=3; # retry 3 times by default 14 | local __sleep__=0; # don't sleep by default 15 | 16 | # parsing command line options 17 | 18 | while true; do 19 | case "$1" in 20 | 21 | -t|--times) 22 | shift # drop --times from the args 23 | __n__="$1" # $1 contains the number of iterations 24 | shift 25 | ;; 26 | 27 | -s|--sleep) 28 | shift # drop --sleep from the args 29 | __sleep__="$1" # $1 contains the sleep value 30 | shift 31 | ;; 32 | 33 | *) 34 | break; # no more args to parse 35 | 36 | esac 37 | done 38 | 39 | 40 | # executing command 41 | 42 | local __cmd__="$@"; 43 | local __result__="0"; 44 | 45 | for __i__ in $(seq 1 $__n__); do 46 | eval "$__cmd__"; 47 | 48 | __result__="$?"; 49 | # echo "$__result__"; 50 | 51 | if [ $__result__ -eq "0" ]; then 52 | exit 0; # command executed successfully 53 | else 54 | if [[ $__i__ == $__n__ ]]; then 55 | echo "[$__i__/$__n__] Execution Failed with exit status $__result__. No more retries." 56 | 57 | exit $__result__; # exit with the same exit status as the invoked command 58 | else 59 | echo "[$__i__/$__n__] Execution Failed with exit status $__result__. Retrying." 60 | sleep $__sleep__; 61 | fi 62 | fi 63 | done 64 | } 65 | 66 | retry_execution $@ 67 | -------------------------------------------------------------------------------- /cache-cli/cmd/list.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | "github.com/semaphoreci/toolbox/cache-cli/pkg/files" 9 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 10 | "github.com/semaphoreci/toolbox/cache-cli/pkg/utils" 11 | log "github.com/sirupsen/logrus" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | func NewListCommand() *cobra.Command { 16 | cmd := &cobra.Command{ 17 | Use: "list", 18 | Short: "List all keys in the cache.", 19 | Long: ``, 20 | Args: cobra.ArbitraryArgs, 21 | Run: func(cmd *cobra.Command, args []string) { 22 | RunList(cmd, args) 23 | }, 24 | } 25 | 26 | description := fmt.Sprintf( 27 | `Sort keys by a specific field. Possible values are: %v.`, 28 | strings.Join(storage.ValidSortByKeys, ","), 29 | ) 30 | 31 | cmd.Flags().StringP("sort-by", "s", storage.SortByStoreTime, description) 32 | return cmd 33 | } 34 | 35 | func RunList(cmd *cobra.Command, args []string) { 36 | sortBy, err := cmd.Flags().GetString("sort-by") 37 | utils.Check(err) 38 | 39 | storage, err := storage.InitStorageWithConfig(storage.StorageConfig{SortKeysBy: sortBy}) 40 | utils.Check(err) 41 | 42 | keys, err := storage.List() 43 | utils.Check(err) 44 | 45 | if len(keys) == 0 { 46 | log.Info("Cache is empty.") 47 | } else { 48 | log.Info(formatList(keys)) 49 | } 50 | } 51 | 52 | func formatList(keys []storage.CacheKey) string { 53 | formatted := fmt.Sprintf("%-60s %-12s %-22s %-22s\n", "NAME", "SIZE", "STORED AT", "ACCESSED AT") 54 | for _, key := range keys { 55 | formatted += fmt.Sprintf( 56 | "%-60s %-12s %-22s %-22s\n", 57 | key.Name, 58 | files.HumanReadableSize(key.Size), 59 | key.StoredAt.Format(time.RFC822), 60 | key.LastAccessedAt.Format(time.RFC822), 61 | ) 62 | } 63 | 64 | return formatted 65 | } 66 | 67 | func init() { 68 | RootCmd.AddCommand(NewListCommand()) 69 | } 70 | -------------------------------------------------------------------------------- /sem-context/cmd/get.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/semaphoreci/toolbox/sem-context/pkg/flags" 7 | "github.com/semaphoreci/toolbox/sem-context/pkg/utils" 8 | "github.com/semaphoreci/toolbox/sem-context/pkg/validators" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | var getCmd = &cobra.Command{ 13 | Use: "get [key]", 14 | Short: "Get a variable", 15 | Run: RunGetCmd, 16 | } 17 | 18 | func RunGetCmd(cmd *cobra.Command, args []string) { 19 | utils.CheckError(validators.ValidateGetAndDeleteArguments(args)) 20 | key := args[0] 21 | 22 | value, err := SearchForKeyInAllContexts(key) 23 | if value != "" { 24 | fmt.Println(value) 25 | return 26 | } 27 | if err != nil && err.(*utils.Error).ExitCode == 1 && flags.Fallback != "" { 28 | fmt.Println(flags.Fallback) 29 | return 30 | } 31 | utils.CheckError(err) 32 | } 33 | 34 | // Goes from current context all the way to the root context (context<=>pipeline) and 35 | // searches for given key. 36 | func SearchForKeyInAllContexts(key string) (string, error) { 37 | contextHierarchy := utils.GetPipelineContextHierarchy() 38 | for _, contextID := range contextHierarchy { 39 | value, err := Store.Get(key, contextID) 40 | if err == nil { 41 | return value, nil 42 | } 43 | if err.(*utils.Error).ExitCode == 2 { 44 | return "", err 45 | } 46 | deleted, err := Store.CheckIfKeyDeleted(key, contextID) 47 | if err != nil { 48 | utils.CheckError(err) 49 | } 50 | if deleted { 51 | return "", &utils.Error{ErrorMessage: fmt.Sprintf("Cant find the key '%s'", key), ExitCode: 1} 52 | } 53 | } 54 | return "", &utils.Error{ErrorMessage: fmt.Sprintf("Cant find the key '%s'", key), ExitCode: 1} 55 | } 56 | 57 | func init() { 58 | getCmd.Flags().StringVar(&flags.Fallback, "fallback", "", "Default value to be returned if key does not exist.") 59 | RootCmd.AddCommand(getCmd) 60 | } 61 | -------------------------------------------------------------------------------- /test-results/pkg/parsers/helpers.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | 7 | "github.com/semaphoreci/toolbox/test-results/pkg/fileloader" 8 | "github.com/semaphoreci/toolbox/test-results/pkg/parser" 9 | "golang.org/x/text/cases" 10 | "golang.org/x/text/language" 11 | ) 12 | 13 | // LoadPath ... 14 | func LoadPath(path string) (*bytes.Reader, error) { 15 | var reader *bytes.Reader 16 | // Preload path with loader. If nothing is found in file cache - load it up from path. 17 | reader, found := fileloader.Load(path, &bytes.Reader{}) 18 | 19 | if !found { 20 | file, err := os.ReadFile(path) // #nosec 21 | 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | b := bytes.NewReader(file) 27 | reader, _ = fileloader.Load(path, b) 28 | } 29 | return reader, nil 30 | } 31 | 32 | // LoadXML ... 33 | func LoadXML(path string) (*parser.XMLElement, error) { 34 | reader, err := LoadPath(path) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | xmlElement := parser.NewXMLElement() 40 | 41 | err = xmlElement.Parse(reader) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | return &xmlElement, nil 47 | } 48 | 49 | // LoadFile loads a file from the given path 50 | func LoadFile(path string) ([]byte, error) { 51 | // Check file cache first 52 | reader, found := fileloader.Load(path, &bytes.Reader{}) 53 | 54 | if found { 55 | buf := new(bytes.Buffer) 56 | _, err := buf.ReadFrom(reader) 57 | if err != nil { 58 | return nil, err 59 | } 60 | return buf.Bytes(), nil 61 | } 62 | 63 | data, err := os.ReadFile(path) // #nosec 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | // Cache for future use 69 | fileloader.Load(path, bytes.NewReader(data)) 70 | 71 | return data, nil 72 | } 73 | 74 | func Title(s string) string { 75 | caser := cases.Title(language.English) 76 | return caser.String(s) 77 | } 78 | -------------------------------------------------------------------------------- /tests/sem_version_jammy/go.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | @test "sem-version go 1.18.10" { 27 | 28 | sem-version go 1.18.10 29 | run echo ${PATH} 30 | assert_line --partial "$(go env GOPATH)/bin" 31 | run go version 32 | assert_line --partial "go1.18.10" 33 | } 34 | 35 | @test "sem-version go 1.19.9" { 36 | 37 | sem-version go 1.19.9 38 | run echo ${PATH} 39 | assert_line --partial "$(go env GOPATH)/bin" 40 | run go version 41 | assert_line --partial "go1.19.9" 42 | } 43 | 44 | @test "sem-version go 1.20.4" { 45 | 46 | sem-version go 1.20.4 47 | run echo ${PATH} 48 | assert_line --partial "$(go env GOPATH)/bin" 49 | run go version 50 | assert_line --partial "go1.20.4" 51 | } 52 | 53 | @test "sem-version go 1.21.1" { 54 | 55 | sem-version go 1.21.1 56 | run echo ${PATH} 57 | assert_line --partial "$(go env GOPATH)/bin" 58 | run go version 59 | assert_line --partial "go1.21.1" 60 | } 61 | 62 | @test "sem-version go 1.25.0" { 63 | 64 | sem-version go 1.25.0 65 | run echo ${PATH} 66 | assert_line --partial "$(go env GOPATH)/bin" 67 | run go version 68 | assert_line --partial "go1.25.0" 69 | } 70 | -------------------------------------------------------------------------------- /tests/sem_version_noble/go.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | @test "sem-version go 1.18.10" { 27 | 28 | sem-version go 1.18.10 29 | run echo ${PATH} 30 | assert_line --partial "$(go env GOPATH)/bin" 31 | run go version 32 | assert_line --partial "go1.18.10" 33 | } 34 | 35 | @test "sem-version go 1.19.9" { 36 | 37 | sem-version go 1.19.9 38 | run echo ${PATH} 39 | assert_line --partial "$(go env GOPATH)/bin" 40 | run go version 41 | assert_line --partial "go1.19.9" 42 | } 43 | 44 | @test "sem-version go 1.20.4" { 45 | 46 | sem-version go 1.20.4 47 | run echo ${PATH} 48 | assert_line --partial "$(go env GOPATH)/bin" 49 | run go version 50 | assert_line --partial "go1.20.4" 51 | } 52 | 53 | @test "sem-version go 1.21.1" { 54 | 55 | sem-version go 1.21.1 56 | run echo ${PATH} 57 | assert_line --partial "$(go env GOPATH)/bin" 58 | run go version 59 | assert_line --partial "go1.21.1" 60 | } 61 | 62 | @test "sem-version go 1.25.0" { 63 | 64 | sem-version go 1.25.0 65 | run echo ${PATH} 66 | assert_line --partial "$(go env GOPATH)/bin" 67 | run go version 68 | assert_line --partial "go1.25.0" 69 | } 70 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/s3_clear.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aws/aws-sdk-go-v2/service/s3" 8 | "github.com/aws/aws-sdk-go-v2/service/s3/types" 9 | ) 10 | 11 | func (s *S3Storage) Clear() error { 12 | keys, err := s.List() 13 | if err != nil { 14 | return err 15 | } 16 | 17 | if len(keys) == 0 { 18 | return nil 19 | } 20 | 21 | // the s3 DeleteObjects operation only allows up to 1000 keys to be used 22 | chunks := createChunks(keys, 1000) 23 | 24 | for _, chunk := range chunks { 25 | err := s.deleteChunk(chunk) 26 | if err != nil { 27 | return err 28 | } 29 | } 30 | 31 | return nil 32 | } 33 | 34 | func (s *S3Storage) deleteChunk(keys []CacheKey) error { 35 | output, err := s.Client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ 36 | Bucket: &s.Bucket, 37 | Delete: &types.Delete{ 38 | Objects: cacheKeysToObjectIdentifiers(s.Project, keys), 39 | }, 40 | }) 41 | 42 | if err != nil { 43 | return err 44 | } 45 | 46 | if len(output.Errors) > 0 { 47 | firstError := output.Errors[0] 48 | return fmt.Errorf("clear operation failed, some keys might not have been deleted: %s", *firstError.Message) 49 | } 50 | 51 | return nil 52 | } 53 | 54 | func createChunks(keys []CacheKey, chunkSize int) [][]CacheKey { 55 | var chunks [][]CacheKey 56 | for i := 0; i < len(keys); i += chunkSize { 57 | end := i + chunkSize 58 | 59 | if end > len(keys) { 60 | end = len(keys) 61 | } 62 | 63 | chunks = append(chunks, keys[i:end]) 64 | } 65 | 66 | return chunks 67 | } 68 | 69 | func cacheKeysToObjectIdentifiers(project string, keys []CacheKey) []types.ObjectIdentifier { 70 | objectIdentifiers := make([]types.ObjectIdentifier, 0) 71 | for _, key := range keys { 72 | bucketKey := fmt.Sprintf("%s/%s", project, key.Name) 73 | identifier := types.ObjectIdentifier{ 74 | Key: &bucketKey, 75 | } 76 | 77 | objectIdentifiers = append(objectIdentifiers, identifier) 78 | } 79 | 80 | return objectIdentifiers 81 | } 82 | -------------------------------------------------------------------------------- /cache-cli/pkg/storage/sftp_list.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "io/fs" 5 | "sort" 6 | "time" 7 | 8 | "github.com/pkg/sftp" 9 | ) 10 | 11 | func (s *SFTPStorage) List() ([]CacheKey, error) { 12 | files, err := s.SFTPClient.ReadDir(".") 13 | if err != nil { 14 | return nil, err 15 | } 16 | 17 | keys := []CacheKey{} 18 | for _, file := range files { 19 | storedAt := file.ModTime() 20 | keys = append(keys, CacheKey{ 21 | Name: file.Name(), 22 | Size: file.Size(), 23 | StoredAt: &storedAt, 24 | LastAccessedAt: findLastAccessedAt(file), 25 | }) 26 | } 27 | 28 | return s.sortKeys(keys), nil 29 | } 30 | 31 | func (s *SFTPStorage) sortKeys(keys []CacheKey) []CacheKey { 32 | switch s.Config().SortKeysBy { 33 | case SortBySize: 34 | sort.SliceStable(keys, func(i, j int) bool { 35 | return keys[i].Size > keys[j].Size 36 | }) 37 | case SortByAccessTime: 38 | sort.SliceStable(keys, func(i, j int) bool { 39 | return keys[i].LastAccessedAt.After(*keys[j].LastAccessedAt) 40 | }) 41 | default: 42 | sort.SliceStable(keys, func(i, j int) bool { 43 | return keys[i].StoredAt.After(*keys[j].StoredAt) 44 | }) 45 | } 46 | 47 | return keys 48 | } 49 | 50 | // If we can't figure out the access time of the file, 51 | // we fallback to the modification time. 52 | func findLastAccessedAt(fileInfo fs.FileInfo) *time.Time { 53 | mtime := fileInfo.ModTime() 54 | 55 | // Try to get the underlying data source; if nil, fallback to mtime. 56 | ds := fileInfo.Sys() 57 | if ds == nil { 58 | return &mtime 59 | } 60 | 61 | // Try to cast the underlying data source to something we understand; if nil, fallback to mtime. 62 | stat, ok := ds.(*sftp.FileStat) 63 | if !ok { 64 | return &mtime 65 | } 66 | 67 | // atime can also be unset; fallback to mtime. 68 | if stat.Atime == 0 { 69 | mtime := fileInfo.ModTime() 70 | return &mtime 71 | } 72 | 73 | // Otherwise, we use atime. 74 | atime := time.Unix(int64(stat.Atime), 0) 75 | return &atime 76 | } 77 | -------------------------------------------------------------------------------- /test-results/cmd/combine.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/semaphoreci/toolbox/test-results/pkg/cli" 5 | "github.com/semaphoreci/toolbox/test-results/pkg/logger" 6 | "github.com/semaphoreci/toolbox/test-results/pkg/parser" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | // combineCmd represents the combine command 11 | var combineCmd = &cobra.Command{ 12 | Use: "combine ... ]", 13 | Short: "combines multiples json summary files into one", 14 | Long: `Combines multiples json summary files into one"`, 15 | Args: cobra.MinimumNArgs(2), 16 | RunE: func(cmd *cobra.Command, args []string) error { 17 | inputs := args[:len(args)-1] 18 | output := args[len(args)-1] 19 | 20 | err := cli.SetLogLevel(cmd) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | skipCompression, err := cmd.Flags().GetBool("no-compress") 26 | if err != nil { 27 | return err 28 | } 29 | 30 | paths, err := cli.LoadFiles(inputs, ".json") 31 | if err != nil { 32 | return err 33 | } 34 | 35 | result := parser.NewResult() 36 | for _, path := range paths { 37 | inFile, err := cli.CheckFile(path) 38 | if err != nil { 39 | logger.Error(err.Error()) 40 | return err 41 | } 42 | 43 | newResult, err := cli.Load(inFile, cmd) 44 | 45 | if err != nil { 46 | logger.Error(err.Error()) 47 | return err 48 | } 49 | result.Combine(*newResult) 50 | } 51 | 52 | err = cli.DecorateResults(&result, cmd) 53 | if err != nil { 54 | logger.Error("Decorating results failed with error: %v", err) 55 | return err 56 | } 57 | 58 | jsonData, err := cli.Marshal(result) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | _, err = cli.WriteToFilePath(jsonData, output, !skipCompression) 64 | if err != nil { 65 | return err 66 | } 67 | return nil 68 | }, 69 | } 70 | 71 | func init() { 72 | combineCmd.Flags().BoolP("omit-output-for-passed", "o", false, "omit stdout if test passed, defaults to false") 73 | rootCmd.AddCommand(combineCmd) 74 | } 75 | -------------------------------------------------------------------------------- /tests/sem_version_jammy/elixir.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # Elixir 27 | @test "change elixir to 1.14.0" { 28 | sem-version elixir 1.14.0 29 | run elixir --version 30 | assert_line --partial "Elixir 1.14.0" 31 | } 32 | 33 | @test "change elixir to 1.14.5" { 34 | sem-version elixir 1.14.5 35 | run elixir --version 36 | assert_line --partial "Elixir 1.14.5" 37 | } 38 | 39 | @test "change elixir to 1.15.0" { 40 | sem-version elixir 1.15.0 41 | run elixir --version 42 | assert_line --partial "Elixir 1.15.0" 43 | } 44 | 45 | @test "change elixir to 1.15.8" { 46 | sem-version elixir 1.15.8 47 | run elixir --version 48 | assert_line --partial "Elixir 1.15.8" 49 | } 50 | 51 | @test "change elixir to 1.16.3" { 52 | sem-version elixir 1.16.3 53 | run elixir --version 54 | assert_line --partial "Elixir 1.16.3" 55 | } 56 | 57 | @test "change elixir to 1.17.3" { 58 | sem-version elixir 1.17.3 59 | run elixir --version 60 | assert_line --partial "Elixir 1.17.3" 61 | } 62 | 63 | @test "change elixir to 1.18.4" { 64 | sem-version elixir 1.18.4 65 | run elixir --version 66 | assert_line --partial "Elixir 1.18.4" 67 | } 68 | 69 | @test "change elixir to 1.19.0" { 70 | sem-version elixir 1.19.0 71 | run elixir --version 72 | assert_line --partial "Elixir 1.19.0" 73 | } 74 | -------------------------------------------------------------------------------- /tests/sem_version_noble/elixir.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source ~/.phpbrew/bashrc 14 | . /home/semaphore/.nvm/nvm.sh 15 | export PATH="$PATH:/home/semaphore/.yarn/bin" 16 | export KIEX_HOME="$HOME/.kiex" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # Elixir 27 | @test "change elixir to 1.14.0" { 28 | sem-version elixir 1.14.0 29 | run elixir --version 30 | assert_line --partial "Elixir 1.14.0" 31 | } 32 | 33 | @test "change elixir to 1.14.5" { 34 | sem-version elixir 1.14.5 35 | run elixir --version 36 | assert_line --partial "Elixir 1.14.5" 37 | } 38 | 39 | @test "change elixir to 1.15.0" { 40 | sem-version elixir 1.15.0 41 | run elixir --version 42 | assert_line --partial "Elixir 1.15.0" 43 | } 44 | 45 | @test "change elixir to 1.15.8" { 46 | sem-version elixir 1.15.8 47 | run elixir --version 48 | assert_line --partial "Elixir 1.15.8" 49 | } 50 | 51 | @test "change elixir to 1.16.3" { 52 | sem-version elixir 1.16.3 53 | run elixir --version 54 | assert_line --partial "Elixir 1.16.3" 55 | } 56 | 57 | @test "change elixir to 1.17.3" { 58 | sem-version elixir 1.17.3 59 | run elixir --version 60 | assert_line --partial "Elixir 1.17.3" 61 | } 62 | 63 | @test "change elixir to 1.18.4" { 64 | sem-version elixir 1.18.4 65 | run elixir --version 66 | assert_line --partial "Elixir 1.18.4" 67 | } 68 | 69 | @test "change elixir to 1.19.0" { 70 | sem-version elixir 1.19.0 71 | run elixir --version 72 | assert_line --partial "Elixir 1.19.0" 73 | } 74 | -------------------------------------------------------------------------------- /.semaphore/release.yml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | name: Release 3 | agent: 4 | machine: 5 | type: e2-standard-2 6 | os_image: ubuntu2004 7 | blocks: 8 | - name: Release 9 | task: 10 | secrets: 11 | - name: github-release-bot-toolbox 12 | jobs: 13 | - name: Create and Upload assets 14 | commands: 15 | - export GITHUB_TOKEN=$ACCESS_TOKEN 16 | - checkout 17 | - artifact pull workflow bin/linux/amd64/cache -d cache-cli/bin/linux/amd64/cache 18 | - artifact pull workflow bin/linux/arm64/cache -d cache-cli/bin/linux/arm64/cache 19 | - artifact pull workflow bin/darwin/amd64/cache -d cache-cli/bin/darwin/amd64/cache 20 | - artifact pull workflow bin/darwin/arm64/cache -d cache-cli/bin/darwin/arm64/cache 21 | - artifact pull workflow bin/windows/cache.exe -d cache-cli/bin/windows/cache.exe 22 | - artifact pull workflow bin/linux/amd64/sem-context -d sem-context/bin/linux/amd64/sem-context 23 | - artifact pull workflow bin/linux/arm64/sem-context -d sem-context/bin/linux/arm64/sem-context 24 | - artifact pull workflow bin/darwin/amd64/sem-context -d sem-context/bin/darwin/amd64/sem-context 25 | - artifact pull workflow bin/darwin/arm64/sem-context -d sem-context/bin/darwin/arm64/sem-context 26 | - artifact pull workflow bin/windows/sem-context.exe -d sem-context/bin/windows/sem-context.exe 27 | - artifact pull workflow bin/linux/amd64/test-results -d test-results/bin/linux/amd64/test-results 28 | - artifact pull workflow bin/linux/arm64/test-results -d test-results/bin/linux/arm64/test-results 29 | - artifact pull workflow bin/darwin/amd64/test-results -d test-results/bin/darwin/amd64/test-results 30 | - artifact pull workflow bin/darwin/arm64/test-results -d test-results/bin/darwin/arm64/test-results 31 | - artifact pull workflow bin/windows/test-results.exe -d test-results/bin/windows/test-results.exe 32 | - bash release/create.sh -a 33 | - bash release/upload.sh 34 | -------------------------------------------------------------------------------- /cache-cli/pkg/files/download_test.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "runtime" 7 | "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func Test__DownloadFromHTTP(t *testing.T) { 12 | if runtime.GOOS == "windows" { 13 | t.Skip() 14 | } 15 | sftpStorage, err := storage.NewSFTPStorage(storage.SFTPStorageOptions{ 16 | URL: "sftp-server:22", 17 | Username: "tester", 18 | PrivateKeyPath: "/root/.ssh/semaphore_cache_key", 19 | Config: storage.StorageConfig{ 20 | MaxSpace: 1024, 21 | SortKeysBy: storage.SortBySize, 22 | }, 23 | }) 24 | 25 | require.NoError(t, err) 26 | require.NoError(t, sftpStorage.Clear()) 27 | require.NoError(t, sftpStorage.Store("abc", "testdata/test.txt")) 28 | 29 | t.Run("download works", func(t *testing.T) { 30 | f, err := DownloadFromHTTP("http://sftp-server:80", "test", "test", "abc") 31 | require.NoError(t, err) 32 | require.FileExists(t, f.Name()) 33 | 34 | content, err := os.ReadFile(f.Name()) 35 | require.NoError(t, err) 36 | require.Equal(t, "Test 123", string(content)) 37 | }) 38 | 39 | t.Run("download fails if URL is not reachable", func(t *testing.T) { 40 | _, err := DownloadFromHTTP("http://sftp-server:801", "test", "test", "abc") 41 | require.ErrorContains(t, err, "connection refused") 42 | }) 43 | 44 | t.Run("download fails if username is invalid", func(t *testing.T) { 45 | _, err := DownloadFromHTTP("http://sftp-server:80", "wrong", "test", "abc") 46 | require.ErrorContains(t, err, "failed to download file: 401 Unauthorized") 47 | }) 48 | 49 | t.Run("download fails if password is wrong", func(t *testing.T) { 50 | _, err := DownloadFromHTTP("http://sftp-server:80", "test", "wrong", "abc") 51 | require.ErrorContains(t, err, "failed to download file: 401 Unauthorized") 52 | }) 53 | 54 | t.Run("download fails if file does not exist", func(t *testing.T) { 55 | _, err := DownloadFromHTTP("http://sftp-server:80", "test", "test", "does-not-exist") 56 | require.ErrorContains(t, err, "failed to download file: 404 Not Found") 57 | }) 58 | } 59 | -------------------------------------------------------------------------------- /sem-context/pkg/utils/pipeline_context_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "regexp" 6 | "sort" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | const artifactIDregex = "SEMAPHORE_PIPELINE_(.*)_ARTEFACT_ID" 12 | 13 | //For this tool Artifact ID's are used as Context ID's 14 | //Returns slice with context ID's where first element of slice is current pipelines ID, and last element is ID of the first pipeline 15 | //inside the workflow 16 | func GetPipelineContextHierarchy() []string { 17 | contextIDs_map := extractArtifactIdsFromEnvVariables() 18 | 19 | return sortContextIDs(contextIDs_map) 20 | } 21 | 22 | func extractArtifactIdsFromEnvVariables() map[int]string { 23 | contextIDs := make(map[int]string) 24 | 25 | for _, element := range os.Environ() { 26 | env_var := strings.Split(element, "=") 27 | if match, _ := regexp.MatchString(artifactIDregex, env_var[0]); match { 28 | idOrderNum := extractPipelineOrdinalNumber(env_var[0]) 29 | contextIDs[idOrderNum] = env_var[1] 30 | } 31 | } 32 | 33 | return contextIDs 34 | } 35 | 36 | // Env variables that contain artefact id for given pipeline contain info about what is 37 | // the order of execution of all pipelines in the given workflow: 38 | // SEMAPHORE_PIPELINE_1_ARTEFACT_ID=... 39 | // SEMAPHORE_PIPELINE_13_ARTEFACT_ID... 40 | // This function extracts ordinal number of given pipeline from env variable name. 41 | // Artifact ID with the highest ordinal number is artifact id of current pipeline. 42 | func extractPipelineOrdinalNumber(env_var string) int { 43 | re := regexp.MustCompile(artifactIDregex) 44 | substrings := re.FindAllStringSubmatch(env_var, 1) 45 | orderNum, _ := strconv.Atoi(substrings[0][1]) 46 | return orderNum 47 | } 48 | 49 | func sortContextIDs(contextIDs_map map[int]string) []string { 50 | keys := make([]int, 0, len(contextIDs_map)) 51 | for key := range contextIDs_map { 52 | keys = append(keys, key) 53 | } 54 | sort.Sort(sort.Reverse(sort.IntSlice(keys))) 55 | 56 | contextIDs := make([]string, 0, len(keys)) 57 | for _, key := range keys { 58 | contextIDs = append(contextIDs, contextIDs_map[key]) 59 | } 60 | return contextIDs 61 | } 62 | -------------------------------------------------------------------------------- /tests/test-results/junit-sample.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Failure/Error: expect(result).to eq(-1) 14 | 15 | expected: -1 16 | got: 3 17 | 18 | (compared using ==) 19 | ./spec/calculator/adder_spec.rb:11:in `block (2 levels) in <top (required)>' 20 | 21 | 22 | 23 | 24 | 25 | 26 | Failure/Error: expect(result).to eq(3) 32 | 33 | expected: 3 34 | got: -1 35 | 36 | (compared using ==) 37 | ./spec/calculator/subtractor_spec.rb:11:in `block (2 levels) in <top (required)>' 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /cache-cli/pkg/metrics/local_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | assert "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestLogEventWritesInfluxLine(t *testing.T) { 13 | os.Setenv("SEMAPHORE_TOOLBOX_METRICS_ENABLED", "true") 14 | os.Setenv("SEMAPHORE_CACHE_USERNAME", "tester") 15 | os.Setenv("SEMAPHORE_CACHE_URL", "10.0.0.5:1234") 16 | 17 | metricsManager, err := NewLocalMetricsBackend() 18 | assert.Nil(t, err) 19 | 20 | event := CacheEvent{ 21 | Command: CommandStore, 22 | SizeBytes: 2048, 23 | Duration: time.Second, 24 | } 25 | 26 | err = metricsManager.LogEvent(event) 27 | assert.Nil(t, err) 28 | 29 | bytes, err := ioutil.ReadFile(metricsManager.ToolboxMetricsPath) 30 | assert.Nil(t, err) 31 | assert.Contains(t, string(bytes), "usercache,server=10.0.0.5,user=tester,command=store,corrupt=0 size=2048,duration=1000") 32 | 33 | os.Remove(metricsManager.ToolboxMetricsPath) 34 | } 35 | 36 | func TestLogEventMarksCorruption(t *testing.T) { 37 | os.Setenv("SEMAPHORE_TOOLBOX_METRICS_ENABLED", "true") 38 | os.Setenv("SEMAPHORE_CACHE_USERNAME", "") 39 | os.Setenv("SEMAPHORE_CACHE_URL", "") 40 | 41 | metricsManager, err := NewLocalMetricsBackend() 42 | assert.Nil(t, err) 43 | 44 | event := CacheEvent{ 45 | Command: CommandRestore, 46 | Corrupt: true, 47 | } 48 | 49 | err = metricsManager.LogEvent(event) 50 | assert.Nil(t, err) 51 | 52 | bytes, err := ioutil.ReadFile(metricsManager.ToolboxMetricsPath) 53 | assert.Nil(t, err) 54 | assert.Contains(t, string(bytes), "command=restore,corrupt=1") 55 | assert.Contains(t, string(bytes), "size=0,duration=0") 56 | 57 | os.Remove(metricsManager.ToolboxMetricsPath) 58 | } 59 | 60 | func TestLogEventDisabled(t *testing.T) { 61 | os.Setenv("SEMAPHORE_TOOLBOX_METRICS_ENABLED", "false") 62 | metricsManager, err := NewLocalMetricsBackend() 63 | assert.Nil(t, err) 64 | 65 | event := CacheEvent{ 66 | Command: CommandStore, 67 | SizeBytes: 100, 68 | } 69 | 70 | err = metricsManager.LogEvent(event) 71 | assert.Nil(t, err) 72 | 73 | _, err = os.Stat(metricsManager.ToolboxMetricsPath) 74 | assert.NotNil(t, err) 75 | } 76 | -------------------------------------------------------------------------------- /cache-cli/pkg/metrics/local.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | "strings" 9 | ) 10 | 11 | type LocalMetricsManager struct { 12 | ToolboxMetricsPath string 13 | } 14 | 15 | func NewLocalMetricsBackend() (*LocalMetricsManager, error) { 16 | basePath := "/tmp" 17 | if runtime.GOOS == "windows" { 18 | basePath = os.TempDir() 19 | } 20 | 21 | return &LocalMetricsManager{ 22 | ToolboxMetricsPath: filepath.Join(basePath, "toolbox_metrics"), 23 | }, nil 24 | } 25 | 26 | func (b *LocalMetricsManager) Enabled() bool { 27 | return os.Getenv("SEMAPHORE_TOOLBOX_METRICS_ENABLED") == "true" 28 | } 29 | 30 | func (b *LocalMetricsManager) LogEvent(event CacheEvent) error { 31 | if !b.Enabled() { 32 | return nil 33 | } 34 | 35 | return publishEventToFile(b.ToolboxMetricsPath, event) 36 | } 37 | 38 | func publishEventToFile(file string, event CacheEvent) error { 39 | server := event.Server 40 | if server == "" { 41 | server = CacheServerIP() 42 | } 43 | 44 | user := event.User 45 | if user == "" { 46 | user = CacheUsername() 47 | } 48 | 49 | command := event.Command 50 | if command == "" { 51 | command = CommandRestore 52 | } 53 | 54 | corruptValue := 0 55 | if event.Corrupt { 56 | corruptValue = 1 57 | } 58 | 59 | durationMs := event.Duration.Milliseconds() 60 | if durationMs < 0 { 61 | durationMs = 0 62 | } 63 | 64 | // #nosec 65 | f, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | line := fmt.Sprintf( 71 | "%s,server=%s,user=%s,command=%s,corrupt=%d size=%d,duration=%d\n", 72 | MeasurementName, 73 | escapeTagValue(server), 74 | escapeTagValue(user), 75 | escapeTagValue(command), 76 | corruptValue, 77 | event.SizeBytes, 78 | durationMs, 79 | ) 80 | 81 | _, err = f.WriteString(line) 82 | if err != nil { 83 | _ = f.Close() 84 | return err 85 | } 86 | 87 | return f.Close() 88 | } 89 | 90 | func escapeTagValue(value string) string { 91 | if value == "" { 92 | return "unknown" 93 | } 94 | 95 | replacer := strings.NewReplacer(",", "\\,", " ", "\\ ", "=", "\\=") 96 | return replacer.Replace(value) 97 | } 98 | -------------------------------------------------------------------------------- /tests/sem_version_focal/php.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "../support/bats-support/load" 4 | load "../support/bats-assert/load" 5 | 6 | setup() { 7 | source /tmp/.env-* 8 | source /opt/change-erlang-version.sh 9 | source /opt/change-python-version.sh 10 | source /opt/change-go-version.sh 11 | source /opt/change-java-version.sh 12 | source /opt/change-scala-version.sh 13 | source /opt/change-firefox-version.sh 14 | source ~/.phpbrew/bashrc 15 | . /home/semaphore/.nvm/nvm.sh 16 | export PATH="$PATH:/home/semaphore/.yarn/bin" 17 | source "/home/semaphore/.kiex/scripts/kiex" 18 | export PATH="/home/semaphore/.rbenv/bin:$PATH" 19 | export NVM_DIR=/home/semaphore/.nvm 20 | export PHPBREW_HOME=/home/semaphore/.phpbrew 21 | eval "$(rbenv init -)" 22 | 23 | source ~/.toolbox/toolbox 24 | } 25 | 26 | # PHP 27 | 28 | 29 | @test "change php to 7.4.33" { 30 | 31 | run sem-version php 7.4.33 32 | assert_success 33 | source ~/.phpbrew/bashrc 34 | run php -v 35 | assert_line --partial "PHP 7.4.33" 36 | run php -m 37 | assert_line --partial "magick" 38 | assert_line --partial "gd" 39 | assert_line --partial "imap" 40 | } 41 | 42 | @test "change php to 8.0.30" { 43 | 44 | run sem-version php 8.0.30 45 | assert_success 46 | source ~/.phpbrew/bashrc 47 | run php -v 48 | assert_line --partial "PHP 8.0.30" 49 | run php -m 50 | assert_line --partial "gd" 51 | assert_line --partial "imap" 52 | } 53 | 54 | @test "php check composer 8.0.30" { 55 | 56 | run which composer 57 | assert_success 58 | source ~/.phpbrew/bashrc 59 | assert_line --partial "8.0.30" 60 | } 61 | 62 | @test "php check 8.0.30" { 63 | 64 | run sem-version php 8.0.30 65 | assert_success 66 | source ~/.phpbrew/bashrc 67 | assert_line --partial "8.0.30" 68 | run phpbrew ext install iconv 69 | assert_success 70 | } 71 | 72 | @test "php check sources 8.0.30" { 73 | 74 | run sem-version php 8.0.30 75 | assert_success 76 | source ~/.phpbrew/bashrc 77 | 78 | run ls -lah ~/.phpbrew/distfiles/ 79 | assert_line --partial "8.0.30" 80 | 81 | run ls -lah ~/.phpbrew/build/ 82 | assert_line --partial "8.0.30" 83 | 84 | run phpbrew ext install iconv 85 | assert_success 86 | } 87 | -------------------------------------------------------------------------------- /test-results/priv/parsers/junit_phpunit/in.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | --------------------------------------------------------------------------------