├── .github └── workflows │ ├── ci.yml │ ├── job-lint.yml │ └── job-test.yml ├── .gitignore ├── .vscode ├── extensions.json └── settings.json ├── LICENSE ├── Makefile ├── README.md ├── archive.go ├── bytesutil ├── bytes.go ├── bytes_test.go ├── format.go ├── format_test.go ├── json.go ├── lines.go ├── lines_test.go ├── markdown.go ├── sort.go ├── sort_test.go ├── splitter.go ├── splitter_test.go ├── synchronized.go ├── text.go └── text_test.go ├── cache.go ├── cache_test.go ├── capture_linux.go ├── capture_notlinux.go ├── capture_test_linux.go ├── capture_testwrapper_linux.go ├── chdir.go ├── checksum.go ├── checksum_test.go ├── context.go ├── conversion.go ├── conversion_test.go ├── copy.go ├── copy_test.go ├── directory.go ├── download.go ├── env.go ├── env_test.go ├── exists.go ├── exists_notwindows.go ├── exists_test.go ├── exists_windows.go ├── files.go ├── files_notwindows.go ├── files_test.go ├── files_windows.go ├── go.mod ├── go.sum ├── info_darwin.go ├── info_darwin_test.go ├── info_linux.go ├── info_linux_test.go ├── info_windows.go ├── info_windows_test.go ├── io.go ├── limits.go ├── limits_linux.go ├── limits_linux_test.go ├── limits_notlinux.go ├── limittest └── memory.go ├── makefile.go ├── os.go ├── path_notwindows.go ├── path_operator.go ├── path_operator_test.go ├── path_windows.go ├── permission.go ├── permission_test.go ├── platform_notwindows.go ├── platform_windows.go ├── progress.go ├── remove.go ├── scripts └── editor.sh ├── static.go ├── sync.go ├── templateutil ├── defaults.go ├── file.go └── interface.go └── variables.go /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: ["push", "release"] 4 | 5 | concurrency: 6 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 7 | cancel-in-progress: true 8 | 9 | jobs: 10 | lint: 11 | name: "Lint the repository" 12 | uses: ./.github/workflows/job-lint.yml 13 | test: 14 | name: "Run tests and lint artifacts" 15 | needs: 16 | - lint 17 | secrets: inherit 18 | strategy: 19 | fail-fast: false # Run the whole matrix for maximum information. No matter if we fail with one job early. 20 | matrix: 21 | os: 22 | - "macOS-latest" 23 | - "ubuntu-latest" 24 | - "windows-latest" 25 | uses: ./.github/workflows/job-test.yml 26 | with: 27 | os: ${{ matrix.os }} 28 | -------------------------------------------------------------------------------- /.github/workflows/job-lint.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: 3 | 4 | jobs: 5 | lint: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - name: checkout code 9 | uses: actions/checkout@v4 10 | 11 | - name: Set up Go 12 | uses: actions/setup-go@v5 13 | with: 14 | go-version-file: "go.mod" 15 | 16 | - name: Lint 17 | run: make lint 18 | shell: bash # Explicitly use Bash because otherwise failing Windows jobs are not erroring. 19 | -------------------------------------------------------------------------------- /.github/workflows/job-test.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: 3 | inputs: 4 | os: 5 | required: true 6 | type: string 7 | 8 | jobs: 9 | test: 10 | runs-on: ${{ inputs.os }} 11 | steps: 12 | - uses: actions/checkout@v4 13 | 14 | - uses: jlumbroso/free-disk-space@main 15 | if: contains(inputs.os, 'ubuntu') 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v5 19 | with: 20 | go-version-file: "go.mod" 21 | 22 | - name: Set up Git 23 | run: | 24 | git config --global user.name "GitHub Actions Bot" 25 | git config --global user.email "<>" 26 | shell: bash # Explicitly use Bash because otherwise failing Windows jobs are not erroring. 27 | 28 | - name: Build 29 | run: make install 30 | shell: bash # Explicitly use Bash because otherwise failing Windows jobs are not erroring. 31 | 32 | - name: Test 33 | run: make test 34 | shell: bash # Explicitly use Bash because otherwise failing Windows jobs are not erroring. 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins. 2 | /bin 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | 9 | # Test binary, built with `go test -c`. 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE. 13 | *.out 14 | 15 | # Test artifacts 16 | /limittest/memory 17 | 18 | # Go workspace file. 19 | go.work 20 | 21 | # Ignore local configurations. 22 | .envrc 23 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "augustocdias.tasks-shell-input", 4 | "golang.go", 5 | "spadin.memento-inputs", 6 | "Symflower.symflower" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "gopls": { 3 | // Disable analyzers that we are currently not interested in because they do not obey our conventions. 4 | "analyses": { 5 | "simplifycompositelit": false, 6 | }, 7 | // Add parameter placeholders when completing a function. 8 | "usePlaceholders": true, 9 | }, 10 | "symflower.lint.onOpen": true, 11 | "symflower.lint.onSave": true, 12 | } 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Markus Zimmermann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) 2 | 3 | export PACKAGE_BASE := github.com/zimmski/osutil 4 | export UNIT_TEST_TIMEOUT := 480 5 | 6 | ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) 7 | $(eval $(ARGS):;@:) # turn arguments into do-nothing targets 8 | export ARGS 9 | 10 | ifdef ARGS 11 | HAS_ARGS := "1" 12 | PACKAGE := $(ARGS) 13 | else 14 | HAS_ARGS := 15 | PACKAGE := $(PACKAGE_BASE)/... 16 | endif 17 | 18 | ifdef NO_UNIT_TEST_CACHE 19 | export NO_UNIT_TEST_CACHE=-count=1 20 | else 21 | export NO_UNIT_TEST_CACHE= 22 | endif 23 | 24 | .DEFAULT_GOAL := help 25 | 26 | clean: # Clean up artifacts of the development environment to allow for untainted builds, installations and updates. 27 | go clean -i $(PACKAGE) 28 | go clean -i -race $(PACKAGE) 29 | .PHONY: clean 30 | 31 | editor: # Open our default IDE with the project's configuration. 32 | @# WORKAROUND VS.code does not call Delve with absolute paths to files which it needs to set breakpoints. Until either Delve or VS.code have a fix we need to disable "-trimpath" which converts absolute to relative paths of Go builds which is a requirement for reproducible builds. 33 | GOFLAGS="$(GOFLAGS) -trimpath=false" $(ROOT_DIR)/scripts/editor.sh 34 | .PHONY: editor 35 | 36 | help: # Show this help message. 37 | @grep -E '^[a-zA-Z-][a-zA-Z0-9.-]*?:.*?# (.+)' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?# "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 38 | .PHONY: help 39 | 40 | install: 41 | go install -v $(PACKAGE) 42 | .PHONY: install 43 | 44 | install-all: install install-tools-linting install-tools-testing 45 | .PHONY: install-all 46 | 47 | lint: 48 | go tool github.com/kisielk/errcheck ./... 49 | go vet ./... 50 | .PHONY: lint 51 | 52 | test: # [ 0 && data[i-1] == '\r' { 34 | ls = append(ls, uint(i)-1) 35 | } else { 36 | ls = append(ls, uint(i)) 37 | } 38 | data = data[i+1:] 39 | } 40 | 41 | return ls 42 | } 43 | 44 | // PrefixLines prefixes every non-empty line with the given prefix. 45 | func PrefixLines(data []byte, prefix []byte) (result []byte) { 46 | for l := range Split(data, '\n') { 47 | if len(result) > 0 { 48 | result = append(result, '\n') 49 | } 50 | if len(l) > 0 { 51 | result = append(result, prefix...) 52 | result = append(result, l...) 53 | } 54 | } 55 | 56 | return result 57 | } 58 | -------------------------------------------------------------------------------- /bytesutil/lines_test.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestLineLengthsForFile(t *testing.T) { 10 | assert.Equal( 11 | t, 12 | []uint{0, 1, 2, 1}, 13 | LineLengths([]byte("a\naa\r\na")), 14 | ) 15 | } 16 | 17 | func TestPrefixLines(t *testing.T) { 18 | type testCase struct { 19 | Name string 20 | 21 | Data []byte 22 | Prefix []byte 23 | 24 | ExpectedResult []byte 25 | } 26 | 27 | validate := func(t *testing.T, tc *testCase) { 28 | t.Run(tc.Name, func(t *testing.T) { 29 | tc.Data = TrimIndentations(tc.Data) 30 | tc.ExpectedResult = TrimIndentations(tc.ExpectedResult) 31 | 32 | actualResult := PrefixLines(tc.Data, tc.Prefix) 33 | 34 | assert.Equal(t, tc.ExpectedResult, actualResult) 35 | }) 36 | } 37 | 38 | validate(t, &testCase{ 39 | Name: "Prefix lines", 40 | 41 | Data: []byte(` 42 | a 43 | b 44 | c 45 | `), 46 | Prefix: []byte("- "), 47 | 48 | ExpectedResult: []byte(` 49 | - a 50 | - b 51 | - c 52 | `), 53 | }) 54 | 55 | validate(t, &testCase{ 56 | Name: "Prefix with empty lines", 57 | 58 | Data: []byte(` 59 | a 60 | 61 | b 62 | 63 | c 64 | `), 65 | Prefix: []byte("- "), 66 | 67 | ExpectedResult: []byte(` 68 | - a 69 | 70 | - b 71 | 72 | - c 73 | `), 74 | }) 75 | } 76 | -------------------------------------------------------------------------------- /bytesutil/markdown.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | 7 | pkgerrors "github.com/pkg/errors" 8 | "github.com/yuin/goldmark" 9 | ) 10 | 11 | // RenderMarkdownFileToHTMLFile reads in the Markdown file, renders it as HTML and writes that output in the HTML file. 12 | func RenderMarkdownFileToHTMLFile(markdownFilePath string, htmlFilePath string) (err error) { 13 | data, err := os.ReadFile(markdownFilePath) 14 | if err != nil { 15 | return pkgerrors.Wrap(err, markdownFilePath) 16 | } 17 | 18 | var html bytes.Buffer 19 | if err := goldmark.Convert(data, &html); err != nil { 20 | return pkgerrors.Wrap(err, markdownFilePath) 21 | } 22 | 23 | if err := os.WriteFile(htmlFilePath, html.Bytes(), 0640); err != nil { 24 | return pkgerrors.Wrap(err, html.String()) 25 | } 26 | 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /bytesutil/sort.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "sort" 5 | "strings" 6 | ) 7 | 8 | // SortLines splits the given string into lines, sorts them and then returns the sorted lines as a combined string. 9 | func SortLines(s string) (sorted string) { 10 | lines := strings.Split(s, "\n") 11 | 12 | sort.Strings(lines) 13 | 14 | return strings.Join(lines, "\n") 15 | } 16 | 17 | // SortLinesAndTrimSpace sorts the lines of the given string and removes all leading and trailing whitespaces. 18 | func SortLinesAndTrimSpace(s string) (sorted string) { 19 | s = SortLines(s) 20 | s = strings.TrimSpace(s) 21 | 22 | return s 23 | } 24 | -------------------------------------------------------------------------------- /bytesutil/sort_test.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestSortLines(t *testing.T) { 11 | type testCase struct { 12 | In string 13 | Out string 14 | } 15 | 16 | validate := func(name string, tc testCase) { 17 | assert.Equal(t, tc.Out, SortLines(tc.In)) 18 | } 19 | 20 | validate("One Line", testCase{ 21 | In: "abc", 22 | Out: "abc", 23 | }) 24 | validate("Unsorted Multiple Lines", testCase{ 25 | In: strings.TrimSpace(StringTrimIndentations(` 26 | c 27 | b 28 | a 29 | d 30 | `)), 31 | Out: strings.TrimSpace(StringTrimIndentations(` 32 | a 33 | b 34 | c 35 | d 36 | `)), 37 | }) 38 | validate("Sorted Multiple Lines", testCase{ 39 | In: strings.TrimSpace(StringTrimIndentations(` 40 | a 41 | b 42 | c 43 | d 44 | `)), 45 | Out: strings.TrimSpace(StringTrimIndentations(` 46 | a 47 | b 48 | c 49 | d 50 | `)), 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /bytesutil/splitter.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "bytes" 5 | ) 6 | 7 | // Split iteratively splits argument s and returns the split items over the returned channel. 8 | func Split(s []byte, sep byte) <-chan []byte { 9 | ch := make(chan []byte) 10 | 11 | go func() { 12 | for { 13 | end := bytes.IndexByte(s, sep) 14 | if end == -1 { 15 | end = len(s) 16 | } 17 | 18 | ch <- s[:end] 19 | 20 | if end == len(s) { 21 | break 22 | } 23 | 24 | s = s[end+1:] 25 | } 26 | 27 | close(ch) 28 | }() 29 | 30 | return ch 31 | } 32 | -------------------------------------------------------------------------------- /bytesutil/splitter_test.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestSplitter(t *testing.T) { 10 | type testCase struct { 11 | Data string 12 | Expected []string 13 | } 14 | 15 | validate := func(name string, tc testCase) { 16 | var got []string 17 | for line := range Split([]byte(tc.Data), '\n') { 18 | got = append(got, string(line)) 19 | } 20 | 21 | assert.Equal(t, tc.Expected, got, "Split %q", tc.Data) 22 | } 23 | 24 | validate("some items", testCase{ 25 | Data: "a\nb\nc", 26 | Expected: []string{ 27 | "a", 28 | "b", 29 | "c", 30 | }, 31 | }) 32 | 33 | validate("empty item at the beginning", testCase{ 34 | Data: "\nc", 35 | Expected: []string{ 36 | "", 37 | "c", 38 | }, 39 | }) 40 | 41 | validate("empty item at the end", testCase{ 42 | Data: "c\n", 43 | Expected: []string{ 44 | "c", 45 | "", 46 | }, 47 | }) 48 | 49 | validate("empty string", testCase{ 50 | Data: "", 51 | Expected: []string{ 52 | "", 53 | }, 54 | }) 55 | } 56 | -------------------------------------------------------------------------------- /bytesutil/synchronized.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "sync" 7 | ) 8 | 9 | // SynchronizedBuffer holds a concurrency-safe buffer. 10 | type SynchronizedBuffer struct { 11 | lock sync.Mutex 12 | 13 | b bytes.Buffer 14 | } 15 | 16 | // Bytes calls "Bytes" of "bytes.Buffer". 17 | func (b *SynchronizedBuffer) Bytes() []byte { 18 | b.lock.Lock() 19 | defer b.lock.Unlock() 20 | 21 | return b.b.Bytes() 22 | } 23 | 24 | // Cap calls "Cap" of "bytes.Buffer". 25 | func (b *SynchronizedBuffer) Cap() int { 26 | b.lock.Lock() 27 | defer b.lock.Unlock() 28 | 29 | return b.b.Cap() 30 | } 31 | 32 | // Grow calls "Grow" of "bytes.Buffer". 33 | func (b *SynchronizedBuffer) Grow(n int) { 34 | b.lock.Lock() 35 | defer b.lock.Unlock() 36 | 37 | b.b.Grow(n) 38 | } 39 | 40 | // Len calls "Len" of "bytes.Buffer". 41 | func (b *SynchronizedBuffer) Len() int { 42 | b.lock.Lock() 43 | defer b.lock.Unlock() 44 | 45 | return b.b.Len() 46 | } 47 | 48 | // Next calls "Next" of "bytes.Buffer". 49 | func (b *SynchronizedBuffer) Next(n int) []byte { 50 | b.lock.Lock() 51 | defer b.lock.Unlock() 52 | 53 | return b.b.Next(n) 54 | } 55 | 56 | // Read calls "Read" of "bytes.Buffer". 57 | func (b *SynchronizedBuffer) Read(p []byte) (n int, err error) { 58 | b.lock.Lock() 59 | defer b.lock.Unlock() 60 | 61 | return b.b.Read(p) 62 | } 63 | 64 | // ReadByte calls "ReadByte" of "bytes.Buffer". 65 | func (b *SynchronizedBuffer) ReadByte() (byte, error) { 66 | b.lock.Lock() 67 | defer b.lock.Unlock() 68 | 69 | return b.b.ReadByte() 70 | } 71 | 72 | // ReadBytes calls "ReadBytes" of "bytes.Buffer". 73 | func (b *SynchronizedBuffer) ReadBytes(delim byte) (line []byte, err error) { 74 | b.lock.Lock() 75 | defer b.lock.Unlock() 76 | 77 | return b.b.ReadBytes(delim) 78 | } 79 | 80 | // ReadFrom calls "ReadFrom" of "bytes.Buffer". 81 | func (b *SynchronizedBuffer) ReadFrom(r io.Reader) (n int64, err error) { 82 | b.lock.Lock() 83 | defer b.lock.Unlock() 84 | 85 | return b.b.ReadFrom(r) 86 | } 87 | 88 | // ReadRune calls "ReadRune" of "bytes.Buffer". 89 | func (b *SynchronizedBuffer) ReadRune() (r rune, size int, err error) { 90 | b.lock.Lock() 91 | defer b.lock.Unlock() 92 | 93 | return b.b.ReadRune() 94 | } 95 | 96 | // ReadString calls "ReadString" of "bytes.Buffer". 97 | func (b *SynchronizedBuffer) ReadString(delim byte) (line string, err error) { 98 | b.lock.Lock() 99 | defer b.lock.Unlock() 100 | 101 | return b.b.ReadString(delim) 102 | } 103 | 104 | // Reset calls "Reset" of "bytes.Buffer". 105 | func (b *SynchronizedBuffer) Reset() { 106 | b.lock.Lock() 107 | defer b.lock.Unlock() 108 | 109 | b.b.Reset() 110 | } 111 | 112 | // String calls "String" of "bytes.Buffer". 113 | func (b *SynchronizedBuffer) String() string { 114 | b.lock.Lock() 115 | defer b.lock.Unlock() 116 | 117 | return b.b.String() 118 | } 119 | 120 | // Truncate calls "Truncate" of "bytes.Buffer". 121 | func (b *SynchronizedBuffer) Truncate(n int) { 122 | b.lock.Lock() 123 | defer b.lock.Unlock() 124 | 125 | b.b.Truncate(n) 126 | } 127 | 128 | // UnreadByte calls "UnreadByte" of "bytes.Buffer". 129 | func (b *SynchronizedBuffer) UnreadByte() error { 130 | b.lock.Lock() 131 | defer b.lock.Unlock() 132 | 133 | return b.b.UnreadByte() 134 | } 135 | 136 | // UnreadRune calls "UnreadRune" of "bytes.Buffer". 137 | func (b *SynchronizedBuffer) UnreadRune() error { 138 | b.lock.Lock() 139 | defer b.lock.Unlock() 140 | 141 | return b.b.UnreadRune() 142 | } 143 | 144 | // Write calls "Write" of "bytes.Buffer". 145 | func (b *SynchronizedBuffer) Write(p []byte) (n int, err error) { 146 | b.lock.Lock() 147 | defer b.lock.Unlock() 148 | 149 | return b.b.Write(p) 150 | } 151 | 152 | // WriteByte calls "WriteByte" of "bytes.Buffer". 153 | func (b *SynchronizedBuffer) WriteByte(c byte) error { 154 | b.lock.Lock() 155 | defer b.lock.Unlock() 156 | 157 | return b.b.WriteByte(c) 158 | } 159 | 160 | // WriteRune calls "WriteRune" of "bytes.Buffer". 161 | func (b *SynchronizedBuffer) WriteRune(r rune) (n int, err error) { 162 | b.lock.Lock() 163 | defer b.lock.Unlock() 164 | 165 | return b.b.WriteRune(r) 166 | } 167 | 168 | // WriteString calls "WriteString" of "bytes.Buffer". 169 | func (b *SynchronizedBuffer) WriteString(s string) (n int, err error) { 170 | b.lock.Lock() 171 | defer b.lock.Unlock() 172 | 173 | return b.b.WriteString(s) 174 | } 175 | 176 | // WriteTo calls "WriteTo" of "bytes.Buffer". 177 | func (b *SynchronizedBuffer) WriteTo(w io.Writer) (n int64, err error) { 178 | b.lock.Lock() 179 | defer b.lock.Unlock() 180 | 181 | return b.b.WriteTo(w) 182 | } 183 | -------------------------------------------------------------------------------- /bytesutil/text.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "crypto/sha256" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "log" 11 | "os" 12 | "regexp" 13 | "strings" 14 | 15 | pkgerrors "github.com/pkg/errors" 16 | "github.com/zimmski/osutil" 17 | ) 18 | 19 | var errTrimIndentationsLastLine = errors.New("last line of input must be indented exactly one less than all other lines") 20 | var errTrimIndentationsMissingStartingNewline = errors.New("input must start with a newline") 21 | 22 | // TrimIndentations removes indentations that were added for a cleaner code style 23 | func TrimIndentations(s []byte) []byte { 24 | if len(s) == 0 { 25 | return s 26 | } 27 | 28 | // Check if the beginning starts with a new line 29 | if s[0] != '\n' { 30 | panic(errTrimIndentationsMissingStartingNewline) 31 | } 32 | 33 | i := 1 34 | for ; i < len(s) && s[i] == '\n'; i++ { 35 | } 36 | 37 | indentationCount := 0 38 | for ; i < len(s); i++ { 39 | if s[i] != '\t' { 40 | break 41 | } 42 | 43 | indentationCount++ 44 | } 45 | 46 | // Check if there is an indentation at all 47 | if indentationCount == 0 { 48 | // If there is no indentation we simply remove the first character from the string, which is a new line character to fullfil the convention. 49 | return s[1:] 50 | } 51 | 52 | // Check if the last line has exactly one less indentation 53 | if s[len(s)-indentationCount] != '\n' { 54 | panic(errTrimIndentationsLastLine) 55 | } 56 | for i = len(s) - indentationCount + 1; i < len(s); i++ { 57 | if s[i] != '\t' { 58 | panic(errTrimIndentationsLastLine) 59 | } 60 | } 61 | 62 | var b bytes.Buffer 63 | 64 | // Only use the data beginning from the second line to the second to last line 65 | lineNumber := 0 66 | for line := range Split(s[1:len(s)-indentationCount], '\n') { 67 | lineNumber++ 68 | if len(line) == 0 { 69 | b.WriteByte('\n') 70 | 71 | continue 72 | } 73 | 74 | // Check if there is enough indentation for each line 75 | if len(line) < indentationCount { 76 | panic(fmt.Errorf("missing indentation in line %v", i+1)) 77 | } 78 | 79 | b.Write(line[indentationCount:]) 80 | b.WriteByte('\n') 81 | } 82 | 83 | return b.Bytes() 84 | } 85 | 86 | // StringTrimIndentations removes indentations that were added for a cleaner code style 87 | func StringTrimIndentations(s string) string { 88 | return string(TrimIndentations([]byte(s))) 89 | } 90 | 91 | // PrefixContinuationLinesWith indents every line except the first one with the given amount of whitespace. 92 | func PrefixContinuationLinesWith(paragraph string, prefix string) string { 93 | endsWithNewLine := strings.HasSuffix(paragraph, "\n") 94 | paragraph = strings.ReplaceAll(paragraph, "\n", "\n"+prefix) 95 | if endsWithNewLine { 96 | paragraph = strings.TrimSuffix(paragraph, prefix) 97 | } 98 | 99 | return paragraph 100 | } 101 | 102 | // RemoveLine looks up every line with the given search string in the input and returns an output removing all the selected lines. 103 | func RemoveLine(in string, search string) (out string) { 104 | return regexp.MustCompile(`(?m)[\r\n]+^.*`+search+`.*$`).ReplaceAllString(in, "") 105 | } 106 | 107 | // WordAfterFirstMatch returns the next word in the string after the given substring, or the empty string if it does not exist. 108 | func WordAfterFirstMatch(str string, substring string) string { 109 | separator := " " 110 | substring = substring + separator 111 | offset := strings.Index(str, substring) 112 | if offset < 0 { 113 | return "" 114 | } 115 | 116 | word := str[offset+len(substring):] 117 | remainder := strings.Index(word, separator) 118 | if remainder < 0 { 119 | return word 120 | } 121 | 122 | return word[:remainder] 123 | } 124 | 125 | var rewriteWebsiteContentURLReplace = regexp.MustCompile(`(action|href|poster|src)="(/.*?)"`) 126 | var rewriteWebsiteContentURLReplaceSingleQuote = regexp.MustCompile(`(action|href|poster|src)='(/.*?)'`) 127 | var rewriteWebsiteContentJSURLReplace = regexp.MustCompile(`(url\()'(/.*?)'`) 128 | 129 | // RewriteWebsiteContent replaces all URLs and URIs to the given ones, and gives all URLs and URIs also a hash so they invalidate their cache when their content changes. 130 | func RewriteWebsiteContent(data string, defaultURL string, url string, uriPrefix string, fileHashes map[string]string) (dataReplaced string) { 131 | // Rewrite URIs and URLs to use the correct schema, domain and path prefix. 132 | hasNonDefaultURI := uriPrefix != "" && uriPrefix != "/" 133 | if hasNonDefaultURI { 134 | uriPrefix = strings.TrimSuffix(uriPrefix, "/") 135 | 136 | data = rewriteWebsiteContentURLReplace.ReplaceAllString(data, "${1}=\""+uriPrefix+"${2}\"") 137 | data = rewriteWebsiteContentURLReplaceSingleQuote.ReplaceAllString(data, "${1}=\""+uriPrefix+"${2}\"") 138 | data = rewriteWebsiteContentJSURLReplace.ReplaceAllString(data, "${1}'"+uriPrefix+"${2}'") 139 | } 140 | if url != defaultURL { 141 | data = strings.ReplaceAll(data, defaultURL, strings.TrimSuffix(url, "/")) 142 | } 143 | 144 | // Rewrite URIs and URLs to have a fingerprint so we do not hit the cache if the content has changed. 145 | data = rewriteWebsiteContentURLReplace.ReplaceAllStringFunc(data, func(match string) string { 146 | m := rewriteWebsiteContentURLReplace.FindStringSubmatch(match) 147 | 148 | p := m[2] 149 | if hasNonDefaultURI { 150 | p = strings.TrimPrefix(p, uriPrefix) 151 | } 152 | 153 | hash, ok := fileHashes[p] 154 | if !ok || hash == "" { 155 | return match 156 | } 157 | 158 | return m[1] + "=\"" + m[2] + "?" + hash[:6] + "\"" 159 | }) 160 | data = rewriteWebsiteContentURLReplaceSingleQuote.ReplaceAllStringFunc(data, func(match string) string { 161 | m := rewriteWebsiteContentURLReplaceSingleQuote.FindStringSubmatch(match) 162 | 163 | p := m[2] 164 | if hasNonDefaultURI { 165 | p = strings.TrimPrefix(p, uriPrefix) 166 | } 167 | 168 | hash, ok := fileHashes[p] 169 | if !ok || hash == "" { 170 | return match 171 | } 172 | 173 | return m[1] + "=\"" + m[2] + "?" + hash[:6] + "\"" 174 | }) 175 | data = rewriteWebsiteContentJSURLReplace.ReplaceAllStringFunc(data, func(match string) string { 176 | m := rewriteWebsiteContentJSURLReplace.FindStringSubmatch(match) 177 | 178 | p := m[2] 179 | if hasNonDefaultURI { 180 | p = strings.TrimPrefix(p, uriPrefix) 181 | } 182 | 183 | hash, ok := fileHashes[p] 184 | if !ok || hash == "" { 185 | return match 186 | } 187 | 188 | return m[1] + "'" + m[2] + "?" + hash[:6] + "'" 189 | }) 190 | 191 | return data 192 | } 193 | 194 | // RewriteWebsiteContentDirectory replaces all URLs and URIs to the given ones, and gives all URLs and URIs also a hash so they invalidate their cache when their content changes. 195 | func RewriteWebsiteContentDirectory(contentDirectoryPath string, defaultURL string, url string, uriPrefix string, staticFiles map[string]*osutil.StaticFile) (err error) { 196 | rewriteURIPrefix := uriPrefix != "" && uriPrefix != "/" 197 | rewriteURL := url != defaultURL 198 | 199 | if !rewriteURIPrefix && !rewriteURL { 200 | return nil 201 | } 202 | 203 | log.Printf("Rewriting %s directory", contentDirectoryPath) 204 | 205 | var b2 bytes.Buffer 206 | hash := sha256.New() 207 | 208 | fileHashes := make(map[string]string, len(staticFiles)) 209 | for filePath, file := range staticFiles { 210 | fileHashes[filePath] = file.Hash 211 | } 212 | 213 | for filePath, file := range staticFiles { 214 | if file.Directory || (!strings.HasSuffix(filePath, ".html") && !strings.HasSuffix(filePath, ".xml") && !strings.HasSuffix(filePath, ".css") && !strings.HasSuffix(filePath, "/robots.txt")) { 215 | continue 216 | } 217 | 218 | data := file.Data 219 | 220 | // Is the file compressed? 221 | if file.Size != 0 { 222 | var b bytes.Buffer 223 | 224 | reader, err := gzip.NewReader(strings.NewReader(data)) 225 | if err != nil { 226 | return err 227 | } 228 | 229 | _, err = b.ReadFrom(reader) 230 | if err != nil { 231 | return err 232 | } 233 | 234 | data = b.String() 235 | } 236 | 237 | dataOriginal := data 238 | data = RewriteWebsiteContent(data, defaultURL, url, uriPrefix, fileHashes) 239 | 240 | if data != dataOriginal { 241 | compressedWriter, _ := gzip.NewWriterLevel(&b2, gzip.BestCompression) 242 | writer := io.MultiWriter(compressedWriter, hash) 243 | if _, err := writer.Write([]byte(data)); err != nil { 244 | return err 245 | } 246 | if err := compressedWriter.Close(); err != nil { 247 | return err 248 | } 249 | 250 | // Should the file be saved uncompressed? 251 | if file.Size == 0 { 252 | file.Data = data 253 | } else { 254 | file.Data = b2.String() 255 | file.Size = len(data) 256 | } 257 | file.Hash = fmt.Sprintf("%x", hash.Sum(nil)) 258 | 259 | staticFiles[filePath] = file 260 | fileHashes[filePath] = file.Hash 261 | 262 | log.Printf("File %s was rewritten", filePath) 263 | 264 | b2.Reset() 265 | hash.Reset() 266 | } 267 | } 268 | 269 | return nil 270 | } 271 | 272 | // SearchAndReplaceFile searches for occurrences of a given pattern in a file and replaces them accordingly. 273 | // Capturing groups can be referenced in the replace string by using $, i.e. $1 is the first capturing group. 274 | func SearchAndReplaceFile(filePath string, search *regexp.Regexp, replace string) (err error) { 275 | content, err := os.ReadFile(filePath) 276 | if err != nil { 277 | return pkgerrors.Wrap(err, filePath) 278 | } 279 | info, err := os.Stat(filePath) 280 | if err != nil { 281 | return pkgerrors.Wrap(err, filePath) 282 | } 283 | 284 | replaced := search.ReplaceAllString(string(content), replace) 285 | 286 | if err := os.WriteFile(filePath, []byte(replaced), info.Mode().Perm()); err != nil { 287 | return pkgerrors.Wrap(err, filePath) 288 | } 289 | 290 | return nil 291 | } 292 | 293 | // whiteSpaceRe matches only whitespace content. 294 | var whiteSpaceRe = regexp.MustCompile(`^[\s\t\n]*$`) 295 | 296 | // IsWhitespace checks if the given string consists of only whitespace. 297 | func IsWhitespace(data string) (isWhitespace bool) { 298 | return whiteSpaceRe.MatchString(data) 299 | } 300 | 301 | // GuardedBlocks extracts blocks of consecutive lines that are guarded by the given begin and end lines. 302 | // The guarding lines are included in the results. If no end guard is given, the start guard is used as end guard as well. 303 | func GuardedBlocks(data string, begin *regexp.Regexp, end *regexp.Regexp) (blocks []string) { 304 | if end == nil { 305 | end = begin 306 | } 307 | 308 | var block strings.Builder 309 | inBlock := false 310 | lines := strings.Split(data, "\n") 311 | for i, line := range lines { 312 | if begin.MatchString(line) && !inBlock { 313 | inBlock = true 314 | 315 | block.WriteString(line) 316 | block.WriteString("\n") 317 | } else if end.MatchString(line) && inBlock { 318 | inBlock = false 319 | 320 | block.WriteString(line) 321 | if i != len(lines)-1 { 322 | block.WriteString("\n") 323 | } 324 | 325 | blocks = append(blocks, block.String()) 326 | block = strings.Builder{} 327 | } else if inBlock { 328 | block.WriteString(line) 329 | block.WriteString("\n") 330 | } 331 | } 332 | 333 | return blocks 334 | } 335 | 336 | // Itemize itemizes a collection with the marker passed as parameter. 337 | func Itemize[S fmt.Stringer](collection []S, marker string) string { 338 | var sb strings.Builder 339 | 340 | for i, item := range collection { 341 | sb.WriteString(marker) 342 | sb.WriteString(" ") 343 | sb.WriteString(item.String()) 344 | if i < len(collection)-1 { 345 | sb.WriteString("\n") 346 | } 347 | } 348 | 349 | return sb.String() 350 | } 351 | -------------------------------------------------------------------------------- /bytesutil/text_test.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestRemoveLine(t *testing.T) { 13 | type testCase struct { 14 | Name string 15 | 16 | In string 17 | Search string 18 | 19 | ExpectedOut string 20 | } 21 | 22 | validate := func(t *testing.T, tc *testCase) { 23 | t.Run(tc.Name, func(t *testing.T) { 24 | actualOut := RemoveLine(StringTrimIndentations(tc.In), tc.Search) 25 | 26 | assert.Equal(t, StringTrimIndentations(tc.ExpectedOut), actualOut) 27 | }) 28 | } 29 | 30 | validate(t, &testCase{ 31 | Name: "Empty string", 32 | }) 33 | 34 | validate(t, &testCase{ 35 | Name: "No match", 36 | 37 | In: ` 38 | a 39 | b 40 | c 41 | `, 42 | Search: "d", 43 | 44 | ExpectedOut: ` 45 | a 46 | b 47 | c 48 | `, 49 | }) 50 | 51 | validate(t, &testCase{ 52 | Name: "Multiple matches", 53 | 54 | In: ` 55 | a 56 | b 57 | d 58 | c 59 | d 60 | `, 61 | Search: "d", 62 | 63 | ExpectedOut: ` 64 | a 65 | b 66 | c 67 | `, 68 | }) 69 | } 70 | 71 | func TestTrimIndentations(t *testing.T) { 72 | type testCase struct { 73 | Data string 74 | Expected string 75 | Error error 76 | } 77 | 78 | validate := func(name string, tc testCase) { 79 | defer func() { 80 | assert.Equal(t, tc.Error, recover()) 81 | }() 82 | 83 | assert.Equal(t, tc.Expected, string(TrimIndentations([]byte(tc.Data)))) 84 | } 85 | 86 | validate("normal source code indentation", testCase{ 87 | Data: ` 88 | this line gives the indentation for the rest of the data 89 | this will be trimmed 90 | 91 | above is an empty line 92 | one more indentation here 93 | below is the last line which has one less indentation 94 | `, 95 | Expected: `this line gives the indentation for the rest of the data 96 | this will be trimmed 97 | 98 | above is an empty line 99 | one more indentation here 100 | below is the last line which has one less indentation 101 | `, 102 | }) 103 | 104 | validate("start with blank lines", testCase{ 105 | Data: ` 106 | 107 | blank line above 108 | still valid 109 | `, 110 | Expected: ` 111 | blank line above 112 | still valid 113 | `, 114 | }) 115 | 116 | validate("ignore the content if it is not formatted to our convention", testCase{ 117 | Data: `does not matter`, 118 | Expected: `does not matter`, 119 | Error: errTrimIndentationsMissingStartingNewline, 120 | }) 121 | 122 | validate("if there is not indentation at all we do not trim anything except the first new line character", testCase{ 123 | Data: ` 124 | does not matter`, 125 | Expected: `does not matter`, 126 | }) 127 | 128 | validate("empty", testCase{ 129 | Data: ``, 130 | Expected: ``, 131 | }) 132 | } 133 | 134 | func TestWordAfterFirstMatch(t *testing.T) { 135 | type testCase struct { 136 | Name string 137 | 138 | Str string 139 | Substring string 140 | 141 | ExpectedString string 142 | } 143 | 144 | validate := func(t *testing.T, tc *testCase) { 145 | t.Run(tc.Name, func(t *testing.T) { 146 | actualString := WordAfterFirstMatch(tc.Str, tc.Substring) 147 | 148 | assert.Equal(t, tc.ExpectedString, actualString) 149 | }) 150 | } 151 | 152 | t.Run("Word exists", func(t *testing.T) { 153 | validate(t, &testCase{ 154 | Name: "Word comes last", 155 | 156 | Str: "We love Symflower", 157 | Substring: "love", 158 | 159 | ExpectedString: "Symflower", 160 | }) 161 | validate(t, &testCase{ 162 | Name: "Word in the middle", 163 | 164 | Str: "We love Symflower a lot", 165 | Substring: "love", 166 | 167 | ExpectedString: "Symflower", 168 | }) 169 | }) 170 | 171 | t.Run("Word does not exist", func(t *testing.T) { 172 | validate(t, &testCase{ 173 | Name: "Not a substring", 174 | 175 | Str: "We love Symflower", 176 | Substring: "abc", 177 | 178 | ExpectedString: "", 179 | }) 180 | validate(t, &testCase{ 181 | Name: "No subsequent word", 182 | 183 | Str: "We love Symflower", 184 | Substring: "Symflower", 185 | 186 | ExpectedString: "", 187 | }) 188 | }) 189 | 190 | validate(t, &testCase{ 191 | Name: "Empty substring", 192 | 193 | Str: "We love Symflower", 194 | Substring: "", 195 | 196 | ExpectedString: "love", 197 | }) 198 | } 199 | 200 | func TestRewriteWebsiteContent(t *testing.T) { 201 | type testCase struct { 202 | Name string 203 | 204 | Data string 205 | DefaultURL string 206 | URL string 207 | URIPrefix string 208 | FileHashes map[string]string 209 | 210 | ExpectedDataReplaced string 211 | } 212 | 213 | validate := func(t *testing.T, tc *testCase) { 214 | t.Run(tc.Name, func(t *testing.T) { 215 | actualDataReplaced := RewriteWebsiteContent(tc.Data, tc.DefaultURL, tc.URL, tc.URIPrefix, tc.FileHashes) 216 | 217 | assert.Equal(t, tc.ExpectedDataReplaced, actualDataReplaced) 218 | }) 219 | } 220 | 221 | validate(t, &testCase{ 222 | Name: "Non-default URL and default URI prefix", 223 | 224 | Data: StringTrimIndentations(` 225 | http://symflower-website/en/ 226 | `), 227 | DefaultURL: "http://symflower-website", 228 | URL: "https://symflower.com/", 229 | URIPrefix: "/", 230 | FileHashes: map[string]string{}, 231 | 232 | ExpectedDataReplaced: StringTrimIndentations(` 233 | https://symflower.com/en/ 234 | `), 235 | }) 236 | 237 | validate(t, &testCase{ 238 | Name: "Non-default URL and non-default URI prefix", 239 | 240 | Data: StringTrimIndentations(` 241 | http://symflower-website/en/ 242 | `), 243 | DefaultURL: "http://symflower-website", 244 | URL: "https://symflower.com/", 245 | URIPrefix: "/foobar/", 246 | FileHashes: map[string]string{}, 247 | 248 | ExpectedDataReplaced: StringTrimIndentations(` 249 | https://symflower.com/en/ 250 | `), 251 | }) 252 | } 253 | 254 | func TestGuardedBlock(t *testing.T) { 255 | type testCase struct { 256 | Name string 257 | 258 | Data string 259 | Begin string 260 | End string 261 | 262 | ExpectedBlocks []string 263 | } 264 | 265 | validate := func(t *testing.T, tc *testCase) { 266 | t.Run(tc.Name, func(t *testing.T) { 267 | beginRe, err := regexp.Compile(tc.Begin) 268 | require.NoError(t, err) 269 | var endRe *regexp.Regexp 270 | if tc.End != "" { 271 | endRe, err = regexp.Compile(tc.End) 272 | require.NoError(t, err) 273 | } 274 | data := tc.Data 275 | if strings.HasPrefix(data, "\n") { 276 | data = StringTrimIndentations(tc.Data) 277 | } 278 | 279 | actualBlocks := GuardedBlocks(data, beginRe, endRe) 280 | 281 | assert.Equal(t, tc.ExpectedBlocks, actualBlocks) 282 | }) 283 | } 284 | 285 | validate(t, &testCase{ 286 | Name: "No Block", 287 | 288 | Data: ` 289 | DATA 290 | `, 291 | Begin: "begin", 292 | End: "end", 293 | 294 | ExpectedBlocks: nil, 295 | }) 296 | 297 | validate(t, &testCase{ 298 | Name: "Identic Start and End Guards", 299 | 300 | Data: ` 301 | begin 302 | DATA 303 | begin 304 | `, 305 | Begin: "begin", 306 | 307 | ExpectedBlocks: []string{ 308 | "begin\nDATA\nbegin\n", 309 | }, 310 | }) 311 | 312 | validate(t, &testCase{ 313 | Name: "Different Start and End Guards", 314 | 315 | Data: ` 316 | begin 317 | DATA 318 | end 319 | `, 320 | Begin: "begin", 321 | End: "end", 322 | 323 | ExpectedBlocks: []string{ 324 | "begin\nDATA\nend\n", 325 | }, 326 | }) 327 | 328 | validate(t, &testCase{ 329 | Name: "Multiple Blocks", 330 | 331 | Data: ` 332 | begin 333 | DATA1 334 | end 335 | 336 | begin 337 | DATA2 338 | end 339 | `, 340 | Begin: "begin", 341 | End: "end", 342 | 343 | ExpectedBlocks: []string{ 344 | "begin\nDATA1\nend\n", 345 | "begin\nDATA2\nend\n", 346 | }, 347 | }) 348 | 349 | validate(t, &testCase{ 350 | Name: "Unopened Block", 351 | 352 | Data: ` 353 | DATA1 354 | end 355 | 356 | begin 357 | DATA2 358 | end 359 | `, 360 | Begin: "begin", 361 | End: "end", 362 | 363 | ExpectedBlocks: []string{ 364 | "begin\nDATA2\nend\n", 365 | }, 366 | }) 367 | 368 | validate(t, &testCase{ 369 | Name: "Unclosed Block", 370 | 371 | Data: ` 372 | begin 373 | DATA1 374 | end 375 | 376 | begin 377 | DATA2 378 | `, 379 | Begin: "begin", 380 | End: "end", 381 | 382 | ExpectedBlocks: []string{ 383 | "begin\nDATA1\nend\n", 384 | }, 385 | }) 386 | 387 | validate(t, &testCase{ 388 | Name: "Duplicated Begin Guard", 389 | 390 | Data: ` 391 | begin 392 | begin 393 | DATA 394 | end 395 | `, 396 | Begin: "begin", 397 | End: "end", 398 | 399 | ExpectedBlocks: []string{ 400 | "begin\nbegin\nDATA\nend\n", 401 | }, 402 | }) 403 | 404 | validate(t, &testCase{ 405 | Name: "Duplicated End Guard", 406 | 407 | Data: ` 408 | begin 409 | DATA 410 | end 411 | end 412 | `, 413 | Begin: "begin", 414 | End: "end", 415 | 416 | ExpectedBlocks: []string{ 417 | "begin\nDATA\nend\n", 418 | }, 419 | }) 420 | 421 | validate(t, &testCase{ 422 | Name: "No final Newline", 423 | 424 | Data: "begin\nDATA\nbegin", 425 | Begin: "begin", 426 | 427 | ExpectedBlocks: []string{ 428 | "begin\nDATA\nbegin", 429 | }, 430 | }) 431 | } 432 | -------------------------------------------------------------------------------- /cache.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "encoding/gob" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | // CacheObjectType defines a unique type for a cache object. 15 | // This type should be descriptive to the object that is cached. 16 | type CacheObjectType string 17 | 18 | // CacheObjectWrite write data with the given unique identifier and type to the cache. 19 | // The meta data will be written as a human-readable data to identify the cache object. 20 | func CacheObjectWrite(cachePath string, identifier string, cacheObjectType CacheObjectType, data any, meta map[string]string) (err error) { 21 | cacheObjectPathRelative := cacheObjectPath(identifier) 22 | cacheObjectPath := filepath.Join(cachePath, cacheObjectPathRelative) 23 | 24 | if err := os.MkdirAll(cacheObjectPath, 0755); err != nil { 25 | return err 26 | } 27 | 28 | dataFile, err := os.Create(filepath.Join(cacheObjectPath, string(cacheObjectType)+".gob")) 29 | if err != nil { 30 | return err 31 | } 32 | defer func() { 33 | if e := dataFile.Close(); e != nil { 34 | if err == nil { 35 | err = e 36 | } else { 37 | err = errors.Join(err, e) 38 | } 39 | } 40 | }() 41 | if err := gob.NewEncoder(dataFile).Encode(data); err != nil { 42 | return err 43 | } 44 | if err := dataFile.Sync(); err != nil { 45 | return err 46 | } 47 | 48 | metaFile, err := os.Create(filepath.Join(cacheObjectPath, string(cacheObjectType)+".json")) 49 | if err != nil { 50 | return err 51 | } 52 | defer func() { 53 | if e := metaFile.Close(); e != nil { 54 | if err == nil { 55 | err = e 56 | } else { 57 | err = errors.Join(err, e) 58 | } 59 | } 60 | }() 61 | if err := json.NewEncoder(metaFile).Encode(meta); err != nil { 62 | return err 63 | } 64 | if err := metaFile.Sync(); err != nil { 65 | return err 66 | } 67 | 68 | return nil 69 | } 70 | 71 | // CacheObjectRead reads data with the given unique identifier and type from the cache. 72 | func CacheObjectRead(cachePath string, identifier string, cacheObjectType CacheObjectType, data any) (exists bool, err error) { 73 | cacheObjectPathRelative := cacheObjectPath(identifier) 74 | cacheObjectPath := filepath.Join(cachePath, cacheObjectPathRelative) 75 | 76 | raw, err := os.ReadFile(filepath.Join(cacheObjectPath, string(cacheObjectType)+".gob")) 77 | if err != nil { 78 | return false, nil 79 | } 80 | 81 | if err := gob.NewDecoder(bytes.NewReader(raw)).Decode(data); err != nil { 82 | return false, err 83 | } 84 | 85 | return true, nil 86 | } 87 | 88 | // cacheObjectPath returns the relative object path for the given identifier. 89 | func cacheObjectPath(identifier string) (cacheObjectPathRelative string) { 90 | checksum := fmt.Sprintf("%x", sha256.Sum256([]byte(identifier))) 91 | 92 | return filepath.Join(checksum[0:2], checksum[2:]) 93 | } 94 | -------------------------------------------------------------------------------- /cache_test.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestCacheObject(t *testing.T) { 10 | temporaryPath := t.TempDir() 11 | 12 | dataToBeCached := map[string]string{ 13 | "A": "1", 14 | "B": "2", 15 | } 16 | identifier := "some-identifier" 17 | typ := CacheObjectType("some-type") 18 | 19 | { 20 | var dataToBeRead map[string]string 21 | exists, err := CacheObjectRead(temporaryPath, identifier, typ, &dataToBeRead) 22 | assert.NoError(t, err) 23 | assert.False(t, exists) 24 | } 25 | 26 | assert.NoError(t, CacheObjectWrite(temporaryPath, identifier, typ, dataToBeCached, nil)) 27 | 28 | { 29 | var dataToBeRead map[string]string 30 | exists, err := CacheObjectRead(temporaryPath, identifier, typ, &dataToBeRead) 31 | assert.NoError(t, err) 32 | assert.True(t, exists) 33 | assert.Equal(t, dataToBeCached, dataToBeRead) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /capture_linux.go: -------------------------------------------------------------------------------- 1 | //go:build cgo 2 | 3 | package osutil 4 | 5 | /* 6 | #include 7 | #include 8 | */ 9 | import "C" 10 | 11 | import ( 12 | "bytes" 13 | "io" 14 | "os" 15 | "os/signal" 16 | "sync" 17 | "syscall" 18 | ) 19 | 20 | var lockStdFileDescriptorsSwapping sync.Mutex 21 | var lockStdFileWithCGoDescriptorsSwapping sync.Mutex 22 | 23 | // Capture captures stderr and stdout of a given function call. 24 | func Capture(call func()) (output []byte, err error) { 25 | lockStdFileDescriptorsSwapping.Lock() 26 | 27 | r, w, err := os.Pipe() 28 | if err != nil { 29 | return nil, err 30 | } 31 | defer func() { 32 | e := r.Close() 33 | if e != nil { 34 | err = e 35 | } 36 | if w != nil { 37 | e = w.Close() 38 | if err != nil { 39 | err = e 40 | } 41 | } 42 | }() 43 | 44 | originalStdErr, originalStdOut := os.Stderr, os.Stdout 45 | release := func() { 46 | os.Stderr, os.Stdout = originalStdErr, originalStdOut 47 | } 48 | defer func() { 49 | lockStdFileDescriptorsSwapping.Lock() 50 | 51 | if w != nil { 52 | release() 53 | } 54 | 55 | lockStdFileDescriptorsSwapping.Unlock() 56 | }() 57 | os.Stderr, os.Stdout = w, w 58 | 59 | out := make(chan []byte) 60 | go func() { 61 | defer func() { 62 | // If there is a panic in the function call, copying from "r" does not work anymore. 63 | _ = recover() 64 | }() 65 | 66 | var b bytes.Buffer 67 | 68 | _, err := io.Copy(&b, r) 69 | if err != nil { 70 | panic(err) 71 | } 72 | 73 | out <- b.Bytes() 74 | }() 75 | 76 | lockStdFileDescriptorsSwapping.Unlock() 77 | 78 | call() 79 | 80 | lockStdFileDescriptorsSwapping.Lock() 81 | 82 | err = w.Close() 83 | if err != nil { 84 | lockStdFileDescriptorsSwapping.Unlock() 85 | 86 | return nil, err 87 | } 88 | w = nil 89 | 90 | release() 91 | 92 | lockStdFileDescriptorsSwapping.Unlock() 93 | 94 | return <-out, err 95 | } 96 | 97 | // CaptureWithCGo captures stderr and stdout as well as stderr and stdout of C of a given function call. 98 | // Currently this function cannot be nested. 99 | func CaptureWithCGo(call func()) (output []byte, err error) { 100 | // FIXME At the moment this function does not work with nested calls (recursively). This might be because of the signal handler or the way we clone the file descriptors. I really do not know. Since we do not need recursive calls right now we can postpone this for later. https://$INTERNAL/symflower/symflower/-/issues/85 101 | lockStdFileWithCGoDescriptorsSwapping.Lock() 102 | defer lockStdFileWithCGoDescriptorsSwapping.Unlock() 103 | 104 | lockStdFileDescriptorsSwapping.Lock() 105 | 106 | r, w, err := os.Pipe() 107 | if err != nil { 108 | return nil, err 109 | } 110 | defer func() { 111 | e := r.Close() 112 | if e != nil { 113 | err = e 114 | } 115 | if w != nil { 116 | e = w.Close() 117 | if err != nil { 118 | err = e 119 | } 120 | } 121 | }() 122 | 123 | originalStdout, err := syscall.Dup(syscall.Stdout) 124 | if err != nil { 125 | lockStdFileDescriptorsSwapping.Unlock() 126 | 127 | return nil, err 128 | } 129 | originalStderr, err := syscall.Dup(syscall.Stderr) 130 | if err != nil { 131 | lockStdFileDescriptorsSwapping.Unlock() 132 | 133 | return nil, err 134 | } 135 | release := func() { 136 | if e := syscall.Dup2(originalStdout, syscall.Stdout); e != nil { 137 | err = e 138 | } 139 | if e := syscall.Close(originalStdout); e != nil { 140 | err = e 141 | } 142 | if e := syscall.Dup2(originalStderr, syscall.Stderr); e != nil { 143 | err = e 144 | } 145 | if e := syscall.Close(originalStderr); e != nil { 146 | err = e 147 | } 148 | } 149 | defer func() { 150 | lockStdFileDescriptorsSwapping.Lock() 151 | 152 | if w != nil { 153 | release() 154 | } 155 | 156 | lockStdFileDescriptorsSwapping.Unlock() 157 | }() 158 | 159 | // WORKAROUND Since Go 1.10 `go test` can hang (randomly) if a subprocess has another subprocess that got the original STDOUT/STDERR if the defined STDOUT/STDERR are not a file from the Go side. This is a somewhat incomplete description of this bug. More details can be found here https://github.com/golang/go/issues/24050 and here https://github.com/golang/go/issues/23019. This bug occurs randomly not just with `go test` but anywhere the same APIs are used. However, the only time this happens with the current function is when a parent process kills the currently running process but not when we have, e.g. a panic in the call we are capturing. This is already handled by the defer calls. Since the exiting is not handled, we have to set up a signal handler to take care of the cleanup. 160 | 161 | exitSignalHandler := make(chan bool) 162 | sigs := make(chan os.Signal, 10) 163 | signal.Notify(sigs, syscall.SIGCHLD) 164 | defer func() { 165 | signal.Stop(sigs) 166 | exitSignalHandler <- true 167 | }() 168 | 169 | go func() { 170 | select { 171 | case <-sigs: 172 | _ = syscall.Close(originalStdout) 173 | _ = syscall.Close(originalStderr) 174 | case <-exitSignalHandler: 175 | } 176 | }() 177 | 178 | if e := syscall.Dup2(int(w.Fd()), syscall.Stdout); e != nil { 179 | lockStdFileDescriptorsSwapping.Unlock() 180 | 181 | return nil, e 182 | } 183 | if e := syscall.Dup2(int(w.Fd()), syscall.Stderr); e != nil { 184 | lockStdFileDescriptorsSwapping.Unlock() 185 | 186 | return nil, e 187 | } 188 | 189 | out := make(chan []byte) 190 | go func() { 191 | defer func() { 192 | // If there is a panic in the function call, copying from "r" does not work anymore. 193 | _ = recover() 194 | }() 195 | 196 | var b bytes.Buffer 197 | 198 | _, err := io.Copy(&b, r) 199 | if err != nil { 200 | panic(err) 201 | } 202 | 203 | out <- b.Bytes() 204 | }() 205 | 206 | lockStdFileDescriptorsSwapping.Unlock() 207 | 208 | call() 209 | 210 | lockStdFileDescriptorsSwapping.Lock() 211 | 212 | C.fflush(C.stderr) 213 | C.fflush(C.stdout) 214 | 215 | if err = w.Close(); err != nil { 216 | lockStdFileDescriptorsSwapping.Unlock() 217 | 218 | return nil, err 219 | } 220 | w = nil 221 | 222 | if err = syscall.Close(syscall.Stdout); err != nil { 223 | lockStdFileDescriptorsSwapping.Unlock() 224 | 225 | return nil, err 226 | } 227 | if err = syscall.Close(syscall.Stderr); err != nil { 228 | lockStdFileDescriptorsSwapping.Unlock() 229 | 230 | return nil, err 231 | } 232 | 233 | release() 234 | 235 | lockStdFileDescriptorsSwapping.Unlock() 236 | 237 | return <-out, err 238 | } 239 | -------------------------------------------------------------------------------- /capture_notlinux.go: -------------------------------------------------------------------------------- 1 | //go:build !linux || !cgo 2 | 3 | package osutil 4 | 5 | // Capture captures stderr and stdout of a given function call. 6 | func Capture(call func()) (output []byte, err error) { 7 | panic("not implemented") // WORKAROUND Implement this function for MacOS and Windows when it is actual needed. Until then we can cross-compile even if the function is only mentioned in a package. https://$INTERNAL/symflower/symflower/-/issues/3575 8 | } 9 | -------------------------------------------------------------------------------- /capture_test_linux.go: -------------------------------------------------------------------------------- 1 | //go:build cgo 2 | 3 | package osutil 4 | 5 | import ( 6 | "bytes" 7 | "fmt" 8 | "os" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestCapture(t *testing.T) { 17 | assert.NoError(t, SetRLimitFiles(10, func(limit uint64) { 18 | // Use at least one more file descriptor than our current limit, so we make sure that there are no file descriptor leaks. 19 | for i := 0; i <= int(limit); i++ { 20 | testCapture(t) 21 | } 22 | })) 23 | } 24 | func TestCaptureRecursive(t *testing.T) { 25 | assert.NoError(t, SetRLimitFiles(10, func(limit uint64) { 26 | // Use at least one more file descriptor than our current limit, so we make sure that there are no file descriptor leaks. 27 | for i := 0; i <= int(limit); i++ { 28 | out, err := Capture(func() { 29 | _, err := fmt.Fprintf(os.Stdout, "1") 30 | require.NoError(t, err) 31 | _, err = fmt.Fprintf(os.Stderr, "2") 32 | require.NoError(t, err) 33 | _, err = fmt.Fprintf(os.Stdout, "3") 34 | require.NoError(t, err) 35 | _, err = fmt.Fprintf(os.Stdout, "4") 36 | require.NoError(t, err) 37 | 38 | out, err := Capture(func() { 39 | _, err := fmt.Fprintf(os.Stdout, "A") 40 | require.NoError(t, err) 41 | _, err = fmt.Fprintf(os.Stderr, "B") 42 | require.NoError(t, err) 43 | _, err = fmt.Fprintf(os.Stdout, "C") 44 | require.NoError(t, err) 45 | _, err = fmt.Fprintf(os.Stderr, "D") 46 | require.NoError(t, err) 47 | 48 | }) 49 | assert.NoError(t, err) 50 | 51 | assert.Equal(t, "ABCD", string(out)) 52 | }) 53 | assert.NoError(t, err) 54 | 55 | assert.Equal(t, "1234", string(out)) 56 | } 57 | })) 58 | } 59 | 60 | func TestCaptureWithPanic(t *testing.T) { 61 | assert.NoError(t, SetRLimitFiles(10, func(limit uint64) { 62 | // Use at least one more file descriptor than our current limit, so we make sure that there are no file descriptor leaks. 63 | for i := 0; i <= int(limit); i++ { 64 | assert.Panics(t, func() { 65 | _, _ = Capture(func() { 66 | fmt.Println("abc") 67 | 68 | panic("stop") 69 | }) 70 | }) 71 | } 72 | })) 73 | } 74 | func TestCaptureWithHugeOutput(t *testing.T) { 75 | // Huge output to test buffering and piping. 76 | 77 | out, err := Capture(func() { 78 | for i := 0; i < 1024; i++ { 79 | fmt.Println(strings.Repeat("a", 1024)) 80 | } 81 | }) 82 | assert.NoError(t, err) 83 | 84 | assert.NotEqual(t, bytes.Repeat([]byte("a"), 1024*1024), out) 85 | } 86 | 87 | func TestCaptureWithCGo(t *testing.T) { 88 | assert.NoError(t, SetRLimitFiles(10, func(limit uint64) { 89 | // Use at least one more file descriptor than our current limit, so we make sure that there are no file descriptor leaks. 90 | for i := 0; i <= int(limit); i++ { 91 | testCaptureWithCGo(t) 92 | } 93 | })) 94 | } 95 | 96 | func TestCaptureWithCGoWithPanic(t *testing.T) { 97 | assert.NoError(t, SetRLimitFiles(10, func(limit uint64) { 98 | // Use at least one more file descriptor than our current limit, so we make sure that there are no file descriptor leaks. 99 | for i := 0; i <= int(limit); i++ { 100 | assert.Panics(t, func() { 101 | _, _ = CaptureWithCGo(func() { 102 | fmt.Println("abc") 103 | 104 | panic("stop") 105 | }) 106 | }) 107 | } 108 | })) 109 | } 110 | func TestCaptureWithCGoWithHugeOutput(t *testing.T) { 111 | // Huge output to test buffering and piping. 112 | 113 | out, err := CaptureWithCGo(func() { 114 | for i := 0; i < 1024; i++ { 115 | fmt.Println(strings.Repeat("a", 1024)) 116 | } 117 | }) 118 | assert.NoError(t, err) 119 | 120 | assert.NotEqual(t, bytes.Repeat([]byte("a"), 1024*1024), out) 121 | } 122 | -------------------------------------------------------------------------------- /capture_testwrapper_linux.go: -------------------------------------------------------------------------------- 1 | //go:build cgo 2 | 3 | package osutil 4 | 5 | /* 6 | #include 7 | 8 | void printSomething() { 9 | printf("C\n"); 10 | } 11 | */ 12 | import "C" 13 | 14 | import ( 15 | "fmt" 16 | "testing" 17 | 18 | "github.com/stretchr/testify/assert" 19 | ) 20 | 21 | func testCapture(t *testing.T) { 22 | out, err := Capture(func() { 23 | fmt.Println("Go") 24 | C.printSomething() 25 | }) 26 | assert.NoError(t, err) 27 | 28 | assert.Contains(t, string(out), "Go") 29 | assert.NotContains(t, string(out), "C") 30 | } 31 | 32 | func testCaptureWithCGo(t *testing.T) { 33 | out, err := CaptureWithCGo(func() { 34 | fmt.Println("Go") 35 | C.printSomething() 36 | }) 37 | assert.NoError(t, err) 38 | 39 | assert.Contains(t, string(out), "Go") 40 | assert.Contains(t, string(out), "C") 41 | } 42 | -------------------------------------------------------------------------------- /chdir.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // Chdir temporarily changes to the given working directory while calling the given function. 8 | func Chdir(workingDirectory string, call func() error) (err error) { 9 | var owd string 10 | 11 | owd, err = os.Getwd() 12 | if err != nil { 13 | return err 14 | } 15 | defer func() { 16 | e := os.Chdir(owd) 17 | if err == nil { 18 | err = e 19 | } 20 | }() 21 | err = os.Chdir(workingDirectory) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | return call() 27 | } 28 | -------------------------------------------------------------------------------- /checksum.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "fmt" 8 | "io" 9 | "os" 10 | "path/filepath" 11 | "syscall" 12 | ) 13 | 14 | // WriteChecksumForPath computes a checksum of a file or directory and writes it to the given file. 15 | func WriteChecksumForPath(path string, checksumFile string) error { 16 | digest, err := ChecksumForPath(path) 17 | if err != nil { 18 | panic(err) 19 | } 20 | if err := os.WriteFile(checksumFile, []byte(fmt.Sprintf("%x\n", digest)), 0644); err != nil { 21 | return err 22 | } 23 | 24 | return nil 25 | } 26 | 27 | // ValidateChecksumForPath computes a checksum of a file or directory and returns an error if it does not match the checksum stored in the given file. 28 | func ValidateChecksumForPath(path string, checksumFile string) (valid bool, err error) { 29 | contents, err := os.ReadFile(checksumFile) 30 | if err != nil { 31 | return false, err 32 | } 33 | 34 | digest, err := ChecksumForPath(path) 35 | if err != nil { 36 | return false, err 37 | } 38 | 39 | return fmt.Sprintf("%x\n", digest) == string(contents), nil 40 | } 41 | 42 | // ChecksumForPath computes a checksum of a file or directory. 43 | func ChecksumForPath(path string) (digest []byte, err error) { 44 | hash := md5.New() 45 | 46 | if err := filepath.Walk(path, func(file string, info os.FileInfo, err error) error { 47 | if err != nil { 48 | return err 49 | } 50 | if info.IsDir() { 51 | return nil 52 | } 53 | 54 | contents, err := os.ReadFile(file) 55 | if err != nil { 56 | // Even though "filepath.Walk" does not recurse into symlinks, it still invokes us with symlink itself. We can still include symlinked files in the checksum computation. Skip symlinks to directories and broken symlinks instead of failing to compute a checksum. 57 | if info.Mode()&os.ModeSymlink != 0 { 58 | if pe, ok := err.(*os.PathError); ok && (pe.Err == syscall.EISDIR || pe.Err == syscall.ENOENT) { 59 | return nil 60 | } 61 | } 62 | return err 63 | } 64 | 65 | relativePath, err := filepath.Rel(path, file) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | for _, data := range [][]byte{ 71 | []byte(relativePath), 72 | []byte{'\x00'}, 73 | contents, 74 | []byte{'\x00'}, 75 | } { 76 | if _, err := hash.Write(data); err != nil { 77 | return err 78 | } 79 | } 80 | 81 | return nil 82 | }); err != nil { 83 | return nil, err 84 | } 85 | 86 | return hash.Sum(nil), nil 87 | } 88 | 89 | // ChecksumsSHA256ForFiles creates checksum-files with SHA-256 recursively for all files in a directory. 90 | func ChecksumsSHA256ForFiles(filePath string) (err error) { 91 | return filepath.WalkDir(filePath, func(path string, info os.DirEntry, err error) error { 92 | if err != nil { 93 | return err 94 | } 95 | 96 | // Do not generate checksums for directories, but walk into them. 97 | if info.IsDir() { 98 | return nil 99 | } 100 | 101 | // Generate checksum of file. 102 | file, err := os.Open(path) 103 | defer func() { 104 | if e := file.Close(); e != nil { 105 | err = fmt.Errorf("error during closing of file: %v, %v", e, err) 106 | } 107 | }() 108 | if err != nil { 109 | return err 110 | } 111 | checksum := sha256.New() 112 | if _, err := io.Copy(checksum, file); err != nil { 113 | return err 114 | } 115 | 116 | // Write checksum to checksum-file. 117 | if err := os.WriteFile(path+".sha256", []byte(hex.EncodeToString(checksum.Sum(nil))), 0644); err != nil { 118 | return err 119 | } 120 | 121 | return nil 122 | }) 123 | } 124 | -------------------------------------------------------------------------------- /checksum_test.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func checkTearError(t *testing.T, err error) { 12 | if !assert.NoError(t, err) { 13 | t.FailNow() 14 | } 15 | } 16 | 17 | func TestChecksumForPath(t *testing.T) { 18 | if IsWindows() { 19 | t.SkipNow() // TODO Implement symlink handling under Windows or make this test case compatible with Windows. https://$INTERNAL/symflower/symflower/-/issues/3637 20 | } 21 | 22 | temporaryPath, err := os.MkdirTemp("", "TestChecksumForPath") 23 | checkTearError(t, err) 24 | defer func() { 25 | checkTearError(t, os.RemoveAll(temporaryPath)) 26 | }() 27 | 28 | checkTearError(t, os.Symlink(".", filepath.Join(temporaryPath, "symlinkToDirectory"))) 29 | digestForEmptyDirectory, err := ChecksumForPath(temporaryPath) 30 | assert.NoError(t, err) 31 | assert.NotEmpty(t, digestForEmptyDirectory) 32 | 33 | checkTearError(t, os.Symlink("broken target", filepath.Join(temporaryPath, "symlink with broken target"))) 34 | _, err = ChecksumForPath(temporaryPath) 35 | assert.NoError(t, err) 36 | 37 | checkTearError(t, os.WriteFile(filepath.Join(temporaryPath, "some file"), []byte{}, 0600)) 38 | checkTearError(t, os.WriteFile(filepath.Join(temporaryPath, "symlink or file"), []byte{}, 0600)) 39 | digestWithFile, err := ChecksumForPath(temporaryPath) 40 | assert.NoError(t, err) 41 | 42 | checkTearError(t, os.Remove(filepath.Join(temporaryPath, "symlink or file"))) 43 | checkTearError(t, os.Symlink("some file", filepath.Join(temporaryPath, "symlink or file"))) 44 | digestWithSymlink, err := ChecksumForPath(temporaryPath) 45 | assert.NoError(t, err) 46 | 47 | assert.NotEqual(t, digestForEmptyDirectory, digestWithFile) 48 | assert.Equal(t, digestWithFile, digestWithSymlink) 49 | } 50 | -------------------------------------------------------------------------------- /context.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | ) 10 | 11 | // ContextWithInterrupt returns a context which can be interrupted by signals "SIGTERM" and "SIGQUIT". 12 | // If the signal is sent once, then the returned context is cancelled. If multiple signals are sent, then the program terminates via "os.Exit(1)". 13 | func ContextWithInterrupt(ctx context.Context, logWriter io.Writer) (contextWithInterrupt context.Context, cancelContext context.CancelFunc) { 14 | contextWithInterrupt, cancelContext = context.WithCancel(ctx) 15 | 16 | c := make(chan os.Signal, 10) 17 | signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) 18 | go func() { 19 | count := 0 20 | for { 21 | switch <-c { 22 | case os.Interrupt, os.Kill: 23 | _, err := io.WriteString(logWriter, "Received termination signal") 24 | if err != nil { 25 | panic(err) 26 | } 27 | count++ 28 | 29 | if count == 1 { 30 | _, err := io.WriteString(logWriter, "Canceling analysis and writing results") 31 | if err != nil { 32 | panic(err) 33 | } 34 | 35 | cancelContext() 36 | } else { 37 | _, err := io.WriteString(logWriter, "Exiting immediately") 38 | if err != nil { 39 | panic(err) 40 | } 41 | 42 | //revive:disable:deep-exit 43 | os.Exit(1) 44 | //revive:enable:deep-exit 45 | } 46 | } 47 | } 48 | }() 49 | 50 | return contextWithInterrupt, cancelContext 51 | } 52 | -------------------------------------------------------------------------------- /conversion.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | // AnySliceToTypeSlice returns a slice of the designated type with the values from the given "any" slice that match the type. 4 | func AnySliceToTypeSlice[T any](anySlice []any) (typeSlice []T) { 5 | for _, v := range anySlice { 6 | if x, ok := v.(T); ok { 7 | typeSlice = append(typeSlice, x) 8 | } 9 | } 10 | 11 | return typeSlice 12 | } 13 | -------------------------------------------------------------------------------- /conversion_test.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestAnySliceToTypeSlice(t *testing.T) { 10 | t.Run("String slice", func(t *testing.T) { 11 | type testCase struct { 12 | Name string 13 | 14 | AnySlice []any 15 | 16 | ExpectedTypeSlice []string 17 | } 18 | 19 | validate := func(t *testing.T, tc *testCase) { 20 | t.Run(tc.Name, func(t *testing.T) { 21 | actualTypeSlice := AnySliceToTypeSlice[string](tc.AnySlice) 22 | 23 | assert.Equal(t, tc.ExpectedTypeSlice, actualTypeSlice) 24 | }) 25 | } 26 | 27 | validate(t, &testCase{ 28 | Name: "Nil", 29 | 30 | AnySlice: nil, 31 | 32 | ExpectedTypeSlice: nil, 33 | }) 34 | 35 | validate(t, &testCase{ 36 | Name: "Convert any slice to string slice", 37 | 38 | AnySlice: []any{ 39 | "foo", 40 | "bar", 41 | }, 42 | 43 | ExpectedTypeSlice: []string{ 44 | "foo", 45 | "bar", 46 | }, 47 | }) 48 | 49 | validate(t, &testCase{ 50 | Name: "Only values that match the type", 51 | 52 | AnySlice: []any{ 53 | "foo", 54 | "bar", 55 | 12, 56 | 145.66, 57 | }, 58 | 59 | ExpectedTypeSlice: []string{ 60 | "foo", 61 | "bar", 62 | }, 63 | }) 64 | }) 65 | t.Run("Integer slice", func(t *testing.T) { 66 | type testCase struct { 67 | Name string 68 | 69 | AnySlice []any 70 | 71 | ExpectedTypeSlice []int 72 | } 73 | 74 | validate := func(t *testing.T, tc *testCase) { 75 | t.Run(tc.Name, func(t *testing.T) { 76 | actualTypeSlice := AnySliceToTypeSlice[int](tc.AnySlice) 77 | 78 | assert.Equal(t, tc.ExpectedTypeSlice, actualTypeSlice) 79 | }) 80 | } 81 | 82 | validate(t, &testCase{ 83 | Name: "Nil", 84 | 85 | AnySlice: nil, 86 | 87 | ExpectedTypeSlice: nil, 88 | }) 89 | 90 | validate(t, &testCase{ 91 | Name: "Convert any slice to int slice", 92 | 93 | AnySlice: []any{ 94 | 15, 95 | 22, 96 | }, 97 | 98 | ExpectedTypeSlice: []int{ 99 | 15, 100 | 22, 101 | }, 102 | }) 103 | 104 | validate(t, &testCase{ 105 | Name: "Only values that match the type", 106 | 107 | AnySlice: []any{ 108 | "foo", 109 | "bar", 110 | 12, 111 | "13", 112 | 145.66, 113 | }, 114 | 115 | ExpectedTypeSlice: []int{ 116 | 12, 117 | }, 118 | }) 119 | }) 120 | } 121 | -------------------------------------------------------------------------------- /copy.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "compress/gzip" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/termie/go-shutil" 12 | ) 13 | 14 | // CopyFile copies a file from src to dst. 15 | func CopyFile(src string, dst string) (err error) { 16 | s, err := os.Open(src) 17 | if err != nil { 18 | return err 19 | } 20 | defer func() { 21 | e := s.Close() 22 | if err == nil { 23 | err = e 24 | } 25 | }() 26 | 27 | d, err := os.Create(dst) 28 | if err != nil { 29 | // In case the file is a symlink, we need to remove the file before we can write to it. 30 | if _, e := os.Lstat(dst); e == nil { 31 | if e := os.Remove(dst); e != nil { 32 | return e 33 | } 34 | d, err = os.Create(dst) 35 | if err != nil { 36 | return err 37 | } 38 | } else { 39 | return err 40 | } 41 | } 42 | defer func() { 43 | e := d.Close() 44 | if err == nil { 45 | err = e 46 | } 47 | }() 48 | 49 | _, err = io.Copy(d, s) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | i, err := os.Stat(src) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | return os.Chmod(dst, i.Mode()) 60 | } 61 | 62 | // CopyFileCompressed reads the file src and writes a compressed version to dst. 63 | // The compression level can be gzip.DefaultCompression, gzip.NoCompression, gzip.HuffmanOnly or any integer value between gzip.BestSpeed and gzip.BestCompression inclusive. 64 | func CopyFileCompressed(src string, dst string, compressionLevel int) (err error) { 65 | s, err := os.Open(src) 66 | if err != nil { 67 | return err 68 | } 69 | defer func() { 70 | e := s.Close() 71 | if err == nil { 72 | err = e 73 | } 74 | }() 75 | 76 | d, err := os.Create(dst) 77 | if err != nil { 78 | // In case the file is a symlink, we need to remove the file before we can write to it. 79 | if _, e := os.Lstat(dst); e == nil { 80 | if e := os.Remove(dst); e != nil { 81 | return e 82 | } 83 | d, err = os.Create(dst) 84 | if err != nil { 85 | return err 86 | } 87 | } else { 88 | return err 89 | } 90 | } 91 | defer func() { 92 | e := d.Close() 93 | if err == nil { 94 | err = e 95 | } 96 | }() 97 | 98 | gzipWriter, err := gzip.NewWriterLevel(d, compressionLevel) 99 | if err != nil { 100 | return err 101 | } 102 | defer func() { 103 | e := gzipWriter.Close() 104 | if err == nil { 105 | err = e 106 | } 107 | }() 108 | 109 | if _, err := io.Copy(gzipWriter, s); err != nil { 110 | return err 111 | } 112 | 113 | i, err := os.Stat(src) 114 | if err != nil { 115 | return err 116 | } 117 | 118 | return os.Chmod(dst, i.Mode()) 119 | } 120 | 121 | // CopyTree copies a whole file system tree from the source path to destination path. 122 | func CopyTree(sourcePath string, destinationPath string) (err error) { 123 | return shutil.CopyTree(sourcePath, destinationPath, nil) 124 | } 125 | 126 | // CompressDirectory reads the directory srcDirectory and writes a compressed version to archive. 127 | func CompressDirectory(srcDirectory string, archive string) (err error) { 128 | archiveFile, err := os.Create(archive) 129 | if err != nil { 130 | return err 131 | } 132 | defer func() { 133 | e := archiveFile.Close() 134 | if err == nil { 135 | err = e 136 | } 137 | }() 138 | 139 | zipWriter := zip.NewWriter(archiveFile) 140 | defer func() { 141 | e := zipWriter.Close() 142 | if err == nil { 143 | err = e 144 | } 145 | }() 146 | 147 | return filepath.WalkDir(srcDirectory, func(path string, info os.DirEntry, err error) error { 148 | if err != nil { 149 | return err 150 | } 151 | if info.IsDir() { 152 | return nil 153 | } 154 | 155 | file, err := os.Open(path) 156 | if err != nil { 157 | return err 158 | } 159 | defer func() { 160 | e := file.Close() 161 | if err == nil { 162 | err = e 163 | } 164 | }() 165 | 166 | relativePath, err := filepath.Rel(srcDirectory, path) 167 | if err != nil { 168 | return err 169 | } 170 | 171 | zipFileWriter, err := zipWriter.Create(relativePath) 172 | if err != nil { 173 | return err 174 | } 175 | 176 | if _, err := io.Copy(zipFileWriter, file); err != nil { 177 | return err 178 | } 179 | 180 | return nil 181 | }) 182 | } 183 | 184 | // Uncompress extracts the given archive into the given destination. 185 | func Uncompress(archive io.Reader, dstDirectory string) (err error) { 186 | data, err := io.ReadAll(archive) 187 | if err != nil { 188 | return err 189 | } 190 | byteReader := bytes.NewReader(data) 191 | 192 | zipReader, err := zip.NewReader(byteReader, byteReader.Size()) 193 | if err != nil { 194 | return err 195 | } 196 | 197 | for _, zipFile := range zipReader.File { 198 | zipReaderFile, err := zipFile.Open() 199 | if err != nil { 200 | return err 201 | } 202 | 203 | destinationPath := filepath.Join(dstDirectory, zipFile.Name) 204 | if err := os.MkdirAll(filepath.Dir(destinationPath), 0700); err != nil { 205 | return err 206 | } 207 | destinationFile, err := os.Create(destinationPath) 208 | if err != nil { 209 | return err 210 | } 211 | 212 | if _, err := io.Copy(destinationFile, zipReaderFile); err != nil { 213 | _ = destinationFile.Close() 214 | _ = zipReaderFile.Close() 215 | 216 | return err 217 | } 218 | 219 | if err := destinationFile.Close(); err != nil { 220 | _ = zipReaderFile.Close() 221 | 222 | return err 223 | } 224 | if err := zipReaderFile.Close(); err != nil { 225 | return err 226 | } 227 | 228 | if err := os.Chmod(destinationPath, zipFile.Mode()); err != nil { 229 | return err 230 | } 231 | } 232 | 233 | return nil 234 | } 235 | -------------------------------------------------------------------------------- /copy_test.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestCopyFile(t *testing.T) { 11 | src := "copy.go" 12 | dst := "copy.go.tmp" 13 | 14 | assert.NoError(t, CopyFile(src, dst)) 15 | 16 | s, err := os.ReadFile(src) 17 | assert.NoError(t, err) 18 | 19 | d, err := os.ReadFile(dst) 20 | assert.NoError(t, err) 21 | 22 | assert.Equal(t, s, d) 23 | 24 | assert.NoError(t, os.Remove(dst)) 25 | } 26 | -------------------------------------------------------------------------------- /directory.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // MkdirAll creates a directory named path, along with any necessary parents, and returns nil, or else returns an error. 8 | func MkdirAll(path string) error { 9 | return os.MkdirAll(path, 0750) 10 | } 11 | -------------------------------------------------------------------------------- /download.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "net" 8 | "net/http" 9 | "net/http/cookiejar" 10 | "os" 11 | "time" 12 | ) 13 | 14 | // HTTPClient defines an HTTP client with sane default settings. 15 | var HTTPClient *http.Client = func() *http.Client { 16 | c := &http.Client{ 17 | // The timeout defaults of the default client are terrible. See https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/ for details. 18 | Transport: &http.Transport{ 19 | Dial: (&net.Dialer{ 20 | Timeout: 30 * time.Second, 21 | KeepAlive: 30 * time.Second, 22 | }).Dial, 23 | TLSHandshakeTimeout: 10 * time.Second, 24 | ResponseHeaderTimeout: 10 * time.Second, 25 | ExpectContinueTimeout: 1 * time.Second, 26 | }, 27 | Timeout: 0, // This timeout includes the whole process of downloading a file. Hence, big files always run into a timeout so we are setting the timeout granularly. 28 | } 29 | c.Jar, _ = cookiejar.New(nil) 30 | 31 | return c 32 | }() 33 | 34 | // DownloadFile downloads a file from the URL to the file path. 35 | func DownloadFile(url string, filePath string) (err error) { 36 | resp, err := HTTPClient.Get(url) 37 | if err != nil { 38 | return err 39 | } 40 | defer func() { 41 | if e := resp.Body.Close(); e != nil { 42 | err = errors.Join(err, e) 43 | } 44 | }() 45 | 46 | out, err := os.Create(filePath) 47 | if err != nil { 48 | return err 49 | } 50 | defer func() { 51 | if e := out.Close(); e != nil { 52 | err = errors.Join(err, e) 53 | } 54 | }() 55 | 56 | _, err = io.Copy(out, resp.Body) 57 | 58 | return err 59 | } 60 | 61 | // DownloadFileWithProgress downloads a file from the URL to the file path while printing a progress to STDOUT. 62 | func DownloadFileWithProgress(url string, filePath string) (err error) { 63 | request, err := http.NewRequest("GET", url, nil) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | response, err := HTTPClient.Do(request) 69 | if err != nil { 70 | return err 71 | } 72 | defer func() { 73 | if e := response.Body.Close(); e != nil { 74 | if err != nil { 75 | err = errors.Join(err, e) 76 | } else { 77 | err = e 78 | } 79 | } 80 | }() 81 | if response.StatusCode != http.StatusOK { 82 | return fmt.Errorf("downloading file failed with status code %d: %s", response.StatusCode, response.Status) 83 | } 84 | 85 | file, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY, 0644) 86 | if err != nil { 87 | return err 88 | } 89 | defer func() { 90 | if e := file.Close(); e != nil { 91 | if err != nil { 92 | err = errors.Join(err, e) 93 | } else { 94 | err = e 95 | } 96 | } 97 | }() 98 | 99 | pg := ProgressBarBytes(os.Stdout, int(response.ContentLength), "downloading") 100 | defer func() { 101 | if e := pg.Close(); e != nil { 102 | if err != nil { 103 | err = errors.Join(err, e) 104 | } else { 105 | err = e 106 | } 107 | } 108 | }() 109 | 110 | if _, err := io.Copy(io.MultiWriter(file, pg), response.Body); err != nil { 111 | return err 112 | } 113 | 114 | return nil 115 | } 116 | -------------------------------------------------------------------------------- /env.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | // EnvironMap returns a map of the current environment variables. 10 | func EnvironMap() (environMap map[string]string) { 11 | environ := os.Environ() 12 | environMap = make(map[string]string, len(environ)) 13 | for _, e := range environ { 14 | kv := strings.SplitN(e, "=", 2) 15 | environMap[kv[0]] = kv[1] 16 | } 17 | 18 | return environMap 19 | } 20 | 21 | // EnvOrDefault returns the environment variable with the given key, or the default value if the key is not defined. 22 | func EnvOrDefault(key string, defaultValue string) (value string) { 23 | if v, ok := os.LookupEnv(key); ok { 24 | return v 25 | } 26 | 27 | return defaultValue 28 | } 29 | 30 | // RequireEnv returns the environment variable with the given key, or an error if the key is not defined. 31 | func RequireEnv(key string) (value string, err error) { 32 | if v, ok := os.LookupEnv(key); ok { 33 | return v, nil 34 | } 35 | 36 | return "", fmt.Errorf("environment variable %q needs to be set", key) 37 | } 38 | 39 | // IsEnvEnabled checks if the environment variable is enabled. 40 | // By default an environment variable is considered enabled if it is set to "1", "true", "on" or "yes". Further such values can be provided as well. Capitalization is ignored. 41 | func IsEnvEnabled(key string, additionalEnabledValues ...string) bool { 42 | value, ok := os.LookupEnv(key) 43 | if !ok { 44 | return false 45 | } 46 | value = strings.ToLower(value) 47 | 48 | if value == "1" || value == "true" || value == "on" || value == "yes" { 49 | return true 50 | } 51 | 52 | for _, match := range additionalEnabledValues { 53 | if value == match { 54 | return true 55 | } 56 | } 57 | 58 | return false 59 | } 60 | -------------------------------------------------------------------------------- /env_test.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestIsEnvEnabled(t *testing.T) { 12 | type testCase struct { 13 | Value string 14 | EnabledValues []string 15 | 16 | Expected bool 17 | } 18 | envTestName := "_OSUTIL_TEST_ENVIRONMENT_VARIABLE" 19 | 20 | validate := func(t *testing.T, tc *testCase) { 21 | t.Run(tc.Value, func(t *testing.T) { 22 | require.NoError(t, os.Setenv(envTestName, tc.Value)) 23 | defer func() { 24 | require.NoError(t, os.Unsetenv(envTestName)) 25 | }() 26 | 27 | actual := IsEnvEnabled(envTestName, tc.EnabledValues...) 28 | 29 | assert.Equal(t, tc.Expected, actual) 30 | }) 31 | } 32 | 33 | validate(t, &testCase{ 34 | Value: "1", 35 | Expected: true, 36 | }) 37 | validate(t, &testCase{ 38 | Value: "0", 39 | Expected: false, 40 | }) 41 | validate(t, &testCase{ 42 | Value: "", 43 | Expected: false, 44 | }) 45 | validate(t, &testCase{ 46 | Value: "yes", 47 | Expected: true, 48 | }) 49 | validate(t, &testCase{ 50 | Value: "on", 51 | Expected: true, 52 | }) 53 | validate(t, &testCase{ 54 | Value: "true", 55 | Expected: true, 56 | }) 57 | t.Run("Other Cases", func(t *testing.T) { 58 | validate(t, &testCase{ 59 | Value: "True", 60 | Expected: true, 61 | }) 62 | validate(t, &testCase{ 63 | Value: "TRUE", 64 | Expected: true, 65 | }) 66 | }) 67 | t.Run("Other Values", func(t *testing.T) { 68 | validate(t, &testCase{ 69 | Value: "positive", 70 | EnabledValues: []string{"positive"}, 71 | Expected: true, 72 | }) 73 | validate(t, &testCase{ 74 | Value: "negative", 75 | EnabledValues: []string{"positive"}, 76 | Expected: false, 77 | }) 78 | }) 79 | } 80 | -------------------------------------------------------------------------------- /exists.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "errors" 5 | "io/fs" 6 | "os" 7 | ) 8 | 9 | var ( 10 | // ErrNotADirectory indicates that the given directory does not exist. 11 | ErrNotADirectory = errors.New("not a directory") 12 | // ErrNotAFile indicates thate the given file does not exist. 13 | ErrNotAFile = errors.New("not a file") 14 | ) 15 | 16 | // Stat retuns a FileInfo structure describing the given file. 17 | func Stat(filePath string) (os.FileInfo, error) { 18 | return os.Stat(filePath) 19 | } 20 | 21 | // DirExists checks if a directory exists. 22 | func DirExists(filePath string) error { 23 | fi, err := Stat(filePath) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | if !fi.Mode().IsDir() { 29 | return ErrNotADirectory 30 | } 31 | 32 | return nil 33 | } 34 | 35 | // FileExists checks if a file exists while following symlinks. 36 | func FileExists(filePath string) error { 37 | fi, err := Stat(filePath) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | if fi.Mode().IsDir() { 43 | return ErrNotAFile 44 | } 45 | 46 | return nil 47 | } 48 | 49 | // RemoveFileIfExists checks if a file exists, and removes the file if it does exist. 50 | // Symlinks are not followed, since they are files and should be removable by this function. 51 | func RemoveFileIfExists(filePath string) error { 52 | if _, err := os.Lstat(filePath); err != nil { 53 | if errors.Is(err, fs.ErrNotExist) { 54 | return nil 55 | } 56 | 57 | return err 58 | } 59 | 60 | return os.Remove(filePath) 61 | } 62 | 63 | // FileOrSymlinkExists checks if a file exists while not following symlinks. 64 | func FileOrSymlinkExists(filepath string) error { 65 | fi, err := os.Lstat(filepath) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | if fi.Mode().IsDir() { 71 | return ErrNotAFile 72 | } 73 | 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /exists_notwindows.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package osutil 4 | 5 | import ( 6 | "syscall" 7 | ) 8 | 9 | // ErrDirectoryNotEmpty indicates that a directory is not empty. 10 | var ErrDirectoryNotEmpty = syscall.ENOTEMPTY 11 | -------------------------------------------------------------------------------- /exists_test.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestDirExists(t *testing.T) { 13 | assert.NoError(t, DirExists("/")) 14 | assert.NoError(t, DirExists("../osutil")) 15 | 16 | assert.Error(t, DirExists("hey")) 17 | 18 | assert.Equal(t, ErrNotADirectory, DirExists("exists.go")) 19 | } 20 | 21 | func TestRemoveFileIfExists(t *testing.T) { 22 | t.Run("File", func(t *testing.T) { 23 | temporaryPath := t.TempDir() 24 | 25 | filePath := filepath.Join(temporaryPath, "plain.txt") 26 | require.NoError(t, os.WriteFile(filePath, nil, 0600)) 27 | 28 | assert.NoError(t, RemoveFileIfExists(filePath)) 29 | assert.NoFileExists(t, filePath) 30 | }) 31 | 32 | t.Run("Empty directory", func(t *testing.T) { 33 | temporaryPath := t.TempDir() 34 | 35 | filePath := filepath.Join(temporaryPath, "plain") 36 | require.NoError(t, os.MkdirAll(filePath, 0700)) 37 | 38 | assert.NoError(t, RemoveFileIfExists(filePath)) 39 | assert.NoDirExists(t, filePath) 40 | }) 41 | 42 | t.Run("Non-empty directory", func(t *testing.T) { 43 | temporaryPath := t.TempDir() 44 | 45 | filePath := filepath.Join(temporaryPath, "plain/subdir") 46 | require.NoError(t, os.MkdirAll(filePath, 0700)) 47 | 48 | assert.ErrorIs(t, RemoveFileIfExists(filepath.Dir(filePath)), ErrDirectoryNotEmpty) 49 | assert.DirExists(t, filePath) 50 | }) 51 | 52 | t.Run("Symlink", func(t *testing.T) { 53 | temporaryPath := t.TempDir() 54 | 55 | filePath := filepath.Join(temporaryPath, "plain.txt") 56 | require.NoError(t, os.WriteFile(filePath, nil, 0600)) 57 | 58 | linkFilePath := filepath.Join(temporaryPath, "symlink") 59 | require.NoError(t, os.Symlink(filePath, linkFilePath)) 60 | 61 | assert.NoError(t, RemoveFileIfExists(linkFilePath)) 62 | assert.FileExists(t, filePath) 63 | assert.NoFileExists(t, linkFilePath) 64 | }) 65 | } 66 | -------------------------------------------------------------------------------- /exists_windows.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "syscall" 5 | ) 6 | 7 | // ErrDirectoryNotEmpty indicates that a directory is not empty. 8 | var ErrDirectoryNotEmpty = syscall.ERROR_DIR_NOT_EMPTY 9 | -------------------------------------------------------------------------------- /files.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | // CanonicalizeAndEvaluateSymlinks returns the path after canonicalizing it and the evaluation of any symbolic links. 11 | func CanonicalizeAndEvaluateSymlinks(path string) (resolvedPath string, err error) { 12 | path, err = filepath.Abs(path) 13 | if err != nil { 14 | return "", err 15 | } 16 | 17 | return filepath.EvalSymlinks(path) 18 | } 19 | 20 | // DirectoriesRecursive returns all subdirectories of the given path including the given path. 21 | func DirectoriesRecursive(directoryPath string) (directories []string, err error) { 22 | if err := filepath.WalkDir(directoryPath, func(path string, d os.DirEntry, err error) error { 23 | if err != nil { 24 | return err 25 | } 26 | 27 | if d.IsDir() { 28 | directories = append(directories, path) 29 | } 30 | 31 | return nil 32 | }); err != nil { 33 | return nil, err 34 | } 35 | 36 | return directories, nil 37 | } 38 | 39 | // FilesRecursive returns all files in a given path and its subpaths. 40 | func FilesRecursive(path string) (files []string, err error) { 41 | var fs []string 42 | 43 | err = filepath.Walk(path, func(path string, f os.FileInfo, err error) error { 44 | if err != nil { 45 | return err 46 | } 47 | 48 | if f.IsDir() { 49 | return nil 50 | } 51 | 52 | fs = append(fs, path) 53 | 54 | return err 55 | }) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | return fs, nil 61 | } 62 | 63 | // ForEachFile walks through the given path and calls the given callback with every file. 64 | func ForEachFile(path string, handle func(filePath string) error) error { 65 | return filepath.WalkDir(path, func(filePath string, d os.DirEntry, err error) error { 66 | if err != nil { 67 | return err 68 | } 69 | 70 | if d.IsDir() { 71 | return nil 72 | } 73 | 74 | return handle(filePath) 75 | }) 76 | } 77 | 78 | // FilePathsByHierarchy sorts file paths by their hierarchy. 79 | type FilePathsByHierarchy []string 80 | 81 | // Len is the number of elements in the collection. 82 | func (s FilePathsByHierarchy) Len() int { 83 | return len(s) 84 | } 85 | 86 | // Swap swaps the elements with indexes i and j. 87 | func (s FilePathsByHierarchy) Swap(i, j int) { 88 | s[i], s[j] = s[j], s[i] 89 | } 90 | 91 | // Less reports whether the element with index i must sort before the element with index j. 92 | func (s FilePathsByHierarchy) Less(i, j int) bool { 93 | si := strings.Split(s[i], string(os.PathSeparator)) 94 | sj := strings.Split(s[j], string(os.PathSeparator)) 95 | 96 | if len(si) != len(sj) { 97 | return len(si) < len(sj) 98 | } 99 | 100 | for i, sie := range si { 101 | if c := sie < sj[i]; c { 102 | return c 103 | } 104 | } 105 | 106 | return false 107 | } 108 | 109 | // AppendToFile opens the named file. If the file does not exist it is created. 110 | func AppendToFile(name string) (*os.File, error) { 111 | return os.OpenFile(name, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) 112 | } 113 | 114 | // FileChange changes the content of a file. 115 | func FileChange(filePath string, change func(data []byte) (changed []byte, err error)) error { 116 | data, err := os.ReadFile(filePath) 117 | if err != nil { 118 | return err 119 | } 120 | 121 | data, err = change(data) 122 | if err != nil { 123 | return err 124 | } 125 | 126 | if err := os.WriteFile(filePath, data, 0000); err != nil { // The permission is invalid and will be ignored, as we know the specified file already exists. 127 | return err 128 | } 129 | 130 | return nil 131 | } 132 | 133 | // ReplaceVariablesInFile replaces all variables in a file. 134 | // A variable in a file has the syntax `{{$key}}` and which is then replaced by its value. 135 | func ReplaceVariablesInFile(filePath string, variables map[string]string) (err error) { 136 | d, err := os.ReadFile(filePath) 137 | if err != nil { 138 | return err 139 | } 140 | 141 | for k, v := range variables { 142 | d = bytes.ReplaceAll(d, []byte(k), []byte(v)) 143 | } 144 | 145 | return os.WriteFile(filePath, d, 0) 146 | } 147 | 148 | // WriteFile writes the given content into the given file while creating all necessary directories in between. 149 | func WriteFile(filePath string, content []byte) (err error) { 150 | if err := MkdirAll(filepath.Dir(filePath)); err != nil { 151 | return err 152 | } 153 | 154 | return os.WriteFile(filePath, content, 0644) 155 | } 156 | -------------------------------------------------------------------------------- /files_notwindows.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package osutil 4 | 5 | const ( 6 | // LineEnding holds the line ending for text files. 7 | LineEnding = "\n" 8 | ) 9 | -------------------------------------------------------------------------------- /files_test.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "math/rand" 5 | "os" 6 | "path/filepath" 7 | "sort" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestFilesRecursive(t *testing.T) { 14 | path, err := os.MkdirTemp("", "os-util") 15 | assert.NoError(t, err) 16 | 17 | assert.NoError(t, os.MkdirAll(filepath.Join(path, "a", "b"), 0750)) 18 | assert.NoError(t, os.WriteFile(filepath.Join(path, "c.txt"), []byte("foobar"), 0640)) 19 | assert.NoError(t, os.WriteFile(filepath.Join(path, "a", "d.txt"), []byte("foobar"), 0640)) 20 | assert.NoError(t, os.WriteFile(filepath.Join(path, "a", "b", "e.txt"), []byte("foobar"), 0640)) 21 | 22 | fs, err := FilesRecursive(path) 23 | assert.NoError(t, err) 24 | 25 | sort.Strings(fs) 26 | 27 | assert.Equal( 28 | t, 29 | []string{ 30 | filepath.Join(path, "a", "b", "e.txt"), 31 | filepath.Join(path, "a", "d.txt"), 32 | filepath.Join(path, "c.txt"), 33 | }, 34 | fs, 35 | ) 36 | } 37 | 38 | func TestFilePathsByHierarchy(t *testing.T) { 39 | type testCase struct { 40 | Name string 41 | 42 | ExpectedSortedFilePaths []string 43 | } 44 | 45 | validate := func(t *testing.T, tc *testCase) { 46 | t.Run(tc.Name, func(t *testing.T) { 47 | for i := 0; i < 1000; i++ { 48 | actualSortedFilePaths := make([]string, len(tc.ExpectedSortedFilePaths)) 49 | copy(actualSortedFilePaths, tc.ExpectedSortedFilePaths) 50 | 51 | rand.Shuffle(len(actualSortedFilePaths), func(i, j int) { 52 | actualSortedFilePaths[i], actualSortedFilePaths[j] = actualSortedFilePaths[j], actualSortedFilePaths[i] 53 | }) 54 | sort.Sort(FilePathsByHierarchy(actualSortedFilePaths)) 55 | 56 | assert.Equal(t, tc.ExpectedSortedFilePaths, actualSortedFilePaths) 57 | } 58 | }) 59 | } 60 | 61 | validate(t, &testCase{ 62 | ExpectedSortedFilePaths: []string{ 63 | "a", 64 | "a b", 65 | "ab", 66 | "b", 67 | filepath.Join("a", "b"), 68 | filepath.Join("b c", "c"), 69 | filepath.Join("a", "b", " "), 70 | filepath.Join("a", "b", "c"), 71 | }, 72 | }) 73 | } 74 | 75 | func TestAppendtoFile(t *testing.T) { 76 | path, err := os.MkdirTemp("", "os-util") 77 | assert.NoError(t, err) 78 | 79 | file := filepath.Join(path, "test.log") 80 | f, err := AppendToFile(file) 81 | assert.NoError(t, err) 82 | 83 | _, err = f.WriteString("Test") 84 | assert.NoError(t, err) 85 | 86 | assert.NoError(t, f.Close()) 87 | 88 | f, err = AppendToFile(file) 89 | assert.NoError(t, err) 90 | 91 | _, err = f.WriteString("Blub") 92 | assert.NoError(t, err) 93 | 94 | assert.NoError(t, f.Close()) 95 | 96 | actual, err := os.ReadFile(file) 97 | assert.NoError(t, err) 98 | 99 | assert.Equal(t, "TestBlub", string(actual)) 100 | } 101 | -------------------------------------------------------------------------------- /files_windows.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | const ( 4 | // LineEnding holds the line ending for text files. 5 | LineEnding = "\r\n" 6 | ) 7 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/zimmski/osutil 2 | 3 | go 1.24.1 4 | 5 | require ( 6 | github.com/avast/retry-go v3.0.0+incompatible 7 | github.com/pkg/errors v0.9.1 8 | github.com/schollz/progressbar/v3 v3.18.0 9 | github.com/stretchr/testify v1.10.0 10 | github.com/symflower/pretty v1.0.0 11 | github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae 12 | github.com/ulikunitz/xz v0.5.12 13 | github.com/yuin/goldmark v1.7.8 14 | golang.org/x/sys v0.31.0 15 | ) 16 | 17 | require ( 18 | github.com/bitfield/gotestdox v0.2.2 // indirect 19 | github.com/davecgh/go-spew v1.1.1 // indirect 20 | github.com/dnephin/pflag v1.0.7 // indirect 21 | github.com/fatih/color v1.17.0 // indirect 22 | github.com/fsnotify/fsnotify v1.8.0 // indirect 23 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect 24 | github.com/kisielk/errcheck v1.9.0 // indirect 25 | github.com/kr/pretty v0.3.1 // indirect 26 | github.com/kr/text v0.2.0 // indirect 27 | github.com/mattn/go-colorable v0.1.13 // indirect 28 | github.com/mattn/go-isatty v0.0.20 // indirect 29 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect 30 | github.com/pmezard/go-difflib v1.0.0 // indirect 31 | github.com/rivo/uniseg v0.4.7 // indirect 32 | golang.org/x/mod v0.23.0 // indirect 33 | golang.org/x/sync v0.11.0 // indirect 34 | golang.org/x/term v0.30.0 // indirect 35 | golang.org/x/text v0.17.0 // indirect 36 | golang.org/x/tools v0.30.0 // indirect 37 | gopkg.in/yaml.v3 v3.0.1 // indirect 38 | gotest.tools/gotestsum v1.12.1 // indirect 39 | ) 40 | 41 | tool ( 42 | github.com/kisielk/errcheck 43 | gotest.tools/gotestsum 44 | ) 45 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= 2 | github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= 3 | github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= 4 | github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY= 5 | github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= 6 | github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= 7 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= 11 | github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= 12 | github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= 13 | github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= 14 | github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 15 | github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 16 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 17 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 18 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= 19 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= 20 | github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3M= 21 | github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= 22 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 23 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 24 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 25 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 26 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 27 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 28 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 29 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 30 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 31 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 32 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 33 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= 34 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= 35 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 36 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 37 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 38 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 39 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 40 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 41 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 42 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 43 | github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= 44 | github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= 45 | github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= 46 | github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= 47 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 48 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 49 | github.com/symflower/pretty v1.0.0 h1:wYSv0CBazyyzHNiGTwjkLzcmUQUFjRafEyWf3A7LJCk= 50 | github.com/symflower/pretty v1.0.0/go.mod h1:6/K5MZq/CgT9l/l6RRgiIGI8j/nkxJMe2mYgiln+q/o= 51 | github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae h1:vgGSvdW5Lqg+I1aZOlG32uyE6xHpLdKhZzcTEktz5wM= 52 | github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae/go.mod h1:quDq6Se6jlGwiIKia/itDZxqC5rj6/8OdFyMMAwTxCs= 53 | github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= 54 | github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= 55 | github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA= 56 | github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= 57 | github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= 58 | github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= 59 | golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= 60 | golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= 61 | golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= 62 | golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 63 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 64 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 65 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 66 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 67 | golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= 68 | golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= 69 | golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= 70 | golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= 71 | golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= 72 | golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= 73 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 74 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 75 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 76 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 77 | gotest.tools/gotestsum v1.12.1 h1:dvcxFBTFR1QsQmrCQa4k/vDXow9altdYz4CjdW+XeBE= 78 | gotest.tools/gotestsum v1.12.1/go.mod h1:mwDmLbx9DIvr09dnAoGgQPLaSXszNpXpWo2bsQge5BE= 79 | gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= 80 | gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= 81 | -------------------------------------------------------------------------------- /info_darwin.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | 3 | package osutil 4 | 5 | import ( 6 | "bytes" 7 | "os" 8 | "os/exec" 9 | "strings" 10 | "syscall" 11 | 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | const ( 16 | // KernelVersionIdentifier holds the identifier for the kernel version OS information. 17 | KernelVersionIdentifier = "KernelVersion" 18 | // OperatingSystemBuildIdentifier holds the identifier for the OS build OS information. 19 | OperatingSystemBuildIdentifier = "BuildVersion" 20 | // OperatingSystemIdentifier holds the identifier for the OS name OS information. 21 | OperatingSystemIdentifier = "ProductName" 22 | // OperatingSystemVersionIdentifier holds the identifier for the OS version OS information. 23 | OperatingSystemVersionIdentifier = "ProductVersion" 24 | ) 25 | 26 | // Info returns a list of OS relevant information. 27 | func Info() (info map[string]string, err error) { 28 | info = map[string]string{} 29 | 30 | kernelVersion, err := syscall.Sysctl("kern.osrelease") 31 | if err != nil { 32 | return nil, errors.Wrap(err, "failed to query kernel version") 33 | } 34 | info[KernelVersionIdentifier] = strings.TrimSpace(kernelVersion) 35 | 36 | softwareVersionsCommand := exec.Command("sw_vers") 37 | var softwareVersions bytes.Buffer 38 | softwareVersionsCommand.Stdout = &softwareVersions 39 | if err := softwareVersionsCommand.Run(); err != nil { 40 | return nil, errors.Wrap(err, "failed to query software versions") 41 | } 42 | for _, line := range strings.Split(softwareVersions.String(), "\n") { 43 | ls := strings.SplitN(line, ":", 2) 44 | if len(ls) == 1 { // Ignore empty lines 45 | continue 46 | } 47 | info[strings.TrimSpace(ls[0])] = strings.TrimSpace(ls[1]) 48 | } 49 | 50 | info[EnvironmentPathIdentifier] = os.Getenv(EnvironmentPathIdentifier) 51 | 52 | return info, err 53 | } 54 | -------------------------------------------------------------------------------- /info_darwin_test.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | 3 | package osutil 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestInfo(t *testing.T) { 12 | info, err := Info() 13 | assert.NoError(t, err) 14 | 15 | assert.NotEmpty(t, info[OperatingSystemVersionIdentifier]) 16 | assert.NotEmpty(t, info[OperatingSystemIdentifier]) 17 | assert.NotEmpty(t, info[OperatingSystemBuildIdentifier]) 18 | assert.NotEmpty(t, info[KernelVersionIdentifier]) 19 | assert.NotEmpty(t, info[EnvironmentPathIdentifier]) 20 | } 21 | -------------------------------------------------------------------------------- /info_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package osutil 4 | 5 | import ( 6 | "bytes" 7 | "os" 8 | "os/exec" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | const ( 16 | // KernelVersionIdentifier holds the identifier for the kernel version OS information. 17 | KernelVersionIdentifier = "KernelVersion" 18 | // OperatingSystemIdentifier holds the identifier for the OS name OS information. 19 | OperatingSystemIdentifier = "ProductName" 20 | // OperatingSystemVersionIdentifier holds the identifier for the OS version OS information. 21 | OperatingSystemVersionIdentifier = "ProductVersion" 22 | ) 23 | 24 | // Info returns a list of OS relevant information. 25 | func Info() (info map[string]string, err error) { 26 | info = map[string]string{} 27 | 28 | kernelVersionCommand := exec.Command("uname", "-r") 29 | var kernelVersion bytes.Buffer 30 | kernelVersionCommand.Stdout = &kernelVersion 31 | if err := kernelVersionCommand.Run(); err != nil { 32 | return nil, errors.Wrap(err, "failed to query kernel versions") 33 | } 34 | info[KernelVersionIdentifier] = strings.TrimSpace(kernelVersion.String()) 35 | 36 | osReleaseData, osReleaseError := os.ReadFile("/etc/os-release") 37 | if osReleaseError != nil { 38 | osReleaseData, err = os.ReadFile("/etc/lsb-release") 39 | if err != nil { 40 | return nil, errors.Wrap(err, "failed to query /etc/os-release") 41 | } 42 | } 43 | for _, line := range strings.Split(string(osReleaseData), "\n") { 44 | ls := strings.SplitN(line, "=", 2) 45 | if len(ls) == 1 { // Ignore empty lines 46 | continue 47 | } 48 | k := strings.TrimSpace(ls[0]) 49 | v := strings.TrimSpace(ls[1]) 50 | if v[0] == '"' { 51 | v, err = strconv.Unquote(v) 52 | if err != nil { 53 | return nil, errors.Wrap(err, "failed to unquote data in /etc/os-release") 54 | } 55 | } 56 | switch k { 57 | case "NAME": 58 | info[OperatingSystemIdentifier] = v 59 | case "VERSION_ID": 60 | info[OperatingSystemVersionIdentifier] = v 61 | } 62 | } 63 | 64 | info[EnvironmentPathIdentifier] = os.Getenv(EnvironmentPathIdentifier) 65 | 66 | return info, err 67 | } 68 | -------------------------------------------------------------------------------- /info_linux_test.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package osutil 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestInfo(t *testing.T) { 12 | info, err := Info() 13 | assert.NoError(t, err) 14 | 15 | assert.NotEmpty(t, info[OperatingSystemVersionIdentifier]) 16 | assert.NotEmpty(t, info[OperatingSystemIdentifier]) 17 | assert.NotEmpty(t, info[KernelVersionIdentifier]) 18 | assert.NotEmpty(t, info[EnvironmentPathIdentifier]) 19 | } 20 | -------------------------------------------------------------------------------- /info_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | 3 | package osutil 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | "strconv" 9 | 10 | "github.com/pkg/errors" 11 | "golang.org/x/sys/windows/registry" 12 | ) 13 | 14 | const ( 15 | // OperatingSystemBuildIdentifier holds the identifier for the OS build OS information. 16 | OperatingSystemBuildIdentifier = "BuildVersion" 17 | // OperatingSystemIdentifier holds the identifier for the OS name OS information. 18 | OperatingSystemIdentifier = "ProductName" 19 | // OperatingSystemVersionIdentifier holds the identifier for the OS version OS information. 20 | OperatingSystemVersionIdentifier = "ProductVersion" 21 | // OperatingSystemVersionMajorIdentifier holds the identifier for the OS major version OS information. 22 | OperatingSystemVersionMajorIdentifier = "ProductVersionMajor" 23 | // OperatingSystemVersionMinorIdentifier holds the identifier for the OS minor version OS information. 24 | OperatingSystemVersionMinorIdentifier = "ProductVersionMinor" 25 | ) 26 | 27 | const ( 28 | registryKeyBuildUpdateVersion = "UBR" 29 | registryKeyBuildVersion = "CurrentBuildNumber" 30 | registryKeyProductMajorVersion = "CurrentMajorVersionNumber" 31 | registryKeyProductVersionMinor = "CurrentMinorVersionNumber" 32 | registryKeyProductName = "ProductName" 33 | registryKeyProductVersion = "CurrentVersion" 34 | ) 35 | 36 | // Info returns a list of OS relevant information. 37 | func Info() (info map[string]string, err error) { 38 | k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE) 39 | if err != nil { 40 | return nil, errors.Wrap(err, "failed to open registry") 41 | } 42 | defer k.Close() 43 | 44 | info = map[string]string{} 45 | 46 | info[OperatingSystemBuildIdentifier], _, err = k.GetStringValue(registryKeyBuildVersion) 47 | if err != nil { 48 | return nil, fmt.Errorf("failed to query key %s: %w", registryKeyBuildVersion, err) 49 | } 50 | { 51 | data, _, err := k.GetIntegerValue(registryKeyBuildUpdateVersion) 52 | if err != nil && err != registry.ErrNotExist { 53 | return nil, fmt.Errorf("failed to query key %s: %w", registryKeyBuildUpdateVersion, err) 54 | } else if err == nil { 55 | info[OperatingSystemBuildIdentifier] += "." + strconv.FormatUint(data, 10) 56 | } 57 | } 58 | 59 | info[OperatingSystemIdentifier], _, err = k.GetStringValue(registryKeyProductName) 60 | if err != nil { 61 | return nil, fmt.Errorf("failed to query key %s: %w", registryKeyProductName, err) 62 | } 63 | 64 | info[OperatingSystemVersionIdentifier], _, err = k.GetStringValue(registryKeyProductVersion) 65 | if err != nil { 66 | return nil, fmt.Errorf("failed to query key %s: %w", registryKeyProductVersion, err) 67 | } 68 | { 69 | data, _, err := k.GetIntegerValue(registryKeyProductMajorVersion) 70 | if err != nil { 71 | return nil, fmt.Errorf("failed to query key %s: %w", registryKeyProductMajorVersion, err) 72 | } 73 | info[OperatingSystemVersionMajorIdentifier] = strconv.FormatUint(data, 10) 74 | } 75 | { 76 | data, _, err := k.GetIntegerValue(registryKeyProductVersionMinor) 77 | if err != nil { 78 | return nil, fmt.Errorf("failed to query key %s: %w", registryKeyProductVersionMinor, err) 79 | } 80 | info[OperatingSystemVersionMinorIdentifier] = strconv.FormatUint(data, 10) 81 | } 82 | 83 | info[EnvironmentPathIdentifier] = os.Getenv(EnvironmentPathIdentifier) 84 | 85 | return info, err 86 | } 87 | -------------------------------------------------------------------------------- /info_windows_test.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | 3 | package osutil 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestInfo(t *testing.T) { 12 | info, err := Info() 13 | assert.NoError(t, err) 14 | 15 | assert.NotEmpty(t, info[OperatingSystemBuildIdentifier]) 16 | assert.NotEmpty(t, info[OperatingSystemIdentifier]) 17 | assert.NotEmpty(t, info[OperatingSystemVersionIdentifier]) 18 | assert.NotEmpty(t, info[OperatingSystemVersionMajorIdentifier]) 19 | assert.NotEmpty(t, info[OperatingSystemVersionMinorIdentifier]) 20 | assert.NotEmpty(t, info[EnvironmentPathIdentifier]) 21 | } 22 | -------------------------------------------------------------------------------- /io.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | // InMemoryStream allows to read and write to an in-memory stream. 9 | type InMemoryStream struct { 10 | // reader holds the reading end of the stream. 11 | reader io.ReadCloser 12 | // writer holds the writing end of the stream. 13 | writer io.WriteCloser 14 | } 15 | 16 | var _ io.ReadWriteCloser = (*InMemoryStream)(nil) 17 | 18 | func NewInMemoryStream(reader io.ReadCloser, writer io.WriteCloser) *InMemoryStream { 19 | return &InMemoryStream{ 20 | reader: reader, 21 | writer: writer, 22 | } 23 | } 24 | 25 | // Read reads from the stream until the given buffer is full. 26 | func (s *InMemoryStream) Read(buffer []byte) (n int, err error) { 27 | return s.reader.Read(buffer) 28 | } 29 | 30 | // Write writes the given data to the stream. 31 | func (s *InMemoryStream) Write(data []byte) (n int, err error) { 32 | return s.writer.Write(data) 33 | } 34 | 35 | // Close closes the stream. 36 | func (s *InMemoryStream) Close() error { 37 | if err := s.reader.Close(); err != nil { 38 | return err 39 | } 40 | if err := s.writer.Close(); err != nil { 41 | return err 42 | } 43 | 44 | return nil 45 | } 46 | 47 | // StandardStream allows to read from STDIN and write to STDOUT. 48 | type StandardStream struct{} 49 | 50 | var _ io.ReadWriteCloser = (*StandardStream)(nil) 51 | 52 | // Read reads from the stream until the given buffer is full. 53 | func (s *StandardStream) Read(buffer []byte) (n int, err error) { 54 | return os.Stdin.Read(buffer) 55 | } 56 | 57 | // Write writes the given data to the stream. 58 | func (s *StandardStream) Write(data []byte) (n int, err error) { 59 | return os.Stdout.Write(data) 60 | } 61 | 62 | // Close closes the stream. 63 | func (s *StandardStream) Close() error { 64 | if err := os.Stdin.Close(); err != nil { 65 | return err 66 | } 67 | if err := os.Stdout.Close(); err != nil { 68 | return err 69 | } 70 | 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /limits.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // ProcessTreeLimits holds limits to apply to the current process's resource usage, including the resource usage of its descendant processes. 8 | type ProcessTreeLimits struct { 9 | // MaxMemoryInMiB holds the limit for the memory usage of the current process and all descendants in 1024-based mebibytes. 10 | // Zero means no limit. 11 | MaxMemoryInMiB uint 12 | // OnOutOfMemory may or may not run when the memory limit is reached, depending on the enforcement strategy. If it runs, the process will not be killed automatically and the function should end the process. 13 | OnMemoryLimitReached func(currentMemoryInMiB uint, maxMemoryInMiB uint) 14 | // WatchdogInterval holds the amount of time to sleep between checks if the limits have been exceeded, if a watchdog strategy is used. 15 | // The default is two seconds. 16 | WatchdogInterval time.Duration 17 | } 18 | -------------------------------------------------------------------------------- /limits_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package osutil 4 | 5 | import ( 6 | "bytes" 7 | "errors" 8 | "fmt" 9 | "os" 10 | "os/exec" 11 | "strconv" 12 | "syscall" 13 | "time" 14 | ) 15 | 16 | // EnforceProcessTreeLimits constrains the current process and all descendant processes to the specified limits. The current process exits when the limits are exceeded. 17 | func EnforceProcessTreeLimits(limits ProcessTreeLimits) { 18 | if limits.MaxMemoryInMiB <= 0 { 19 | return 20 | } 21 | 22 | var watchdogInterval time.Duration 23 | if limits.WatchdogInterval == 0 { 24 | watchdogInterval = 2 * time.Second 25 | } else { 26 | watchdogInterval = limits.WatchdogInterval 27 | } 28 | 29 | go func() { 30 | for { 31 | memoryUsageInKiB, err := getProcessTreeMemoryUsage() 32 | if err != nil { 33 | panic(fmt.Errorf("Failed to check memory usage: %w", err)) 34 | } 35 | 36 | currentMemoryInMiB := memoryUsageInKiB / 1024 37 | if currentMemoryInMiB > limits.MaxMemoryInMiB { 38 | limits.OnMemoryLimitReached(currentMemoryInMiB, limits.MaxMemoryInMiB) 39 | } 40 | time.Sleep(watchdogInterval) 41 | } 42 | }() 43 | } 44 | 45 | // getProcessTreeMemoryUsage returns the total memory usage in KiB from the current process and all child processes. 46 | // 47 | // REMARK This is currently a rough approximation. Memory shared between descendant processes is counted multiple times. 48 | func getProcessTreeMemoryUsage() (memoryUsageInKiB uint, err error) { 49 | psCmd := exec.Command("ps", "-H", "-o", "rss=", strconv.Itoa(os.Getpid())) 50 | psOutput, err := psCmd.CombinedOutput() 51 | if err != nil { 52 | return 0, err 53 | } 54 | lines := bytes.Split(bytes.TrimSpace(psOutput), []byte("\n")) 55 | for _, line := range lines { 56 | line = bytes.TrimSpace(line) 57 | if len(line) == 0 { 58 | return 0, errors.New("Encountered empty line in \"ps\" output") 59 | } 60 | rss, err := strconv.Atoi(string(line)) 61 | if err != nil { 62 | return 0, err 63 | } 64 | memoryUsageInKiB += uint(rss) 65 | } 66 | 67 | return memoryUsageInKiB, nil 68 | } 69 | 70 | // SetRLimitFiles temporarily changes the file descriptor resource limit while calling the given function. 71 | func SetRLimitFiles(limit uint64, call func(limit uint64)) (err error) { 72 | var tmp syscall.Rlimit 73 | if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &tmp); err != nil { 74 | return nil 75 | } 76 | defer func() { 77 | if err == nil { 78 | err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &tmp) 79 | } 80 | }() 81 | 82 | if err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &syscall.Rlimit{ 83 | Cur: limit, 84 | Max: tmp.Max, 85 | }); err != nil { 86 | return err 87 | } 88 | 89 | call(limit) 90 | 91 | return nil 92 | } 93 | -------------------------------------------------------------------------------- /limits_linux_test.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package osutil_test 4 | 5 | import ( 6 | "bytes" 7 | "errors" 8 | "io/fs" 9 | "os" 10 | "os/exec" 11 | "strconv" 12 | "testing" 13 | 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func TestEnforceProcessTreeLimitsMemory(t *testing.T) { 18 | type testCase struct { 19 | Name string 20 | 21 | MemoryLimitInMiB uint 22 | MinMemoryToAllocateInMiB uint 23 | 24 | ValidateOutput func(t *testing.T, programErr error, minMemoryAllocatedInMiB uint) 25 | } 26 | 27 | validate := func(t *testing.T, tc *testCase) { 28 | t.Run(tc.Name, func(t *testing.T) { 29 | assert.NoError(t, exec.Command("go", "build", "-v", "-trimpath", "-o", "limittest/memory", "limittest/memory.go").Run()) 30 | 31 | cmd := exec.Command("limittest/memory", strconv.Itoa(int(tc.MemoryLimitInMiB)), strconv.Itoa(int(tc.MinMemoryToAllocateInMiB))) 32 | out, programmErr := cmd.CombinedOutput() 33 | 34 | lines := bytes.Split(out, []byte("\n")) 35 | assert.True(t, len(lines) > 1) 36 | mem, err := strconv.Atoi(string(lines[len(lines)-2])) 37 | assert.NoError(t, err) 38 | 39 | tc.ValidateOutput(t, programmErr, uint(mem)) // REMARK The number of loop iterations in the test program is not an accurate metric for how much memory was used but it's an alright sanity check for low memory usage numbers. 40 | }) 41 | } 42 | 43 | validate(t, &testCase{ 44 | Name: "Limit hit", 45 | 46 | MemoryLimitInMiB: 100, 47 | MinMemoryToAllocateInMiB: 1000, 48 | 49 | ValidateOutput: func(t *testing.T, programErr error, mem uint) { 50 | if err, ok := programErr.(*exec.ExitError); ok { 51 | assert.Equal(t, 5, err.ExitCode()) 52 | } else { 53 | assert.Fail(t, "Error was not an ExitError: %v", err) 54 | } 55 | 56 | err := os.Remove("success.txt") 57 | if err == nil { 58 | assert.Fail(t, "Expected success.txt to not exist but it existed") 59 | } 60 | assert.True(t, errors.Is(err, fs.ErrNotExist), "Unexpected error from removing success.txt: %v", err) 61 | 62 | assert.False(t, mem > 300, "Expected memory usage to stay below 300MiB but was at least %dMiB", mem) 63 | }, 64 | }) 65 | 66 | validate(t, &testCase{ 67 | Name: "No limit hit", 68 | 69 | MemoryLimitInMiB: 1000, 70 | MinMemoryToAllocateInMiB: 100, 71 | 72 | ValidateOutput: func(t *testing.T, programErr error, mem uint) { 73 | assert.NoError(t, programErr) 74 | 75 | assert.NoError(t, os.Remove("success.txt")) 76 | 77 | assert.Equal(t, uint(100), mem) 78 | }, 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /limits_notlinux.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | 3 | package osutil 4 | 5 | // EnforceProcessTreeLimits constrains the current process and all descendant processes to the specified limits. The current process exits when the limits are exceeded. 6 | func EnforceProcessTreeLimits(limits ProcessTreeLimits) { 7 | // WORKAROUND Implement this function for MacOS and Windows when it is actual needed. Until then we can cross-compile even if the function is only mentioned in a package. https://$INTERNAL/symflower/symflower/-/issues/3592 8 | } 9 | -------------------------------------------------------------------------------- /limittest/memory.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/zimmski/osutil" 10 | ) 11 | 12 | func main() { 13 | memoryLimitInMiB, err := strconv.Atoi(os.Args[1]) 14 | if err != nil { 15 | panic(err) 16 | } 17 | memoryToAllocateInMiB, err := strconv.Atoi(os.Args[2]) 18 | if err != nil { 19 | panic(err) 20 | } 21 | 22 | osutil.EnforceProcessTreeLimits(osutil.ProcessTreeLimits{ 23 | MaxMemoryInMiB: uint(memoryLimitInMiB), 24 | OnMemoryLimitReached: func(currentMemoryInMiB uint, maxMemoryInMiB uint) { 25 | os.Exit(5) 26 | }, 27 | }) 28 | 29 | // Consume requested memory at roughly 20 MiB/s. 30 | mebibytes := []byte{} 31 | for i := 0; i < memoryToAllocateInMiB; i++ { 32 | time.Sleep(50 * time.Millisecond) 33 | mebibytes = append(mebibytes, make([]byte, 1024*1024)...) 34 | fmt.Printf("%d\n", i+1) 35 | } 36 | 37 | // Write a flag. 38 | if err := os.WriteFile("success.txt", []byte("process finished successfully"), 0640); err != nil { 39 | panic(err) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /makefile.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "regexp" 7 | ) 8 | 9 | // MakeFileCopyTargets copyies Make targets of a Makefile to another Makefile that can have a manually-writen parts until a `# REMARK Do not edit` line. 10 | func MakeFileCopyTargets(sourceMakefilePath string, destinationMakefilePath string, makeTargets []string) (err error) { 11 | copiedMakefile, err := os.ReadFile(destinationMakefilePath) 12 | if err != nil { 13 | return err 14 | } 15 | 16 | // Only keep the manual written part of the Makefile. 17 | copiedMakefile = regexp.MustCompile(`(?ms)(\A.+# REMARK Do not edit.+?\n).+\z`).ReplaceAll(copiedMakefile, []byte("$1")) 18 | 19 | // Copy the Make targets from the original Makefile that should be included in the copied Makefile. 20 | originalMakefile, err := os.ReadFile(sourceMakefilePath) 21 | if err != nil { 22 | return err 23 | } 24 | for _, targetName := range makeTargets { 25 | matches := regexp.MustCompile(`(?ms)(` + targetName + `: .+?\.PHONY:.+?\n)`).FindSubmatch(originalMakefile) 26 | if matches == nil { 27 | return fmt.Errorf("could not find Make target %q", targetName) 28 | } 29 | 30 | copiedMakefile = append(copiedMakefile, []byte("\n")...) 31 | copiedMakefile = append(copiedMakefile, matches[1]...) 32 | } 33 | 34 | if err := os.WriteFile(destinationMakefilePath, copiedMakefile, 0644); err != nil { 35 | return err 36 | } 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /os.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import "runtime" 4 | 5 | // IsDarwin returns whether the operating system is Darwin. 6 | func IsDarwin() bool { 7 | return runtime.GOOS == Darwin 8 | } 9 | 10 | // IsLinux returns whether the operating system is Linux. 11 | func IsLinux() bool { 12 | return runtime.GOOS == Linux 13 | } 14 | 15 | // IsWindows returns whether the operating system is Windows. 16 | func IsWindows() bool { 17 | return runtime.GOOS == Windows 18 | } 19 | 20 | // IsArchitectureARMWith32Bit returns wheter the operating system runs on ARM with 32 bits. 21 | func IsArchitectureARMWith32Bit() bool { 22 | return runtime.GOARCH == "arm" 23 | } 24 | 25 | // IsArchitectureARMWith64Bit returns wheter the operating system runs on ARM with 64 bits. 26 | func IsArchitectureARMWith64Bit() bool { 27 | return runtime.GOARCH == "arm64" 28 | } 29 | 30 | // IsArchitectureX86With32Bit returns wheter the operating system runs on x86 with 32 bits. 31 | func IsArchitectureX86With32Bit() bool { 32 | return runtime.GOARCH == "386" 33 | } 34 | 35 | // IsArchitectureX86With64Bit returns wheter the operating system runs on x86 with 64 bits. 36 | func IsArchitectureX86With64Bit() bool { 37 | return runtime.GOARCH == "amd64" 38 | } 39 | -------------------------------------------------------------------------------- /path_notwindows.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package osutil 4 | 5 | const ( 6 | // EnvironmentPathIdentifier holds the environment variable identifier for the "PATH" variable. 7 | EnvironmentPathIdentifier = "PATH" 8 | ) 9 | -------------------------------------------------------------------------------- /path_operator.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | // GreatestCommonDirectory computes the greatest common part of the given paths. 9 | // The resulting string must be the prefix of all the given paths. 10 | func GreatestCommonDirectory(paths []string) string { 11 | if len(paths) == 0 { 12 | return "" 13 | } 14 | 15 | greatestCommonDirectory := paths[0] 16 | for i := 1; i < len(paths); i++ { 17 | current := paths[i] 18 | 19 | minLength := len(greatestCommonDirectory) 20 | if len(current) < minLength { 21 | minLength = len(current) 22 | } 23 | 24 | currentCommonIndex := 0 25 | for i := 0; i < minLength; i++ { 26 | if current[i] != greatestCommonDirectory[i] { 27 | break 28 | } 29 | 30 | if current[i] == os.PathSeparator { 31 | currentCommonIndex = i 32 | } else if i == minLength-1 { 33 | currentCommonIndex = minLength 34 | } 35 | } 36 | 37 | greatestCommonDirectory = greatestCommonDirectory[:currentCommonIndex] 38 | } 39 | 40 | return greatestCommonDirectory 41 | } 42 | 43 | // EnvironmentPathList returns the list of file paths contained in the "PATH" environment variable. 44 | func EnvironmentPathList() (filePaths []string) { 45 | path := os.Getenv(EnvironmentPathIdentifier) 46 | 47 | return strings.Split(path, string(os.PathListSeparator)) 48 | } 49 | 50 | // RemoveFromEnvironmentPathBySearchTerm returns the content of the "PATH" environment variable where file paths containing the given search terms are removed. 51 | func RemoveFromEnvironmentPathBySearchTerm(searchTerms ...string) (newEnvironmentPath string) { 52 | filePathsOld := EnvironmentPathList() 53 | var filePathsNew []string 54 | PATH: 55 | for _, p := range filePathsOld { 56 | for _, term := range searchTerms { 57 | if strings.Contains(p, term) { 58 | continue PATH 59 | } 60 | } 61 | 62 | filePathsNew = append(filePathsNew, p) 63 | } 64 | 65 | return strings.Join(filePathsNew, string(os.PathListSeparator)) 66 | } 67 | -------------------------------------------------------------------------------- /path_operator_test.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestGreatestCommonDirectory(t *testing.T) { 11 | type testCase struct { 12 | Name string 13 | 14 | Paths []string 15 | 16 | ExpectedString string 17 | } 18 | 19 | validate := func(t *testing.T, tc *testCase) { 20 | t.Run(tc.Name, func(t *testing.T) { 21 | for i, p := range tc.Paths { 22 | tc.Paths[i] = filepath.FromSlash(p) 23 | } 24 | tc.ExpectedString = filepath.FromSlash(tc.ExpectedString) 25 | 26 | actualString := GreatestCommonDirectory(tc.Paths) 27 | 28 | assert.Equal(t, tc.ExpectedString, actualString) 29 | }) 30 | } 31 | 32 | validate(t, &testCase{ 33 | Name: "Equal paths", 34 | 35 | Paths: []string{ 36 | "pkg/a", 37 | "pkg/a", 38 | }, 39 | 40 | ExpectedString: "pkg/a", 41 | }) 42 | validate(t, &testCase{ 43 | Name: "Equal paths with trailing slash for one path", 44 | 45 | Paths: []string{ 46 | "pkg/a", 47 | "pkg/a/", 48 | }, 49 | 50 | ExpectedString: "pkg/a", 51 | }) 52 | validate(t, &testCase{ 53 | Name: "Equal paths with trailing slash for both paths", 54 | 55 | Paths: []string{ 56 | "pkg/a/", 57 | "pkg/a/", 58 | }, 59 | 60 | ExpectedString: "pkg/a", 61 | }) 62 | validate(t, &testCase{ 63 | Name: "Simple with common part", 64 | 65 | Paths: []string{ 66 | "pkg/a", 67 | "pkg/b", 68 | }, 69 | 70 | ExpectedString: "pkg", 71 | }) 72 | validate(t, &testCase{ 73 | Name: "Simple without common start", 74 | 75 | Paths: []string{ 76 | "pkg/a", 77 | "other/a", 78 | }, 79 | 80 | ExpectedString: "", 81 | }) 82 | validate(t, &testCase{ 83 | Name: "Different number of path components", 84 | 85 | Paths: []string{ 86 | "pkg/a/b", 87 | "pkg/c", 88 | }, 89 | 90 | ExpectedString: "pkg", 91 | }) 92 | validate(t, &testCase{ 93 | Name: "Multiple common parts", 94 | 95 | Paths: []string{ 96 | "same/pkg/until/now", 97 | "same/pkg/until/then", 98 | }, 99 | 100 | ExpectedString: "same/pkg/until", 101 | }) 102 | } 103 | -------------------------------------------------------------------------------- /path_windows.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | const ( 4 | // EnvironmentPathIdentifier holds the environment variable identifier for the "PATH" variable. 5 | EnvironmentPathIdentifier = "Path" 6 | ) 7 | -------------------------------------------------------------------------------- /permission.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "io/fs" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | // DirectoryPermissionOfParent looks at parent directory of the given path and returns a directory permission based on the permission of the parent. 10 | // The returned permission copies the read, write and execute permissions. 11 | func DirectoryPermissionOfParent(path string) (permission fs.FileMode, err error) { 12 | path, err = filepath.Abs(path) 13 | if err != nil { 14 | return 0, err 15 | } 16 | var s fs.FileInfo 17 | for { 18 | if path != "/" { 19 | path = filepath.Dir(path) 20 | } 21 | 22 | s, err = os.Stat(path) 23 | if err != nil { 24 | continue 25 | } 26 | 27 | break 28 | } 29 | 30 | permission |= s.Mode() & 0777 31 | 32 | return permission, nil 33 | } 34 | 35 | // FilePermissionOfParent looks at parent directory of the given path and returns a file permission based on the permission of the parent. 36 | // The returned permission copies the read and write permissions but not the execute permission. 37 | func FilePermissionOfParent(path string) (permission fs.FileMode, err error) { 38 | path, err = filepath.Abs(path) 39 | if err != nil { 40 | return 0, err 41 | } 42 | var s fs.FileInfo 43 | for { 44 | if path != "/" { 45 | path = filepath.Dir(path) 46 | } 47 | 48 | s, err = os.Stat(path) 49 | if err != nil { 50 | continue 51 | } 52 | 53 | break 54 | } 55 | 56 | permission |= s.Mode() & 0666 57 | 58 | return permission, nil 59 | } 60 | -------------------------------------------------------------------------------- /permission_test.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "fmt" 5 | "io/fs" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestDirectoryPermissionOfParent(t *testing.T) { 15 | type testCase struct { 16 | // Name holds the name of the test case. 17 | Name string 18 | 19 | // Path holds the relative directory path that should be created and used. 20 | Path string 21 | // Permission holds the directory permission that should be applied and that we should read back. 22 | Permission fs.FileMode 23 | } 24 | 25 | validate := func(t *testing.T, tc testCase) { 26 | t.Run(tc.Name, func(t *testing.T) { 27 | path, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", "-")) 28 | assert.NoError(t, err) 29 | defer func() { 30 | assert.NoError(t, os.RemoveAll(path)) 31 | }() 32 | 33 | p := path + "/" + tc.Path 34 | assert.NoError(t, os.MkdirAll(p, tc.Permission)) 35 | assert.NoError(t, os.Chmod(filepath.Dir(p), tc.Permission)) // The "umask" usually removes some bits of the permission, we do want to have all of them. 36 | permission, err := DirectoryPermissionOfParent(p) 37 | assert.NoError(t, err) 38 | if IsWindows() { 39 | assert.Equal(t, fmt.Sprintf("%o", 0777), fmt.Sprintf("%o", permission)) // TODO Implement file permission handling for Windows. https://$INTERNAL/symflower/symflower/-/issues/3637 40 | } else { 41 | assert.Equal(t, fmt.Sprintf("%o", tc.Permission), fmt.Sprintf("%o", permission)) 42 | } 43 | }) 44 | } 45 | 46 | validate(t, testCase{ 47 | Name: "User", 48 | 49 | Path: "user/child", 50 | Permission: 0700, 51 | }) 52 | 53 | validate(t, testCase{ 54 | Name: "Group", 55 | 56 | Path: "group/child", 57 | Permission: 0770, 58 | }) 59 | 60 | validate(t, testCase{ 61 | Name: "All", 62 | 63 | Path: "all/child", 64 | Permission: 0777, 65 | }) 66 | } 67 | 68 | func TestFilePermissionOfParent(t *testing.T) { 69 | type testCase struct { 70 | // Name holds the name of the test case. 71 | Name string 72 | 73 | // Path holds the relative file path that should be created and used. 74 | Path string 75 | // Permission holds the directory permission that should be applied. 76 | Permission fs.FileMode 77 | 78 | // FilePermission holds the file permission that we want to read back. 79 | FilePermission fs.FileMode 80 | } 81 | 82 | validate := func(t *testing.T, tc testCase) { 83 | t.Run(tc.Name, func(t *testing.T) { 84 | path, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", "-")) 85 | assert.NoError(t, err) 86 | defer func() { 87 | assert.NoError(t, os.RemoveAll(path)) 88 | }() 89 | 90 | directoryPath := filepath.Dir(path + "/" + tc.Path) 91 | assert.NoError(t, os.MkdirAll(directoryPath, tc.Permission)) 92 | assert.NoError(t, os.Chmod(directoryPath, tc.Permission)) // The "umask" usually removes some bits of the permission, we do want to have all of them. 93 | permission, err := FilePermissionOfParent(path + "/" + tc.Path) 94 | assert.NoError(t, err) 95 | if IsWindows() { 96 | assert.Equal(t, fmt.Sprintf("%o", 0666), fmt.Sprintf("%o", permission)) // TODO Implement file permission handling for Windows. https://$INTERNAL/symflower/symflower/-/issues/3637 97 | } else { 98 | assert.Equal(t, fmt.Sprintf("%o", tc.FilePermission), fmt.Sprintf("%o", permission)) 99 | } 100 | }) 101 | } 102 | 103 | validate(t, testCase{ 104 | Name: "User", 105 | 106 | Path: "user/child", 107 | Permission: 0700, 108 | 109 | FilePermission: 0600, 110 | }) 111 | 112 | validate(t, testCase{ 113 | Name: "Group", 114 | 115 | Path: "group/child", 116 | Permission: 0770, 117 | 118 | FilePermission: 0660, 119 | }) 120 | 121 | validate(t, testCase{ 122 | Name: "All", 123 | 124 | Path: "all/child", 125 | Permission: 0777, 126 | 127 | FilePermission: 0666, 128 | }) 129 | } 130 | -------------------------------------------------------------------------------- /platform_notwindows.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package osutil 4 | 5 | // BatchFileExtension returns the common file extension of a batch file. 6 | func BatchFileExtension() (extension string) { 7 | return "" 8 | } 9 | 10 | // BinaryExtension returns the common file extension of a binary. 11 | func BinaryExtension() (extension string) { 12 | return "" 13 | } 14 | 15 | // CommandFileExtension returns the common file extension of a command file. 16 | func CommandFileExtension() (extension string) { 17 | return "" 18 | } 19 | -------------------------------------------------------------------------------- /platform_windows.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | // BatchFileExtension returns the common file extension of a batch file. 4 | func BatchFileExtension() (extension string) { 5 | return ".bat" 6 | } 7 | 8 | // BinaryExtension returns the common file extension of a binary. 9 | func BinaryExtension() (extension string) { 10 | return ".exe" 11 | } 12 | 13 | // CommandFileExtension returns the common file extension of a command file. 14 | func CommandFileExtension() (extension string) { 15 | return ".cmd" 16 | } 17 | -------------------------------------------------------------------------------- /progress.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "time" 7 | 8 | "github.com/schollz/progressbar/v3" 9 | ) 10 | 11 | var defaultProgressBarOptions = []progressbar.Option{ 12 | progressbar.OptionFullWidth(), 13 | progressbar.OptionSetRenderBlankState(false), // Print the progress bar when the first data arrives, not on initialization. This allows the user to print log entries until that first data, i.e. only one progress bar show until then. 14 | progressbar.OptionSetTheme(progressbar.Theme{ 15 | Saucer: "=", 16 | SaucerHead: ">", 17 | SaucerPadding: " ", 18 | BarStart: "[", 19 | BarEnd: "]", 20 | }), 21 | progressbar.OptionSetWidth(80), 22 | progressbar.OptionShowCount(), 23 | progressbar.OptionSpinnerType(14), 24 | progressbar.OptionThrottle(65 * time.Millisecond), 25 | } 26 | 27 | // ProgressBar returns a progress bar for counting items with sane defaults that prints its updates to the given writer. 28 | func ProgressBar(stream io.Writer, max int, description ...string) (progress *progressbar.ProgressBar) { 29 | var d string 30 | if len(description) > 0 { 31 | d = description[0] 32 | } 33 | 34 | os := []progressbar.Option{ 35 | progressbar.OptionOnCompletion(func() { 36 | if _, err := fmt.Fprint(stream, "\n"); err != nil { 37 | panic(err) 38 | } 39 | }), 40 | progressbar.OptionSetDescription(d), 41 | progressbar.OptionShowIts(), 42 | progressbar.OptionSetWriter(stream), 43 | } 44 | os = append(os, defaultProgressBarOptions...) 45 | 46 | return progressbar.NewOptions(max, os...) 47 | } 48 | 49 | // ProgressBarBytes returns a progress bar for counting bytes with sane defaults that prints its updates to the given writer. 50 | func ProgressBarBytes(stream io.Writer, length int, description ...string) (progress *progressbar.ProgressBar) { 51 | var d string 52 | if len(description) > 0 { 53 | d = description[0] 54 | } 55 | 56 | os := []progressbar.Option{ 57 | progressbar.OptionOnCompletion(func() { 58 | if _, err := fmt.Fprint(stream, "\n"); err != nil { 59 | panic(err) 60 | } 61 | }), 62 | progressbar.OptionSetDescription(d), 63 | progressbar.OptionShowBytes(true), 64 | progressbar.OptionSetWriter(stream), 65 | } 66 | os = append(os, defaultProgressBarOptions...) 67 | 68 | return progressbar.NewOptions(length, os...) 69 | } 70 | 71 | // ActivityIndicator prints a spinning activity indicator to the given stream until the indicator is stopped. 72 | func ActivityIndicator(stream io.Writer, description ...string) (stopIndicator func()) { 73 | var d string 74 | if len(description) > 0 { 75 | d = description[0] 76 | } 77 | 78 | os := []progressbar.Option{ 79 | progressbar.OptionOnCompletion(func() { 80 | if _, err := fmt.Fprint(stream, "\n"); err != nil { 81 | panic(err) 82 | } 83 | }), 84 | progressbar.OptionSetDescription(d), 85 | progressbar.OptionSetWriter(stream), 86 | progressbar.OptionSpinnerType(14), 87 | progressbar.OptionThrottle(65 * time.Millisecond), 88 | } 89 | 90 | const unknownLength = -1 91 | indicator := progressbar.NewOptions(unknownLength, os...) 92 | runChannel := make(chan struct{}) 93 | go func() { 94 | for { 95 | select { 96 | case <-runChannel: 97 | return 98 | default: 99 | if err := indicator.Add(1); err != nil { 100 | panic(err) 101 | } 102 | } 103 | 104 | time.Sleep(100 * time.Millisecond) 105 | } 106 | }() 107 | 108 | return func() { 109 | close(runChannel) 110 | _ = indicator.Finish() 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /remove.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "time" 8 | 9 | "github.com/avast/retry-go" 10 | ) 11 | 12 | // RemoveTemporaryDirectory removes the given temporary directory path from disk with special handling for Windows. 13 | // The reason we need special handling is because Windows seems to be colossally stupid when it comes to handling the open-ess of files and directories. https://$INTERNAL/symflower/symflower/-/merge_requests/2399#note_293837. 14 | func RemoveTemporaryDirectory(directoryPath string) { 15 | isRetryableError := func(err error) bool { 16 | return IsWindows() && (strings.Contains(err.Error(), "Access is denied") || strings.Contains(err.Error(), "The process cannot access the file because it is being used by another process.")) 17 | } 18 | 19 | if err := retry.Do( 20 | func() error { 21 | return os.RemoveAll(directoryPath) 22 | }, 23 | retry.Attempts(3), 24 | retry.Delay(2*time.Second), 25 | retry.LastErrorOnly(true), 26 | retry.RetryIf(func(err error) bool { 27 | // On Windows we sometimes receive an access denied error which happens because a process has exited but is not yet cleaned up by the operating system and still has a handler to a file we want to delete. In this case we wait a while and try again to remove the directory. 28 | return isRetryableError(err) 29 | }), 30 | ); err != nil { 31 | if !isRetryableError(err) { 32 | if err != nil { 33 | panic(err) 34 | } 35 | } 36 | 37 | // At this point we have given our best to delete the directory. The only chance we now have is to end this processes we are currently in and then delete the directory. See https://$INTERNAL/symflower/symflower/-/merge_requests/2399#note_293837 for details. The only thing we can now do is to log the error, so the processes higher up in the process tree can know about this tragedy and deal with it accordingly. 38 | fmt.Fprintf(os.Stderr, "cannot remove temporary directory: %s\n", err) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /scripts/editor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exuo pipefail 4 | 5 | export VSCODE_OPTIONS=${VSCODE_OPTIONS:-""} 6 | 7 | code $VSCODE_OPTIONS $ROOT_DIR 8 | -------------------------------------------------------------------------------- /static.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "os" 5 | "regexp" 6 | "time" 7 | ) 8 | 9 | // StaticFile holds a single file or directory in-memory. 10 | type StaticFile struct { 11 | Data string 12 | Mime string 13 | Mtime time.Time 14 | // Size is the size before compression. 15 | // If 0, it means the data is uncompressed. 16 | Size int 17 | // Hash is a SHA-256 hash of the file contents, which is used for the Etag, and useful for caching. 18 | Hash string 19 | // Directory determines if this file is a directory. 20 | Directory bool 21 | } 22 | 23 | // RewriteStaticIndexFile rewrites a `github.com/bouk/staticfiles` index file to be extendable by replacing inlined code to common code. 24 | func RewriteStaticIndexFile(filePath string) (err error) { 25 | data, err := os.ReadFile(filePath) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | data = regexp.MustCompile(`(?s)(\t"golang.org/x/text/language"\n)`).ReplaceAll(data, []byte("\t\"github.com/zimmski/osutil\"\n$1")) 31 | data = regexp.MustCompile(`(?s)type StaticFilesFile struct {.+?}\n\s+`).ReplaceAll(data, []byte("")) 32 | data = regexp.MustCompile(`StaticFilesFile`).ReplaceAll(data, []byte("osutil.StaticFile")) 33 | 34 | if err := os.WriteFile(filePath, data, 0); err != nil { 35 | return err 36 | } 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /sync.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | // SyncedMap holds a synchronized map. 8 | type SyncedMap[K comparable, V any] struct { 9 | mutex sync.RWMutex 10 | m map[K]V 11 | } 12 | 13 | // NewSyncedMap returns a nw synchronized map. 14 | func NewSyncedMap[K comparable, V any]() (syncedMap *SyncedMap[K, V]) { 15 | return &SyncedMap[K, V]{ 16 | m: map[K]V{}, 17 | } 18 | } 19 | 20 | // Delete removes the entry with the given key from the map. 21 | func (m *SyncedMap[K, V]) Delete(key K) { 22 | m.mutex.Lock() 23 | defer m.mutex.Unlock() 24 | 25 | delete(m.m, key) 26 | } 27 | 28 | // Get returns the value for the given key. 29 | func (m *SyncedMap[K, V]) Get(key K) (value V, ok bool) { 30 | m.mutex.RLock() 31 | defer m.mutex.RUnlock() 32 | 33 | value, ok = m.m[key] 34 | 35 | return value, ok 36 | } 37 | 38 | // Set sets the given value for the given key. 39 | func (m *SyncedMap[K, V]) Set(key K, value V) { 40 | m.mutex.Lock() 41 | defer m.mutex.Unlock() 42 | 43 | m.m[key] = value 44 | } 45 | -------------------------------------------------------------------------------- /templateutil/defaults.go: -------------------------------------------------------------------------------- 1 | package templateutil 2 | 3 | import ( 4 | "encoding/base64" 5 | "text/template" 6 | "unicode" 7 | 8 | "github.com/symflower/pretty" 9 | "github.com/zimmski/osutil/bytesutil" 10 | ) 11 | 12 | // DefaultFuncMap holds common template functions. 13 | var DefaultFuncMap = template.FuncMap{ 14 | "base64": func(in string) string { 15 | return base64.StdEncoding.EncodeToString([]byte(in)) 16 | }, 17 | "prefixContinuationLinesWith": bytesutil.PrefixContinuationLinesWith, 18 | "lowerFirst": func(s string) string { 19 | return string(unicode.ToLower(rune(s[0]))) + s[1:] 20 | }, 21 | "pretty": func(data any) string { 22 | return pretty.Sprintf("%# v", data) 23 | }, 24 | "prettyLazy": func(data any) string { 25 | return pretty.LazySprintf("%# v", data) 26 | }, 27 | "quote": func(data any) string { 28 | return pretty.Sprintf("%q", data) 29 | }, 30 | } 31 | 32 | // MergeFuncMaps returns all functions of "a" and all functions of "b" in a new function mapping. 33 | // For entries that are defined in both maps the entry defined in b is chosen. 34 | func MergeFuncMaps(a template.FuncMap, b template.FuncMap) template.FuncMap { 35 | c := template.FuncMap{} 36 | 37 | for n, f := range a { 38 | c[n] = f 39 | } 40 | for n, f := range b { 41 | c[n] = f 42 | } 43 | 44 | return c 45 | } 46 | -------------------------------------------------------------------------------- /templateutil/file.go: -------------------------------------------------------------------------------- 1 | package templateutil 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "os" 7 | "path/filepath" 8 | "text/template" 9 | 10 | pkgerrors "github.com/pkg/errors" 11 | "github.com/symflower/pretty" 12 | ) 13 | 14 | // WriteTemplateToFile executes a template with the given data and saves the result into a file. 15 | func WriteTemplateToFile(filePath string, tmpl Template, data any) error { 16 | var driver bytes.Buffer 17 | 18 | err := tmpl.Execute(&driver, data) 19 | if err != nil { 20 | return pkgerrors.Wrap(err, pretty.LazySprintf("%# v", data)) 21 | } 22 | 23 | err = os.WriteFile(filePath, driver.Bytes(), 0640) 24 | if err != nil { 25 | return pkgerrors.Wrap(err, filePath) 26 | } 27 | 28 | return nil 29 | } 30 | 31 | // RewriteFileAsTemplate read in a file, execute it as a template with the given data and save the result into the same file. 32 | func RewriteFileAsTemplate(filePath string, funcMap template.FuncMap, data any) (err error) { 33 | tmpl, err := template.New(filepath.Base(filePath)).Funcs(funcMap).ParseFiles(filePath) // REMARK Use the file name as template identifier because otherwise `template.ParseFiles` fails. 34 | if err != nil { 35 | return pkgerrors.Wrap(err, filePath) 36 | } 37 | 38 | f, err := os.Create(filePath) 39 | if err != nil { 40 | return pkgerrors.Wrap(err, filePath) 41 | } 42 | defer func() { 43 | if e := f.Close(); e != nil { 44 | err = errors.Join(err, e) 45 | } 46 | }() 47 | 48 | if err := tmpl.Execute(f, data); err != nil { 49 | return pkgerrors.Wrap(err, filePath) 50 | } 51 | 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /templateutil/interface.go: -------------------------------------------------------------------------------- 1 | package templateutil 2 | 3 | import "io" 4 | 5 | type Template interface { 6 | // Execute applies the template to the specified data object and writes the output to the writer. 7 | Execute(wr io.Writer, data any) (err error) 8 | } 9 | -------------------------------------------------------------------------------- /variables.go: -------------------------------------------------------------------------------- 1 | package osutil 2 | 3 | const ( 4 | // Darwin holds the value of GOOS on MacOS. 5 | Darwin = "darwin" 6 | // Linux holds the value of GOOS on Linux. 7 | Linux = "linux" 8 | // Windows holds the value of GOOS on Windows. 9 | Windows = "windows" 10 | ) 11 | --------------------------------------------------------------------------------