├── for_tests.txt ├── .gitpod.yml ├── .gitignore ├── README.md ├── log.go ├── file_test.go ├── .github └── workflows │ └── build.yml ├── util_test.go ├── compress_test.go ├── go.mod ├── mime.go ├── LICENSE ├── string_test.go ├── duration.go ├── git.go ├── cmd.go ├── http.go ├── file_walk.go ├── deploy.go ├── string.go ├── wc.go ├── util.go ├── compress.go ├── minio.go ├── go.sum └── file.go /for_tests.txt: -------------------------------------------------------------------------------- 1 | line 1 2 | lien 2 3 | line 3 4 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | tasks: 2 | - init: go get && go build ./... && go test ./... 3 | command: go run 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## This has moved to https://github.com/kjk/common (`u`) 2 | 3 | Go utility functions that I use in multiple projects. 4 | 5 | Documentation: 6 | * [api docs](https://godoc.org/github.com/kjk/u) 7 | 8 | It contains functions to help writing equivalent of wc -l in Go: 9 | * [example use](https://presstige.io/p/Using-Go-instead-of-bash-for-scripts-6b51885c1f6940aeb40476000d0eb0fc#cd603cb2-0887-4f28-9d14-e46a5e5319c5) 10 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | ) 7 | 8 | var ( 9 | LogFile io.Writer 10 | ) 11 | 12 | // a centralized place allows us to tweak logging, if need be 13 | func Logf(format string, args ...interface{}) { 14 | if len(args) == 0 { 15 | fmt.Print(format) 16 | if LogFile != nil { 17 | _, _ = fmt.Fprint(LogFile, format) 18 | } 19 | return 20 | } 21 | fmt.Printf(format, args...) 22 | if LogFile != nil { 23 | _, _ = fmt.Fprintf(LogFile, format, args...) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /file_test.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestFile(t *testing.T) { 10 | assert.True(t, FileExists("file.go")) 11 | assert.False(t, FileExists("file_that_doesnt_exist.go")) 12 | assert.True(t, DirExists(".")) 13 | assert.False(t, DirExists("dir_that_doesnt_exist")) 14 | 15 | lines, err := ReadLinesFromFile("for_tests.txt") 16 | assert.Nil(t, err) 17 | assert.Equal(t, 3, len(lines)) 18 | assert.Equal(t, "line 1", lines[0]) 19 | } 20 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push, pull_request] 3 | jobs: 4 | build: 5 | name: Build 6 | runs-on: ubuntu-latest 7 | steps: 8 | - name: Set up Go 1.15 9 | uses: actions/setup-go@v2 10 | with: 11 | go-version: 1.15 12 | 13 | - name: Check out source code 14 | uses: actions/checkout@v2 15 | 16 | - name: Test 17 | run: go test -v ./... 18 | 19 | - name: Staticcheck 20 | run: | 21 | go get -u honnef.co/go/tools/cmd/staticcheck 22 | staticcheck ./... 23 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func testEncodeBase64(t *testing.T, n int) { 10 | s := EncodeBase64(n) 11 | n2, err := DecodeBase64(s) 12 | assert.Nil(t, err) 13 | assert.Equal(t, n, n2) 14 | } 15 | 16 | func TestEncodeBase64(t *testing.T) { 17 | testEncodeBase64(t, 1404040) 18 | testEncodeBase64(t, 0) 19 | testEncodeBase64(t, 1) 20 | testEncodeBase64(t, 35) 21 | testEncodeBase64(t, 36) 22 | testEncodeBase64(t, 37) 23 | testEncodeBase64(t, 123413343) 24 | _, err := DecodeBase64("azasdf!") 25 | assert.Error(t, err) 26 | } 27 | -------------------------------------------------------------------------------- /compress_test.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func testGzip(t *testing.T, path string) { 14 | d, err := ioutil.ReadFile(path) 15 | assert.Nil(t, err) 16 | 17 | dstPath := path + ".gz" 18 | err = GzipFile(dstPath, path) 19 | defer os.Remove(dstPath) 20 | assert.Nil(t, err) 21 | r, err := OpenFileMaybeCompressed(dstPath) 22 | assert.Nil(t, err) 23 | defer r.Close() 24 | var dst bytes.Buffer 25 | _, err = io.Copy(&dst, r) 26 | assert.Nil(t, err) 27 | d2 := dst.Bytes() 28 | assert.Equal(t, d, d2) 29 | } 30 | 31 | func TestGzip(t *testing.T) { 32 | testGzip(t, "compress.go") 33 | } 34 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kjk/u 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/dustin/go-humanize v1.0.0 7 | github.com/json-iterator/go v1.1.10 // indirect 8 | github.com/kjk/atomicfile v0.0.0-20190916063300-2d5c7d7d05bf 9 | github.com/klauspost/cpuid v1.3.1 // indirect 10 | github.com/minio/minio-go/v6 v6.0.57 11 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 12 | github.com/modern-go/reflect2 v1.0.1 // indirect 13 | github.com/stretchr/objx v0.1.1 // indirect 14 | github.com/stretchr/testify v1.3.0 15 | golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 // indirect 16 | golang.org/x/net v0.0.0-20201024042810-be3efd7ff127 // indirect 17 | golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd // indirect 18 | gopkg.in/ini.v1 v1.62.0 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /mime.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "path/filepath" 5 | "strings" 6 | ) 7 | 8 | /* additions to mime */ 9 | 10 | var mimeTypes = map[string]string{ 11 | // this is a list from go's mime package 12 | ".css": "text/css; charset=utf-8", 13 | ".gif": "image/gif", 14 | ".htm": "text/html; charset=utf-8", 15 | ".html": "text/html; charset=utf-8", 16 | ".jpg": "image/jpeg", 17 | ".js": "application/javascript", 18 | ".wasm": "application/wasm", 19 | ".pdf": "application/pdf", 20 | ".png": "image/png", 21 | ".svg": "image/svg+xml", 22 | ".xml": "text/xml; charset=utf-8", 23 | 24 | // those are my additions 25 | ".txt": "text/plain", 26 | ".exe": "application/octet-stream", 27 | ".json": "application/json", 28 | } 29 | 30 | func MimeTypeFromFileName(path string) string { 31 | ext := strings.ToLower(filepath.Ext(path)) 32 | mt := mimeTypes[ext] 33 | if mt != "" { 34 | return mt 35 | } 36 | // if not given, default to this 37 | return "application/octet-stream" 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014-2017 Krzysztof Kowalczyk 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /string_test.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestStringsRemoveFirst(t *testing.T) { 10 | tests := [][]string{ 11 | nil, nil, 12 | []string{"a"}, []string{}, 13 | []string{"a", "b"}, []string{"b"}, 14 | } 15 | n := len(tests) / 2 16 | for i := 0; i < n; i++ { 17 | got := StringsRemoveFirst(tests[i*2]) 18 | exp := tests[i*2+1] 19 | assert.Equal(t, exp, got) 20 | } 21 | } 22 | 23 | func TestRemoveDuplicateStrings(t *testing.T) { 24 | // note: the fact that arrays are sorted after RemoveDuplicateString 25 | // is accidental. We could make the tests more robust by writing 26 | // doStringArraysHaveTheSameContent(a1, a2 []string) 27 | tests := [][]string{ 28 | nil, nil, 29 | []string{"a"}, []string{"a"}, 30 | []string{"b", "a"}, []string{"a", "b"}, 31 | []string{"a", "a"}, []string{"a"}, 32 | []string{"a", "b", "a"}, []string{"a", "b"}, 33 | []string{"ab", "ba", "ab", "ab", "cd"}, []string{"ab", "ba", "cd"}, 34 | } 35 | n := len(tests) / 2 36 | for i := 0; i < n; i++ { 37 | got := RemoveDuplicateStrings(tests[i*2]) 38 | exp := tests[i*2+1] 39 | assert.Equal(t, exp, got) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /duration.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | ) 7 | 8 | // time.Duration with a better string representation 9 | type FormattedDuration time.Duration 10 | 11 | func (d FormattedDuration) String() string { 12 | return FormatDuration(time.Duration(d)) 13 | } 14 | 15 | // FormatDuration formats duration in a more human friendly way 16 | // than time.Duration.String() 17 | func FormatDuration(d time.Duration) string { 18 | s := d.String() 19 | if strings.HasSuffix(s, "µs") { 20 | // for µs we don't want fractions 21 | parts := strings.Split(s, ".") 22 | if len(parts) > 1 { 23 | return parts[0] + " µs" 24 | } 25 | return strings.ReplaceAll(s, "µs", " µs") 26 | } else if strings.HasSuffix(s, "ms") { 27 | // for ms we only want 2 digit fractions 28 | parts := strings.Split(s, ".") 29 | //fmt.Printf("fmtDur: '%s' => %#v\n", s, parts) 30 | if len(parts) > 1 { 31 | s2 := parts[1] 32 | if len(s2) > 4 { 33 | // 2 for "ms" and 2+ for fraction 34 | res := parts[0] + "." + s2[:2] + " ms" 35 | //fmt.Printf("fmtDur: s2: '%s', res: '%s'\n", s2, res) 36 | return res 37 | } 38 | } 39 | return strings.ReplaceAll(s, "ms", " ms") 40 | } 41 | return s 42 | } 43 | -------------------------------------------------------------------------------- /git.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | "strings" 7 | ) 8 | 9 | func GitPullMust(dir string) { 10 | cmd := exec.Command("git", "pull") 11 | if dir != "" { 12 | cmd.Dir = dir 13 | } 14 | RunCmdMust(cmd) 15 | } 16 | 17 | func GitStatusMust(dir string) string { 18 | cmd := exec.Command("git", "status") 19 | if dir != "" { 20 | cmd.Dir = dir 21 | } 22 | return RunCmdMust(cmd) 23 | } 24 | 25 | func IsGitClean(dir string) bool { 26 | s := GitStatusMust(dir) 27 | expected1 := []string{ 28 | "On branch master", 29 | "Your branch is up to date with 'origin/master'.", 30 | "nothing to commit, working tree clean", 31 | } 32 | expected2 := []string{ 33 | "On branch main", 34 | "Your branch is up to date with 'origin/main'.", 35 | "nothing to commit, working tree clean", 36 | } 37 | { 38 | hasAll := true 39 | for _, exp := range expected1 { 40 | if !strings.Contains(s, exp) { 41 | //Logf("Git repo in '%s' not clean.\nDidn't find '%s' in output of git status:\n%s\n", dir, exp, s) 42 | hasAll = false 43 | } 44 | } 45 | if hasAll { 46 | return true 47 | } 48 | } 49 | { 50 | hasAll := true 51 | for _, exp := range expected2 { 52 | if !strings.Contains(s, exp) { 53 | hasAll = false 54 | } 55 | } 56 | if hasAll { 57 | return true 58 | } 59 | } 60 | Logf("Git repo in '%s' not clean.\nGit status:\n%s\n", dir, s) 61 | return false 62 | } 63 | 64 | func EnsureGitClean(dir string) { 65 | if !IsGitClean(dir) { 66 | Must(fmt.Errorf("git repo in '%s' is not clean", dir)) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /cmd.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "runtime" 9 | "strings" 10 | ) 11 | 12 | // FmtCmdShort formats exec.Cmd in a short way 13 | func FmtCmdShort(cmd exec.Cmd) string { 14 | cmd.Path = filepath.Base(cmd.Path) 15 | return cmd.String() 16 | } 17 | 18 | // RunCmdLoggedMust runs a command and returns its stdout 19 | // Shows output as it happens 20 | func RunCmdLoggedMust(cmd *exec.Cmd) string { 21 | cmd.Stdout = os.Stdout 22 | cmd.Stderr = os.Stderr 23 | return RunCmdMust(cmd) 24 | } 25 | 26 | // RunCmdMust runs a command and returns its stdout 27 | func RunCmdMust(cmd *exec.Cmd) string { 28 | fmt.Printf("> %s\n", FmtCmdShort(*cmd)) 29 | canCapture := (cmd.Stdout == nil) && (cmd.Stderr == nil) 30 | if canCapture { 31 | out, err := cmd.CombinedOutput() 32 | if err == nil { 33 | if len(out) > 0 { 34 | fmt.Printf("Output:\n%s\n", string(out)) 35 | } 36 | return string(out) 37 | } 38 | fmt.Printf("cmd '%s' failed with '%s'. Output:\n%s\n", cmd, err, string(out)) 39 | Must(err) 40 | return string(out) 41 | } 42 | err := cmd.Run() 43 | if err == nil { 44 | return "" 45 | } 46 | fmt.Printf("cmd '%s' failed with '%s'\n", cmd, err) 47 | Must(err) 48 | return "" 49 | } 50 | 51 | func OpenNotepadWithFileMust(path string) { 52 | cmd := exec.Command("notepad.exe", path) 53 | err := cmd.Start() 54 | Must(err) 55 | } 56 | 57 | func OpenCodeDiffMust(path1, path2 string) { 58 | if runtime.GOOS == "darwin" { 59 | path1 = strings.Replace(path1, ".\\", "./", -1) 60 | path2 = strings.Replace(path2, ".\\", "./", -1) 61 | } 62 | cmd := exec.Command("code", "--new-window", "--diff", path1, path2) 63 | fmt.Printf("> %s\n", FmtCmdShort(*cmd)) 64 | err := cmd.Start() 65 | Must(err) 66 | } 67 | -------------------------------------------------------------------------------- /http.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | // Request.RemoteAddress contains port, which we want to remove i.e.: 9 | // "[::1]:58292" => "[::1]" 10 | func ipAddrFromRemoteAddr(s string) string { 11 | idx := strings.LastIndex(s, ":") 12 | if idx == -1 { 13 | return s 14 | } 15 | return s[:idx] 16 | } 17 | 18 | // RequestGetRemoteAddress returns ip address of the client making the request, 19 | // taking into account http proxies 20 | func RequestGetRemoteAddress(r *http.Request) string { 21 | hdr := r.Header 22 | hdrRealIP := hdr.Get("X-Real-Ip") 23 | hdrForwardedFor := hdr.Get("X-Forwarded-For") 24 | if hdrRealIP == "" && hdrForwardedFor == "" { 25 | return ipAddrFromRemoteAddr(r.RemoteAddr) 26 | } 27 | if hdrForwardedFor != "" { 28 | // X-Forwarded-For is potentially a list of addresses separated with "," 29 | parts := strings.Split(hdrForwardedFor, ",") 30 | for i, p := range parts { 31 | parts[i] = strings.TrimSpace(p) 32 | } 33 | // TODO: should return first non-local address 34 | return parts[0] 35 | } 36 | return hdrRealIP 37 | } 38 | 39 | // RequestGetProtocol returns protocol under which the request is being served i.e. "http" or "https" 40 | func RequestGetProtocol(r *http.Request) string { 41 | hdr := r.Header 42 | // X-Forwarded-Proto is set by proxies e.g. CloudFlare 43 | forwardedProto := strings.TrimSpace(strings.ToLower(hdr.Get("X-Forwarded-Proto"))) 44 | if forwardedProto != "" { 45 | if forwardedProto == "http" || forwardedProto == "https" { 46 | return forwardedProto 47 | } 48 | } 49 | if r.TLS != nil { 50 | return "https" 51 | } 52 | return "http" 53 | } 54 | 55 | // RequestGetFullHost returns full host name e.g. "https://blog.kowalczyk.info/" 56 | func RequestGetFullHost(r *http.Request) string { 57 | return RequestGetProtocol(r) + "://" + r.Host 58 | } 59 | -------------------------------------------------------------------------------- /file_walk.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "sync/atomic" 8 | ) 9 | 10 | // FileWalkEntry describes a single file from FileWalk 11 | type FileWalkEntry struct { 12 | Dir string 13 | FileInfo os.FileInfo 14 | } 15 | 16 | // Path returns full path of the file 17 | func (e *FileWalkEntry) Path() string { 18 | return filepath.Join(e.Dir, e.FileInfo.Name()) 19 | } 20 | 21 | // FileWalk describes a file traversal 22 | type FileWalk struct { 23 | startDir string 24 | FilesChan chan *FileWalkEntry 25 | askedToStop int32 26 | } 27 | 28 | // Stop stops file traversal 29 | func (ft *FileWalk) Stop() { 30 | atomic.StoreInt32(&ft.askedToStop, 1) 31 | // drain the channel 32 | for range ft.FilesChan { 33 | } 34 | } 35 | 36 | func fileWalkWorker(ft *FileWalk) { 37 | toVisit := []string{ft.startDir} 38 | defer close(ft.FilesChan) 39 | 40 | for len(toVisit) > 0 { 41 | shouldStop := atomic.LoadInt32(&ft.askedToStop) 42 | if shouldStop > 0 { 43 | return 44 | } 45 | // would be more efficient to shift by one and 46 | // chop off at the end 47 | dir := toVisit[0] 48 | toVisit = StringsRemoveFirst(toVisit) 49 | 50 | files, err := ioutil.ReadDir(dir) 51 | // TODO: should I send errors as well? 52 | if err != nil { 53 | continue 54 | } 55 | for _, fi := range files { 56 | path := filepath.Join(dir, fi.Name()) 57 | mode := fi.Mode() 58 | if mode.IsDir() { 59 | toVisit = append(toVisit, path) 60 | } else if mode.IsRegular() { 61 | fte := &FileWalkEntry{ 62 | Dir: dir, 63 | FileInfo: fi, 64 | } 65 | ft.FilesChan <- fte 66 | } 67 | } 68 | } 69 | } 70 | 71 | // StartFileWalk starts a file traversal from startDir 72 | func StartFileWalk(startDir string) *FileWalk { 73 | // buffered channel so that 74 | ch := make(chan *FileWalkEntry, 1024*64) 75 | ft := &FileWalk{ 76 | startDir: startDir, 77 | FilesChan: ch, 78 | } 79 | go fileWalkWorker(ft) 80 | return ft 81 | } 82 | -------------------------------------------------------------------------------- /deploy.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | ) 10 | 11 | var ( 12 | ServerIPAddress string 13 | IdentityFilePath string 14 | ) 15 | 16 | func SshInteractive(user string) { 17 | panicIfServerInfoNotSet() 18 | cmd := exec.Command("ssh", "-i", IdentityFilePath, user) 19 | cmd.Stdin = os.Stdin 20 | RunCmdLoggedMust(cmd) 21 | } 22 | 23 | func LoginAsRoot() { 24 | user := fmt.Sprintf("root@%s", ServerIPAddress) 25 | SshInteractive(user) 26 | } 27 | 28 | // "-o StrictHostKeyChecking=no" is for the benefit of CI which start 29 | // fresh environment 30 | func ScpCopy(localSrcPath string, serverDstPath string) { 31 | panicIfServerInfoNotSet() 32 | cmd := exec.Command("scp", "-o", "StrictHostKeyChecking=no", "-i", IdentityFilePath, localSrcPath, serverDstPath) 33 | RunCmdLoggedMust(cmd) 34 | } 35 | 36 | // "-o StrictHostKeyChecking=no" is for the benefit of CI which start 37 | // fresh environment 38 | func SshExec(user string, script string) { 39 | panicIfServerInfoNotSet() 40 | cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=no", "-i", IdentityFilePath, user) 41 | r := bytes.NewBufferString(script) 42 | cmd.Stdin = r 43 | RunCmdLoggedMust(cmd) 44 | } 45 | 46 | func MakeExecScript(name string) string { 47 | script := fmt.Sprintf(` 48 | chmod ug+x %s 49 | %s 50 | rm %s 51 | `, name, name, name) 52 | return script 53 | } 54 | 55 | func panicIfServerInfoNotSet() { 56 | PanicIf(IdentityFilePath == "", "IdentityFilePath not set") 57 | PanicIf(!FileExists(IdentityFilePath), "IdentityFilePath '%s' doesn't exist", IdentityFilePath) 58 | PanicIf(ServerIPAddress == "", "ServerIPAddress not set") 59 | } 60 | 61 | // CopyAndExecServerScript copies a given script to the server and executes 62 | // it under a given user name 63 | func CopyAndExecServerScript(scriptLocalPath, user string) { 64 | panicIfServerInfoNotSet() 65 | PanicIf(!FileExists(scriptLocalPath), "script file '%s' doesn't exist", scriptLocalPath) 66 | 67 | serverAndUser := fmt.Sprintf("%s@%s", user, ServerIPAddress) 68 | scriptBaseName := filepath.Base(scriptLocalPath) 69 | scriptServerPath := "/root/" + scriptBaseName 70 | if user != "root" { 71 | scriptServerPath = "/home/" + user + "/" + scriptBaseName 72 | } 73 | { 74 | serverDstPath := fmt.Sprintf("%s:%s", serverAndUser, scriptServerPath) 75 | ScpCopy(scriptLocalPath, serverDstPath) 76 | } 77 | { 78 | script := MakeExecScript(scriptServerPath) 79 | SshExec(serverAndUser, script) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /string.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "sort" 7 | ) 8 | 9 | // FmtArgs formats args as a string. First argument should be format string 10 | // and the rest are arguments to the format 11 | func FmtArgs(args ...interface{}) string { 12 | if len(args) == 0 { 13 | return "" 14 | } 15 | format := args[0].(string) 16 | if len(args) == 1 { 17 | return format 18 | } 19 | return fmt.Sprintf(format, args[1:]...) 20 | } 21 | 22 | // FmtSmart avoids formatting if only format is given 23 | func FmtSmart(format string, args ...interface{}) string { 24 | if len(args) == 0 { 25 | return format 26 | } 27 | return fmt.Sprintf(format, args...) 28 | } 29 | 30 | // StringInSlice returns true if a string is present in slice 31 | func StringInSlice(a []string, toCheck string) bool { 32 | for _, s := range a { 33 | if s == toCheck { 34 | return true 35 | } 36 | } 37 | return false 38 | } 39 | 40 | // StringsRemoveFirst removes first sstring from the slice 41 | func StringsRemoveFirst(a []string) []string { 42 | n := len(a) 43 | if n > 0 { 44 | copy(a[:n-1], a[1:]) 45 | a = a[:n-1] 46 | } 47 | return a 48 | } 49 | 50 | // StringRemoveFromSlice removes a given string from a slice of strings 51 | // returns a (potentially) new slice and true if was removed 52 | func StringRemoveFromSlice(a []string, toRemove string) ([]string, bool) { 53 | n := len(a) 54 | if n == 0 { 55 | return nil, false 56 | } 57 | res := make([]string, 0, n) 58 | for _, s := range a { 59 | if s != toRemove { 60 | res = append(res, s) 61 | } 62 | } 63 | didRemove := len(res) != len(a) 64 | if !didRemove { 65 | return a, false 66 | } 67 | return res, true 68 | } 69 | 70 | // RemoveDuplicateStrings removes duplicate strings from an array of strings. 71 | // It's optimized for the case of no duplicates. It modifes a in place. 72 | func RemoveDuplicateStrings(a []string) []string { 73 | if len(a) < 2 { 74 | return a 75 | } 76 | // sort and remove dupplicates (which are now grouped) 77 | sort.Strings(a) 78 | writeIdx := 1 79 | for i := 1; i < len(a); i++ { 80 | if a[i-1] == a[i] { 81 | continue 82 | } 83 | if writeIdx != i { 84 | a[writeIdx] = a[i] 85 | } 86 | writeIdx++ 87 | } 88 | return a[:writeIdx] 89 | } 90 | 91 | // NormalizeNewLines changes CR and CRLF into LF 92 | func NormalizeNewlines(d []byte) []byte { 93 | // replace CR LF \r\n (windows) with LF \n (unix) 94 | d = bytes.Replace(d, []byte{13, 10}, []byte{10}, -1) 95 | // replace CF \r (mac) with LF \n (unix) 96 | d = bytes.Replace(d, []byte{13}, []byte{10}, -1) 97 | return d 98 | } 99 | 100 | const ( 101 | indentStr = " " 102 | ) 103 | 104 | // IndentStr returns an indentation string which has (2*n) spaces 105 | func IndentStr(n int) string { 106 | if n == 0 { 107 | return "" 108 | } 109 | n = n * 2 110 | if len(indentStr) >= n { 111 | return indentStr[:n] 112 | } 113 | s := indentStr 114 | for len(s) < n { 115 | s += " " 116 | } 117 | return s 118 | } 119 | -------------------------------------------------------------------------------- /wc.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | // LineCount describes line count for a file 12 | type LineCount struct { 13 | Name string 14 | Ext string 15 | LineCount int 16 | } 17 | 18 | // LineStats gathers line count info for files 19 | type LineStats struct { 20 | FileToCount map[string]*LineCount 21 | } 22 | 23 | // NewLineStats returns new LineStats 24 | func NewLineStats() *LineStats { 25 | return &LineStats{ 26 | FileToCount: map[string]*LineCount{}, 27 | } 28 | } 29 | 30 | func statsPerExt(fileToCount map[string]*LineCount) []*LineCount { 31 | extToCount := map[string]*LineCount{} 32 | for _, wc := range fileToCount { 33 | ext := wc.Ext 34 | extWc := extToCount[ext] 35 | if extWc == nil { 36 | extWc = &LineCount{ 37 | Ext: ext, 38 | } 39 | extToCount[ext] = extWc 40 | } 41 | extWc.LineCount += wc.LineCount 42 | } 43 | var res []*LineCount 44 | for _, wc := range extToCount { 45 | res = append(res, wc) 46 | } 47 | sort.Slice(res, func(i, j int) bool { 48 | wc1 := res[i] 49 | wc2 := res[j] 50 | return wc1.LineCount < wc2.LineCount 51 | }) 52 | return res 53 | } 54 | 55 | type FilterFunc func(string) bool 56 | 57 | func MakeAllowedFileFilterForExts(exts ...string) FilterFunc { 58 | for i, ext := range exts { 59 | exts[i] = strings.ToLower(ext) 60 | } 61 | 62 | return func(path string) bool { 63 | fext := strings.ToLower(filepath.Ext(path)) 64 | for _, ext := range exts { 65 | if ext == fext { 66 | return true 67 | } 68 | } 69 | return false 70 | } 71 | } 72 | 73 | func MakeExcludeDirsFilter(dirs ...string) FilterFunc { 74 | return func(path string) bool { 75 | // path starts as a file path 76 | // we only compare directory names 77 | path = filepath.Dir(path) 78 | for len(path) > 0 { 79 | if path == "." { 80 | return true 81 | } 82 | name := filepath.Base(path) 83 | for _, dir := range dirs { 84 | if name == dir { 85 | return false 86 | } 87 | } 88 | path = filepath.Dir(path) 89 | } 90 | return true 91 | } 92 | } 93 | 94 | func MakeFilterOr(filters ...FilterFunc) FilterFunc { 95 | return func(name string) bool { 96 | for _, f := range filters { 97 | if f(name) { 98 | return true 99 | } 100 | } 101 | return false 102 | } 103 | } 104 | 105 | func MakeFilterAnd(filters ...FilterFunc) FilterFunc { 106 | return func(name string) bool { 107 | for _, f := range filters { 108 | if !f(name) { 109 | return false 110 | } 111 | } 112 | return true 113 | } 114 | } 115 | 116 | // FileLineCount returns number of lines in a file 117 | func FileLineCount(path string) (int, error) { 118 | d, err := ioutil.ReadFile(path) 119 | if err != nil { 120 | return 0, err 121 | } 122 | if len(d) == 0 { 123 | return 0, nil 124 | } 125 | d = NormalizeNewlines(d) 126 | nLines := 1 127 | n := len(d) 128 | for i := 0; i < n; i++ { 129 | if d[i] == 10 { 130 | nLines++ 131 | } 132 | } 133 | return nLines, nil 134 | } 135 | 136 | func (s *LineStats) CalcInDir(dir string, allowedFileFilter func(name string) bool, recur bool) error { 137 | files, err := ioutil.ReadDir(dir) 138 | if err != nil { 139 | return err 140 | } 141 | for _, fi := range files { 142 | name := fi.Name() 143 | path := filepath.Join(dir, name) 144 | if fi.IsDir() { 145 | if recur { 146 | s.CalcInDir(path, allowedFileFilter, recur) 147 | } 148 | continue 149 | } 150 | if !fi.Mode().IsRegular() { 151 | continue 152 | } 153 | if !allowedFileFilter(path) { 154 | continue 155 | } 156 | lineCount, err := FileLineCount(path) 157 | if err != nil { 158 | return err 159 | } 160 | s.FileToCount[path] = &LineCount{ 161 | Name: fi.Name(), 162 | Ext: strings.ToLower(filepath.Ext(name)), 163 | LineCount: lineCount, 164 | } 165 | } 166 | return nil 167 | } 168 | 169 | func PrintLineStats(stats *LineStats) { 170 | var files []string 171 | for k := range stats.FileToCount { 172 | files = append(files, k) 173 | } 174 | sort.Strings(files) 175 | total := 0 176 | for _, f := range files { 177 | wc := stats.FileToCount[f] 178 | fmt.Printf("% 6d %s\n", wc.LineCount, f) 179 | total += wc.LineCount 180 | } 181 | fmt.Printf("\nPer extension:\n") 182 | wcPerExt := statsPerExt(stats.FileToCount) 183 | for _, wc := range wcPerExt { 184 | fmt.Printf("%d %s\n", wc.LineCount, wc.Ext) 185 | } 186 | fmt.Printf("\ntotal: %d\n", total) 187 | } 188 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "crypto/sha1" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "os/signal" 10 | "runtime" 11 | "strings" 12 | "syscall" 13 | "time" 14 | ) 15 | 16 | var ( 17 | errInvalidBase64 = errors.New("invalid base64 value") 18 | ) 19 | 20 | func Must(err error) { 21 | if err != nil { 22 | panic(err) 23 | } 24 | } 25 | 26 | func Assert(ok bool, format string, args ...interface{}) { 27 | if ok { 28 | return 29 | } 30 | if len(args) == 0 { 31 | panic(format) 32 | } 33 | panic(fmt.Sprintf(format, args...)) 34 | } 35 | 36 | // PanicIf panics if cond is true 37 | func PanicIf(cond bool, args ...interface{}) { 38 | if !cond { 39 | return 40 | } 41 | if len(args) == 0 { 42 | panic("condition failed") 43 | } 44 | format := args[0].(string) 45 | if len(args) == 1 { 46 | panic(format) 47 | } 48 | panic(fmt.Sprintf(format, args[1:]...)) 49 | } 50 | 51 | func panicWithMsg(defaultMsg string, args ...interface{}) { 52 | s := FmtArgs(args...) 53 | if s == "" { 54 | s = defaultMsg 55 | } 56 | fmt.Printf("%s\n", s) 57 | panic(s) 58 | } 59 | 60 | // PanicIfErr panics if err is not nil 61 | func PanicIfErr(err error, args ...interface{}) { 62 | if err == nil { 63 | return 64 | } 65 | panicWithMsg(err.Error(), args...) 66 | } 67 | 68 | // IsLinux returns true if running on linux 69 | func IsLinux() bool { 70 | return runtime.GOOS == "linux" 71 | } 72 | 73 | // IsMac returns true if running on mac 74 | func IsMac() bool { 75 | return runtime.GOOS == "darwin" 76 | } 77 | 78 | // UserHomeDir returns $HOME diretory of the user 79 | func UserHomeDirMust() string { 80 | s, err := os.UserHomeDir() 81 | Must(err) 82 | return s 83 | } 84 | 85 | // ExpandTildeInPath converts ~ to $HOME 86 | func ExpandTildeInPath(s string) string { 87 | if strings.HasPrefix(s, "~") { 88 | return UserHomeDirMust() + s[1:] 89 | } 90 | return s 91 | } 92 | 93 | // Sha1HexOfBytes returns 40-byte hex sha1 of bytes 94 | func Sha1HexOfBytes(data []byte) string { 95 | return fmt.Sprintf("%x", Sha1OfBytes(data)) 96 | } 97 | 98 | // Sha1OfBytes returns 20-byte sha1 of bytes 99 | func Sha1OfBytes(data []byte) []byte { 100 | h := sha1.New() 101 | h.Write(data) 102 | return h.Sum(nil) 103 | } 104 | 105 | // DurationToString converts duration to a string 106 | func DurationToString(d time.Duration) string { 107 | minutes := int(d.Minutes()) % 60 108 | hours := int(d.Hours()) 109 | days := hours / 24 110 | hours = hours % 24 111 | if days > 0 { 112 | return fmt.Sprintf("%dd %dhr", days, hours) 113 | } 114 | if hours > 0 { 115 | return fmt.Sprintf("%dhr %dm", hours, minutes) 116 | } 117 | return fmt.Sprintf("%dm", minutes) 118 | } 119 | 120 | // TimeSinceNowAsString returns string version of time since a ginve timestamp 121 | func TimeSinceNowAsString(t time.Time) string { 122 | return DurationToString(time.Since(t)) 123 | } 124 | 125 | // UtcNow returns current time in UTC 126 | func UtcNow() time.Time { 127 | return time.Now().UTC() 128 | } 129 | 130 | const base64Chars = "0123456789abcdefghijklmnopqrstuvwxyz" 131 | 132 | // EncodeBase64 encodes n as base64 133 | func EncodeBase64(n int) string { 134 | var buf [16]byte 135 | size := 0 136 | for { 137 | buf[size] = base64Chars[n%36] 138 | size++ 139 | if n < 36 { 140 | break 141 | } 142 | n /= 36 143 | } 144 | end := size - 1 145 | for i := 0; i < end; i++ { 146 | b := buf[i] 147 | buf[i] = buf[end] 148 | buf[end] = b 149 | end-- 150 | } 151 | return string(buf[:size]) 152 | } 153 | 154 | // DecodeBase64 decodes base64 string 155 | func DecodeBase64(s string) (int, error) { 156 | n := 0 157 | for _, c := range s { 158 | n *= 36 159 | i := strings.IndexRune(base64Chars, c) 160 | if i == -1 { 161 | return 0, errInvalidBase64 162 | } 163 | n += i 164 | } 165 | return n, nil 166 | } 167 | 168 | // OpenBrowsers open web browser with a given url 169 | // (can be http:// or file://) 170 | func OpenBrowser(url string) error { 171 | var err error 172 | switch runtime.GOOS { 173 | case "linux": 174 | err = exec.Command("xdg-open", url).Start() 175 | case "windows": 176 | err = exec.Command("rundll32", "url.dll,FileProtocolHandler", url).Start() 177 | case "darwin": 178 | err = exec.Command("open", url).Start() 179 | default: 180 | err = fmt.Errorf("unsupported platform") 181 | } 182 | return err 183 | } 184 | 185 | // WaitForCtrlC waits until a user presses Ctrl-C 186 | func WaitForCtrlC() { 187 | c := make(chan os.Signal, 2) 188 | signal.Notify(c, os.Interrupt /* SIGINT */, syscall.SIGTERM) 189 | <-c 190 | } 191 | -------------------------------------------------------------------------------- /compress.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "archive/zip" 5 | "compress/bzip2" 6 | "compress/gzip" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | ) 14 | 15 | // implement io.ReadCloser over os.File wrapped with io.Reader. 16 | // io.Closer goes to os.File, io.Reader goes to wrapping reader 17 | type readerWrappedFile struct { 18 | f *os.File 19 | r io.Reader 20 | } 21 | 22 | func (rc *readerWrappedFile) Close() error { 23 | return rc.f.Close() 24 | } 25 | 26 | func (rc *readerWrappedFile) Read(p []byte) (int, error) { 27 | return rc.r.Read(p) 28 | } 29 | 30 | // OpenFileMaybeCompressed opens a file that might be compressed with gzip 31 | // or bzip2. 32 | // TODO: could sniff file content instead of checking file extension 33 | func OpenFileMaybeCompressed(path string) (io.ReadCloser, error) { 34 | ext := strings.ToLower(filepath.Ext(path)) 35 | f, err := os.Open(path) 36 | if err != nil { 37 | return nil, err 38 | } 39 | if ext == ".gz" { 40 | r, err := gzip.NewReader(f) 41 | if err != nil { 42 | f.Close() 43 | return nil, err 44 | } 45 | rc := &readerWrappedFile{ 46 | f: f, 47 | r: r, 48 | } 49 | return rc, nil 50 | } 51 | if ext == ".bz2" { 52 | r := bzip2.NewReader(f) 53 | rc := &readerWrappedFile{ 54 | f: f, 55 | r: r, 56 | } 57 | return rc, nil 58 | } 59 | return f, nil 60 | } 61 | 62 | // ReadFileMaybeCompressed reads file. Ungzips if it's gzipped. 63 | func ReadFileMaybeCompressed(path string) ([]byte, error) { 64 | r, err := OpenFileMaybeCompressed(path) 65 | if err != nil { 66 | return nil, err 67 | } 68 | defer r.Close() 69 | return ioutil.ReadAll(r) 70 | } 71 | 72 | // WriteFileGzipped writes data to a path, using best gzip compression 73 | func WriteFileGzipped(path string, data []byte) error { 74 | f, err := os.Create(path) 75 | if err != nil { 76 | return err 77 | } 78 | w, err := gzip.NewWriterLevel(f, gzip.BestCompression) 79 | if err != nil { 80 | return err 81 | } 82 | _, err = w.Write(data) 83 | if err != nil { 84 | f.Close() 85 | os.Remove(path) 86 | return err 87 | } 88 | err = w.Close() 89 | if err != nil { 90 | f.Close() 91 | os.Remove(path) 92 | return err 93 | } 94 | err = f.Close() 95 | if err != nil { 96 | os.Remove(path) 97 | return err 98 | } 99 | return nil 100 | } 101 | 102 | // GzipFile compresses srcPath with gzip and saves as dstPath 103 | func GzipFile(dstPath, srcPath string) error { 104 | fSrc, err := os.Open(srcPath) 105 | if err != nil { 106 | return err 107 | } 108 | defer fSrc.Close() 109 | fDst, err := os.Create(dstPath) 110 | if err != nil { 111 | return err 112 | } 113 | defer fDst.Close() 114 | w, err := gzip.NewWriterLevel(fDst, gzip.BestCompression) 115 | if err != nil { 116 | return err 117 | } 118 | _, err = io.Copy(w, fSrc) 119 | if err != nil { 120 | return err 121 | } 122 | return w.Close() 123 | } 124 | 125 | // CreateZipWithDirContent creates a zip file with the content of a directory. 126 | // The names of files inside the zip file are relatitve to dirToZip e.g. 127 | // if dirToZip is foo and there is a file foo/bar.txt, the name in the zip 128 | // will be bar.txt 129 | func CreateZipWithDirContent(zipFilePath, dirToZip string) error { 130 | if isDir, err := PathIsDir(dirToZip); err != nil || !isDir { 131 | // TODO: should return an error if err == nil && !isDir 132 | return err 133 | } 134 | zf, err := os.Create(zipFilePath) 135 | if err != nil { 136 | //fmt.Printf("Failed to os.Create() %s, %s\n", zipFilePath, err.Error()) 137 | return err 138 | } 139 | defer zf.Close() 140 | zipWriter := zip.NewWriter(zf) 141 | // TODO: is the order of defer here can create problems? 142 | // TODO: need to check error code returned by Close() 143 | defer zipWriter.Close() 144 | 145 | //fmt.Printf("Walk root: %s\n", config.LocalDir) 146 | err = filepath.Walk(dirToZip, func(pathToZip string, info os.FileInfo, err error) error { 147 | if err != nil { 148 | //fmt.Printf("WalkFunc() received err %s from filepath.Wath()\n", err.Error()) 149 | return err 150 | } 151 | //fmt.Printf("%s\n", path) 152 | isDir, err := PathIsDir(pathToZip) 153 | if err != nil { 154 | //fmt.Printf("PathIsDir() for %s failed with %s\n", pathToZip, err.Error()) 155 | return err 156 | } 157 | if isDir { 158 | return nil 159 | } 160 | toZipReader, err := os.Open(pathToZip) 161 | if err != nil { 162 | //fmt.Printf("os.Open() %s failed with %s\n", pathToZip, err.Error()) 163 | return err 164 | } 165 | defer toZipReader.Close() 166 | 167 | zipName := pathToZip[len(dirToZip)+1:] // +1 for '/' in the path 168 | inZipWriter, err := zipWriter.Create(zipName) 169 | if err != nil { 170 | //fmt.Printf("Error in zipWriter(): %s\n", err.Error()) 171 | return err 172 | } 173 | _, err = io.Copy(inZipWriter, toZipReader) 174 | if err != nil { 175 | return err 176 | } 177 | //fmt.Printf("Added %s to zip file\n", pathToZip) 178 | return nil 179 | }) 180 | return err 181 | } 182 | 183 | func ReadZipFileMust(path string) map[string][]byte { 184 | r, err := zip.OpenReader(path) 185 | Must(err) 186 | defer CloseNoError(r) 187 | res := map[string][]byte{} 188 | for _, f := range r.File { 189 | rc, err := f.Open() 190 | Must(err) 191 | d, err := ioutil.ReadAll(rc) 192 | Must(err) 193 | _ = rc.Close() 194 | res[f.Name] = d 195 | } 196 | return res 197 | } 198 | 199 | func zipAddFile(zw *zip.Writer, zipName string, path string) { 200 | zipName = filepath.ToSlash(zipName) 201 | d, err := ioutil.ReadFile(path) 202 | Must(err) 203 | w, err := zw.Create(zipName) 204 | Must(err) 205 | _, err = w.Write(d) 206 | Must(err) 207 | fmt.Printf(" added %s from %s\n", zipName, path) 208 | } 209 | 210 | func zipDirRecur(zw *zip.Writer, baseDir string, dirToZip string) { 211 | dir := filepath.Join(baseDir, dirToZip) 212 | files, err := ioutil.ReadDir(dir) 213 | Must(err) 214 | for _, fi := range files { 215 | if fi.IsDir() { 216 | zipDirRecur(zw, baseDir, filepath.Join(dirToZip, fi.Name())) 217 | } else if fi.Mode().IsRegular() { 218 | zipName := filepath.Join(dirToZip, fi.Name()) 219 | path := filepath.Join(baseDir, zipName) 220 | zipAddFile(zw, zipName, path) 221 | } else { 222 | path := filepath.Join(baseDir, fi.Name()) 223 | s := fmt.Sprintf("%s is not a dir or regular file", path) 224 | panic(s) 225 | } 226 | } 227 | } 228 | 229 | // toZip is a list of files and directories in baseDir 230 | // Directories are added recursively 231 | func CreateZipFile(dst string, baseDir string, toZip ...string) { 232 | os.Remove(dst) 233 | if len(toZip) == 0 { 234 | panic("must provide toZip args") 235 | } 236 | fmt.Printf("Creating zip file %s\n", dst) 237 | w, err := os.Create(dst) 238 | Must(err) 239 | defer CloseNoError(w) 240 | zw := zip.NewWriter(w) 241 | Must(err) 242 | for _, name := range toZip { 243 | path := filepath.Join(baseDir, name) 244 | fi, err := os.Stat(path) 245 | Must(err) 246 | if fi.IsDir() { 247 | zipDirRecur(zw, baseDir, name) 248 | } else if fi.Mode().IsRegular() { 249 | zipAddFile(zw, name, path) 250 | } else { 251 | s := fmt.Sprintf("%s is not a dir or regular file", path) 252 | panic(s) 253 | } 254 | } 255 | err = zw.Close() 256 | Must(err) 257 | } 258 | -------------------------------------------------------------------------------- /minio.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/kjk/atomicfile" 11 | "github.com/minio/minio-go/v6" 12 | ) 13 | 14 | // MinioClient represents s3/spaces etc. client 15 | type MinioClient struct { 16 | StorageKey string 17 | StorageSecret string 18 | Bucket string 19 | Endpoint string // e.g. "nyc3.digitaloceanspaces.com" 20 | Secure bool 21 | client *minio.Client 22 | } 23 | 24 | // EnsureCondfigured will panic if client not configured 25 | func (c *MinioClient) EnsureConfigured() { 26 | PanicIf(c.StorageKey == "", "minio storage key not set") 27 | PanicIf(c.StorageSecret == "", "minio storage secret not set") 28 | PanicIf(c.Bucket == "", "minio bucket not set") 29 | PanicIf(c.Endpoint == "", "minio endpoint not set") 30 | } 31 | 32 | // URLBase returns base of url under which files are accesible 33 | // (if public it's a publicly available file) 34 | func (c *MinioClient) URLBase() string { 35 | return fmt.Sprintf("https://%s.%s/", c.Bucket, c.Endpoint) 36 | } 37 | 38 | // GetClient returns a (cached) minio client 39 | func (c *MinioClient) GetClient() (*minio.Client, error) { 40 | if c.client != nil { 41 | return c.client, nil 42 | } 43 | 44 | var err error 45 | c.client, err = minio.New(c.Endpoint, c.StorageKey, c.StorageSecret, c.Secure) 46 | return c.client, err 47 | } 48 | 49 | // ListRemoveFiles returns a list of files under a given prefix 50 | func (c *MinioClient) ListRemoteFiles(prefix string) ([]*minio.ObjectInfo, error) { 51 | var res []*minio.ObjectInfo 52 | client, err := c.GetClient() 53 | if err != nil { 54 | return nil, err 55 | } 56 | doneCh := make(chan struct{}) 57 | defer close(doneCh) 58 | 59 | files := client.ListObjectsV2(c.Bucket, prefix, true, doneCh) 60 | for oi := range files { 61 | oic := oi 62 | res = append(res, &oic) 63 | } 64 | return res, nil 65 | } 66 | 67 | // IsMinioNotExistsError returns true if an error indicates that a key 68 | // doesn't exist in storage 69 | func IsMinioNotExistsError(err error) bool { 70 | if err == nil { 71 | return false 72 | } 73 | return err.Error() == "The specified key does not exist." 74 | } 75 | 76 | // SetPublicObjectMetadata sets options that mark object as public 77 | // for doing put operation 78 | func SetPublicObjectMetadata(opts *minio.PutObjectOptions) { 79 | if opts.UserMetadata == nil { 80 | opts.UserMetadata = map[string]string{} 81 | } 82 | opts.UserMetadata["x-amz-acl"] = "public-read" 83 | } 84 | 85 | func (c *MinioClient) UploadReaderPublic(remotePath string, r io.Reader, size int64, contentType string) error { 86 | return c.UploadReader(remotePath, r, size, true, contentType) 87 | } 88 | 89 | func (c *MinioClient) UploadReaderPrivate(remotePath string, r io.Reader, size int64, contentType string) error { 90 | return c.UploadReader(remotePath, r, size, false, contentType) 91 | } 92 | 93 | func (c *MinioClient) UploadData(remotePath string, d []byte, opts minio.PutObjectOptions) error { 94 | client, err := c.GetClient() 95 | if err != nil { 96 | return err 97 | } 98 | r := bytes.NewReader(d) 99 | size := int64(len(d)) 100 | _, err = client.PutObject(c.Bucket, remotePath, r, size, opts) 101 | return err 102 | } 103 | 104 | func (c *MinioClient) UploadReader(remotePath string, r io.Reader, size int64, public bool, contentType string) error { 105 | PanicIf(remotePath[0] == '/', "name '%s' shouldn't start with '/'", remotePath) 106 | 107 | if contentType == "" { 108 | contentType = MimeTypeFromFileName(remotePath) 109 | } 110 | //timeStart := time.Now() 111 | //sizeStr := humanize.Bytes(uint64(size)) 112 | //fmt.Printf("Uploading '%s' of size %s and type %s as public.", remotePath, sizeStr, contentType) 113 | client, err := c.GetClient() 114 | if err != nil { 115 | return err 116 | } 117 | opts := minio.PutObjectOptions{ 118 | ContentType: contentType, 119 | } 120 | if public { 121 | SetPublicObjectMetadata(&opts) 122 | } 123 | opts.ContentType = contentType 124 | _, err = client.PutObject(c.Bucket, remotePath, r, size, opts) 125 | if err != nil { 126 | return err 127 | } 128 | //fmt.Printf(" Took %s.\n", time.Since(timeStart)) 129 | return nil 130 | } 131 | 132 | func (c *MinioClient) DownloadFileAsData(remotePath string) ([]byte, error) { 133 | client, err := c.GetClient() 134 | if err != nil { 135 | return nil, err 136 | } 137 | opts := minio.GetObjectOptions{} 138 | obj, err := client.GetObject(c.Bucket, remotePath, opts) 139 | if err != nil { 140 | return nil, err 141 | } 142 | defer obj.Close() 143 | var buf bytes.Buffer 144 | _, err = io.Copy(&buf, obj) 145 | if err != nil { 146 | return nil, err 147 | } 148 | return buf.Bytes(), nil 149 | } 150 | 151 | func (c *MinioClient) DownloadFileAtomically(dstPath string, remotePath string) error { 152 | client, err := c.GetClient() 153 | if err != nil { 154 | return err 155 | } 156 | opts := minio.GetObjectOptions{} 157 | obj, err := client.GetObject(c.Bucket, remotePath, opts) 158 | if err != nil { 159 | return err 160 | } 161 | defer obj.Close() 162 | 163 | // ensure there's a dir for destination file 164 | dir := filepath.Dir(dstPath) 165 | err = os.MkdirAll(dir, 0755) 166 | if err != nil { 167 | return err 168 | } 169 | 170 | f, err := atomicfile.New(dstPath) 171 | if err != nil { 172 | return err 173 | } 174 | defer f.Close() 175 | _, err = io.Copy(f, obj) 176 | if err != nil { 177 | return err 178 | } 179 | return f.Close() 180 | } 181 | 182 | func (c *MinioClient) UploadFilePublic(remotePath string, filePath string) error { 183 | return c.UploadFile(remotePath, filePath, true) 184 | } 185 | 186 | func (c *MinioClient) UploadFilePrivate(remotePath string, filePath string) error { 187 | return c.UploadFile(remotePath, filePath, false) 188 | } 189 | 190 | func (c *MinioClient) UploadFile(remotePath string, filePath string, public bool) error { 191 | stat, err := os.Stat(filePath) 192 | if err != nil { 193 | return err 194 | } 195 | size := stat.Size() 196 | f, err := os.Open(filePath) 197 | if err != nil { 198 | return err 199 | } 200 | defer func() { 201 | f.Close() 202 | }() 203 | return c.UploadReaderPublic(remotePath, f, size, "") 204 | } 205 | 206 | func (c *MinioClient) UploadDataPublic(remotePath string, d []byte) error { 207 | r := bytes.NewBuffer(d) 208 | return c.UploadReaderPublic(remotePath, r, int64(len(d)), "") 209 | } 210 | 211 | func (c *MinioClient) UploadStringPublic(remotePath string, s string) error { 212 | r := bytes.NewBufferString(s) 213 | return c.UploadReaderPublic(remotePath, r, int64(len(s)), "") 214 | } 215 | 216 | func (c *MinioClient) UploadStringPrivate(remotePath string, s string) error { 217 | r := bytes.NewBufferString(s) 218 | return c.UploadReaderPrivate(remotePath, r, int64(len(s)), "") 219 | } 220 | 221 | func (c *MinioClient) StatObject(remotePath string) (minio.ObjectInfo, error) { 222 | client, err := c.GetClient() 223 | if err != nil { 224 | return minio.ObjectInfo{}, err 225 | } 226 | var opts minio.StatObjectOptions 227 | return client.StatObject(c.Bucket, remotePath, opts) 228 | } 229 | 230 | func (c *MinioClient) Delete(remotePath string) error { 231 | client, err := c.GetClient() 232 | if err != nil { 233 | return err 234 | } 235 | err = client.RemoveObject(c.Bucket, remotePath) 236 | if IsMinioNotExistsError(err) { 237 | return nil 238 | } 239 | return nil 240 | } 241 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 5 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 6 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 7 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= 8 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 9 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 10 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 11 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= 12 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 13 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 14 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 15 | github.com/kjk/atomicfile v0.0.0-20190916063300-2d5c7d7d05bf h1:HuwmGC6wEC0CdGC3QUSBnAfQzydOiR4HeTZCySsHpHY= 16 | github.com/kjk/atomicfile v0.0.0-20190916063300-2d5c7d7d05bf/go.mod h1:+YlBbo63AHA3uS6tdRhd42B+I1lV7H7+aqDhwTRl5rs= 17 | github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= 18 | github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 19 | github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= 20 | github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= 21 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 22 | github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= 23 | github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= 24 | github.com/minio/minio-go/v6 v6.0.44 h1:CVwVXw+uCOcyMi7GvcOhxE8WgV+Xj8Vkf2jItDf/EGI= 25 | github.com/minio/minio-go/v6 v6.0.44/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= 26 | github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw= 27 | github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= 28 | github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= 29 | github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= 30 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 31 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 32 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 33 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 34 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 35 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 36 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 37 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 38 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 39 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 40 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 41 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 42 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 43 | github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= 44 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= 45 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 46 | github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= 47 | github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 48 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 49 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 50 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 51 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 52 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 53 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 54 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 55 | golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= 56 | golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 57 | golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc= 58 | golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 59 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 60 | golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E= 61 | golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 62 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 63 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 64 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= 65 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 66 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= 67 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 68 | golang.org/x/net v0.0.0-20201024042810-be3efd7ff127 h1:pZPp9+iYUqwYKLjht0SDBbRCRK/9gAXDy7pz5fRDpjo= 69 | golang.org/x/net v0.0.0-20201024042810-be3efd7ff127/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 70 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 71 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 72 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= 73 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 74 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8 h1:JA8d3MPx/IToSyXZG/RhwYEtfrKO1Fxrqe8KrkiLXKM= 75 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 76 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 77 | golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc= 78 | golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 79 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 80 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 81 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 82 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 83 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 84 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 85 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 86 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 87 | gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= 88 | gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 89 | gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= 90 | gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 91 | gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= 92 | gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 93 | -------------------------------------------------------------------------------- /file.go: -------------------------------------------------------------------------------- 1 | package u 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "crypto/sha1" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | 14 | "github.com/dustin/go-humanize" 15 | ) 16 | 17 | // PathExists returns true if a filesystem path exists 18 | // Treats any error (e.g. lack of access due to permissions) as non-existence 19 | func PathExists(path string) bool { 20 | _, err := os.Stat(path) 21 | return err == nil 22 | } 23 | 24 | // FileExists returns true if a given path exists and is a file 25 | func FileExists(path string) bool { 26 | st, err := os.Stat(path) 27 | return err == nil && !st.IsDir() && st.Mode().IsRegular() 28 | } 29 | 30 | // DirExists returns true if a given path exists and is a directory 31 | func DirExists(path string) bool { 32 | st, err := os.Stat(path) 33 | return err == nil && st.IsDir() 34 | } 35 | 36 | // PathIsDir returns true if a path exists and is a directory 37 | // Returns false, nil if a path exists and is not a directory (e.g. a file) 38 | // Returns undefined, error if there was an error e.g. because a path doesn't exists 39 | func PathIsDir(path string) (isDir bool, err error) { 40 | fi, err := os.Stat(path) 41 | if err != nil { 42 | return false, err 43 | } 44 | return fi.IsDir(), nil 45 | } 46 | 47 | // GetFileSize returns size of the file 48 | func GetFileSize(path string) (int64, error) { 49 | fi, err := os.Lstat(path) 50 | if err != nil { 51 | return 0, err 52 | } 53 | return fi.Size(), nil 54 | } 55 | 56 | // CreateDir creates a directory if it doesn't exist 57 | func CreateDir(dir string) error { 58 | return os.MkdirAll(dir, 0755) 59 | } 60 | 61 | // CreateDirMust creates a directory. Panics on error 62 | func CreateDirMust(path string) string { 63 | err := CreateDir(path) 64 | Must(err) 65 | return path 66 | } 67 | 68 | // CreateDirForFile creates intermediary directories for a file 69 | func CreateDirForFile(path string) error { 70 | dir := filepath.Dir(path) 71 | return CreateDir(dir) 72 | } 73 | 74 | // CreateDirForFileMust is like CreateDirForFile. Panics on error. 75 | func CreateDirForFileMust(path string) string { 76 | dir := filepath.Dir(path) 77 | CreateDirMust(dir) 78 | return dir 79 | } 80 | 81 | // WriteFileCreateDirMust is like ioutil.WriteFile() but also creates 82 | // intermediary directories 83 | func WriteFileCreateDirMust(d []byte, path string) error { 84 | if err := CreateDir(filepath.Dir(path)); err != nil { 85 | return err 86 | } 87 | return ioutil.WriteFile(path, d, 0644) 88 | } 89 | 90 | // WriteFileMust writes data to a file 91 | func WriteFileMust(path string, data []byte) { 92 | err := ioutil.WriteFile(path, data, 0644) 93 | Must(err) 94 | } 95 | 96 | // ReadFileMust reads data from a file 97 | func ReadFileMust(path string) []byte { 98 | d, err := ioutil.ReadFile(path) 99 | Must(err) 100 | return d 101 | } 102 | 103 | // CloseNoError is like io.Closer Close() but ignores an error 104 | // use as: defer CloseNoError(f) 105 | func CloseNoError(f io.Closer) { 106 | _ = f.Close() 107 | } 108 | 109 | // ListFilesInDir returns a list of files in a directory 110 | func ListFilesInDir(dir string, recursive bool) []string { 111 | files := make([]string, 0) 112 | filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { 113 | if err != nil { 114 | return err 115 | } 116 | isDir, err := PathIsDir(path) 117 | if err != nil { 118 | return err 119 | } 120 | if isDir { 121 | if recursive || path == dir { 122 | return nil 123 | } 124 | return filepath.SkipDir 125 | } 126 | files = append(files, path) 127 | return nil 128 | }) 129 | return files 130 | } 131 | 132 | // RemoveFilesInDirMust removes all files in a directory 133 | // (but not sub-directories) 134 | func RemoveFilesInDirMust(dir string) { 135 | if !DirExists(dir) { 136 | return 137 | } 138 | files, err := ioutil.ReadDir(dir) 139 | Must(err) 140 | for _, fi := range files { 141 | if !fi.Mode().IsRegular() { 142 | continue 143 | } 144 | path := filepath.Join(dir, fi.Name()) 145 | err = os.Remove(path) 146 | Must(err) 147 | } 148 | } 149 | 150 | // RemoveFileLogged removes a file and logs the action 151 | func RemoveFileLogged(path string) { 152 | err := os.Remove(path) 153 | if err == nil { 154 | Logf("RemoveFileLogged('%s')\n", path) 155 | return 156 | } 157 | if os.IsNotExist(err) { 158 | // TODO: maybe should print note 159 | return 160 | } 161 | Logf("os.Remove('%s') failed with '%s'\n", path, err) 162 | } 163 | 164 | // CopyFile copies a file from src to dst 165 | func CopyFile(dst, src string) error { 166 | fsrc, err := os.Open(src) 167 | if err != nil { 168 | return err 169 | } 170 | defer fsrc.Close() 171 | fdst, err := os.Create(dst) 172 | if err != nil { 173 | return err 174 | } 175 | defer fdst.Close() 176 | if _, err = io.Copy(fdst, fsrc); err != nil { 177 | return err 178 | } 179 | return nil 180 | } 181 | 182 | // CopyFileMust copies a file from src to dst 183 | func CopyFileMust(dst, src string) { 184 | Must(CopyFile(dst, src)) 185 | } 186 | 187 | // ReadLinesFromReader reads all lines from io.Reader. Newlines are not included. 188 | func ReadLinesFromReader(r io.Reader) ([]string, error) { 189 | res := make([]string, 0) 190 | scanner := bufio.NewScanner(r) 191 | for scanner.Scan() { 192 | res = append(res, scanner.Text()) 193 | } 194 | 195 | if err := scanner.Err(); err != nil { 196 | return res, err 197 | } 198 | return res, nil 199 | } 200 | 201 | // ReadLinesFromFile reads all lines from a file. Newlines are not included. 202 | func ReadLinesFromFile(path string) ([]string, error) { 203 | f, err := os.Open(path) 204 | if err != nil { 205 | return nil, err 206 | } 207 | defer f.Close() 208 | return ReadLinesFromReader(f) 209 | } 210 | 211 | // Sha1OfFile returns 20-byte sha1 of file content 212 | func Sha1OfFile(path string) ([]byte, error) { 213 | f, err := os.Open(path) 214 | if err != nil { 215 | //fmt.Printf("os.Open(%s) failed with %s\n", path, err.Error()) 216 | return nil, err 217 | } 218 | defer f.Close() 219 | h := sha1.New() 220 | _, err = io.Copy(h, f) 221 | if err != nil { 222 | //fmt.Printf("io.Copy() failed with %s\n", err.Error()) 223 | return nil, err 224 | } 225 | return h.Sum(nil), nil 226 | } 227 | 228 | // Sha1HexOfFile returns 40-byte hex sha1 of file content 229 | func Sha1HexOfFile(path string) (string, error) { 230 | sha1, err := Sha1OfFile(path) 231 | if err != nil { 232 | return "", err 233 | } 234 | return fmt.Sprintf("%x", sha1), nil 235 | } 236 | 237 | // PathMatchesExtensions returns true if path matches any of the extensions 238 | func PathMatchesExtensions(path string, extensions []string) bool { 239 | if len(extensions) == 0 { 240 | return true 241 | } 242 | ext := strings.ToLower(filepath.Ext(path)) 243 | for _, allowed := range extensions { 244 | if ext == allowed { 245 | return true 246 | } 247 | } 248 | return false 249 | } 250 | 251 | // DeleteFilesIf deletes a files in a given directory if shouldDelete callback 252 | // returns true 253 | func DeleteFilesIf(dir string, shouldDelete func(os.FileInfo) bool) error { 254 | files, err := ioutil.ReadDir(dir) 255 | if err != nil { 256 | return err 257 | } 258 | for _, fi := range files { 259 | if fi.IsDir() || !fi.Mode().IsRegular() { 260 | continue 261 | } 262 | if shouldDelete(fi) { 263 | path := filepath.Join(dir, fi.Name()) 264 | err = os.Remove(path) 265 | // Maybe: keep deleting? 266 | if err != nil { 267 | return err 268 | } 269 | } 270 | } 271 | return nil 272 | } 273 | 274 | // CurrDirAbsMust returns absolute path of the current directory 275 | func CurrDirAbsMust() string { 276 | dir, err := filepath.Abs(".") 277 | Must(err) 278 | return dir 279 | } 280 | 281 | // we are executed for do/ directory so top dir is parent dir 282 | func CdUpDir(dirName string) { 283 | startDir := CurrDirAbsMust() 284 | dir := startDir 285 | for { 286 | // we're already in top directory 287 | if filepath.Base(dir) == dirName && DirExists(dir) { 288 | err := os.Chdir(dir) 289 | Must(err) 290 | return 291 | } 292 | parentDir := filepath.Dir(dir) 293 | PanicIf(dir == parentDir, "invalid startDir: '%s', dir: '%s'", startDir, dir) 294 | dir = parentDir 295 | } 296 | } 297 | 298 | func FmtSizeHuman(size int64) string { 299 | return humanize.Bytes(uint64(size)) 300 | } 301 | 302 | func PrintFileSize(path string) { 303 | st, err := os.Stat(path) 304 | if err != nil { 305 | fmt.Printf("File '%s' doesn't exist\n", path) 306 | return 307 | } 308 | fmt.Printf("'%s': %s\n", path, FmtSizeHuman(st.Size())) 309 | } 310 | 311 | func AreFilesEuqalMust(path1, path2 string) bool { 312 | d1 := ReadFileMust(path1) 313 | d2 := ReadFileMust(path2) 314 | return bytes.Equal(d1, d2) 315 | } 316 | 317 | func FilesSameSize(path1, path2 string) bool { 318 | s1, err := GetFileSize(path1) 319 | if err != nil { 320 | return false 321 | } 322 | s2, err := GetFileSize(path2) 323 | if err != nil { 324 | return false 325 | } 326 | return s1 == s2 327 | } 328 | 329 | func DirCopyRecur(dstDir, srcDir string, shouldCopyFn func(path string) bool) ([]string, error) { 330 | err := CreateDir(dstDir) 331 | if err != nil { 332 | return nil, err 333 | } 334 | fileInfos, err := ioutil.ReadDir(srcDir) 335 | if err != nil { 336 | return nil, err 337 | } 338 | var allCopied []string 339 | for _, fi := range fileInfos { 340 | name := fi.Name() 341 | if fi.IsDir() { 342 | dst := filepath.Join(dstDir, name) 343 | src := filepath.Join(srcDir, name) 344 | copied, err := DirCopyRecur(dst, src, shouldCopyFn) 345 | if err != nil { 346 | return nil, err 347 | } 348 | allCopied = append(allCopied, copied...) 349 | continue 350 | } 351 | 352 | src := filepath.Join(srcDir, name) 353 | dst := filepath.Join(dstDir, name) 354 | shouldCopy := true 355 | if shouldCopyFn != nil { 356 | shouldCopy = shouldCopyFn(src) 357 | } 358 | if !shouldCopy { 359 | continue 360 | } 361 | CopyFileMust(dst, src) 362 | allCopied = append(allCopied, src) 363 | } 364 | return allCopied, nil 365 | } 366 | 367 | func DirCopyRecurMust(dstDir, srcDir string, shouldCopyFn func(path string) bool) []string { 368 | copied, err := DirCopyRecur(dstDir, srcDir, shouldCopyFn) 369 | Must(err) 370 | return copied 371 | } 372 | --------------------------------------------------------------------------------