├── README.md ├── frozen_reader.go ├── go.mod ├── frozen_reader_test.go ├── timeout_reader_test.go ├── .github ├── workflows │ ├── build-test.yml │ ├── lint-test.yml │ └── codeql-analysis.yml └── dependabot.yml ├── timeout_reader.go ├── LICENSE.md ├── go.sum ├── file.go └── file_test.go /README.md: -------------------------------------------------------------------------------- 1 | # fileutil 2 | The package contains various helpers to interact with files -------------------------------------------------------------------------------- /frozen_reader.go: -------------------------------------------------------------------------------- 1 | package fileutil 2 | 3 | import ( 4 | "io" 5 | "math" 6 | "time" 7 | ) 8 | 9 | // FrozenReader is a reader that never returns 10 | type FrozenReader struct{} 11 | 12 | // Read into the buffer 13 | func (reader FrozenReader) Read(p []byte) (n int, err error) { 14 | time.Sleep(math.MaxInt32 * time.Second) 15 | return 0, io.EOF 16 | } 17 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/projectdiscovery/fileutil 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d 7 | github.com/pkg/errors v0.9.1 8 | github.com/projectdiscovery/stringsutil v0.0.2 9 | github.com/stretchr/testify v1.8.1 10 | golang.org/x/net v0.1.0 // indirect 11 | gopkg.in/yaml.v3 v3.0.1 12 | ) 13 | -------------------------------------------------------------------------------- /frozen_reader_test.go: -------------------------------------------------------------------------------- 1 | package fileutil 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestFrozenReader(t *testing.T) { 11 | forever := func() { 12 | wrappedStdin := FrozenReader{} 13 | _, err := io.Copy(os.Stdout, wrappedStdin) 14 | if err != nil { 15 | return 16 | } 17 | } 18 | go forever() 19 | <-time.After(10 * time.Second) 20 | } 21 | -------------------------------------------------------------------------------- /timeout_reader_test.go: -------------------------------------------------------------------------------- 1 | package fileutil 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestTimeoutReader(t *testing.T) { 13 | wrappedStdin := TimeoutReader{ 14 | Reader: FrozenReader{}, 15 | Timeout: time.Duration(2 * time.Second), 16 | } 17 | _, err := io.Copy(os.Stdout, wrappedStdin) 18 | require.NotNil(t, err) 19 | } 20 | -------------------------------------------------------------------------------- /.github/workflows/build-test.yml: -------------------------------------------------------------------------------- 1 | name: 🔨 Build Test 2 | on: 3 | push: 4 | pull_request: 5 | workflow_dispatch: 6 | 7 | 8 | jobs: 9 | build: 10 | name: Test Builds 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Set up Go 14 | uses: actions/setup-go@v3 15 | with: 16 | go-version: 1.18 17 | 18 | - name: Check out code 19 | uses: actions/checkout@v3 20 | 21 | - name: Test 22 | run: go test ./... 23 | 24 | # TODO: create examples folder 25 | # - name: Build 26 | # run: go build . -------------------------------------------------------------------------------- /.github/workflows/lint-test.yml: -------------------------------------------------------------------------------- 1 | name: 🙏🏻 Lint Test 2 | on: 3 | push: 4 | pull_request: 5 | workflow_dispatch: 6 | 7 | jobs: 8 | lint: 9 | name: Lint Test 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout code 13 | uses: actions/checkout@v3 14 | - name: Set up Go 15 | uses: actions/setup-go@v3 16 | with: 17 | go-version: 1.18 18 | - name: Run golangci-lint 19 | uses: golangci/golangci-lint-action@v3.3.0 20 | with: 21 | version: latest 22 | args: --timeout 5m 23 | working-directory: . 24 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: 🚨 CodeQL Analysis 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | branches: 7 | - dev 8 | 9 | jobs: 10 | analyze: 11 | name: Analyze 12 | runs-on: ubuntu-latest 13 | permissions: 14 | actions: read 15 | contents: read 16 | security-events: write 17 | 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | language: [ 'go' ] 22 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 23 | 24 | steps: 25 | - name: Checkout repository 26 | uses: actions/checkout@v3 27 | 28 | # Initializes the CodeQL tools for scanning. 29 | - name: Initialize CodeQL 30 | uses: github/codeql-action/init@v2 31 | with: 32 | languages: ${{ matrix.language }} 33 | 34 | - name: Autobuild 35 | uses: github/codeql-action/autobuild@v2 36 | 37 | - name: Perform CodeQL Analysis 38 | uses: github/codeql-action/analyze@v2 -------------------------------------------------------------------------------- /timeout_reader.go: -------------------------------------------------------------------------------- 1 | package fileutil 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "time" 7 | ) 8 | 9 | var TimeoutError = errors.New("Timeout") 10 | 11 | // TimeoutReader is a reader wrapper that stops waiting after Timeout 12 | type TimeoutReader struct { 13 | Timeout time.Duration 14 | Reader io.Reader 15 | timeoutchan chan struct{} 16 | datachan chan struct{} 17 | } 18 | 19 | // Read into the buffer 20 | func (reader TimeoutReader) Read(p []byte) (n int, err error) { 21 | if reader.datachan == nil { 22 | reader.datachan = make(chan struct{}) 23 | } 24 | if reader.timeoutchan == nil { 25 | reader.timeoutchan = make(chan struct{}) 26 | } 27 | go func() { 28 | // if timeout is zero behaves like a normal reader 29 | if reader.Timeout > 0 { 30 | time.Sleep(reader.Timeout) 31 | reader.timeoutchan <- struct{}{} 32 | } 33 | }() 34 | go func() { 35 | n, err = reader.Reader.Read(p) 36 | reader.datachan <- struct{}{} 37 | }() 38 | 39 | select { 40 | case <-reader.timeoutchan: 41 | err = TimeoutError 42 | return 43 | case <-reader.datachan: 44 | return 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 ProjectDiscovery, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | 9 | # Maintain dependencies for GitHub Actions 10 | - package-ecosystem: "github-actions" 11 | directory: "/" 12 | schedule: 13 | interval: "weekly" 14 | target-branch: "main" 15 | commit-message: 16 | prefix: "chore" 17 | include: "scope" 18 | 19 | # Maintain dependencies for go modules 20 | - package-ecosystem: "gomod" 21 | directory: "/" 22 | schedule: 23 | interval: "weekly" 24 | target-branch: "main" 25 | commit-message: 26 | prefix: "chore" 27 | include: "scope" 28 | 29 | # Maintain dependencies for docker 30 | - package-ecosystem: "docker" 31 | directory: "/" 32 | schedule: 33 | interval: "weekly" 34 | target-branch: "main" 35 | commit-message: 36 | prefix: "chore" 37 | include: "scope" -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= 2 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= 3 | github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= 4 | github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= 9 | github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= 10 | github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= 11 | github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= 12 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 13 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 h1:yo7hCL47BOHl8X/aMmPeRQwiqUrH6TZ2WjgqItaSPcc= 17 | github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= 18 | github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= 19 | github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= 20 | github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= 21 | github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= 22 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 23 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 24 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 25 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 26 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 27 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 28 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 29 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 30 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 31 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 32 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 33 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 34 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 35 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 36 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 37 | golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= 38 | golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= 39 | golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= 40 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 41 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 42 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 43 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 44 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 45 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 46 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 47 | golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 48 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 49 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 50 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 51 | golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 52 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 53 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 54 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 55 | golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 56 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 57 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 58 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 59 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 60 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 61 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 62 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 63 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 64 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 65 | -------------------------------------------------------------------------------- /file.go: -------------------------------------------------------------------------------- 1 | package fileutil 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "crypto/tls" 7 | "debug/elf" 8 | "encoding/json" 9 | "io" 10 | "io/fs" 11 | "net/http" 12 | "os" 13 | "path/filepath" 14 | "regexp" 15 | "strings" 16 | "time" 17 | 18 | "github.com/asaskevich/govalidator" 19 | "github.com/pkg/errors" 20 | "github.com/projectdiscovery/stringsutil" 21 | "gopkg.in/yaml.v3" 22 | ) 23 | 24 | // FileExists checks if the file exists in the provided path 25 | func FileExists(filename string) bool { 26 | info, err := os.Stat(filename) 27 | if os.IsNotExist(err) { 28 | return false 29 | } 30 | if err != nil { 31 | return false 32 | } 33 | return !info.IsDir() 34 | } 35 | 36 | // FolderExists checks if the folder exists 37 | func FolderExists(foldername string) bool { 38 | info, err := os.Stat(foldername) 39 | if os.IsNotExist(err) { 40 | return false 41 | } 42 | if err != nil { 43 | return false 44 | } 45 | return info.IsDir() 46 | } 47 | 48 | type FileFilters struct { 49 | OlderThan time.Duration 50 | Prefix string 51 | Suffix string 52 | RegexPattern string 53 | CustomCheck func(filename string) bool 54 | Callback func(filename string) error 55 | } 56 | 57 | func DeleteFilesOlderThan(folder string, filter FileFilters) error { 58 | startScan := time.Now() 59 | return filepath.WalkDir(folder, func(osPathname string, de fs.DirEntry, err error) error { 60 | if err != nil { 61 | return nil 62 | } 63 | if osPathname == "" { 64 | return nil 65 | } 66 | if de.IsDir() { 67 | return nil 68 | } 69 | fileInfo, err := os.Stat(osPathname) 70 | if err != nil { 71 | return nil 72 | } 73 | fileName := fileInfo.Name() 74 | if filter.Prefix != "" && !strings.HasPrefix(fileName, filter.Prefix) { 75 | return nil 76 | } 77 | if filter.Suffix != "" && !strings.HasSuffix(fileName, filter.Suffix) { 78 | return nil 79 | } 80 | if filter.RegexPattern != "" { 81 | regex, err := regexp.Compile(filter.RegexPattern) 82 | if err != nil { 83 | return err 84 | } 85 | if !regex.MatchString(fileName) { 86 | return nil 87 | } 88 | } 89 | if filter.CustomCheck != nil && !filter.CustomCheck(osPathname) { 90 | return nil 91 | } 92 | if fileInfo.ModTime().Add(filter.OlderThan).Before(startScan) { 93 | if filter.Callback != nil { 94 | return filter.Callback(osPathname) 95 | } else { 96 | os.RemoveAll(osPathname) 97 | } 98 | } 99 | return nil 100 | }, 101 | ) 102 | } 103 | 104 | // DownloadFile to specified path 105 | func DownloadFile(filepath string, url string) error { 106 | tr := &http.Transport{ 107 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 108 | } 109 | client := &http.Client{Transport: tr} 110 | resp, err := client.Get(url) 111 | if err != nil { 112 | return err 113 | } 114 | defer resp.Body.Close() 115 | out, err := os.Create(filepath) 116 | if err != nil { 117 | return err 118 | } 119 | defer out.Close() 120 | _, err = io.Copy(out, resp.Body) 121 | return err 122 | } 123 | 124 | // CreateFolders in the list 125 | func CreateFolders(paths ...string) error { 126 | for _, path := range paths { 127 | if err := CreateFolder(path); err != nil { 128 | return err 129 | } 130 | } 131 | 132 | return nil 133 | } 134 | 135 | // CreateFolder path 136 | func CreateFolder(path string) error { 137 | return os.MkdirAll(path, 0700) 138 | } 139 | 140 | // HasStdin determines if the user has piped input 141 | func HasStdin() bool { 142 | stat, err := os.Stdin.Stat() 143 | if err != nil { 144 | return false 145 | } 146 | 147 | mode := stat.Mode() 148 | 149 | isPipedFromChrDev := (mode & os.ModeCharDevice) == 0 150 | isPipedFromFIFO := (mode & os.ModeNamedPipe) != 0 151 | 152 | return isPipedFromChrDev || isPipedFromFIFO 153 | } 154 | 155 | // ReadFileWithReader and stream on a channel 156 | func ReadFileWithReader(r io.Reader) (chan string, error) { 157 | out := make(chan string) 158 | go func() { 159 | defer close(out) 160 | scanner := bufio.NewScanner(r) 161 | for scanner.Scan() { 162 | out <- scanner.Text() 163 | } 164 | }() 165 | 166 | return out, nil 167 | } 168 | 169 | // ReadFileWithReader with specific buffer size and stream on a channel 170 | func ReadFileWithReaderAndBufferSize(r io.Reader, maxCapacity int) (chan string, error) { 171 | out := make(chan string) 172 | go func() { 173 | defer close(out) 174 | scanner := bufio.NewScanner(r) 175 | buf := make([]byte, maxCapacity) 176 | scanner.Buffer(buf, maxCapacity) 177 | for scanner.Scan() { 178 | out <- scanner.Text() 179 | } 180 | }() 181 | 182 | return out, nil 183 | } 184 | 185 | // ReadFile with filename 186 | func ReadFile(filename string) (chan string, error) { 187 | if !FileExists(filename) { 188 | return nil, errors.New("file doesn't exist") 189 | } 190 | out := make(chan string) 191 | go func() { 192 | defer close(out) 193 | f, err := os.Open(filename) 194 | if err != nil { 195 | return 196 | } 197 | defer f.Close() 198 | scanner := bufio.NewScanner(f) 199 | for scanner.Scan() { 200 | out <- scanner.Text() 201 | } 202 | }() 203 | 204 | return out, nil 205 | } 206 | 207 | // ReadFile with filename and specific buffer size 208 | func ReadFileWithBufferSize(filename string, maxCapacity int) (chan string, error) { 209 | if !FileExists(filename) { 210 | return nil, errors.New("file doesn't exist") 211 | } 212 | out := make(chan string) 213 | go func() { 214 | defer close(out) 215 | f, err := os.Open(filename) 216 | if err != nil { 217 | return 218 | } 219 | defer f.Close() 220 | scanner := bufio.NewScanner(f) 221 | buf := make([]byte, maxCapacity) 222 | scanner.Buffer(buf, maxCapacity) 223 | for scanner.Scan() { 224 | out <- scanner.Text() 225 | } 226 | }() 227 | 228 | return out, nil 229 | } 230 | 231 | // GetTempFileName generate a temporary file name 232 | func GetTempFileName() (string, error) { 233 | tmpfile, err := os.CreateTemp("", "") 234 | if err != nil { 235 | return "", err 236 | } 237 | tmpFileName := tmpfile.Name() 238 | if err := tmpfile.Close(); err != nil { 239 | return tmpFileName, err 240 | } 241 | err = os.RemoveAll(tmpFileName) 242 | return tmpFileName, err 243 | } 244 | 245 | // CopyFile from source to destination 246 | func CopyFile(src, dst string) error { 247 | if !FileExists(src) { 248 | return errors.New("source file doesn't exist") 249 | } 250 | srcFile, err := os.Open(src) 251 | if err != nil { 252 | return err 253 | } 254 | defer srcFile.Close() 255 | 256 | dstFile, err := os.Create(dst) 257 | if err != nil { 258 | return err 259 | } 260 | defer dstFile.Close() 261 | 262 | _, err = io.Copy(dstFile, srcFile) 263 | if err != nil { 264 | return err 265 | } 266 | 267 | return dstFile.Sync() 268 | } 269 | 270 | type EncodeType uint8 271 | 272 | const ( 273 | YAML EncodeType = iota 274 | JSON 275 | ) 276 | 277 | func Unmarshal(encodeType EncodeType, data []byte, obj interface{}) error { 278 | switch { 279 | case FileExists(string(data)): 280 | dataFile, err := os.Open(string(data)) 281 | if err != nil { 282 | return err 283 | } 284 | defer dataFile.Close() 285 | return UnmarshalFromReader(encodeType, dataFile, obj) 286 | default: 287 | return UnmarshalFromReader(encodeType, bytes.NewReader(data), obj) 288 | } 289 | } 290 | 291 | func UnmarshalFromReader(encodeType EncodeType, r io.Reader, obj interface{}) error { 292 | switch encodeType { 293 | case YAML: 294 | return yaml.NewDecoder(r).Decode(obj) 295 | case JSON: 296 | return json.NewDecoder(r).Decode(obj) 297 | default: 298 | return errors.New("unsopported encode type") 299 | } 300 | } 301 | 302 | func Marshal(encodeType EncodeType, data []byte, obj interface{}) error { 303 | isFilePath, _ := govalidator.IsFilePath(string(data)) 304 | switch { 305 | case isFilePath: 306 | dataFile, err := os.Create(string(data)) 307 | if err != nil { 308 | return err 309 | } 310 | defer dataFile.Close() 311 | return MarshalToWriter(encodeType, dataFile, obj) 312 | default: 313 | return MarshalToWriter(encodeType, bytes.NewBuffer(data), obj) 314 | } 315 | } 316 | 317 | func MarshalToWriter(encodeType EncodeType, r io.Writer, obj interface{}) error { 318 | switch encodeType { 319 | case YAML: 320 | return yaml.NewEncoder(r).Encode(obj) 321 | case JSON: 322 | return json.NewEncoder(r).Encode(obj) 323 | default: 324 | return errors.New("unsopported encode type") 325 | } 326 | } 327 | 328 | func ExecutableName() string { 329 | executablePath, err := os.Executable() 330 | if err == nil { 331 | executablePath = os.Args[0] 332 | } 333 | executableNameWithExt := filepath.Base(executablePath) 334 | return stringsutil.TrimSuffixAny(executableNameWithExt, filepath.Ext(executableNameWithExt)) 335 | } 336 | 337 | // RemoveAll specified paths, returning those that caused error 338 | func RemoveAll(paths ...string) (errored map[string]error) { 339 | errored = make(map[string]error) 340 | for _, path := range paths { 341 | if err := os.RemoveAll(path); err != nil { 342 | errored[path] = err 343 | } 344 | } 345 | return 346 | } 347 | 348 | // UseMusl checks if the specified elf file uses musl 349 | func UseMusl(path string) (bool, error) { 350 | file, err := os.Open(path) 351 | if err != nil { 352 | return false, err 353 | } 354 | defer file.Close() 355 | elfFile, err := elf.NewFile(file) 356 | if err != nil { 357 | return false, err 358 | } 359 | importedLibraries, err := elfFile.ImportedLibraries() 360 | if err != nil { 361 | return false, err 362 | } 363 | for _, importedLibrary := range importedLibraries { 364 | if stringsutil.ContainsAny(importedLibrary, "libc.musl-") { 365 | return true, nil 366 | } 367 | } 368 | return false, nil 369 | } 370 | 371 | // IsReadable verify file readability 372 | func IsReadable(fileName string) (bool, error) { 373 | return HasPermission(fileName, os.O_RDONLY) 374 | } 375 | 376 | // IsWriteable verify file writeability 377 | func IsWriteable(fileName string) (bool, error) { 378 | return HasPermission(fileName, os.O_WRONLY) 379 | } 380 | 381 | // HasPermission checks if the file has the requested permission 382 | func HasPermission(fileName string, permission int) (bool, error) { 383 | file, err := os.OpenFile(fileName, permission, 0666) 384 | if err != nil { 385 | if os.IsPermission(err) { 386 | return false, errors.Wrap(err, "permission error") 387 | } 388 | return false, err 389 | } 390 | file.Close() 391 | 392 | return true, nil 393 | } 394 | -------------------------------------------------------------------------------- /file_test.go: -------------------------------------------------------------------------------- 1 | package fileutil 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "runtime" 9 | "strings" 10 | "testing" 11 | "time" 12 | 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestFileExists(t *testing.T) { 17 | tests := map[string]bool{ 18 | "file.go": true, 19 | "aaa.bbb": false, 20 | "/": false, 21 | } 22 | for fpath, mustExist := range tests { 23 | exist := FileExists(fpath) 24 | require.Equalf(t, mustExist, exist, "invalid \"%s\": %s", fpath, exist) 25 | } 26 | } 27 | 28 | func TestFolderExists(t *testing.T) { 29 | tests := map[string]bool{ 30 | ".": true, 31 | "../fileutil": true, 32 | "aabb": false, 33 | } 34 | for fpath, mustExist := range tests { 35 | exist := FolderExists(fpath) 36 | require.Equalf(t, mustExist, exist, "invalid \"%s\"", fpath) 37 | } 38 | } 39 | 40 | func TestDeleteFilesOlderThan(t *testing.T) { 41 | // create a temporary folder with a couple of files 42 | fo, err := os.MkdirTemp("", "") 43 | require.Nil(t, err, "couldn't create folder: %s", err) 44 | ttl := time.Duration(5 * time.Second) 45 | 46 | // defer temporary folder removal 47 | defer os.RemoveAll(fo) 48 | checkFolderErr := func(err error) { 49 | require.Nil(t, err, "couldn't create folder: %s", err) 50 | } 51 | checkFiles := func(fileName string) { 52 | require.False(t, FileExists(fileName), "file \"%s\" still exists", fileName) 53 | } 54 | createFile := func() string { 55 | fi, err := os.CreateTemp(fo, "") 56 | require.Nil(t, err, "couldn't create f: %s", err) 57 | fName := fi.Name() 58 | fi.Close() 59 | return fName 60 | } 61 | t.Run("prefix props test", func(t *testing.T) { 62 | fName := createFile() 63 | fileInfo, _ := os.Stat(fName) 64 | // sleep for 5 seconds 65 | time.Sleep(5 * time.Second) 66 | // delete files older than 5 seconds 67 | filter := FileFilters{ 68 | OlderThan: ttl, 69 | Prefix: fileInfo.Name(), 70 | } 71 | err = DeleteFilesOlderThan(fo, filter) 72 | checkFolderErr(err) 73 | checkFiles(fName) 74 | }) 75 | t.Run("suffix props test", func(t *testing.T) { 76 | fName := createFile() 77 | fileInfo, _ := os.Stat(fName) 78 | // sleep for 5 seconds 79 | time.Sleep(5 * time.Second) 80 | // delete files older than 5 seconds 81 | filter := FileFilters{ 82 | OlderThan: ttl, 83 | Suffix: string(fileInfo.Name()[len(fileInfo.Name())-1]), 84 | } 85 | err = DeleteFilesOlderThan(fo, filter) 86 | checkFolderErr(err) 87 | checkFiles(fName) 88 | }) 89 | t.Run("regex pattern props test", func(t *testing.T) { 90 | fName := createFile() 91 | fName1 := createFile() 92 | 93 | // sleep for 5 seconds 94 | time.Sleep(5 * time.Second) 95 | // delete files older than 5 seconds 96 | filter := FileFilters{ 97 | OlderThan: ttl, 98 | RegexPattern: "[0-9]", 99 | } 100 | err = DeleteFilesOlderThan(fo, filter) 101 | checkFolderErr(err) 102 | checkFiles(fName) 103 | checkFiles(fName1) 104 | }) 105 | t.Run("custom check props test", func(t *testing.T) { 106 | fName := createFile() 107 | fName1 := createFile() 108 | 109 | // sleep for 5 seconds 110 | time.Sleep(5 * time.Second) 111 | // delete files older than 5 seconds 112 | filter := FileFilters{ 113 | OlderThan: ttl, 114 | CustomCheck: func(filename string) bool { 115 | return true 116 | }, 117 | } 118 | err = DeleteFilesOlderThan(fo, filter) 119 | checkFolderErr(err) 120 | checkFiles(fName) 121 | checkFiles(fName1) 122 | }) 123 | t.Run("custom check props negative test", func(t *testing.T) { 124 | fName := createFile() 125 | // sleep for 5 seconds 126 | time.Sleep(5 * time.Second) 127 | // delete files older than 5 seconds 128 | filter := FileFilters{ 129 | OlderThan: ttl, 130 | CustomCheck: func(filename string) bool { 131 | return false // should not delete the file 132 | }, 133 | } 134 | err = DeleteFilesOlderThan(fo, filter) 135 | checkFolderErr(err) 136 | require.True(t, FileExists(fName), "file \"%s\" should exists", fName) 137 | }) 138 | t.Run("callback props test", func(t *testing.T) { 139 | fName := createFile() 140 | fName1 := createFile() 141 | 142 | // sleep for 5 seconds 143 | time.Sleep(5 * time.Second) 144 | // delete files older than 5 seconds 145 | filter := FileFilters{ 146 | OlderThan: ttl, 147 | CustomCheck: func(filename string) bool { 148 | return true 149 | }, 150 | Callback: func(filename string) error { 151 | t.Log("deleting file manually") 152 | return os.Remove(filename) 153 | }, 154 | } 155 | err = DeleteFilesOlderThan(fo, filter) 156 | checkFolderErr(err) 157 | checkFiles(fName) 158 | checkFiles(fName1) 159 | }) 160 | } 161 | 162 | func TestDownloadFile(t *testing.T) { 163 | // attempt to download http://ipv4.download.thinkbroadband.com/5MB.zip to temp folder 164 | tmpfile, err := os.CreateTemp("", "") 165 | require.Nil(t, err, "couldn't create folder: %s", err) 166 | fname := tmpfile.Name() 167 | 168 | os.Remove(fname) 169 | 170 | err = DownloadFile(fname, "http://ipv4.download.thinkbroadband.com/5MB.zip") 171 | require.Nil(t, err, "couldn't download file: %s", err) 172 | 173 | require.True(t, FileExists(fname), "file \"%s\" doesn't exists", fname) 174 | 175 | // remove the downloaded file 176 | os.Remove(fname) 177 | } 178 | 179 | func tmpFolderName(s string) string { 180 | return filepath.Join(os.TempDir(), s) 181 | } 182 | func TestCreateFolders(t *testing.T) { 183 | tests := []string{ 184 | tmpFolderName("a"), 185 | tmpFolderName("b"), 186 | } 187 | err := CreateFolders(tests...) 188 | require.Nil(t, err, "couldn't download file: %s", err) 189 | 190 | for _, folder := range tests { 191 | fexists := FolderExists(folder) 192 | require.True(t, fexists, "folder %s doesn't exist", fexists) 193 | } 194 | 195 | // remove folders 196 | for _, folder := range tests { 197 | os.Remove(folder) 198 | } 199 | } 200 | 201 | func TestCreateFolder(t *testing.T) { 202 | tst := tmpFolderName("a") 203 | err := CreateFolder(tst) 204 | require.Nil(t, err, "couldn't download file: %s", err) 205 | 206 | fexists := FolderExists(tst) 207 | require.True(t, fexists, "folder %s doesn't exist", fexists) 208 | 209 | os.Remove(tst) 210 | } 211 | 212 | func TestHasStdin(t *testing.T) { 213 | require.False(t, HasStdin(), "stdin in test") 214 | } 215 | 216 | func TestReadFile(t *testing.T) { 217 | fileContent := `test 218 | test1 219 | test2` 220 | f, err := os.CreateTemp("", "") 221 | require.Nil(t, err, "couldn't create file: %s", err) 222 | fname := f.Name() 223 | _, _ = f.Write([]byte(fileContent)) 224 | f.Close() 225 | defer os.Remove(fname) 226 | 227 | fileContentLines := strings.Split(fileContent, "\n") 228 | // compare file lines 229 | c, err := ReadFile(fname) 230 | require.Nil(t, err, "couldn't open file: %s", err) 231 | i := 0 232 | for line := range c { 233 | require.Equal(t, fileContentLines[i], line, "lines don't match") 234 | i++ 235 | } 236 | } 237 | 238 | func TestReadFileWithBufferSize(t *testing.T) { 239 | fileContent := `test 240 | test1 241 | test2` 242 | f, err := os.CreateTemp("", "") 243 | require.Nil(t, err, "couldn't create file: %s", err) 244 | fname := f.Name() 245 | _, _ = f.Write([]byte(fileContent)) 246 | f.Close() 247 | defer os.Remove(fname) 248 | 249 | fileContentLines := strings.Split(fileContent, "\n") 250 | // compare file lines 251 | c, err := ReadFileWithBufferSize(fname, 64*1024) 252 | require.Nil(t, err, "couldn't open file: %s", err) 253 | i := 0 254 | for line := range c { 255 | require.Equal(t, fileContentLines[i], line, "lines don't match") 256 | i++ 257 | } 258 | } 259 | 260 | func TestPermissions(t *testing.T) { 261 | f, err := os.CreateTemp("", "") 262 | require.Nil(t, err, "couldn't create file: %s", err) 263 | fname := f.Name() 264 | f.Close() 265 | defer os.Remove(fname) 266 | 267 | ok, err := IsReadable(fname) 268 | require.True(t, ok) 269 | require.Nil(t, err) 270 | ok, err = IsWriteable(fname) 271 | require.True(t, ok) 272 | require.Nil(t, err) 273 | } 274 | 275 | func TestUseMusl(t *testing.T) { 276 | executablePath, err := os.Executable() 277 | require.Nil(t, err) 278 | _, err = UseMusl(executablePath) 279 | switch runtime.GOOS { 280 | case "windows", "darwin": 281 | require.NotNil(t, err) 282 | case "linux": 283 | require.Nil(t, err) 284 | } 285 | } 286 | 287 | func TestReadFileWithReader(t *testing.T) { 288 | fileContent := `test 289 | test1 290 | test2` 291 | f, err := os.CreateTemp("", "output") 292 | require.Nil(t, err, "couldn't create file: %s", err) 293 | fname := f.Name() 294 | _, _ = f.Write([]byte(fileContent)) 295 | f.Close() 296 | defer os.Remove(fname) 297 | fileContentLines := strings.Split(fileContent, "\n") 298 | f, err = os.Open(fname) 299 | require.Nil(t, err, "couldn't create file: %s", err) 300 | // compare file lines 301 | c, _ := ReadFileWithReader(f) 302 | i := 0 303 | for line := range c { 304 | require.Equal(t, fileContentLines[i], line, "lines don't match") 305 | i++ 306 | } 307 | f.Close() 308 | } 309 | 310 | func TestReadFileWithReaderAndBufferSize(t *testing.T) { 311 | fileContent := `test 312 | test1 313 | test2` 314 | f, err := os.CreateTemp("", "") 315 | require.Nil(t, err, "couldn't create file: %s", err) 316 | fname := f.Name() 317 | _, _ = f.Write([]byte(fileContent)) 318 | f.Close() 319 | defer os.Remove(fname) 320 | fileContentLines := strings.Split(fileContent, "\n") 321 | f, err = os.Open(fname) 322 | require.Nil(t, err, "couldn't create file: %s", err) 323 | // compare file lines 324 | c, _ := ReadFileWithReaderAndBufferSize(f, 64*1024) 325 | i := 0 326 | for line := range c { 327 | require.Equal(t, fileContentLines[i], line, "lines don't match") 328 | i++ 329 | } 330 | f.Close() 331 | } 332 | 333 | func TestCopyFile(t *testing.T) { 334 | fileContent := `test 335 | test1 336 | test2` 337 | f, err := os.CreateTemp("", "") 338 | require.Nil(t, err, "couldn't create file: %s", err) 339 | fname := f.Name() 340 | _, _ = f.Write([]byte(fileContent)) 341 | f.Close() 342 | defer os.Remove(fname) 343 | fnameCopy := fmt.Sprintf("%s-copy", f.Name()) 344 | err = CopyFile(fname, fnameCopy) 345 | require.Nil(t, err, "couldn't copy file: %s", err) 346 | require.True(t, FileExists(fnameCopy), "file \"%s\" doesn't exists", fnameCopy) 347 | os.Remove(fnameCopy) 348 | } 349 | 350 | func TestGetTempFileName(t *testing.T) { 351 | fname, _ := GetTempFileName() 352 | defer os.Remove(fname) 353 | require.NotEmpty(t, fname) 354 | } 355 | 356 | func TestUnmarshal(t *testing.T) { 357 | type TestStruct struct { 358 | Name string `json:"name"` 359 | } 360 | var ts TestStruct 361 | err := Unmarshal(JSON, []byte(`{"name":"test"}`), &ts) 362 | require.Nil(t, err) 363 | require.Equal(t, "test", ts.Name) 364 | } 365 | 366 | func TestMarshal(t *testing.T) { 367 | type TestStruct struct { 368 | Name string `json:"name"` 369 | } 370 | ts := TestStruct{Name: "test"} 371 | fs, err := GetTempFileName() 372 | require.Nil(t, err) 373 | defer RemoveAll(fs) 374 | err = Marshal(JSON, []byte(fs), ts) 375 | require.Nil(t, err) 376 | data, err := os.ReadFile(fs) 377 | require.Nil(t, err) 378 | require.Equal(t, `{"name":"test"}`, strings.TrimSpace(string(data))) 379 | } 380 | 381 | func TestUnmarshalFromReader(t *testing.T) { 382 | type TestStruct struct { 383 | Name string `json:"name"` 384 | } 385 | var ts TestStruct 386 | err := UnmarshalFromReader(JSON, strings.NewReader(`{"name":"test"}`), &ts) 387 | require.Nil(t, err) 388 | require.Equal(t, "test", ts.Name) 389 | } 390 | 391 | func TestMarshalToWriter(t *testing.T) { 392 | type TestStruct struct { 393 | Name string `json:"name"` 394 | } 395 | ts := TestStruct{Name: "test"} 396 | var data []byte 397 | buf := bytes.NewBuffer(data) 398 | err := MarshalToWriter(JSON, buf, ts) 399 | require.Nil(t, err) 400 | require.Equal(t, `{"name":"test"}`, strings.TrimSpace(buf.String())) 401 | } 402 | 403 | func TestExecutableName(t *testing.T) { 404 | require.NotEmpty(t, ExecutableName()) 405 | } 406 | 407 | func TestRemoveAll(t *testing.T) { 408 | tmpdir, err := os.MkdirTemp("", "") 409 | require.Nil(t, err, "couldn't create folder: %s", err) 410 | f, err := os.CreateTemp(tmpdir, "") 411 | require.Nil(t, err, "couldn't create file: %s", err) 412 | f.Close() 413 | errs := RemoveAll(tmpdir) 414 | require.Equal(t, 0, len(errs), "couldn't remove folder: %s", errs) 415 | } 416 | --------------------------------------------------------------------------------