├── .gitignore ├── LICENSE ├── README.md ├── examples ├── readers │ ├── big_test.txt │ └── main.go └── typed_reader │ ├── big_test.json │ └── main.go ├── go.mod ├── internal └── buffer │ └── read.go ├── paopao └── json.go ├── siopao ├── api.go ├── api_copy.go ├── api_delete.go ├── api_directory.go ├── api_hash.go ├── api_move.go ├── api_reading.go ├── api_streaming.go ├── api_writing.go ├── core_close.go ├── core_open.go ├── core_read_write.go ├── core_write.go └── siopao_test.go └── streaming ├── core_writer.go ├── reader.go ├── text_reader.go ├── typed_reader.go └── writer.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .tests/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Shindou Mihou 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # siopao 2 | 3 | *Simplified file operations for Golang.* siopao is a library built from the experimental joke project 4 | [`go-bun`](https://github.com/ShindouMihou/go-bun) in order to bring some simpler abstractions over common, simple 5 | file operations. As its roots, siopao was inspired [`bun.sh`](https://bun.sh)'s incredible developer-experience 6 | for file operations. 7 | 8 | ## demo 9 | ```go 10 | package main 11 | 12 | import ( 13 | "fmt" 14 | "github.com/ShindouMihou/siopao/siopao" 15 | "log" 16 | ) 17 | 18 | type Hello struct { 19 | World string `json:"world"` 20 | } 21 | 22 | func main() { 23 | // Opening a file interface, this does not open the file yet as the file is only opened 24 | // when needed to prevent unnecessary leaking of resources. 25 | file := siopao.Open("test.json") 26 | 27 | // Overwriting (or writing) content to file. 28 | if err := file.Overwrite(Hello{World: "hello world"}); err != nil { 29 | log.Fatalln(err) 30 | } 31 | 32 | // Unmarshalling file to Json. 33 | var hello Hello 34 | if err := file.Json(&hello); err != nil { 35 | log.Fatalln(err) 36 | } 37 | fmt.Println(hello.World) 38 | } 39 | 40 | ``` 41 | 42 | ## installation 43 | ```go 44 | go get github.com/ShindouMihou/siopao 45 | ``` 46 | 47 | view documentation over at your local ide or through [`pkg.go.dev`](https://pkg.go.dev/github.com/ShindouMihou/siopao). 48 | 49 | ## file io 50 | siopao supports the following file methods: 51 | - [x] `File.Write(any)`: writes to file, appends if it exists. anything other than string, `io.Reader`, `bufio.Reader` and byte array is translated to json. 52 | - [x] `File.Overwrite(any)`: overwrites the file with the new contents, similar to the above and marshals anything else to json. 53 | - [x] `File.WriteMarshal(marshaler, any)`: writes to file, appends if it exists. anything other than string and byte array is marshaled using the provided marshaller. 54 | - [x] `File.OverwriteMarshal(marshaler, any)`: overwrites the file with the new contents, similar to the above and marshals anything else to the provided marshaller. 55 | - [x] `File.Text`: reads the file contents and into a string. 56 | - [x] `File.Json(any)`: reads the file contents as a json and unmarshals into the type. 57 | - [x] `File.Unmarshal(unmarshaler, any)`: reads the file contents and unmarshals into the type. 58 | - [x] `File.Bytes`: reads the file contents and into a byte array. 59 | - [x] `File.Reader`: returns a [`Reader`](#reader) of the file. 60 | - [x] `File.TextReader`: returns a [`TextReader`](#textreader) of the file. 61 | - [x] `File.Writer(overwrite)`: returns a [`Writer`](#write-streams) of the file, creates the file if needed. 62 | - [x] `File.WriterSize(overwrite, buffer_size)`: returns a [`Writer`](#write-streams) with a specified buffer size of the file, creates the file if needed. 63 | - [x] `File.Copy(dest)`: copies the file to the destination path. 64 | - [x] `File.CopyAndHash(kind, dest)`: copies the file to the destination while creating a hash of the content. 65 | - [x] `File.Checksum(kind)`: gets the checksum of the file, `kind` can be `sha512`, `sha256` or `md5`. 66 | - [x] `File.Move(dest)`: moves the file's path to the new path, can change folder and file name. 67 | - [x] `File.Rename(name)`: renames the file's name, works like `File.Move` but keeps the file in the same folder. 68 | - [x] `File.MoveTo(dir)`: moves the file to a new directory, the opposite of `File.Rename`, keeps the file name and extension, but changes the folder. 69 | - [x] `File.DeleteRecursively`: deletes the file or folder. if it's a folder and has contents, deletes the contents recursively. 70 | - [x] `File.Delete`: deletes the file or empty folder. if it's a folder and has contents, errors out. 71 | - [x] `File.MkdirParent`: makes all the directory of the path, includes the path itself if it is a directory. 72 | - [x] `File.IsDir`: checks whether the path is a directory, this is cached. 73 | - [x] `File.UncachedIsDir`: checks whether the path is a directory, this is uncached and results in a system call all the time. 74 | - [x] `File.Recurse`: recursively looks into the items inside the directory, can also go down levels deep when `nested` is `true`. 75 | 76 | all the `File` methods except the ones that opens a stream will lazily open the file, which means that we open the file when needed and close it 77 | immediately after being used, as such, it is recommended to use the streaming methods when needing to write multiple times to the file. 78 | 79 | 80 | ## read streams 81 | 82 | siopao also has simplified streaming that helps with stream reading. 83 | 84 | > **Warning** 85 | > 86 | > All methods in the readers will close the File, which means that these are not reusable. Although, you can create 87 | > the reader again through the same way using the [`File`](#file-io) interface as the `Reader` creation methods will open the file 88 | > once again. 89 | 90 | ### typedreader 91 | a streaming reader that is intended to be used for json arrays with each line being a one-line json object. 92 | can be created using `streaming.NewTypedReader[T any](reader)`. 93 | - [x] `Lines`: reads each line and transform it into the type before adding them to an array. 94 | - [x] `WithUnmarshaler`: sets the unmarshaler of reader, defaults to json. 95 | 96 | ### reader 97 | the base streaming reader that handles with bytes. 98 | - [x] `Lines`: reads each line and creates an array of `[]bytes`. this also caches the array into the reader, you can empty it using `empty` 99 | - [x] `Count`: counts all the lines in the file, this calls `Lines` and counts the cache if there is one already. 100 | - [x] `EachLine`: reads each line and performs an action upon that line, **the line's byte array will be overridden on each next line** 101 | - [x] `EachChar`: reads each char and preforms an action upon that char. 102 | - [x] `EachImmutableLine`: reads each line and performs an action upon that line, slower than the prior method, but the line's value is never overridden on each next line. 103 | - [x] `Empty`: dereferences the cache if there is any. 104 | 105 | ### textreader 106 | a simple streaming reader that handles with strings. it wraps around [`reader`](#reader). 107 | - [x] `Lines`: reads each line and creates an array of string. this also caches the array into the reader, you can empty it using `empty` 108 | - [x] `Count`: counts all the lines in the file, this calls `Lines` and counts the cache if there is one already. 109 | - [x] `EachChar`: reads each char and preforms an action upon that char. 110 | - [x] `EachLine`: reads each line and performs an action upon that line. 111 | - [x] `Empty`: dereferences the cache if there is any. 112 | 113 | ## write streams 114 | 115 | siopao also has simplified streaming that helps with streamwriting. 116 | 117 | > **Warning** 118 | > 119 | > It is your responsibility to close the buffer when it comes to writing, when possible, use 120 | > `Close` to flush the buffer and close the file to prevent anything crazy happening. 121 | 122 | - `Writer`: the all-around streaming writer, defaults to json for anything other than bytes and string. 123 | - [x] `AlwaysAppendNewLine`: sets the writer to always append a new line on each new write. 124 | - [x] `Write(any)`: similar to the [`File.Write`](#file-io) but pushes to the buffer, this marshals anything other than bytes, `io.Reader`, `bufio.Reader` and string to json. 125 | - [x] `WriteMarshal(any)`: similar to the [`File.WriteMarshal`](#file-io) but pushes to the buffer, this marshals anything other than bytes and string with the provided marshaller. 126 | - [x] `Flush`: flushes the buffer. 127 | - [x] `End`: flushes the buffer and closes the file. similar to bun's `FileSink.end`. 128 | - [x] `Close`: closes the file, but does not flush the buffer, this is risky. 129 | - [x] `Reset`: whatever the heck `bufio.Writer.Reset` does. 130 | 131 | ## i hate stdlib json! 132 | 133 | then don't use stdlib json! siopao allows you to change the marshaller to any stdlib-json compatible 134 | marshallers such as [`sonic`](https://github.com/bytedance/sonic). you can change it by changing the values in `paopao` package: 135 | ```go 136 | paopao.Marshal = sonic.Marshal 137 | paopao.Unmarshal = sonic.Unmarshal 138 | ``` 139 | 140 | ## concurrency 141 | 142 | siopao prior to v1.0.4 was written without concurrency in mind. the `File` instance carried a pointer to a `*os.File` and 143 | that was being shared by all operations performed by the `File` instance. that's unsafe. i know. and since v1.0.4, the internal 144 | code has been restructured to open a `*os.File` internally, preventing any sharing of the `*os.File` instance, making `File` 145 | concurrent-safe from v1.0.4 onwards. 146 | 147 | ## license 148 | 149 | siopao is licensed under the MIT license. you are free to redistribute, use and do other related actions under the MIT 150 | license. this is permanent and irrevocable. -------------------------------------------------------------------------------- /examples/readers/big_test.txt: -------------------------------------------------------------------------------- 1 | hello world 2 | hello world 3 | hello world 4 | hello world 5 | hello world 6 | hello world 7 | hello world 8 | hello world 9 | hello world 10 | hello world 11 | hello world 12 | hello world 13 | hello world 14 | hello world 15 | hello world 16 | hello world 17 | hello world 18 | hello world 19 | hello world 20 | hello world 21 | hello world 22 | hello world 23 | hello world 24 | hello world 25 | hello world 26 | hello world 27 | hello world 28 | hello world 29 | hello world 30 | hello world 31 | hello world 32 | hello world 33 | hello world 34 | hello world 35 | hello world 36 | hello world 37 | hello world 38 | hello world 39 | hello world 40 | hello world 41 | hello world 42 | hello world 43 | hello world 44 | hello world 45 | hello world 46 | hello world 47 | hello world 48 | hello world 49 | hello world 50 | hello world 51 | hello world 52 | hello world 53 | hello world 54 | hello world 55 | hello world 56 | hello world 57 | hello world 58 | hello world 59 | hello world 60 | hello world 61 | hello world 62 | hello world 63 | hello world 64 | hello world 65 | hello world 66 | hello world 67 | hello world 68 | hello world 69 | hello world 70 | hello world 71 | hello world 72 | hello world 73 | hello world 74 | hello world 75 | hello world 76 | hello world 77 | hello world 78 | hello world 79 | hello world 80 | hello world 81 | hello world 82 | hello world 83 | hello world 84 | hello world 85 | hello world 86 | hello world 87 | hello world 88 | hello world 89 | hello world 90 | hello world 91 | hello world 92 | hello world 93 | hello world 94 | hello world 95 | hello world 96 | hello world 97 | hello world 98 | hello world 99 | hello world 100 | hello world 101 | hello world 102 | hello world 103 | hello world 104 | hello world 105 | hello world 106 | hello world 107 | hello world 108 | hello world 109 | hello world 110 | hello world 111 | hello world 112 | hello world 113 | hello world 114 | hello world 115 | hello world 116 | hello world 117 | hello world 118 | hello world 119 | hello world 120 | hello world 121 | hello world 122 | hello world 123 | hello world 124 | hello world 125 | hello world 126 | hello world 127 | hello world 128 | hello world 129 | hello world 130 | hello world 131 | hello world 132 | hello world 133 | hello world 134 | hello world 135 | hello world 136 | hello world 137 | hello world 138 | hello world 139 | hello world 140 | hello world 141 | hello world 142 | hello world 143 | hello world 144 | hello world 145 | hello world 146 | hello world 147 | hello world 148 | hello world 149 | hello world 150 | hello world 151 | hello world 152 | hello world 153 | hello world 154 | hello world 155 | hello world 156 | hello world 157 | hello world 158 | hello world 159 | hello world 160 | hello world 161 | hello world 162 | hello world 163 | hello world 164 | hello world 165 | hello world 166 | hello world 167 | hello world 168 | hello world 169 | hello world 170 | hello world 171 | hello world 172 | hello world 173 | hello world 174 | hello world 175 | hello world 176 | hello world 177 | hello world 178 | hello world 179 | hello world 180 | hello world 181 | hello world 182 | hello world 183 | hello world 184 | hello world 185 | hello world -------------------------------------------------------------------------------- /examples/readers/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ShindouMihou/siopao/siopao" 6 | "log" 7 | ) 8 | 9 | func main() { 10 | file := siopao.Open("examples/readers/big_test.txt") 11 | reader, err := file.TextReader() 12 | if err != nil { 13 | log.Fatalln(err) 14 | } 15 | size, err := reader.Count() 16 | if err != nil { 17 | log.Fatalln(err) 18 | } 19 | fmt.Println("A total of", size, "hellos!") 20 | } 21 | -------------------------------------------------------------------------------- /examples/typed_reader/big_test.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"world":"hello world"}, 3 | {"world":"hello world"}, 4 | {"world":"hello world"}, 5 | {"world":"hello world"}, 6 | {"world":"hello world"}, 7 | {"world":"hello world"}, 8 | {"world":"hello world"}, 9 | {"world":"hello world"}, 10 | {"world":"hello world"}, 11 | {"world":"hello world"}, 12 | {"world":"hello world"}, 13 | {"world":"hello world"}, 14 | {"world":"hello world"}, 15 | {"world":"hello world"}, 16 | {"world":"hello world"}, 17 | {"world":"hello world"}, 18 | {"world":"hello world"}, 19 | {"world":"hello world"}, 20 | {"world":"hello world"}, 21 | {"world":"hello world"}, 22 | {"world":"hello world"}, 23 | {"world":"hello world"}, 24 | {"world":"hello world"}, 25 | {"world":"hello world"}, 26 | {"world":"hello world"}, 27 | {"world":"hello world"}, 28 | {"world":"hello world"}, 29 | {"world":"hello world"}, 30 | {"world":"hello world"}, 31 | {"world":"hello world"}, 32 | {"world":"hello world"}, 33 | {"world":"hello world"}, 34 | {"world":"hello world"}, 35 | {"world":"hello world"}, 36 | {"world":"hello world"}, 37 | {"world":"hello world"}, 38 | {"world":"hello world"}, 39 | {"world":"hello world"}, 40 | {"world":"hello world"}, 41 | {"world":"hello world"}, 42 | {"world":"hello world"}, 43 | {"world":"hello world"}, 44 | {"world":"hello world"}, 45 | {"world":"hello world"}, 46 | {"world":"hello world"}, 47 | {"world":"hello world"}, 48 | {"world":"hello world"}, 49 | {"world":"hello world"}, 50 | {"world":"hello world"}, 51 | {"world":"hello world"}, 52 | {"world":"hello world"}, 53 | {"world":"hello world"}, 54 | {"world":"hello world"}, 55 | {"world":"hello world"}, 56 | {"world":"hello world"}, 57 | {"world":"hello world"}, 58 | {"world":"hello world"}, 59 | {"world":"hello world"}, 60 | {"world":"hello world"}, 61 | {"world":"hello world"}, 62 | {"world":"hello world"}, 63 | {"world":"hello world"}, 64 | {"world":"hello world"}, 65 | {"world":"hello world"}, 66 | {"world":"hello world"}, 67 | {"world":"hello world"}, 68 | {"world":"hello world"}, 69 | {"world":"hello world"}, 70 | {"world":"hello world"}, 71 | {"world":"hello world"}, 72 | {"world":"hello world"}, 73 | {"world":"hello world"}, 74 | {"world":"hello world"}, 75 | {"world":"hello world"}, 76 | {"world":"hello world"}, 77 | {"world":"hello world"}, 78 | {"world":"hello world"}, 79 | {"world":"hello world"}, 80 | {"world":"hello world"}, 81 | {"world":"hello world"}, 82 | {"world":"hello world"}, 83 | {"world":"hello world"}, 84 | {"world":"hello world"}, 85 | {"world":"hello world"}, 86 | {"world":"hello world"}, 87 | {"world":"hello world"}, 88 | {"world":"hello world"}, 89 | {"world":"hello world"}, 90 | {"world":"hello world"}, 91 | {"world":"hello world"}, 92 | {"world":"hello world"}, 93 | {"world":"hello world"}, 94 | {"world":"hello world"}, 95 | {"world":"hello world"}, 96 | {"world":"hello world"}, 97 | {"world":"hello world"}, 98 | {"world":"hello world"}, 99 | {"world":"hello world"}, 100 | {"world":"hello world"}, 101 | {"world":"hello world"}, 102 | {"world":"hello world"}, 103 | {"world":"hello world"}, 104 | {"world":"hello world"}, 105 | {"world":"hello world"}, 106 | {"world":"hello world"}, 107 | {"world":"hello world"}, 108 | {"world":"hello world"}, 109 | {"world":"hello world"}, 110 | {"world":"hello world"}, 111 | {"world":"hello world"}, 112 | {"world":"hello world"}, 113 | {"world":"hello world"}, 114 | {"world":"hello world"}, 115 | {"world":"hello world"}, 116 | {"world":"hello world"}, 117 | {"world":"hello world"}, 118 | {"world":"hello world"}, 119 | {"world":"hello world"}, 120 | {"world":"hello world"}, 121 | {"world":"hello world"}, 122 | {"world":"hello world"}, 123 | {"world":"hello world"}, 124 | {"world":"hello world"}, 125 | {"world":"hello world"}, 126 | {"world":"hello world"}, 127 | {"world":"hello world"}, 128 | {"world":"hello world"}, 129 | {"world":"hello world"}, 130 | {"world":"hello world"}, 131 | {"world":"hello world"}, 132 | {"world":"hello world"}, 133 | {"world":"hello world"}, 134 | {"world":"hello world"}, 135 | {"world":"hello world"}, 136 | {"world":"hello world"}, 137 | {"world":"hello world"}, 138 | {"world":"hello world"}, 139 | {"world":"hello world"}, 140 | {"world":"hello world"}, 141 | {"world":"hello world"}, 142 | {"world":"hello world"}, 143 | {"world":"hello world"}, 144 | {"world":"hello world"}, 145 | {"world":"hello world"}, 146 | {"world":"hello world"}, 147 | {"world":"hello world"}, 148 | {"world":"hello world"}, 149 | {"world":"hello world"}, 150 | {"world":"hello world"}, 151 | {"world":"hello world"}, 152 | {"world":"hello world"}, 153 | {"world":"hello world"}, 154 | {"world":"hello world"}, 155 | {"world":"hello world"}, 156 | {"world":"hello world"}, 157 | {"world":"hello world"}, 158 | {"world":"hello world"}, 159 | {"world":"hello world"}, 160 | {"world":"hello world"}, 161 | {"world":"hello world"}, 162 | {"world":"hello world"}, 163 | {"world":"hello world"}, 164 | {"world":"hello world"}, 165 | {"world":"hello world"}, 166 | {"world":"hello world"}, 167 | {"world":"hello world"}, 168 | {"world":"hello world"}, 169 | {"world":"hello world"}, 170 | {"world":"hello world"}, 171 | {"world":"hello world"}, 172 | {"world":"hello world"}, 173 | {"world":"hello world"}, 174 | {"world":"hello world"}, 175 | {"world":"hello world"}, 176 | {"world":"hello world"}, 177 | {"world":"hello world"}, 178 | {"world":"hello world"}, 179 | {"world":"hello world"}, 180 | {"world":"hello world"}, 181 | {"world":"hello world"}, 182 | {"world":"hello world"}, 183 | {"world":"hello world"}, 184 | {"world":"hello world"}, 185 | {"world":"hello world"}, 186 | {"world":"hello world"} 187 | ] -------------------------------------------------------------------------------- /examples/typed_reader/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ShindouMihou/siopao/siopao" 6 | "github.com/ShindouMihou/siopao/streaming" 7 | "log" 8 | ) 9 | 10 | type Hello struct { 11 | World string `json:"world"` 12 | } 13 | 14 | func main() { 15 | file := siopao.Open("examples/typed_reader/big_test.json") 16 | reader, err := file.Reader() 17 | if err != nil { 18 | log.Fatalln(err) 19 | } 20 | hellos, err := streaming.NewTypedReader[Hello](reader).Lines() 21 | if err != nil { 22 | log.Fatalln(err) 23 | } 24 | fmt.Println("A total of", len(hellos), "hellos!") 25 | } 26 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ShindouMihou/siopao 2 | 3 | go 1.20 4 | -------------------------------------------------------------------------------- /internal/buffer/read.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | ) 7 | 8 | func Read(buf io.Reader, size uint16, fn func(bytes []byte) error) error { 9 | buffer := make([]byte, 0, size) 10 | for { 11 | n, err := io.ReadFull(buf, buffer[:cap(buffer)]) 12 | buffer = buffer[:n] 13 | if err != nil { 14 | if err == io.EOF { 15 | break 16 | } 17 | if !errors.Is(err, io.ErrUnexpectedEOF) { 18 | return err 19 | } 20 | } 21 | if err := fn(buffer); err != nil { 22 | return err 23 | } 24 | } 25 | return nil 26 | } 27 | -------------------------------------------------------------------------------- /paopao/json.go: -------------------------------------------------------------------------------- 1 | package paopao 2 | 3 | import "encoding/json" 4 | 5 | type Marshaller func(v any) ([]byte, error) 6 | type Unmarshaler func(data []byte, v any) error 7 | 8 | var Marshal = json.Marshal 9 | var Unmarshal = json.Unmarshal 10 | -------------------------------------------------------------------------------- /siopao/api.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | type File struct { 4 | path string 5 | isDir int 6 | } 7 | 8 | // Open opens up a new interface with the given file. 9 | // 10 | // siopao.Open will lazily open the file, which means that the file is opened as many times as it is needed and is 11 | // closed immediately after use, unless it is needed by streaming. This prevents unnecessary resources from being 12 | // leaked. 13 | func Open(path string) *File { 14 | return &File{ 15 | path: path, 16 | isDir: -1, 17 | } 18 | } 19 | 20 | // Path gets the path of the file. 21 | func (file *File) Path() string { 22 | return file.path 23 | } 24 | -------------------------------------------------------------------------------- /siopao/api_copy.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha256" 6 | "crypto/sha512" 7 | "encoding/hex" 8 | "errors" 9 | "io" 10 | "os" 11 | ) 12 | 13 | // Copy copies the contents of the given source (file) into the destination. 14 | func (file *File) Copy(dest string) error { 15 | destination := Open(dest) 16 | _, err := write(destination, true, func(destFile *os.File) (*any, error) { 17 | srcFile, err := file.openRead() 18 | if err != nil { 19 | return nil, err 20 | } 21 | defer file.close(srcFile) 22 | _, err = io.Copy(destFile, srcFile) 23 | return nil, err 24 | }) 25 | return err 26 | } 27 | 28 | // CopyWithHash works similar to Copy but also creates a hash of the contents. 29 | func (file *File) CopyWithHash(kind ChecksumKind, dest string) (*string, error) { 30 | destination := Open(dest) 31 | return write(destination, true, func(destFile *os.File) (*string, error) { 32 | srcFile, err := file.openRead() 33 | if err != nil { 34 | return nil, err 35 | } 36 | defer file.close(srcFile) 37 | hsh := sha256.New() 38 | switch kind { 39 | case Sha256Checksum: 40 | hsh = sha256.New() 41 | case Md5Checksum: 42 | hsh = md5.New() 43 | case Sha512Checksum: 44 | hsh = sha512.New() 45 | default: 46 | return nil, errors.New("unsupported checksum kind") 47 | } 48 | teeReader := io.TeeReader(srcFile, hsh) 49 | if _, err = io.Copy(destFile, teeReader); err != nil { 50 | return nil, err 51 | } 52 | sum := hex.EncodeToString(hsh.Sum(nil)) 53 | return &sum, nil 54 | }) 55 | } 56 | -------------------------------------------------------------------------------- /siopao/api_delete.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import "os" 4 | 5 | // Delete deletes the file, or an empty directory. If you need to delete a directory that isn't empty, then use 6 | // DeleteRecursively instead. 7 | func (file *File) Delete() error { 8 | return os.Remove(file.path) 9 | } 10 | 11 | // DeleteRecursively deletes the file or directory and its children, if there are any, simply a short-hand of os.RemoveAll. 12 | func (file *File) DeleteRecursively() error { 13 | return os.RemoveAll(file.path) 14 | } 15 | -------------------------------------------------------------------------------- /siopao/api_directory.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | // IsDir checks whether the file is a directory, when the File comes from a Recurse call, or another call previously 10 | // used `IsDir` then that value will be cached. To not use the cached value, use the UncachedIsDir method instead. 11 | func (file *File) IsDir() (bool, error) { 12 | if file.isDir != -1 { 13 | return file.isDir == 1, nil 14 | } 15 | 16 | _, err := file.UncachedIsDir() 17 | if err != nil { 18 | return false, err 19 | } 20 | 21 | return file.isDir == 1, nil 22 | } 23 | 24 | // UncachedIsDir checks whether the file is a directory without passing through the cache. This is recommended 25 | // to use when the file is frequently changing between a directory, or a file. 26 | func (file *File) UncachedIsDir() (bool, error) { 27 | fileInfo, err := os.Stat(file.path) 28 | if err != nil { 29 | return false, err 30 | } 31 | 32 | if fileInfo.IsDir() { 33 | file.isDir = 1 34 | } else { 35 | file.isDir = 0 36 | } 37 | 38 | return file.isDir == 1, nil 39 | } 40 | 41 | // Recurse recurses through the directory if it's a directory. You can specify whether to recurse 42 | // deep into the directory by setting the nested option to true. 43 | func (file *File) Recurse(nested bool, fn func(file *File)) error { 44 | isDirectory, err := file.IsDir() 45 | if err != nil { 46 | return err 47 | } 48 | if !isDirectory { 49 | return fmt.Errorf("%s is not a directory", file.path) 50 | } 51 | return file.recurse(nested, fn) 52 | } 53 | 54 | // MkdirParent creates the parent folders of the path, this also includes the current 55 | // path if it is a directory already. 56 | func (file *File) MkdirParent() error { 57 | return mkparent(file.path) 58 | } 59 | 60 | func (file *File) recurse(nested bool, fn func(file *File)) error { 61 | files, err := os.ReadDir(file.path) 62 | if err != nil { 63 | return err 64 | } 65 | for _, f := range files { 66 | path := filepath.Join(file.path, f.Name()) 67 | child := Open(path) 68 | 69 | if f.IsDir() { 70 | child.isDir = 1 71 | } else { 72 | child.isDir = 0 73 | } 74 | 75 | fn(child) 76 | if f.IsDir() && nested { 77 | err := child.recurse(nested, fn) 78 | if err != nil { 79 | return err 80 | } 81 | } 82 | } 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /siopao/api_hash.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha256" 6 | "crypto/sha512" 7 | "encoding/hex" 8 | "errors" 9 | "hash" 10 | "io" 11 | ) 12 | 13 | type ChecksumKind string 14 | 15 | const ( 16 | Sha512Checksum ChecksumKind = "sha512" 17 | Sha256Checksum ChecksumKind = "sha256" 18 | Md5Checksum ChecksumKind = "md5" 19 | ) 20 | 21 | // Checksum gets the checksum hash of the file's contents. 22 | func (file *File) Checksum(kind ChecksumKind) (string, error) { 23 | f, err := file.openRead() 24 | if err != nil { 25 | return "", err 26 | } 27 | defer file.close(f) 28 | 29 | var hsh hash.Hash 30 | switch kind { 31 | case Sha256Checksum: 32 | hsh = sha256.New() 33 | case Md5Checksum: 34 | hsh = md5.New() 35 | case Sha512Checksum: 36 | hsh = sha512.New() 37 | default: 38 | return "", errors.New("unsupported checksum kind") 39 | } 40 | if _, err := io.Copy(hsh, f); err != nil { 41 | return "", err 42 | } 43 | return hex.EncodeToString(hsh.Sum(nil)), nil 44 | } 45 | -------------------------------------------------------------------------------- /siopao/api_move.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | // Move renames, or moves the file to another path. This is a more direct approach, and will be able to 9 | // move the file to another folder. If you want to simply rename the file's name, use Rename instead, otherwise, 10 | // if you want to keep the name, but move the folder, use MoveTo instead. 11 | func (file *File) Move(dest string) error { 12 | if err := mkparent(dest); err != nil { 13 | return err 14 | } 15 | return os.Rename(file.path, dest) 16 | } 17 | 18 | // Rename renames the file while keeping the source folder, this is useful when you simply want to rename the 19 | // name of the file, change the extension or something similar. 20 | // 21 | // If you want to move the file into an entirely new folder, use Move instead. 22 | // You can also use MoveTo if you want to move to another folder, but still keep the name. 23 | func (file *File) Rename(name string) error { 24 | dir := filepath.Dir(file.path) 25 | return os.Rename(file.path, filepath.Join(dir, name)) 26 | } 27 | 28 | // MoveTo moves the file to another folder while keeping its name, this is useful when you just want to change 29 | // the folder of the file. 30 | // 31 | // If you want to move the file into an entirely new folder, use Move instead. 32 | // You can also use Rename if you want to rename the file's name. 33 | func (file *File) MoveTo(dir string) error { 34 | base := filepath.Base(file.path) 35 | dest := filepath.Join(dir, base) 36 | if err := mkparent(dest); err != nil { 37 | return err 38 | } 39 | return os.Rename(file.path, dest) 40 | } 41 | -------------------------------------------------------------------------------- /siopao/api_reading.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import ( 4 | "errors" 5 | "github.com/ShindouMihou/siopao/paopao" 6 | "io" 7 | "os" 8 | "reflect" 9 | ) 10 | 11 | // Text reads the file directly as a string, this is not recommended to use when handling big 12 | // files, we recommend using TextReader to stream big files instead. 13 | func (file *File) Text() (string, error) { 14 | bytes, err := file.Bytes() 15 | if err != nil { 16 | return "", err 17 | } 18 | return string(bytes), nil 19 | } 20 | 21 | // Bytes reads the file directly as a byte array, this is not recommend to use when handling big 22 | // files, we recommend using Reader to stream big files instead. 23 | func (file *File) Bytes() ([]byte, error) { 24 | bytes, err := read(file, func(f *os.File) (*[]byte, error) { 25 | bytes, err := io.ReadAll(f) 26 | if err != nil { 27 | return nil, nil 28 | } 29 | return &bytes, nil 30 | }) 31 | if err != nil { 32 | return nil, err 33 | } 34 | return *bytes, nil 35 | } 36 | 37 | // Unmarshal unmarshals the given contents of the file with the given unmarshaler. 38 | func (file *File) Unmarshal(unmarshal paopao.Unmarshaler, t interface{}) error { 39 | if reflect.TypeOf(t).Kind() != reflect.Pointer { 40 | return errors.New("non-pointer kind for value") 41 | } 42 | 43 | if _, err := read[any](file, func(f *os.File) (*any, error) { 44 | bytes, err := io.ReadAll(f) 45 | if err != nil { 46 | return nil, err 47 | } 48 | if err := unmarshal(bytes, t); err != nil { 49 | return nil, err 50 | } 51 | return nil, nil 52 | }); err != nil { 53 | return err 54 | } 55 | return nil 56 | } 57 | 58 | // Json unmarshals the contents of the file into Json using the paopao.Unmarshal. 59 | func (file *File) Json(t interface{}) error { 60 | return file.Unmarshal(paopao.Unmarshal, t) 61 | } 62 | -------------------------------------------------------------------------------- /siopao/api_streaming.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import "github.com/ShindouMihou/siopao/streaming" 4 | 5 | // Reader opens a stream to the file, allowing you to handle big file streaming easily. 6 | // 7 | // This causes the file to be opened, therefore, we recommend using the returned streaming.Reader immediately 8 | // to prevent unnecessary leaking of resources. 9 | func (file *File) Reader() (*streaming.Reader, error) { 10 | f, err := file.openRead() 11 | if err != nil { 12 | return nil, err 13 | } 14 | return streaming.NewReader(f), nil 15 | } 16 | 17 | // TextReader opens a string stream to the file, this is an abstraction over the streaming.Reader to handle 18 | // text (string) instead of bytes. 19 | // 20 | // This causes the file to be opened, therefore, we recommend using the returned streaming.Reader immediately 21 | // to prevent unnecessary leaking of resources. 22 | func (file *File) TextReader() (*streaming.TextReader, error) { 23 | reader, err := file.Reader() 24 | if err != nil { 25 | return nil, err 26 | } 27 | return reader.AsTextReader(), nil 28 | } 29 | 30 | // WriterSize opens a write stream, allowing easier stream writing to the file. Unlike Writer, this opens a writing stream 31 | // with the provided buffer size, although it's more recommended to use Writer unless you need to use a higher buffer size. 32 | // 33 | // This causes the file to be opened, it is up to you to close the streaming.Writer using the methods provided. 34 | // We recommend using streaming.Writer's End method to close the writer as it flushes and closes the file. 35 | func (file *File) WriterSize(overwrite bool, size int) (*streaming.Writer, error) { 36 | f, err := file.openWrite(overwrite) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return streaming.NewWriterSize(f, size), nil 41 | } 42 | 43 | // Writer opens a write stream, allowing easier stream writing to the file. Unlike WriterSize, this opens a writing stream 44 | // with a buffer size of 4,096 bytes, if you need to customize the buffer size, then use WriterSize instead. 45 | // 46 | // This causes the file to be opened, it is up to you to close the streaming.Writer using the methods provided. 47 | // We recommend using streaming.Writer's End method to close the writer as it flushes and closes the file. 48 | func (file *File) Writer(overwrite bool) (*streaming.Writer, error) { 49 | f, err := file.openWrite(overwrite) 50 | if err != nil { 51 | return nil, err 52 | } 53 | return streaming.NewWriter(f), nil 54 | } 55 | -------------------------------------------------------------------------------- /siopao/api_writing.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import "github.com/ShindouMihou/siopao/paopao" 4 | 5 | // Write writes, or appends if the file exists, the content to the file. 6 | // Anything other than string, io.Reader and []byte is marshaled into Json with the paopao.Marshal. 7 | func (file *File) Write(t any) error { 8 | return file.wrtany(false, t) 9 | } 10 | 11 | // Overwrite overwrites the file and writes the content to the file. 12 | // Anything other than string, io.Reader and []byte is marshaled into Json with the paopao.Marshal. 13 | func (file *File) Overwrite(t any) error { 14 | return file.wrtany(true, t) 15 | } 16 | 17 | // WriteMarshal works like Write, but marshals anything other than string and []byte with the provided marshal. 18 | func (file *File) WriteMarshal(marshal paopao.Marshaller, t any) error { 19 | return file.wrtmarshal(marshal, false, t) 20 | } 21 | 22 | // OverwriteMarshal works like Overwrite, but marshals anything other than string and []byte with the provided marshal. 23 | func (file *File) OverwriteMarshal(marshal paopao.Marshaller, t any) error { 24 | return file.wrtmarshal(marshal, true, t) 25 | } 26 | -------------------------------------------------------------------------------- /siopao/core_close.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import "os" 4 | 5 | func (file *File) close(f *os.File) { 6 | // ignore the error, it's likely that it just already called 7 | _ = f.Close() 8 | } 9 | -------------------------------------------------------------------------------- /siopao/core_open.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strings" 7 | ) 8 | 9 | func (file *File) openRead() (*os.File, error) { 10 | f, err := os.Open(file.path) 11 | if err != nil { 12 | return nil, err 13 | } 14 | return f, nil 15 | } 16 | 17 | func (file *File) openWrite(trunc bool) (*os.File, error) { 18 | if err := file.MkdirParent(); err != nil { 19 | return nil, err 20 | } 21 | 22 | var f *os.File 23 | var err error 24 | 25 | if trunc { 26 | f, err = os.Create(file.path) 27 | } else { 28 | f, err = os.OpenFile(file.path, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0666) 29 | } 30 | 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | if trunc { 36 | if err := file.clear(f); err != nil { 37 | return nil, err 38 | } 39 | } 40 | return f, nil 41 | } 42 | 43 | func (file *File) clear(f *os.File) error { 44 | if err := f.Truncate(0); err != nil { 45 | return err 46 | } 47 | if _, err := f.Seek(0, 0); err != nil { 48 | return err 49 | } 50 | return nil 51 | } 52 | 53 | func mkparent(path string) error { 54 | if strings.Contains(path, "\\") || strings.Contains(path, "/") { 55 | if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { 56 | return err 57 | } 58 | } 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /siopao/core_read_write.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import "os" 4 | 5 | func read[T any](file *File, fn func(f *os.File) (*T, error)) (*T, error) { 6 | f, err := file.openRead() 7 | if err != nil { 8 | return nil, err 9 | } 10 | defer file.close(f) 11 | return fn(f) 12 | } 13 | 14 | func write[T any](file *File, trunc bool, fn func(f *os.File) (*T, error)) (*T, error) { 15 | f, err := file.openWrite(trunc) 16 | if err != nil { 17 | return nil, err 18 | } 19 | defer file.close(f) 20 | return fn(f) 21 | } 22 | -------------------------------------------------------------------------------- /siopao/core_write.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import ( 4 | "bufio" 5 | buffer2 "github.com/ShindouMihou/siopao/internal/buffer" 6 | "github.com/ShindouMihou/siopao/paopao" 7 | "io" 8 | "os" 9 | ) 10 | 11 | func (file *File) wrt(trunc bool, bytes []byte) error { 12 | if _, err := write(file, trunc, func(f *os.File) (*any, error) { 13 | if _, err := f.Write(bytes); err != nil { 14 | return nil, err 15 | } 16 | return nil, nil 17 | }); err != nil { 18 | return err 19 | } 20 | return nil 21 | } 22 | 23 | func (file *File) wrtbuffer(trunc bool, buffer io.Reader) error { 24 | if _, err := write(file, trunc, func(f *os.File) (*any, error) { 25 | if err := buffer2.Read(buffer, 4_096, func(bytes []byte) error { 26 | if _, err := f.Write(bytes); err != nil { 27 | return err 28 | } 29 | return nil 30 | }); err != nil { 31 | return nil, err 32 | } 33 | return nil, nil 34 | }); err != nil { 35 | return err 36 | } 37 | return nil 38 | } 39 | 40 | func (file *File) wrtjson(trunc bool, t interface{}) error { 41 | return file.wrtmarshal(paopao.Marshal, trunc, t) 42 | } 43 | 44 | func (file *File) wrtmarshal(marshal paopao.Marshaller, trunc bool, t interface{}) error { 45 | bytes, err := marshal(t) 46 | if err != nil { 47 | return err 48 | } 49 | return file.wrt(trunc, bytes) 50 | } 51 | 52 | func (file *File) wrtany(trunc bool, t any) error { 53 | switch t.(type) { 54 | case string: 55 | return file.wrt(trunc, []byte(t.(string))) 56 | case []byte: 57 | return file.wrt(trunc, t.([]byte)) 58 | case *bufio.Reader: 59 | return file.wrtbuffer(trunc, t.(*bufio.Reader)) 60 | case bufio.Reader: 61 | buffer := t.(bufio.Reader) 62 | return file.wrtbuffer(trunc, &buffer) 63 | case io.Reader: 64 | return file.wrtbuffer(trunc, t.(io.Reader)) 65 | default: 66 | return file.wrtjson(true, t) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /siopao/siopao_test.go: -------------------------------------------------------------------------------- 1 | package siopao 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/ShindouMihou/siopao/streaming" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "sync" 11 | "testing" 12 | ) 13 | 14 | func TestFile_Overwrite(t *testing.T) { 15 | file := Open(".tests/write-01.txt") 16 | if err := file.Overwrite("hello world"); err != nil { 17 | t.Fatal("failed to write to test text file: ", err) 18 | } 19 | } 20 | 21 | func TestFile_Overwrite3(t *testing.T) { 22 | file := Open(".tests/write-01.txt") 23 | if err := file.Overwrite(bufio.NewReader(strings.NewReader("hello world"))); err != nil { 24 | t.Fatal("failed to write to test text file: ", err) 25 | } 26 | } 27 | 28 | func TestFile_Text(t *testing.T) { 29 | file := Open(".tests/write-01.txt") 30 | 31 | text, err := file.Text() 32 | if err != nil { 33 | t.Fatal("failed to read to test text file: ", err) 34 | } 35 | if text != "hello world" { 36 | t.Fatal("test file does not match expected result, got '", text, "' instead of 'hello world'") 37 | } 38 | } 39 | 40 | func TestFile_Reader(t *testing.T) { 41 | file := Open(".tests/reader-01.txt") 42 | writer, err := file.Writer(true) 43 | if err != nil { 44 | t.Fatal("failed to clean up test file.") 45 | } 46 | for i := 0; i < 50; i++ { 47 | if err := writer.Write("hello world\n"); err != nil { 48 | t.Fatal("failed to write to test text file: ", err) 49 | } 50 | } 51 | if err := writer.End(); err != nil { 52 | t.Fatal("failed to close writer") 53 | } 54 | reader, err := file.Reader() 55 | if err != nil { 56 | t.Fatal("failed to open reader") 57 | } 58 | if err := reader.EachLine(func(line []byte) { 59 | if len(line) != len([]byte("hello world")) { 60 | t.Fatal("invalid size: ", len(line), line) 61 | } 62 | }); err != nil { 63 | t.Fatal("failed to read test file: ", err) 64 | } 65 | 66 | // each char test 67 | reader, err = file.Reader() 68 | if err != nil { 69 | t.Fatal("failed to open reader") 70 | } 71 | if err := reader.EachChar(func(char rune) { 72 | if char != 'h' && char != 'e' && char != 'l' && char != 'o' && char != 'w' && char != 'r' && char != 'd' && char != '\n' && char != 32 { 73 | t.Fatal("invalid char read: ", string(char)) 74 | } 75 | }); err != nil { 76 | t.Fatal("failed to read test file: ", err) 77 | } 78 | 79 | if err := os.Remove(".tests/reader-01.txt"); err != nil { 80 | t.Fatal("failed to clean up test file.") 81 | } 82 | } 83 | 84 | func TestFile_Checksum(t *testing.T) { 85 | file := Open(".tests/write-01.txt") 86 | 87 | kinds := []ChecksumKind{Sha512Checksum, Sha256Checksum, Md5Checksum} 88 | for _, kind := range kinds { 89 | checksum, err := file.Checksum(kind) 90 | if err != nil { 91 | t.Fatal("failed to read to test text file: ", err) 92 | } 93 | t.Log(kind, ":", checksum) 94 | } 95 | } 96 | 97 | func TestFile_Copy(t *testing.T) { 98 | file := Open(".tests/write-01.txt") 99 | err := file.Copy(".tests/copy-01.txt") 100 | if err != nil { 101 | t.Fatal("failed to copy to test text file: ", err) 102 | } 103 | } 104 | 105 | func TestFile_CopyWithHash(t *testing.T) { 106 | file := Open(".tests/write-01.txt") 107 | checksum, err := file.CopyWithHash(Md5Checksum, ".tests/copy-02.txt") 108 | if err != nil { 109 | t.Fatal("failed to copy to test text file: ", err) 110 | } 111 | t.Log("checksum of copy: ", checksum) 112 | } 113 | 114 | func TestFile_Rename(t *testing.T) { 115 | file := Open(".tests/write-01.txt") 116 | err := file.Copy(".tests/copy-03.txt") 117 | if err != nil { 118 | t.Fatal("failed to copy to test text file: ", err) 119 | } 120 | file = Open(".tests/copy-03.txt") 121 | if err := file.Rename("rename-01.txt"); err != nil { 122 | t.Fatal("failed to rename test text file: ", err) 123 | } 124 | } 125 | 126 | func TestFile_MoveTo(t *testing.T) { 127 | file := Open(".tests/rename-01.txt") 128 | if err := file.MoveTo(".tests/renamed"); err != nil { 129 | t.Fatal("failed to move to new directory: ", err) 130 | } 131 | } 132 | 133 | func TestFile_Move(t *testing.T) { 134 | file := Open(".tests/renamed/rename-01.txt") 135 | if err := file.Move(".tests/renamed/rename-02.txt"); err != nil { 136 | t.Fatal("failed to force move to new directory: ", err) 137 | } 138 | } 139 | 140 | type Hello struct { 141 | World string `json:"world"` 142 | } 143 | 144 | func TestFile_Overwrite2(t *testing.T) { 145 | file := Open(".tests/write-02.json") 146 | if err := file.Overwrite(Hello{"hello world"}); err != nil { 147 | t.Fatal("failed to write to test json file: ", err) 148 | } 149 | } 150 | 151 | func TestFile_Json(t *testing.T) { 152 | file := Open(".tests/write-02.json") 153 | 154 | var hello Hello 155 | if err := file.Json(&hello); err != nil { 156 | t.Fatal("failed to read to test json file: ", err) 157 | } 158 | if hello.World != "hello world" { 159 | t.Fatal("test file does not match expected result, got '", hello.World, "' instead of 'hello world'") 160 | } 161 | } 162 | 163 | func TestFile_Recurse(t *testing.T) { 164 | file := Open("../examples") 165 | 166 | err := file.Recurse(true, func(file *File) { 167 | t.Log("Found file: ", file.Path(), " {is_dir: ", file.isDir, "}") 168 | }) 169 | if err != nil { 170 | t.Fatal("failed to read to recurse examples folder: ", err) 171 | } 172 | } 173 | 174 | func TestConcurrency(t *testing.T) { 175 | file := Open(".tests/concurrency-01.json") 176 | wg := sync.WaitGroup{} 177 | wg.Add(2) 178 | 179 | written := make(chan int, 1) 180 | go func() { 181 | defer wg.Done() 182 | i := 0 183 | for i < 1000 { 184 | i++ 185 | if err := file.Overwrite("hello " + strconv.Itoa(i)); err != nil { 186 | t.Error(err) 187 | } 188 | 189 | written <- i 190 | } 191 | }() 192 | go func() { 193 | defer wg.Done() 194 | for { 195 | i := <-written 196 | if i == 1000 { 197 | fmt.Println("written ", i) 198 | text, err := file.Text() 199 | if err != nil { 200 | t.Error(err) 201 | return 202 | } 203 | fmt.Println("concurrency: ", text) 204 | } 205 | if i == 1000 { 206 | close(written) 207 | break 208 | } 209 | } 210 | }() 211 | wg.Wait() 212 | } 213 | 214 | func BenchmarkFile_Write(b *testing.B) { 215 | file := Open(".tests/bench-01.txt") 216 | for i := 0; i < b.N; i++ { 217 | if err := file.Write("hello world\n"); err != nil { 218 | b.Fatal("failed to write to test text file: ", err) 219 | } 220 | } 221 | 222 | b.Cleanup(func() { 223 | if err := os.Remove(".tests/bench-01.txt"); err != nil { 224 | b.Fatal("failed to clean up benchmark file.") 225 | } 226 | }) 227 | } 228 | 229 | func BenchmarkFile_Writer(b *testing.B) { 230 | file := Open(".tests/bench-01.txt") 231 | writer, err := file.Writer(true) 232 | if err != nil { 233 | b.Fatal("failed to clean up benchmark file.") 234 | } 235 | b.ResetTimer() 236 | 237 | defer func(writer *streaming.Writer) { 238 | err := writer.End() 239 | if err != nil { 240 | b.Fatal("failed to close writer") 241 | } 242 | }(writer) 243 | 244 | for i := 0; i < b.N; i++ { 245 | if err := writer.Write("hello world\n"); err != nil { 246 | b.Fatal("failed to write to test text file: ", err) 247 | } 248 | } 249 | 250 | b.Cleanup(func() { 251 | if err := os.Remove(".tests/bench-01.txt"); err != nil { 252 | b.Fatal("failed to clean up benchmark file.") 253 | } 254 | }) 255 | } 256 | 257 | func BenchmarkFile_Reader(b *testing.B) { 258 | file := Open(".tests/bench-01.txt") 259 | writer, err := file.Writer(true) 260 | if err != nil { 261 | b.Fatal("failed to clean up benchmark file.") 262 | } 263 | for i := 0; i < b.N; i++ { 264 | if err := writer.Write("hello world\n"); err != nil { 265 | b.Fatal("failed to write to test text file: ", err) 266 | } 267 | } 268 | if err := writer.End(); err != nil { 269 | b.Fatal("failed to close writer") 270 | } 271 | b.ResetTimer() 272 | 273 | for i := 0; i < b.N; i++ { 274 | reader, err := file.Reader() 275 | if err != nil { 276 | b.Fatal("failed to open reader") 277 | } 278 | if err := reader.EachLine(func(line []byte) {}); err != nil { 279 | b.Fatal("failed to read test file: ", err) 280 | } 281 | } 282 | 283 | b.Cleanup(func() { 284 | if err := os.Remove(".tests/bench-01.txt"); err != nil { 285 | b.Fatal("failed to clean up benchmark file.") 286 | } 287 | }) 288 | } 289 | 290 | func BenchmarkFile_Bytes2(b *testing.B) { 291 | file := Open(".tests/bench-01.txt") 292 | writer, err := file.Writer(true) 293 | if err != nil { 294 | b.Fatal("failed to clean up benchmark file.") 295 | } 296 | for i := 0; i < b.N; i++ { 297 | if err := writer.Write("hello world\n"); err != nil { 298 | b.Fatal("failed to write to test text file: ", err) 299 | } 300 | } 301 | if err := writer.End(); err != nil { 302 | b.Fatal("failed to close writer") 303 | } 304 | b.ResetTimer() 305 | 306 | for i := 0; i < b.N; i++ { 307 | if _, err := file.Bytes(); err != nil { 308 | b.Fatal("failed to read test file: ", err) 309 | } 310 | } 311 | 312 | b.Cleanup(func() { 313 | if err := os.Remove(".tests/bench-01.txt"); err != nil { 314 | b.Fatal("failed to clean up benchmark file.") 315 | } 316 | }) 317 | } 318 | 319 | func BenchmarkFile_Overwrite(b *testing.B) { 320 | file := Open(".tests/write-01.txt") 321 | for i := 0; i < b.N; i++ { 322 | if err := file.Overwrite("hello world"); err != nil { 323 | b.Fatal("failed to write to test text file: ", err) 324 | } 325 | } 326 | } 327 | 328 | func BenchmarkFile_Text(b *testing.B) { 329 | file := Open(".tests/write-01.txt") 330 | for i := 0; i < b.N; i++ { 331 | if _, err := file.Text(); err != nil { 332 | b.Fatal("failed to read to test text file: ", err) 333 | } 334 | } 335 | } 336 | 337 | func BenchmarkFile_Bytes(b *testing.B) { 338 | file := Open(".tests/write-01.txt") 339 | for i := 0; i < b.N; i++ { 340 | if _, err := file.Bytes(); err != nil { 341 | b.Fatal("failed to read to test text file: ", err) 342 | } 343 | } 344 | } 345 | -------------------------------------------------------------------------------- /streaming/core_writer.go: -------------------------------------------------------------------------------- 1 | package streaming 2 | 3 | import ( 4 | "github.com/ShindouMihou/siopao/internal/buffer" 5 | "io" 6 | ) 7 | 8 | func (writer *Writer) wrtbuffer(buf io.Reader) error { 9 | return buffer.Read(buf, 4_096, func(bytes []byte) error { 10 | if _, err := writer.file.Write(bytes); err != nil { 11 | return err 12 | } 13 | return nil 14 | }) 15 | } 16 | -------------------------------------------------------------------------------- /streaming/reader.go: -------------------------------------------------------------------------------- 1 | package streaming 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "os" 7 | ) 8 | 9 | type Reader struct { 10 | file *os.File 11 | cache *[][]byte 12 | } 13 | 14 | // NewReader creates a streaming reader for the given file. 15 | func NewReader(file *os.File) *Reader { 16 | return &Reader{file: file} 17 | } 18 | 19 | type LineReader func(line []byte) 20 | type CharReader func(char rune) 21 | 22 | // EachLine reads each line of the file as bytes. Unlike EachImmutableLine, the byte array is reused which means 23 | // it will be overridden each next line, therefore, it is not recommended to store the byte array elsewhere without 24 | // copying. 25 | func (reader *Reader) EachLine(fn LineReader) error { 26 | return reader.eachline(false, fn) 27 | } 28 | 29 | // EachChar reads each char of the file. Unlike EachLine, this will do a char-by-char process, which means everything 30 | // including next line characters will be included. 31 | func (reader *Reader) EachChar(fn CharReader) error { 32 | return reader.eachchar(fn) 33 | } 34 | 35 | // EachImmutableLine reads each line of the file as bytes. Unlike EachLine, there is copying involved which makes this 36 | // slower than the other, but the byte array here won't be overridden each line, allowing you to store the byte array 37 | // elsewhere without extra copying. 38 | func (reader *Reader) EachImmutableLine(fn LineReader) error { 39 | return reader.eachline(true, fn) 40 | } 41 | 42 | // File gets the underlying os.File of the Reader. 43 | func (reader *Reader) File() *os.File { 44 | return reader.file 45 | } 46 | 47 | // Empty dereferences the cache of the reader, if any. A cache will be added when methods such as Count or Lines 48 | // are used as it empties the underlying io.Reader, therefore, if you don't want the cache then it is recommended 49 | // to dereference it. 50 | func (reader *Reader) Empty() { 51 | reader.cache = nil 52 | } 53 | 54 | // Lines returns all the lines of the file, this references the cache if there is any, otherwise loads the file 55 | // and saves the array into the cache. To dereference the cache, simply use the Empty method. Note that this will exhaust 56 | // the underlying io.Reader which means that the reader will no longer be usable. 57 | func (reader *Reader) Lines() ([][]byte, error) { 58 | if reader.cache != nil { 59 | return *reader.cache, nil 60 | } 61 | var lines [][]byte 62 | err := reader.EachImmutableLine(func(line []byte) { 63 | lines = append(lines, line) 64 | }) 65 | if err != nil { 66 | return nil, err 67 | } 68 | reader.cache = &lines 69 | return lines, nil 70 | } 71 | 72 | // Count will count all the lines of the file. Internally, this uses Lines and will exhaust the underlying io.Reader 73 | // which means that the reader will no longer be usable, but unless the cache is dereferenced, you can retrieve all the 74 | // lines using the Lines method afterward. 75 | func (reader *Reader) Count() (int, error) { 76 | if reader.cache == nil { 77 | arr, err := reader.Lines() 78 | if err != nil { 79 | return 0, err 80 | } 81 | return len(arr), nil 82 | } 83 | 84 | return len(*reader.cache), nil 85 | } 86 | 87 | // Close will abruptly close the underlying io.Reader, this is not needed in most cases as all the methods in the Reader 88 | // will close the io.Reader upon completion of the action. 89 | func (reader *Reader) Close() { 90 | _ = reader.file.Close() 91 | } 92 | 93 | func (reader *Reader) eachline(immutable bool, fn LineReader) error { 94 | defer reader.Close() 95 | 96 | scanner := bufio.NewScanner(reader.file) 97 | for scanner.Scan() { 98 | line := scanner.Bytes() 99 | if immutable { 100 | cpy := make([]byte, len(line)) 101 | copy(cpy, line) 102 | line = cpy 103 | } 104 | fn(line) 105 | } 106 | if err := scanner.Err(); err != nil { 107 | return err 108 | } 109 | return nil 110 | } 111 | 112 | func (reader *Reader) eachchar(fn CharReader) error { 113 | defer reader.Close() 114 | 115 | rd := bufio.NewReader(reader.file) 116 | for { 117 | if c, _, err := rd.ReadRune(); err != nil { 118 | if err == io.EOF { 119 | break 120 | } else { 121 | return err 122 | } 123 | } else { 124 | fn(c) 125 | } 126 | } 127 | return nil 128 | } 129 | -------------------------------------------------------------------------------- /streaming/text_reader.go: -------------------------------------------------------------------------------- 1 | package streaming 2 | 3 | type TextReader struct { 4 | reader *Reader 5 | cache *[]string 6 | } 7 | 8 | type TextLineReader func(line string) 9 | 10 | // AsTextReader converts a Reader into a TextReader. 11 | func (reader *Reader) AsTextReader() *TextReader { 12 | return &TextReader{reader: reader} 13 | } 14 | 15 | // Empty dereferences the cache of the reader, if any. A cache will be added when methods such as Count or Lines 16 | // are used as it empties the underlying io.Reader, therefore, if you don't want the cache then it is recommended 17 | // to dereference it. 18 | func (reader *TextReader) Empty() { 19 | reader.cache = nil 20 | } 21 | 22 | // Lines returns all the lines of the file, this references the cache if there is any, otherwise loads the file 23 | // and saves the array into the cache. To dereference the cache, simply use the Empty method. Note that this will exhaust 24 | // the underlying io.Reader which means that the reader will no longer be usable. 25 | func (reader *TextReader) Lines() ([]string, error) { 26 | if reader.cache != nil { 27 | return *reader.cache, nil 28 | } 29 | var lines []string 30 | err := reader.EachLine(func(line string) { 31 | lines = append(lines, line) 32 | }) 33 | if err != nil { 34 | return nil, err 35 | } 36 | reader.cache = &lines 37 | return lines, nil 38 | } 39 | 40 | // Count will count all the lines of the file. Internally, this uses Lines and will exhaust the underlying io.Reader 41 | // which means that the reader will no longer be usable, but unless the cache is dereferenced, you can retrieve all the 42 | // lines using the Lines method afterward. 43 | func (reader *TextReader) Count() (int, error) { 44 | if reader.cache == nil { 45 | arr, err := reader.Lines() 46 | if err != nil { 47 | return 0, err 48 | } 49 | return len(arr), nil 50 | } 51 | 52 | return len(*reader.cache), nil 53 | } 54 | 55 | // EachLine reads each line of the file as a string. 56 | func (reader *TextReader) EachLine(fn TextLineReader) error { 57 | return reader.reader.EachLine(func(line []byte) { 58 | fn(string(line)) 59 | }) 60 | } 61 | 62 | // EachChar reads each char of the file. 63 | func (reader *TextReader) EachChar(fn CharReader) error { 64 | return reader.reader.eachchar(fn) 65 | } 66 | -------------------------------------------------------------------------------- /streaming/typed_reader.go: -------------------------------------------------------------------------------- 1 | package streaming 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "github.com/ShindouMihou/siopao/paopao" 7 | ) 8 | 9 | type TypedReader[T any] struct { 10 | reader *Reader 11 | unmarshal paopao.Unmarshaler 12 | } 13 | 14 | // NewTypedReader creates a TypedReader from a Reader instance, this uses the paopao.Unmarshal as its unmarshaler, 15 | // to change the unmarshaler, use WithUnmarshaler. 16 | func NewTypedReader[T any](reader *Reader) *TypedReader[T] { 17 | return &TypedReader[T]{ 18 | reader: reader, 19 | unmarshal: paopao.Unmarshal, 20 | } 21 | } 22 | 23 | // WithUnmarshaler changes the unmarshaler of the typed reader, allowing you to change it to whichever other 24 | // format that you prefer, or using an even faster unmarshaler. 25 | func (reader *TypedReader[T]) WithUnmarshaler(unmarshaler paopao.Unmarshaler) { 26 | reader.unmarshal = unmarshaler 27 | } 28 | 29 | type TypedLineReader[T any] func(t *T) 30 | 31 | // Lines will read each line and unmarshals it into the given type. Note that this will exhaust the underlying 32 | // io.Reader which means that the reader becomes unusable after using this method. 33 | func (reader *TypedReader[T]) Lines() ([]T, error) { 34 | var arr []T 35 | if err := reader.EachLine(func(t *T) { 36 | arr = append(arr, *t) 37 | }); err != nil { 38 | return nil, err 39 | } 40 | return arr, nil 41 | } 42 | 43 | // EachLine reads each line and unmarshals it into the given type before performing the given function. Note that this 44 | // will exhaust the underlying io.Reader which means that the reader becomes unusable after using this method. 45 | func (reader *TypedReader[T]) EachLine(fn TypedLineReader[T]) error { 46 | defer reader.reader.Close() 47 | scanner := bufio.NewScanner(reader.reader.file) 48 | for scanner.Scan() { 49 | line := scanner.Bytes() 50 | 51 | if len(line) < 2 { 52 | continue 53 | } 54 | 55 | if bytes.EqualFold(line, []byte{'['}) || bytes.EqualFold(line, []byte{']'}) { 56 | continue 57 | } 58 | 59 | end := len(line) 60 | if bytes.HasSuffix(line, []byte{','}) { 61 | end = end - 1 62 | } 63 | 64 | var t T 65 | if err := reader.unmarshal(line[:end], &t); err != nil { 66 | return err 67 | } 68 | 69 | fn(&t) 70 | } 71 | if err := scanner.Err(); err != nil { 72 | return err 73 | } 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /streaming/writer.go: -------------------------------------------------------------------------------- 1 | package streaming 2 | 3 | import ( 4 | "bufio" 5 | "github.com/ShindouMihou/siopao/paopao" 6 | "io" 7 | "os" 8 | ) 9 | 10 | type Writer struct { 11 | file *os.File 12 | writer *bufio.Writer 13 | appendNewLine bool 14 | } 15 | 16 | // NewWriter creates a new Writer from the given os.File, this creates a Writer with a buffer size of 17 | // 4,096 bytes. If you want to create one with a different buffer size, use the NewWriterSize method instead. 18 | func NewWriter(file *os.File) *Writer { 19 | return NewWriterSize(file, 4096) 20 | } 21 | 22 | // NewWriterSize creates a new Writer from the given os.File with a given buffer size. 23 | func NewWriterSize(file *os.File, size int) *Writer { 24 | return &Writer{ 25 | file: file, 26 | writer: bufio.NewWriterSize(file, size), 27 | appendNewLine: false, 28 | } 29 | } 30 | 31 | // AlwaysAppendNewLine will set the Writer to always append a new line for each write. 32 | func (writer *Writer) AlwaysAppendNewLine() *Writer { 33 | writer.appendNewLine = true 34 | return writer 35 | } 36 | 37 | // Write writes the content into the file, note that this does not append a new line for each write 38 | // unless the Writer uses AlwaysAppendNewLine. This marshals anything other than string, bufio.Reader and byte array into the 39 | // paopao.Marshal which is Json by default. 40 | func (writer *Writer) Write(t any) error { 41 | switch t.(type) { 42 | case string: 43 | return writer.write([]byte(t.(string))) 44 | case []byte: 45 | return writer.write(t.([]byte)) 46 | case *bufio.Reader: 47 | return writer.wrtbuffer(t.(*bufio.Reader)) 48 | case bufio.Reader: 49 | buffer := t.(bufio.Reader) 50 | return writer.wrtbuffer(&buffer) 51 | case io.Reader: 52 | return writer.wrtbuffer(t.(io.Reader)) 53 | default: 54 | bytes, err := paopao.Marshal(t) 55 | if err != nil { 56 | return err 57 | } 58 | return writer.write(bytes) 59 | } 60 | } 61 | 62 | // WriteMarshal marshals the content with the given marshaller. Note that string and byte array are also marshalled 63 | // with the given marshaller. 64 | func (writer *Writer) WriteMarshal(marshaller paopao.Marshaller, t any) error { 65 | bytes, err := marshaller(t) 66 | if err != nil { 67 | return err 68 | } 69 | return writer.write(bytes) 70 | } 71 | 72 | // Flush will flush all the buffered contents into the file. It is recommended to use this only when you want 73 | // to push the contents of the file immediately, otherwise use End instead to flush and close the Writer. 74 | func (writer *Writer) Flush() error { 75 | return writer.writer.Flush() 76 | } 77 | 78 | // Close will abruptly close the underlying io.Writer of the Writer. IT IS NOT RECOMMENDED TO USE THIS, PLEASE USE 79 | // End INSTEAD TO FLUSH AND CLOSE THE Writer. 80 | func (writer *Writer) Close() { 81 | _ = writer.file.Close() 82 | } 83 | 84 | // End flushes the contents into the file before closing the underlying io.Writer. 85 | func (writer *Writer) End() error { 86 | defer writer.Close() 87 | return writer.Flush() 88 | } 89 | 90 | // Reset discards any unflushed buffered data, clears any error, and resets buffer to write its output to File. 91 | // i.e. whatever the heck bufio.Writer's Reset method does. 92 | func (writer *Writer) Reset() { 93 | writer.writer.Reset(writer.file) 94 | } 95 | 96 | func (writer *Writer) write(t []byte) error { 97 | if _, err := writer.writer.Write(t); err != nil { 98 | return err 99 | } 100 | if writer.appendNewLine { 101 | return writer.write([]byte{'\n'}) 102 | } 103 | return nil 104 | } 105 | --------------------------------------------------------------------------------