├── go.mod ├── .gitignore ├── Makefile ├── LICENSE ├── example.go ├── .github └── workflows │ └── go.yml ├── README.md ├── utils ├── utils_test.go └── utils.go └── rebuf └── rebuf.go /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/stym06/rebuf 2 | 3 | go 1.22 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | .vscode/ 17 | 18 | # Dependency directories (remove the comment below to include it) 19 | # vendor/ 20 | 21 | # Go workspace file 22 | go.work 23 | go.work.sum 24 | 25 | #data dir 26 | data/ 27 | 28 | #binary 29 | bin/ 30 | 31 | coverage.txt 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile 2 | 3 | # Define environment variable 4 | export TEST_LOG_DIR := "something" 5 | 6 | # Define the output binary directory and name 7 | BINARY_DIR := bin 8 | BINARY_NAME := rebuf 9 | 10 | # Define Go build command 11 | BUILD_CMD := go build -o $(BINARY_DIR)/$(BINARY_NAME) 12 | 13 | # Define Go test command 14 | TEST_CMD := go test -cover -coverprofile=coverage.txt ./... 15 | 16 | .PHONY: all build test 17 | 18 | # Default target 19 | all: build test 20 | 21 | # Build target 22 | build: 23 | @echo "Building the binary..." 24 | $(BUILD_CMD) 25 | @echo "Build complete. Binary is located at $(BINARY_DIR)/$(BINARY_NAME)" 26 | 27 | # Test target 28 | test: 29 | @echo "Running tests..." 30 | $(TEST_CMD) 31 | @echo "Tests complete. Coverage report is available in coverage.txt" 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Satyam Raj 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log/slog" 5 | "os" 6 | "time" 7 | 8 | "github.com/stym06/rebuf/rebuf" 9 | ) 10 | 11 | func writeToStdout(data []byte) error { 12 | slog.Info(string(data)) 13 | return nil 14 | } 15 | 16 | func main() { 17 | 18 | logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) 19 | 20 | //Init the RebufOptions 21 | rebufOptions := &rebuf.RebufOptions{ 22 | LogDir: "/Users/satyamraj/personal/rebuf/data", 23 | FsyncTime: 5 * time.Second, 24 | MaxLogSize: 50, 25 | MaxSegments: 5, 26 | Logger: logger, 27 | } 28 | 29 | //Init Rebuf 30 | rebuf, err := rebuf.Init(rebufOptions) 31 | if err != nil { 32 | logger.Info("Error during Rebuf creation: " + err.Error()) 33 | } 34 | 35 | defer rebuf.Close() 36 | 37 | // Write Bytes 38 | for i := 0; i < 30; i++ { 39 | logger.Info("Writing data: ", "iter", i) 40 | go rebuf.Write([]byte("Hello world")) 41 | time.Sleep(300 * time.Millisecond) 42 | } 43 | 44 | //Replay and write to stdout 45 | rebuf.Replay(writeToStdout) 46 | 47 | if err != nil { 48 | logger.Info(err.Error()) 49 | } 50 | 51 | time.Sleep(30 * time.Second) 52 | 53 | } 54 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: ["main"] 9 | pull_request: 10 | types: [opened, reopened, synchronize] 11 | 12 | permissions: 13 | contents: read 14 | pull-requests: write 15 | 16 | jobs: 17 | unit_tests: 18 | env: 19 | TEST_LOG_DIR: "./test-logs" 20 | 21 | runs-on: ubuntu-latest 22 | steps: 23 | - uses: actions/checkout@v4 24 | 25 | - name: Set up Go 26 | uses: actions/setup-go@v4 27 | with: 28 | go-version: "1.22" 29 | 30 | - name: Build 31 | run: go build -v ./... 32 | 33 | - name: Test 34 | run: go test -cover -coverprofile=coverage.txt ./... 35 | 36 | - name: Archive code coverage results 37 | uses: actions/upload-artifact@v4 38 | with: 39 | name: code-coverage 40 | path: coverage.txt 41 | 42 | code_coverage: 43 | name: "Code coverage report" 44 | if: github.event_name == 'pull_request' # Do not run when workflow is triggered by push to main branch 45 | runs-on: ubuntu-latest 46 | needs: unit_tests # Depends on the artifact uploaded by the "unit_tests" job 47 | steps: 48 | - uses: fgrosse/go-coverage-report@v1.0.2 # Consider using a Git revision for maximum security 49 | with: 50 | coverage-artifact-name: "code-coverage" # can be omitted if you used this default value 51 | coverage-file-name: "coverage.txt" 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rebuf 2 | 3 | [![Go](https://github.com/stym06/rebuf/actions/workflows/go.yml/badge.svg)](https://github.com/stym06/rebuf/actions/workflows/go.yml) 4 | 5 | `rebuf` is a Golang implementation of WAL (Write Ahead||After Logging) which can also be used to log data bytes during a downstream service issue which can later be replayed on-demand 6 | 7 | ## Features 8 | 9 | - Create and replay log data on any filesystem. 10 | - Lightweight and easy to use. 11 | - Efficient storage and retrieval of log data. 12 | 13 | ## Installation 14 | 15 | 1. Clone the repository: `git clone https://github.com/stym06/rebuf.git` 16 | 2. Navigate to the project directory: `cd rebuf` 17 | 3. Install the necessary dependencies by running: `go mod download` 18 | 19 | ## Usage 20 | 21 | ``` 22 | func writeToStdout(data []byte) error { 23 | slog.Info(string(data)) 24 | return nil 25 | } 26 | 27 | func main() { 28 | 29 | logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) 30 | 31 | //Init the RebufOptions 32 | rebufOptions := &rebuf.RebufOptions{ 33 | LogDir: "/Users/satyamraj/personal/rebuf/data", 34 | FsyncTime: 5 * time.Second, 35 | MaxLogSize: 50, 36 | MaxSegments: 5, 37 | Logger: logger, 38 | } 39 | 40 | //Init Rebuf 41 | rebuf, err := rebuf.Init(rebufOptions) 42 | if err != nil { 43 | logger.Info("Error during Rebuf creation: " + err.Error()) 44 | } 45 | 46 | defer rebuf.Close() 47 | 48 | // Write Bytes 49 | for i := 0; i < 30; i++ { 50 | logger.Info("Writing data: ", "iter", i) 51 | go rebuf.Write([]byte("Hello world")) 52 | time.Sleep(300 * time.Millisecond) 53 | } 54 | 55 | //Replay and write to stdout 56 | rebuf.Replay(writeToStdout) 57 | 58 | if err != nil { 59 | logger.Info(err.Error()) 60 | } 61 | 62 | time.Sleep(30 * time.Second) 63 | 64 | } 65 | ``` 66 | 67 | ## License 68 | 69 | This project is licensed under the MIT License. See the `LICENSE` file for more information. 70 | 71 | ## Contact Information 72 | 73 | If you have any questions or concerns, please feel free to reach out to the author on GitHub: [@stym06](https://github.com/stym06). 74 | -------------------------------------------------------------------------------- /utils/utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | ) 9 | 10 | func setupSuite(t testing.TB) func(t testing.TB) { 11 | log.Println("Setting up logDir empty") 12 | 13 | dirPath := os.Getenv("TEST_LOG_DIR") 14 | 15 | if _, err := os.Stat(filepath.Join(dirPath)); err != nil { 16 | if os.IsNotExist(err) { 17 | os.Mkdir(dirPath, 0700) 18 | } 19 | } else { 20 | t.Fatal("Error creating dirPath in setup suite") 21 | } 22 | 23 | // Return a function to teardown the test 24 | return func(tb testing.TB) { 25 | log.Printf("Deleting everything in %v", dirPath) 26 | os.RemoveAll(dirPath) 27 | 28 | } 29 | } 30 | 31 | func createFile(fileName string) (*os.File, error) { 32 | file, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 33 | if err != nil { 34 | return nil, err 35 | } 36 | return file, nil 37 | } 38 | 39 | func TestIsDirectoryEmpty(t *testing.T) { 40 | 41 | teardownSuite := setupSuite(t) 42 | defer teardownSuite(t) 43 | dirPath := os.Getenv("TEST_LOG_DIR") 44 | 45 | t.Run("directory exists without .tmp file", func(t *testing.T) { 46 | empty, err := IsDirectoryEmpty(dirPath) 47 | if err != nil { 48 | t.Fatalf("Error in running IsDirectoryEmpty with %v", dirPath) 49 | } 50 | 51 | //empty should be true 52 | if empty == false { 53 | t.Fatalf("Expected %v. Got %v", false, empty) 54 | } 55 | }) 56 | 57 | t.Run("directory exists with .tmp file", func(t *testing.T) { 58 | 59 | file, err := createFile(filepath.Join(dirPath, "rebuf.tmp")) 60 | if err != nil { 61 | t.Fatalf("Error in creating file %v", file) 62 | } 63 | 64 | empty, err := IsDirectoryEmpty(dirPath) 65 | if err != nil { 66 | t.Fatalf("Error in running IsDirectoryEmpty with %v", dirPath) 67 | } 68 | 69 | //empty should be true 70 | if empty == false { 71 | t.Fatalf("Expected %v. Got %v", false, empty) 72 | } 73 | }) 74 | 75 | t.Run("directory exists with .tmp file and data file", func(t *testing.T) { 76 | 77 | dataFile, err := createFile(filepath.Join(dirPath, "rebuf-1")) 78 | if err != nil { 79 | t.Fatalf("Error in creating file %v", dataFile) 80 | } 81 | 82 | empty, err := IsDirectoryEmpty(dirPath) 83 | if err != nil { 84 | t.Fatalf("Error in running IsDirectoryEmpty with %v", err) 85 | } 86 | 87 | //empty should be false 88 | if empty == false { 89 | t.Fatalf("Expected %v. Got %v", false, empty) 90 | } 91 | }) 92 | } 93 | 94 | func TestGetLatestSegmentId(t *testing.T) { 95 | //test2 96 | } 97 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "strconv" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | func IsDirectoryEmpty(dirPath string) (bool, error) { 12 | // Open the directory 13 | dir, err := os.Open(dirPath) 14 | if err != nil { 15 | return false, err 16 | } 17 | defer dir.Close() 18 | 19 | // Read the directory contents 20 | files, err := dir.ReadDir(1) // Read the first entry 21 | var filteredFiles []os.DirEntry 22 | for _, file := range files { 23 | if !strings.HasSuffix(file.Name(), ".tmp") { 24 | filteredFiles = append(filteredFiles, file) 25 | } 26 | } 27 | if err != nil && err != io.EOF { 28 | return false, err 29 | } 30 | 31 | // If the list of files is empty, the directory is empty 32 | return len(filteredFiles) == 0, nil 33 | } 34 | 35 | func GetLatestSegmentId(logDir string) (int, error) { 36 | files, err := os.ReadDir(logDir) 37 | if err != nil { 38 | return 0, err 39 | } 40 | 41 | // Filter out .tmp files 42 | latestModifiedTime := time.Time{} 43 | var latestFileName string 44 | for _, file := range files { 45 | if strings.HasSuffix(file.Name(), ".tmp") { 46 | continue 47 | } 48 | fileInfo, err := file.Info() 49 | 50 | if err != nil { 51 | return 0, err 52 | } 53 | 54 | if fileInfo.ModTime().After(latestModifiedTime) { 55 | latestModifiedTime = fileInfo.ModTime() 56 | latestFileName = file.Name() 57 | } 58 | } 59 | segmentCount, err := strconv.Atoi(strings.Split(latestFileName, "-")[1]) 60 | if err != nil { 61 | return 0, err 62 | } 63 | return segmentCount, nil 64 | } 65 | 66 | func GetNumSegments(logDir string) (int, error) { 67 | files, err := os.ReadDir(logDir) 68 | if err != nil { 69 | return 0, err 70 | } 71 | return len(files) - 1, nil 72 | } 73 | 74 | func FileSize(f *os.File) (int64, error) { 75 | fi, err := f.Stat() 76 | if err != nil { 77 | return 0, err 78 | } 79 | return fi.Size(), nil 80 | } 81 | 82 | func GetOldestSegmentFile(logDir string) (string, error) { 83 | files, err := os.ReadDir(logDir) 84 | if err != nil { 85 | return "0", err 86 | } 87 | 88 | // Filter out .tmp files 89 | oldestModifedTime := time.Now() 90 | var oldestFileName string 91 | for _, file := range files { 92 | if strings.HasSuffix(file.Name(), ".tmp") { 93 | continue 94 | } 95 | fileInfo, err := file.Info() 96 | 97 | if err != nil { 98 | return "", err 99 | } 100 | 101 | if fileInfo.ModTime().Before(oldestModifedTime) { 102 | oldestModifedTime = fileInfo.ModTime() 103 | oldestFileName = file.Name() 104 | } 105 | } 106 | return oldestFileName, nil 107 | } 108 | -------------------------------------------------------------------------------- /rebuf/rebuf.go: -------------------------------------------------------------------------------- 1 | package rebuf 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "io" 7 | "log/slog" 8 | "os" 9 | "path/filepath" 10 | "strconv" 11 | "sync" 12 | "time" 13 | 14 | "github.com/stym06/rebuf/utils" 15 | ) 16 | 17 | type RebufOptions struct { 18 | LogDir string 19 | MaxLogSize int64 20 | FsyncTime time.Duration 21 | MaxSegments int 22 | Logger *slog.Logger 23 | } 24 | 25 | type Rebuf struct { 26 | logDir string 27 | currentSegmentId int 28 | maxLogSize int64 29 | maxSegments int 30 | segmentCount int 31 | bufWriter *bufio.Writer 32 | logSize int64 33 | tmpLogFile *os.File 34 | ticker time.Ticker 35 | mu sync.Mutex 36 | log *slog.Logger 37 | } 38 | 39 | func Init(options *RebufOptions) (*Rebuf, error) { 40 | 41 | //ensure dir is created 42 | if _, err := os.Stat(filepath.Join(options.LogDir)); err != nil { 43 | if os.IsNotExist(err) { 44 | os.Mkdir(options.LogDir, 0700) 45 | } 46 | } 47 | 48 | //open temp file 49 | tmpLogFileName := filepath.Join(options.LogDir, "rebuf.tmp") 50 | tmpLogFile, err := os.OpenFile(tmpLogFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | rebuf := &Rebuf{ 56 | logDir: options.LogDir, 57 | maxLogSize: options.MaxLogSize, 58 | maxSegments: options.MaxSegments, 59 | tmpLogFile: tmpLogFile, 60 | ticker: *time.NewTicker(options.FsyncTime), 61 | log: options.Logger, 62 | } 63 | 64 | err = rebuf.openExistingOrCreateNew(options.LogDir) 65 | 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | go rebuf.syncPeriodically() 71 | 72 | return rebuf, err 73 | } 74 | 75 | func (rebuf *Rebuf) syncPeriodically() error { 76 | for { 77 | select { 78 | case <-rebuf.ticker.C: 79 | rebuf.mu.Lock() 80 | rebuf.tmpLogFile.Sync() 81 | rebuf.mu.Unlock() 82 | } 83 | } 84 | } 85 | 86 | func (rebuf *Rebuf) Write(data []byte) error { 87 | if rebuf.logSize+int64(len(data))+8 > rebuf.maxLogSize { 88 | 89 | if rebuf.segmentCount > rebuf.maxSegments { 90 | rebuf.log.Info("Reached maxSegments", "segments", rebuf.maxSegments) 91 | 92 | //delete the oldest log file 93 | oldestLogFileName, err := utils.GetOldestSegmentFile(rebuf.logDir) 94 | if err != nil { 95 | return err 96 | } 97 | rebuf.log.Info("Would have deleted ", "oldestLogFileName", oldestLogFileName) 98 | os.Remove(filepath.Join(rebuf.logDir, oldestLogFileName)) 99 | 100 | rebuf.segmentCount-- 101 | } 102 | 103 | rebuf.log.Info("Log size will be greater than", "logsize", rebuf.logSize, "Moving to", rebuf.currentSegmentId+1) 104 | rebuf.bufWriter.Flush() 105 | rebuf.tmpLogFile.Sync() 106 | 107 | // rename this file to current segment count 108 | os.Rename(filepath.Join(rebuf.logDir, "/rebuf.tmp"), filepath.Join(rebuf.logDir, "/rebuf-"+strconv.Itoa(rebuf.currentSegmentId))) 109 | //increase segment count by 1 110 | rebuf.currentSegmentId = rebuf.currentSegmentId + 1 111 | rebuf.segmentCount = rebuf.segmentCount + 1 112 | 113 | //change writer to this temp file 114 | tmpLogFile, err := os.OpenFile(filepath.Join(rebuf.logDir, "rebuf.tmp"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 115 | if err != nil { 116 | return err 117 | } 118 | rebuf.tmpLogFile = tmpLogFile 119 | rebuf.bufWriter = bufio.NewWriter(rebuf.tmpLogFile) 120 | rebuf.logSize = 0 121 | } 122 | 123 | //seek to the end of the file 124 | _, err := rebuf.tmpLogFile.Seek(0, io.SeekEnd) 125 | if err != nil { 126 | return err 127 | } 128 | 129 | //write the size of the byte array and then the byte array itself 130 | size := uint64(len(data)) 131 | //creating a byte array of size 8 and putting the length of `data` into the array 132 | sizeBuf := make([]byte, 8) 133 | binary.BigEndian.PutUint64(sizeBuf, size) 134 | 135 | _, err = rebuf.bufWriter.Write(sizeBuf) 136 | if err != nil { 137 | return err 138 | } 139 | _, err = rebuf.bufWriter.Write(data) 140 | if err != nil { 141 | return err 142 | } 143 | rebuf.logSize = rebuf.logSize + int64(len(data)) + 8 144 | rebuf.bufWriter.Flush() 145 | rebuf.mu.Lock() 146 | defer rebuf.mu.Unlock() 147 | rebuf.tmpLogFile.Sync() 148 | 149 | return err 150 | } 151 | 152 | func (rebuf *Rebuf) openExistingOrCreateNew(logDir string) error { 153 | //check if directory is empty (only containing .tmp file) 154 | empty, err := utils.IsDirectoryEmpty(logDir) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | tmpLogFileName := filepath.Join(logDir, "rebuf.tmp") 160 | tmpLogFile, err := os.OpenFile(tmpLogFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 161 | if err != nil { 162 | return err 163 | } 164 | rebuf.tmpLogFile = tmpLogFile 165 | rebuf.bufWriter = bufio.NewWriter(tmpLogFile) 166 | 167 | if empty { 168 | rebuf.currentSegmentId = 0 169 | rebuf.segmentCount = 0 170 | rebuf.logSize = 0 171 | } else { 172 | rebuf.currentSegmentId, err = utils.GetLatestSegmentId(logDir) 173 | if err != nil { 174 | return err 175 | } 176 | 177 | rebuf.segmentCount, err = utils.GetNumSegments(logDir) 178 | if err != nil { 179 | return err 180 | } 181 | rebuf.logSize, _ = utils.FileSize(rebuf.tmpLogFile) 182 | } 183 | 184 | return nil 185 | } 186 | 187 | func (rebuf *Rebuf) Replay(callbackFn func([]byte) error) error { 188 | files, err := os.ReadDir(rebuf.logDir) 189 | if err != nil { 190 | return err 191 | } 192 | for _, fileInfo := range files { 193 | file, err := os.Open(filepath.Join(rebuf.logDir, fileInfo.Name())) 194 | if err != nil { 195 | return err 196 | } 197 | defer file.Close() 198 | bufReader := bufio.NewReader(file) 199 | 200 | var readBytes []byte 201 | for err == nil { 202 | readBytes, err = bufReader.Peek(8) 203 | if err != nil { 204 | break 205 | } 206 | size := int(binary.BigEndian.Uint64(readBytes)) 207 | _, err := bufReader.Discard(8) 208 | if err != nil { 209 | break 210 | } 211 | 212 | data, err := bufReader.Peek(size) 213 | if err != nil { 214 | break 215 | } 216 | err = callbackFn(data) 217 | if err != nil { 218 | break 219 | } 220 | _, _ = bufReader.Discard(size) 221 | } 222 | 223 | } 224 | return nil 225 | } 226 | 227 | func (rebuf *Rebuf) Close() error { 228 | if rebuf.bufWriter == nil { 229 | return nil 230 | } 231 | 232 | if err := rebuf.bufWriter.Flush(); err != nil { 233 | rebuf.tmpLogFile.Close() 234 | return err 235 | } 236 | 237 | if err := rebuf.tmpLogFile.Sync(); err != nil { 238 | rebuf.tmpLogFile.Close() 239 | return err 240 | } 241 | 242 | rebuf.ticker.Stop() 243 | 244 | return rebuf.tmpLogFile.Close() 245 | } 246 | --------------------------------------------------------------------------------