├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── README.zh-CN.md ├── bin ├── README.md ├── parseAOF_linux_aarch64 ├── parseAOF_linux_x86_64 ├── parseAOF_macos_arm64 ├── parseAOF_macos_x86_64 ├── parseAOF_win_aarch64 └── parseAOF_win_x86_64 ├── data ├── README.md └── appendonly.aof ├── example.png ├── go.mod ├── go.sum └── src ├── global └── global.go ├── main.go ├── parser ├── parser.go └── parser_test.go ├── runner └── runner.go └── writer ├── writer.go └── writer_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Output of the go coverage tool, specifically when used with LiteIDE 9 | *.out 10 | 11 | # others 12 | .DS_Store 13 | .idea/ 14 | *.txt 15 | *.split 16 | *.parsed 17 | .vscode 18 | parseAOF_data -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.16.x 5 | 6 | script: 7 | - bash test.sh 8 | 9 | branches: 10 | only: 11 | - main 12 | - testing -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 WGrape 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GO_Linux_ENV := CGO_ENABLED=0 GOOS=linux 2 | GO_Darwin_ENV := CGO_ENABLED=0 GOOS=darwin 3 | GO_Windows_ENV := CGO_ENABLED=0 GOOS=windows 4 | GO_ENV_AMD64 := GOARCH=amd64 5 | GO_ENV_ARM64:= GOARCH=arm64 6 | DIST_DIR_AMD64 := dist/x86_64 7 | DIST_DIR_ARM64 := dist/aarch64 8 | 9 | build: 10 | $(GO_Linux_ENV) $(GO_ENV_AMD64) go build -o bin/parseAOF_linux_x86_64 ./src 11 | $(GO_Linux_ENV) $(GO_ENV_ARM64) go build -o bin/parseAOF_linux_aarch64 ./src 12 | $(GO_Darwin_ENV) $(GO_ENV_AMD64) go build -o bin/parseAOF_macos_x86_64 ./src 13 | $(GO_Darwin_ENV) $(GO_ENV_ARM64) go build -o bin/parseAOF_macos_arm64 ./src 14 | $(GO_Windows_ENV) $(GO_ENV_AMD64) go build -o bin/parseAOF_win_x86_64 ./src 15 | $(GO_Windows_ENV) $(GO_ENV_ARM64) go build -o bin/parseAOF_win_aarch64 ./src -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 |

6 | 7 | 8 | 9 | 10 |

11 | 12 |

13 |

A simple and fast tool to parse the AOF file of redis

14 |

Document ：中文 / English

15 |

41 | 42 | ## 2、Build 43 | 44 | ```bash 45 | git clone https://github.com/WGrape/parseAOF 46 | cd parseAOF 47 | go mod download 48 | make build 49 | ``` 50 | ## 3、Usage 51 | Run the binary under `bin` dir `parseAOF__` with the path of the aof file 52 | 53 | ```bash 54 | ./bin/parseAOF_macos_arm64 -i ~/Download/appendonly.aof -r 8 55 | ./bin/parseAOF_macos_arm64 -h 56 | parse redis aof to readable 57 | 58 | Usage: 59 | parseAOF [flags] 60 | 61 | Flags: 62 | -h, --help help for parseAOF 63 | -i, --input string input AOF file path 64 | -o, --output string output dir path 65 | -r, --routines int max goroutines (default 8) 66 | ``` 67 | 68 | ### (1) The input file 69 | > Here's an example input file [./data/appendonly.aof](./data/appendonly.aof) for you to test 70 | 71 | Before running, pass the path of the aof file to the ```start.sh``` script, the content is as follows 72 | 73 | ```text 74 | *2 75 | $6 76 | SELECT 77 | $1 78 | 0 79 | ... ... 80 | ``` 81 | 82 | ### (2) The output file 83 | > Here's an example output file [./data/aof.merged](./data/aof.merged) for you to test 84 | 85 | After the parsing is complete, the file [aof.merged](./data/aof.merged) will be generated in the directory of ```data```, the content is as follows 86 | 87 | ```text 88 | --------------------parseAOF | version=0.5.0-------------------- 89 | SELECT 0 90 | set key1 1 91 | set key2 2 92 | set key3 3 93 | sadd key4 1 2 3 4 94 | lpush key5 1 2 3 4 5 95 | zadd key6 1 2 3 4 5 6 96 | ``` 97 | 98 | ### (3) Example 99 | ![example](example.png) 100 | 101 | ## 4、Performance 102 | 103 | - The average speed to parse is ```50000 lines/s``` 104 | - The maximum size of the aof is 1GB 105 | 106 | ### (1) Testing 107 | 108 | | Id | Lines | Size | Cost | CPU | 109 | | --- | :----: | :---: | :---: | :---: | 110 | | 1 | 1,2301,117 | 39MB | 3m50s | <=65% | 111 | | 2 | 3,435,263 | 13MB | 1m12s | <=65% | 112 | | 3 | 357,850 | 8.6MB | 3.47s | <=113% | 113 | -------------------------------------------------------------------------------- /README.zh-CN.md: -------------------------------------------------------------------------------- 1 | ## 目录 2 | - [目录](#目录) 3 | - [1、介绍](#1介绍) 4 | - [(1) 特性](#1-特性) 5 | - [(2) 架构](#2-架构) 6 | - [(3) 原理](#3-原理) 7 | - [2、编译](#2编译) 8 | - [3、使用](#3使用) 9 | - [(1) 输入文件](#1-输入文件) 10 | - [(2) 输出文件](#2-输出文件) 11 | - [(3) 使用示例](#3-使用示例) 12 | - [4、性能\ 26 | 27 | ### (3) 原理 28 | 关于项目的相关原理可以参考[这篇文章](https://github.com/WGrape/Blog/issues/11) 29 | 30 | ## 2、编译 31 | 32 | ```bash 33 | git clone https://github.com/WGrape/parseAOF 34 | cd parseAOF 35 | go mod download 36 | make build 37 | ``` 38 | ## 3、使用 39 | 运行位于`bin`目录下的parseAOF二进制，并制定输入的AOF文件路径 40 | 41 | ```bash 42 | ./bin/parseAOF_macos_arm64 -i ~/Download/appendonly.aof -r 8 43 | ./bin/parseAOF_macos_arm64 -h 44 | parse redis aof to readable 45 | 46 | Usage: 47 | parseAOF [flags] 48 | 49 | Flags: 50 | -h, --help help for parseAOF 51 | -i, --input string input AOF file path 52 | -o, --output string output dir path 53 | -r, --routines int max goroutines (default 8) 54 | ``` 55 | 56 | ### (1) 输入文件 57 | > 为了便于测试，可以使用 [./data/appendonly.aof](./data/appendonly.aof) 这个示例输入文件 58 | 59 | 开始执行前，传递AOF文件路径参数给 ```start.sh``` 脚本, AOF文件内容如下所示 60 | 61 | ```text 62 | *2 63 | $6 64 | SELECT 65 | $1 66 | 0 67 | ... ... 68 | ``` 69 | 70 | ### (2) 输出文件 71 | > 为了便于测试，可以使用 [./data/aof.merged](./data/aof.merged) 这个示例输出文件 72 | 73 | 解析完成后，会在 ```data``` 目录下生成 [aof.merged](./data/aof.merged) 文件，其内容如下所示 74 | 75 | ```text 76 | --------------------parseAOF | version=0.5.0-------------------- 77 | SELECT 0 78 | set key1 1 79 | set key2 2 80 | set key3 3 81 | sadd key4 1 2 3 4 82 | lpush key5 1 2 3 4 5 83 | zadd key6 1 2 3 4 5 6 84 | ``` 85 | 86 | ### (3) 使用示例 87 | 88 | ![example](example.png) 89 | 90 | ## 4、性能(1) 测试 95 | 96 | | Id | Lines | Size | Cost | CPU | 97 | | --- | :----: | :---: | :---: | :---: | 98 | | 1 | 1,2301,117 | 39MB | 3m50s | <=65% | 99 | | 2 | 3,435,263 | 13MB | 1m12s | <=65% | 100 | | 3 | 357,850 | 8.6MB | 3.47s | <=113% | 101 | -------------------------------------------------------------------------------- /bin/README.md: -------------------------------------------------------------------------------- 1 | ### What is this directory 2 | The compiled file is stored here -------------------------------------------------------------------------------- /bin/parseAOF_linux_aarch64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WGrape/parseAOF/0c7218b7c9e0fe3530830d697fd52d3e122ca01c/bin/parseAOF_linux_aarch64 -------------------------------------------------------------------------------- /bin/parseAOF_linux_x86_64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WGrape/parseAOF/0c7218b7c9e0fe3530830d697fd52d3e122ca01c/bin/parseAOF_linux_x86_64 -------------------------------------------------------------------------------- /bin/parseAOF_macos_arm64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WGrape/parseAOF/0c7218b7c9e0fe3530830d697fd52d3e122ca01c/bin/parseAOF_macos_arm64 -------------------------------------------------------------------------------- /bin/parseAOF_macos_x86_64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WGrape/parseAOF/0c7218b7c9e0fe3530830d697fd52d3e122ca01c/bin/parseAOF_macos_x86_64 -------------------------------------------------------------------------------- /bin/parseAOF_win_aarch64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WGrape/parseAOF/0c7218b7c9e0fe3530830d697fd52d3e122ca01c/bin/parseAOF_win_aarch64 -------------------------------------------------------------------------------- /bin/parseAOF_win_x86_64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WGrape/parseAOF/0c7218b7c9e0fe3530830d697fd52d3e122ca01c/bin/parseAOF_win_x86_64 -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | ### What is this directory 2 | The files generated after running are stored here. -------------------------------------------------------------------------------- /data/appendonly.aof: -------------------------------------------------------------------------------- 1 | *2 2 | $6 3 | SELECT 4 | $1 5 | 0 6 | *3 7 | $3 8 | set 9 | $4 10 | key1 11 | $1 12 | 1 13 | *3 14 | $3 15 | set 16 | $4 17 | key2 18 | $1 19 | 2 20 | *3 21 | $3 22 | set 23 | $4 24 | key3 25 | $1 26 | 3 27 | *6 28 | $4 29 | sadd 30 | $4 31 | key4 32 | $1 33 | 1 34 | $1 35 | 2 36 | $1 37 | 3 38 | $1 39 | 4 40 | *7 41 | $5 42 | lpush 43 | $4 44 | key5 45 | $1 46 | 1 47 | $1 48 | 2 49 | $1 50 | 3 51 | $1 52 | 4 53 | $1 54 | 5 55 | *8 56 | $4 57 | zadd 58 | $4 59 | key6 60 | $1 61 | 1 62 | $1 63 | 2 64 | $1 65 | 3 66 | $1 67 | 4 68 | $1 69 | 5 70 | $1 71 | 6 -------------------------------------------------------------------------------- /example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WGrape/parseAOF/0c7218b7c9e0fe3530830d697fd52d3e122ca01c/example.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module parseAOF 2 | 3 | go 1.16 4 | 5 | require github.com/spf13/cobra v1.7.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 2 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 3 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 4 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 5 | github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= 6 | github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= 7 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 8 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 10 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 11 | -------------------------------------------------------------------------------- /src/global/global.go: -------------------------------------------------------------------------------- 1 | package global 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "regexp" 7 | ) 8 | 9 | const PatternOfAOFFile = "^aof\\.split_[a-z0-9]+$" 10 | const PatternOfLineCmdStart = "^\\*\\d$" 11 | const PatternOfLineArgLen = "^\\$\\d$" 12 | 13 | const MatchTypeCmdStart = 1 14 | const MatchTypeArgLen = 2 15 | const MatchTypeArgRaw = 3 16 | 17 | const EmptyString = "" 18 | const WhitespaceString = " " 19 | 20 | const ExitCodeFatal = 1 21 | const DefaultFileMode = 0644 22 | const Delta = 1 23 | 24 | const ConfigDefaultDebug = false 25 | const ConfigDefaultMaxRoutines = 1024 26 | 27 | var RootDir, _ = os.Getwd() 28 | var DataDir = "" 29 | var Debug = false 30 | 31 | func IsAOFFile(fileName string) bool { 32 | matched, _ := regexp.MatchString(PatternOfAOFFile, fileName) 33 | return matched 34 | } 35 | 36 | func GetSplitFilePath(fileName string) string { 37 | return fmt.Sprintf("%s/%s", DataDir, fileName) 38 | } 39 | 40 | func GetParsedFilePath(fileName string) string { 41 | return fmt.Sprintf("%s/%s.parsed", DataDir, fileName) 42 | } 43 | 44 | func LogDebug(text string) { 45 | if Debug { 46 | fmt.Printf(text) 47 | } 48 | } 49 | 50 | func LogInfo(text string) { 51 | fmt.Printf(text) 52 | } 53 | 54 | func LogError(text string) { 55 | fmt.Printf(text) 56 | } 57 | -------------------------------------------------------------------------------- /src/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "parseAOF/src/global" 8 | "parseAOF/src/runner" 9 | "path/filepath" 10 | "sort" 11 | "strings" 12 | "sync" 13 | 14 | "github.com/spf13/cobra" 15 | ) 16 | 17 | var ( 18 | input = "" 19 | output = "" 20 | maxGoRountine = 8 21 | debug = false 22 | rootCmd = &cobra.Command{ 23 | Use: "parseAOF", 24 | Short: "parse redis aof to readable", 25 | Long: "", 26 | Run: func(cmd *cobra.Command, args []string) { 27 | global.Debug = debug 28 | fmt.Println(debug) 29 | Execute(input, output) 30 | }, 31 | } 32 | ) 33 | 34 | func init() { 35 | cobra.OnInitialize() 36 | rootCmd.PersistentFlags().StringVarP(&input, "input", "i", "", "input AOF file path") 37 | rootCmd.PersistentFlags().StringVarP(&output, "output", "o", "", "output dir path") 38 | rootCmd.PersistentFlags().IntVarP(&maxGoRountine, "routines", "r", 8, "max goroutines") 39 | rootCmd.PersistentFlags().BoolVar(&debug, "v", false, "verbose mode for debug") 40 | } 41 | 42 | func main() { 43 | if err := rootCmd.Execute(); err != nil { 44 | fmt.Fprintln(os.Stderr, err) 45 | os.Exit(1) 46 | } 47 | } 48 | 49 | func Execute(input string, output string) { 50 | if input == "" { 51 | return 52 | } 53 | if output != "" { 54 | global.DataDir = output 55 | } else { 56 | // 获取当前可执行文件路径 57 | execPath, err := filepath.Abs(filepath.Dir(os.Args[0])) 58 | if err != nil { 59 | global.LogError(fmt.Sprintf("get current path error: %s\n", err.Error())) 60 | return 61 | } else { 62 | global.DataDir = filepath.Join(execPath, "parseAOF_data") 63 | } 64 | } 65 | splitAOF(input) 66 | 67 | taskChan := make(chan string) 68 | files, _ := ioutil.ReadDir(global.DataDir) 69 | num := maxGoRountine 70 | if len(files) < num { 71 | num = len(files) 72 | } 73 | var wg sync.WaitGroup 74 | for i := 0; i < num; i++ { 75 | wg.Add(1) 76 | go func() { 77 | defer wg.Done() 78 | for fileName := range taskChan { 79 | splitFilePath := global.GetSplitFilePath(fileName) 80 | parsedFilePath := global.GetParsedFilePath(fileName) 81 | err := runner.HandleAOFFile(splitFilePath, parsedFilePath) 82 | if err != nil { 83 | global.LogError(fmt.Sprintf("HandleAOFFile failed: %s | error: %s\n", splitFilePath, err.Error())) 84 | } else { 85 | global.LogInfo(fmt.Sprintf("HandleAOFFile success: %s => %s => %s\n", fileName, splitFilePath, parsedFilePath)) 86 | } 87 | } 88 | }() 89 | } 90 | for _, item := range files { 91 | if item.IsDir() { 92 | continue 93 | } 94 | if global.IsAOFFile(item.Name()) { 95 | taskChan <- item.Name() 96 | } 97 | } 98 | 99 | close(taskChan) 100 | wg.Wait() 101 | mergeParsedFile() 102 | } 103 | 104 | func splitAOF(name string) { 105 | global.LogInfo(fmt.Sprintf("splitAOF Start split file ...\n")) 106 | outputFilePrefix := filepath.Join(global.DataDir, "aof.split_") 107 | startNum := 10000 108 | // 读取文件 109 | fileContent, err := ioutil.ReadFile(name) 110 | if err != nil { 111 | global.LogError(fmt.Sprintf("splitAOF read src file: %s error: %s\n", name, err.Error())) 112 | return 113 | } 114 | _, err = os.Stat(global.DataDir) 115 | if os.IsNotExist(err) { 116 | // create data directory 117 | err = os.MkdirAll(global.DataDir, 0755) 118 | if err != nil { 119 | global.LogError(fmt.Sprintf("mergeParsedFile create merge data directory error: %s\n", err.Error())) 120 | return 121 | } 122 | } 123 | // 计算文件大小 124 | fileSize := len(fileContent) 125 | // 计算切分后的文件数量 126 | splitFileNum := (fileSize + 1024*100 - 1) / (1024 * 100) 127 | // 切分文件并写入输出文件 128 | for i := 0; i < splitFileNum; i++ { 129 | start := i * 1024 * 100 130 | end := (i + 1) * 1024 * 100 131 | if i == splitFileNum-1 { 132 | end = fileSize 133 | } 134 | outputFile := fmt.Sprintf("%s%d", outputFilePrefix, startNum+i) 135 | outputFileContent := fileContent[start:end] 136 | err = ioutil.WriteFile(outputFile, outputFileContent, 0644) 137 | if err != nil { 138 | global.LogError(fmt.Sprintf("splitAOF save split file: %s error: %s\n", outputFile, err.Error())) 139 | return 140 | } 141 | global.LogInfo(fmt.Sprintf("Split file %d: %s\n", i, outputFile)) 142 | } 143 | } 144 | 145 | func mergeParsedFile() { 146 | global.LogInfo(fmt.Sprintf("mergeParsedFile Start merging ...\n")) 147 | // merge files 148 | parsedFiles, _ := ioutil.ReadDir(global.DataDir) 149 | // sort by name a -> z 150 | sort.SliceStable(parsedFiles, func(i, j int) bool { 151 | return parsedFiles[i].Name() < parsedFiles[j].Name() 152 | }) 153 | outputFileName := filepath.Join(global.DataDir, "aof.merged") 154 | merged, err := os.OpenFile(outputFileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 155 | if err != nil { 156 | global.LogError(fmt.Sprintf("mergeParsedFile create merged file error: %s\n", err.Error())) 157 | return 158 | } 159 | defer merged.Close() 160 | for _, item := range parsedFiles { 161 | name := item.Name() 162 | if !strings.HasPrefix(name, "aof.split_") || !strings.HasSuffix(name, ".parsed") || item.IsDir() { 163 | continue 164 | } 165 | content, err := ioutil.ReadFile(filepath.Join(global.DataDir, name)) 166 | if err != nil { 167 | global.LogError(fmt.Sprintf("mergeParsedFile read file: %s | error: %s\n", name, err.Error())) 168 | continue 169 | } 170 | _, err = merged.Write(content) 171 | if err != nil { 172 | global.LogError(fmt.Sprintf("mergeParsedFile append content into merged file error: %s\n", err.Error())) 173 | continue 174 | } 175 | os.Remove(filepath.Join(global.DataDir, name)) 176 | } 177 | global.LogInfo(fmt.Sprintf("Parse AOF success, output file: %s\n", outputFileName)) 178 | } 179 | -------------------------------------------------------------------------------- /src/parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "parseAOF/src/global" 5 | "regexp" 6 | ) 7 | 8 | func MatchLine(content string) int { 9 | var matched = false 10 | if matched, _ = regexp.MatchString(global.PatternOfLineCmdStart, content); matched { 11 | return global.MatchTypeCmdStart 12 | } else if matched, _ = regexp.MatchString(global.PatternOfLineArgLen, content); matched { 13 | return global.MatchTypeArgLen 14 | } else { 15 | return global.MatchTypeArgRaw 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/parser/parser_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "parseAOF/src/global" 6 | "testing" 7 | ) 8 | 9 | func TestMatchLine(t *testing.T) { 10 | realMatchType := MatchLine(global.EmptyString) 11 | if realMatchType != global.MatchTypeArgRaw { 12 | msg := fmt.Sprintf("[TestMatchLine 01] Test failed: %d != %d\n", realMatchType, global.MatchTypeArgRaw) 13 | t.Error(msg) 14 | return 15 | } 16 | fmt.Printf("[TestMatchLine] Test success\n") 17 | } 18 | -------------------------------------------------------------------------------- /src/runner/runner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os" 8 | "parseAOF/src/global" 9 | "parseAOF/src/writer" 10 | "time" 11 | ) 12 | 13 | func HandleAOFFile(splitFilePath string, parsedFilePath string) error { 14 | var splitFileResource *os.File 15 | var parsedFileResource *os.File 16 | var lineNumber = 0 17 | var lineByte []byte 18 | var err error 19 | 20 | splitFileResource, err = os.Open(splitFilePath) 21 | if err != nil { 22 | return err 23 | } 24 | buf := bufio.NewReader(splitFileResource) 25 | parsedFileResource, err = os.OpenFile(parsedFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, global.DefaultFileMode) 26 | if err != nil { 27 | return err 28 | } 29 | defer splitFileResource.Close() 30 | defer parsedFileResource.Close() 31 | 32 | for { 33 | lineNumber++ 34 | lineByte, _, err = buf.ReadLine() 35 | if err == io.EOF { 36 | global.LogDebug(fmt.Sprintf("Line:%d |content=%s|\n", lineNumber, "< 1 { 14 | plainText = fmt.Sprintf("\n") 15 | } else if matchType == global.MatchTypeArgRaw { 16 | plainText = fmt.Sprintf("%s ", content) 17 | } else { 18 | plainText = global.EmptyString 19 | } 20 | return plainText, nil 21 | } 22 | 23 | func AppendFile(f *os.File, content string) (string, error) { 24 | _, err := f.WriteString(content) 25 | if err != nil { 26 | return global.EmptyString, err 27 | } 28 | return content, nil 29 | } 30 | -------------------------------------------------------------------------------- /src/writer/writer_test.go: -------------------------------------------------------------------------------- 1 | package writer 2 | 3 | import ( 4 | "fmt" 5 | "parseAOF/src/global" 6 | "testing" 7 | ) 8 | 9 | func TestTranslateToPlainText(t *testing.T) { 10 | var lineNumber int 11 | var expectContent string 12 | var realContent string 13 | 14 | realContent, _ = TranslateToPlainText(lineNumber, global.EmptyString) 15 | if realContent != global.WhitespaceString { 16 | msg := fmt.Sprintf("[TestWriteFile 01] Test failed: %s != %s\n", realContent, expectContent) 17 | t.Error(msg) 18 | return 19 | } 20 | 21 | realContent, _ = TranslateToPlainText(lineNumber, "$3") 22 | if realContent != global.EmptyString { 23 | msg := fmt.Sprintf("[TestWriteFile 02] Test failed: %s != %s\n", realContent, expectContent) 24 | t.Error(msg) 25 | return 26 | } 27 | 28 | fmt.Printf("[TestWriteFile] Test success\n") 29 | } 30 | --------------------------------------------------------------------------------