├── .github └── workflows │ ├── docker.yml │ ├── release.yml │ └── test.yml ├── Dockerfile ├── LICENSE ├── README.md ├── checksum.go ├── checksum_test.go ├── cmd └── ltx │ ├── apply.go │ ├── checksum.go │ ├── dump.go │ ├── encode_db.go │ ├── list.go │ ├── main.go │ └── verify.go ├── compactor.go ├── compactor_test.go ├── decoder.go ├── decoder_test.go ├── dist └── ltx ├── encoder.go ├── encoder_test.go ├── file_spec.go ├── go.mod ├── go.sum ├── ltx.go └── ltx_test.go /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: 4 | - published 5 | push: 6 | branches: 7 | - main 8 | pull_request: 9 | types: 10 | - opened 11 | - synchronize 12 | - reopened 13 | branches-ignore: 14 | - "dependabot/**" 15 | 16 | name: "Docker Image" 17 | jobs: 18 | docker: 19 | name: "Publish to Docker" 20 | runs-on: ubuntu-latest 21 | env: 22 | PLATFORMS: "linux/amd64" 23 | VERSION: "${{ github.event_name == 'release' && github.event.release.name || '' }}" 24 | 25 | steps: 26 | - uses: actions/checkout@v3 27 | - uses: docker/setup-buildx-action@v2 28 | - uses: docker/login-action@v2 29 | id: login 30 | env: 31 | token_is_present: "${{ secrets.DOCKERHUB_TOKEN && true }}" 32 | if: ${{ env.token_is_present }} 33 | with: 34 | username: ${{ secrets.DOCKERHUB_USERNAME || 'benbjohnson' }} 35 | password: ${{ secrets.DOCKERHUB_TOKEN }} 36 | 37 | - id: meta 38 | uses: docker/metadata-action@v4 39 | with: 40 | images: flyio/ltx 41 | tags: | 42 | type=ref,event=branch 43 | type=ref,event=pr 44 | type=sha 45 | type=sha,format=long 46 | type=semver,pattern={{version}} 47 | type=semver,pattern={{major}}.{{minor}} 48 | 49 | - uses: docker/build-push-action@v3 50 | with: 51 | context: . 52 | push: ${{ steps.login.outcome != 'skipped' }} 53 | platforms: ${{ env.PLATFORMS }} 54 | tags: ${{ steps.meta.outputs.tags }} 55 | labels: ${{ steps.meta.outputs.labels }} 56 | build-args: | 57 | LTX_VERSION=${{ env.VERSION }} 58 | LTX_COMMIT=${{ github.sha }} 59 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: 4 | - created 5 | 6 | name: Release 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | include: 13 | - arch: amd64 14 | os: windows 15 | - arch: amd64 16 | os: darwin 17 | - arch: arm64 18 | os: darwin 19 | - arch: amd64 20 | os: linux 21 | - arch: arm64 22 | os: linux 23 | 24 | env: 25 | GOOS: ${{ matrix.os }} 26 | GOARCH: ${{ matrix.arch }} 27 | 28 | steps: 29 | - uses: actions/checkout@v3 30 | - uses: actions/setup-go@v3 31 | with: 32 | go-version: '1.19' 33 | 34 | - id: release 35 | uses: bruceadams/get-release@v1.2.3 36 | env: 37 | GITHUB_TOKEN: ${{ github.token }} 38 | 39 | - name: Build binary 40 | run: | 41 | rm -rf dist 42 | mkdir -p dist 43 | go build -ldflags "-s -w -extldflags "-static" -X 'main.Version=${{ steps.release.outputs.tag_name }}' -X 'main.Commit=${{ github.sha }}'" -o dist/ltx ./cmd/ltx 44 | cd dist 45 | tar -czvf ltx-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}.tar.gz ltx 46 | 47 | - name: Upload release tarball 48 | uses: actions/upload-release-asset@v1.0.2 49 | env: 50 | GITHUB_TOKEN: ${{ github.token }} 51 | with: 52 | upload_url: ${{ steps.release.outputs.upload_url }} 53 | asset_path: ./dist/ltx-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}.tar.gz 54 | asset_name: ltx-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}.tar.gz 55 | asset_content_type: application/gzip 56 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: "Unit Test" 2 | on: ["push"] 3 | 4 | jobs: 5 | build: 6 | name: Unit Test 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | 11 | - uses: actions/setup-go@v2 12 | with: 13 | go-version: '1.19' 14 | 15 | - uses: actions/cache@v2 16 | with: 17 | path: ~/go/pkg/mod 18 | key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }} 19 | restore-keys: ${{ inputs.os }}-go- 20 | 21 | - name: Run unit tests 22 | run: go test -v ./... 23 | 24 | - name: Build binary 25 | run: go install ./cmd/ltx 26 | 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.19 as builder 2 | 3 | WORKDIR /src/ltx 4 | COPY . . 5 | 6 | ARG LTX_VERSION= 7 | ARG LTX_COMMIT= 8 | 9 | RUN go build -ldflags "-s -w -X 'main.Version=${LTX_VERSION}' -X 'main.Commit=${LTX_COMMIT}' -extldflags '-static'" -o /usr/local/bin/ltx ./cmd/ltx 10 | 11 | 12 | FROM scratch 13 | COPY --from=builder /usr/local/bin/ltx /usr/local/bin/ltx 14 | ENTRYPOINT ["/usr/local/bin/ltx"] 15 | CMD [] 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Lite Transaction File (LTX) 2 | ================================= 3 | 4 | The LTX file format provides a way to store SQLite transactional data in 5 | a way that can be encrypted and compacted and is optimized for performance. 6 | 7 | ## File Format 8 | 9 | An LTX file is composed of several sections: 10 | 11 | 1. Header 12 | 2. Page block 13 | 3. Trailer 14 | 15 | The header contains metadata about the file, the page block contains page 16 | frames, and the trailer contains checksums of the file and the database end state. 17 | 18 | 19 | #### Header 20 | 21 | The header provides information about the number of page frames as well as 22 | database information such as the page size and database size. LTX files 23 | can be compacted together so each file contains the transaction ID (TXID) range 24 | that it represents. A timestamp provides users with a rough approximation of 25 | the time the transaction occurred and the checksum provides a basic integrity 26 | check. 27 | 28 | | Offset | Size | Description | 29 | | -------| ---- | --------------------------------------- | 30 | | 0 | 4 | Magic number. Always "LTX1". | 31 | | 4 | 4 | Flags. Reserved. Always 0. | 32 | | 8 | 4 | Page size, in bytes. | 33 | | 12 | 4 | Size of DB after transaction, in pages. | 34 | | 16 | 4 | Database ID. | 35 | | 20 | 8 | Minimum transaction ID. | 36 | | 28 | 8 | Maximum transaction ID. | 37 | | 36 | 8 | Timestamp (Milliseconds since epoch) | 38 | | 44 | 8 | Pre-apply DB checksum (CRC-ISO-64) | 39 | | 52 | 48 | Reserved. | 40 | 41 | 42 | #### Page block 43 | 44 | This block stores a series of page headers and page data. 45 | 46 | | Offset | Size | Description | 47 | | -------| ---- | --------------------------- | 48 | | 0 | 4 | Page number. | 49 | | 4 | N | Page data. | 50 | 51 | 52 | #### Trailer 53 | 54 | The trailer provides checksum for the LTX file data, a rolling checksum of the 55 | database state after the LTX file is applied, and the checksum of the trailer 56 | itself. 57 | 58 | | Offset | Size | Description | 59 | | -------| ---- | --------------------------------------- | 60 | | 0 | 8 | Post-apply DB checksum (CRC-ISO-64) | 61 | | 8 | 8 | File checksum (CRC-ISO-64) | 62 | 63 | 64 | -------------------------------------------------------------------------------- /checksum.go: -------------------------------------------------------------------------------- 1 | package ltx 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "fmt" 7 | "hash" 8 | "hash/crc64" 9 | "io" 10 | "os" 11 | "strconv" 12 | "sync" 13 | ) 14 | 15 | // Checksum represents an LTX checksum. 16 | type Checksum uint64 17 | 18 | // ChecksumPages updates the provided checksums slice with the checksum of each 19 | // page in the specified file. The first (by page number) error encountered is 20 | // returned along with the number of the last page successfully checksummed. 21 | // Checksums for subsequent pages may be updated, regardless of an error being 22 | // returned. 23 | // 24 | // nWorkers specifies the amount of parallelism to use. A reasonable default 25 | // will be used if nWorkers is 0. 26 | func ChecksumPages(dbPath string, pageSize, nPages, nWorkers uint32, checksums []Checksum) (uint32, error) { 27 | // Based on experimentation on a fly.io machine with a slow SSD, 512Mb is 28 | // where checksumming starts to take >1s. As the database size increases, we 29 | // get more benefit from an increasing number of workers. Doing a bunch of 30 | // benchmarking on fly.io machines of difference sizes with a 1Gb database, 31 | // 24 threads seems to be the sweet spot. 32 | if nWorkers == 0 && pageSize*nPages > 512*1024*1024 { 33 | nWorkers = 24 34 | } 35 | 36 | if nWorkers <= 1 { 37 | return checksumPagesSerial(dbPath, 1, nPages, int64(pageSize), checksums) 38 | } 39 | 40 | perWorker := nPages / nWorkers 41 | if nPages%nWorkers != 0 { 42 | perWorker++ 43 | } 44 | 45 | var ( 46 | wg sync.WaitGroup 47 | rets = make([]uint32, nWorkers) 48 | errs = make([]error, nWorkers) 49 | ) 50 | 51 | for w := uint32(0); w < nWorkers; w++ { 52 | w := w 53 | firstPage := w*perWorker + 1 54 | lastPage := firstPage + perWorker - 1 55 | if lastPage > nPages { 56 | lastPage = nPages 57 | } 58 | 59 | wg.Add(1) 60 | go func() { 61 | rets[w], errs[w] = checksumPagesSerial(dbPath, firstPage, lastPage, int64(pageSize), checksums) 62 | wg.Done() 63 | }() 64 | } 65 | 66 | wg.Wait() 67 | for i, err := range errs { 68 | if err != nil { 69 | return rets[i], err 70 | } 71 | } 72 | 73 | return nPages, nil 74 | } 75 | 76 | func checksumPagesSerial(dbPath string, firstPage, lastPage uint32, pageSize int64, checksums []Checksum) (uint32, error) { 77 | f, err := os.Open(dbPath) 78 | if err != nil { 79 | return firstPage - 1, err 80 | } 81 | 82 | _, err = f.Seek(int64(firstPage-1)*pageSize, io.SeekStart) 83 | if err != nil { 84 | return firstPage - 1, err 85 | } 86 | 87 | buf := make([]byte, pageSize+4) 88 | h := NewHasher() 89 | 90 | for pageNo := firstPage; pageNo <= lastPage; pageNo++ { 91 | binary.BigEndian.PutUint32(buf, pageNo) 92 | 93 | if _, err := io.ReadFull(f, buf[4:]); err != nil { 94 | return pageNo - 1, err 95 | } 96 | 97 | h.Reset() 98 | _, _ = h.Write(buf) 99 | checksums[pageNo-1] = ChecksumFlag | Checksum(h.Sum64()) 100 | } 101 | 102 | return lastPage, nil 103 | } 104 | 105 | // ChecksumPage returns a CRC64 checksum that combines the page number & page data. 106 | func ChecksumPage(pgno uint32, data []byte) Checksum { 107 | return ChecksumPageWithHasher(NewHasher(), pgno, data) 108 | } 109 | 110 | // ChecksumPageWithHasher returns a CRC64 checksum that combines the page number & page data. 111 | func ChecksumPageWithHasher(h hash.Hash64, pgno uint32, data []byte) Checksum { 112 | h.Reset() 113 | _ = binary.Write(h, binary.BigEndian, pgno) 114 | _, _ = h.Write(data) 115 | return ChecksumFlag | Checksum(h.Sum64()) 116 | } 117 | 118 | // ChecksumReader reads an entire database file from r and computes its rolling checksum. 119 | func ChecksumReader(r io.Reader, pageSize int) (Checksum, error) { 120 | data := make([]byte, pageSize) 121 | 122 | var chksum Checksum 123 | for pgno := uint32(1); ; pgno++ { 124 | if _, err := io.ReadFull(r, data); err == io.EOF { 125 | break 126 | } else if err != nil { 127 | return chksum, err 128 | } 129 | chksum = ChecksumFlag | (chksum ^ ChecksumPage(pgno, data)) 130 | } 131 | return chksum, nil 132 | } 133 | 134 | // ParseChecksum parses a 16-character hex string into a checksum. 135 | func ParseChecksum(s string) (Checksum, error) { 136 | if len(s) != 16 { 137 | return 0, fmt.Errorf("invalid formatted checksum length: %q", s) 138 | } 139 | v, err := strconv.ParseUint(s, 16, 64) 140 | if err != nil { 141 | return 0, fmt.Errorf("invalid checksum format: %q", s) 142 | } 143 | return Checksum(v), nil 144 | } 145 | 146 | // String returns c formatted as a fixed-width hex number. 147 | func (c Checksum) String() string { 148 | return fmt.Sprintf("%016x", uint64(c)) 149 | } 150 | 151 | func (c Checksum) MarshalJSON() ([]byte, error) { 152 | return []byte(`"` + c.String() + `"`), nil 153 | } 154 | 155 | func (c *Checksum) UnmarshalJSON(data []byte) (err error) { 156 | var s *string 157 | if err := json.Unmarshal(data, &s); err != nil { 158 | return fmt.Errorf("cannot unmarshal checksum from JSON value") 159 | } 160 | 161 | // Set to zero if value is nil. 162 | if s == nil { 163 | *c = 0 164 | return nil 165 | } 166 | 167 | chksum, err := ParseChecksum(*s) 168 | if err != nil { 169 | return fmt.Errorf("cannot parse checksum from JSON string: %q", *s) 170 | } 171 | *c = Checksum(chksum) 172 | 173 | return nil 174 | } 175 | 176 | // NewHasher returns a new CRC64-ISO hasher. 177 | func NewHasher() hash.Hash64 { 178 | return crc64.New(crc64.MakeTable(crc64.ISO)) 179 | } 180 | -------------------------------------------------------------------------------- /checksum_test.go: -------------------------------------------------------------------------------- 1 | package ltx 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | ) 11 | 12 | func TestChecksumPages(t *testing.T) { 13 | // files divisible into pages 14 | testChecksumPages(t, 1024*4, 4, 1024, 1) 15 | testChecksumPages(t, 1024*4, 4, 1024, 2) 16 | testChecksumPages(t, 1024*4, 4, 1024, 3) 17 | testChecksumPages(t, 1024*4, 4, 1024, 4) 18 | 19 | // short pages 20 | testChecksumPages(t, 1024*3+100, 4, 1024, 1) 21 | testChecksumPages(t, 1024*3+100, 4, 1024, 2) 22 | testChecksumPages(t, 1024*3+100, 4, 1024, 3) 23 | testChecksumPages(t, 1024*3+100, 4, 1024, 4) 24 | 25 | // empty files 26 | testChecksumPages(t, 0, 4, 1024, 1) 27 | testChecksumPages(t, 0, 4, 1024, 2) 28 | testChecksumPages(t, 0, 4, 1024, 3) 29 | testChecksumPages(t, 0, 4, 1024, 4) 30 | } 31 | 32 | func testChecksumPages(t *testing.T, fileSize, nPages, pageSize, nWorkers uint32) { 33 | t.Run(fmt.Sprintf("fileSize=%d,nPages=%d,pageSize=%d,nWorkers=%d", fileSize, nPages, pageSize, nWorkers), func(t *testing.T) { 34 | path := filepath.Join(t.TempDir(), "test.db") 35 | f, err := os.Create(path) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | defer f.Close() 40 | if _, err := io.CopyN(f, rand.Reader, int64(fileSize)); err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | legacyCS := make([]Checksum, nPages) 45 | legacyLastPage, legacyErr := legacyChecksumPages(path, pageSize, nPages, legacyCS) 46 | newCS := make([]Checksum, nPages) 47 | newLastPage, newErr := ChecksumPages(path, pageSize, nPages, nWorkers, newCS) 48 | 49 | if legacyErr != newErr { 50 | t.Fatalf("legacy error: %v, new error: %v", legacyErr, newErr) 51 | } 52 | if legacyLastPage != newLastPage { 53 | t.Fatalf("legacy last page: %d, new last page: %d", legacyLastPage, newLastPage) 54 | } 55 | if len(legacyCS) != len(newCS) { 56 | t.Fatalf("legacy checksums: %d, new checksums: %d", len(legacyCS), len(newCS)) 57 | } 58 | for i := range legacyCS { 59 | if legacyCS[i] != newCS[i] { 60 | t.Fatalf("mismatch at index %d: legacy: %v, new: %v", i, legacyCS[i], newCS[i]) 61 | } 62 | } 63 | }) 64 | } 65 | 66 | // logic copied from litefs repo 67 | func legacyChecksumPages(dbPath string, pageSize, nPages uint32, checksums []Checksum) (uint32, error) { 68 | f, err := os.Open(dbPath) 69 | if err != nil { 70 | return 0, err 71 | } 72 | defer f.Close() 73 | 74 | buf := make([]byte, pageSize) 75 | 76 | for pgno := uint32(1); pgno <= nPages; pgno++ { 77 | offset := int64(pgno-1) * int64(pageSize) 78 | if _, err := readFullAt(f, buf, offset); err != nil { 79 | return pgno - 1, err 80 | } 81 | 82 | checksums[pgno-1] = ChecksumPage(pgno, buf) 83 | } 84 | 85 | return nPages, nil 86 | } 87 | 88 | // copied from litefs/internal 89 | func readFullAt(r io.ReaderAt, buf []byte, off int64) (n int, err error) { 90 | for n < len(buf) && err == nil { 91 | var nn int 92 | nn, err = r.ReadAt(buf[n:], off+int64(n)) 93 | n += nn 94 | } 95 | if n >= len(buf) { 96 | return n, nil 97 | } else if n > 0 && err == io.EOF { 98 | return n, io.ErrUnexpectedEOF 99 | } 100 | return n, err 101 | } 102 | -------------------------------------------------------------------------------- /cmd/ltx/apply.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "os" 9 | 10 | "github.com/superfly/ltx" 11 | ) 12 | 13 | // ApplyCommand represents a command to apply a series of LTX files to a database file. 14 | type ApplyCommand struct{} 15 | 16 | // NewApplyCommand returns a new instance of ApplyCommand. 17 | func NewApplyCommand() *ApplyCommand { 18 | return &ApplyCommand{} 19 | } 20 | 21 | // Run executes the command. 22 | func (c *ApplyCommand) Run(ctx context.Context, args []string) (ret error) { 23 | fs := flag.NewFlagSet("ltx-apply", flag.ContinueOnError) 24 | dbPath := fs.String("db", "", "database path") 25 | fs.Usage = func() { 26 | fmt.Println(` 27 | The apply command applies one or more LTX files to a database file. 28 | 29 | Usage: 30 | 31 | ltx apply [arguments] PATH [PATH...] 32 | 33 | Arguments: 34 | `[1:]) 35 | fs.PrintDefaults() 36 | fmt.Println() 37 | } 38 | if err := fs.Parse(args); err != nil { 39 | return err 40 | } 41 | if fs.NArg() == 0 { 42 | fs.Usage() 43 | return flag.ErrHelp 44 | } else if *dbPath == "" { 45 | return fmt.Errorf("required: -db PATH") 46 | } 47 | 48 | // Open database file. Create if it doesn't exist. 49 | dbFile, err := os.OpenFile(*dbPath, os.O_RDWR|os.O_CREATE, 0o666) 50 | if err != nil { 51 | return err 52 | } 53 | defer func() { _ = dbFile.Close() }() 54 | 55 | // Apply LTX files in order. 56 | for _, filename := range fs.Args() { 57 | if err := c.applyLTXFile(ctx, dbFile, filename); err != nil { 58 | return fmt.Errorf("%s: %s", filename, err) 59 | } 60 | } 61 | 62 | // Sync and close resulting database file. 63 | if err := dbFile.Sync(); err != nil { 64 | return err 65 | } 66 | return dbFile.Close() 67 | } 68 | 69 | func (c *ApplyCommand) applyLTXFile(ctx context.Context, dbFile *os.File, filename string) error { 70 | ltxFile, err := os.Open(filename) 71 | if err != nil { 72 | return err 73 | } 74 | defer func() { _ = ltxFile.Close() }() 75 | 76 | // Read LTX header and verify initial checksum matches. 77 | dec := ltx.NewDecoder(ltxFile) 78 | if err := dec.DecodeHeader(); err != nil { 79 | return fmt.Errorf("decode ltx header: %w", err) 80 | } 81 | 82 | // Read checksum before applying. 83 | if _, err := dbFile.Seek(0, io.SeekStart); err != nil { 84 | return err 85 | } 86 | preApplyChecksum, err := ltx.ChecksumReader(dbFile, int(dec.Header().PageSize)) 87 | if err != nil { 88 | return fmt.Errorf("compute pre-apply checksum: %w", err) 89 | } else if preApplyChecksum != dec.Header().PreApplyChecksum { 90 | return fmt.Errorf("pre-apply checksum mismatch: %s <> %s", preApplyChecksum, dec.Header().PreApplyChecksum) 91 | } 92 | 93 | // Apply each page to the database. 94 | data := make([]byte, dec.Header().PageSize) 95 | for { 96 | var pageHeader ltx.PageHeader 97 | if err := dec.DecodePage(&pageHeader, data); err == io.EOF { 98 | break 99 | } else if err != nil { 100 | return fmt.Errorf("decode ltx page: %w", err) 101 | } 102 | 103 | offset := int64(pageHeader.Pgno-1) * int64(dec.Header().PageSize) 104 | if _, err := dbFile.WriteAt(data, offset); err != nil { 105 | return fmt.Errorf("write database page: %w", err) 106 | } 107 | } 108 | 109 | // Close & verify file, print trailer. 110 | if err := dec.Close(); err != nil { 111 | return fmt.Errorf("close ltx file: %w", err) 112 | } 113 | 114 | // Recalculate database checksum and ensure it matches the LTX checksum. 115 | if _, err := dbFile.Seek(0, io.SeekStart); err != nil { 116 | return err 117 | } 118 | postApplyChecksum, err := ltx.ChecksumReader(dbFile, int(dec.Header().PageSize)) 119 | if err != nil { 120 | return fmt.Errorf("compute post-apply checksum: %w", err) 121 | } else if postApplyChecksum != dec.Trailer().PostApplyChecksum { 122 | return fmt.Errorf("post-apply checksum mismatch: %s <> %s", postApplyChecksum, dec.Trailer().PostApplyChecksum) 123 | } 124 | 125 | return nil 126 | } 127 | -------------------------------------------------------------------------------- /cmd/ltx/checksum.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/binary" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "os" 10 | 11 | "github.com/superfly/ltx" 12 | ) 13 | 14 | // ChecksumCommand represents a command to compute the LTX checksum of a database file. 15 | type ChecksumCommand struct{} 16 | 17 | // NewChecksumCommand returns a new instance of ChecksumCommand. 18 | func NewChecksumCommand() *ChecksumCommand { 19 | return &ChecksumCommand{} 20 | } 21 | 22 | // Run executes the command. 23 | func (c *ChecksumCommand) Run(ctx context.Context, args []string) (ret error) { 24 | fs := flag.NewFlagSet("ltx-checksum", flag.ContinueOnError) 25 | fs.Usage = func() { 26 | fmt.Println(` 27 | The checksum command computes the LTX checksum for a database file. 28 | 29 | Usage: 30 | 31 | ltx checksum PATH 32 | `[1:], 33 | ) 34 | } 35 | if err := fs.Parse(args); err != nil { 36 | return err 37 | } else if fs.NArg() == 0 { 38 | fs.Usage() 39 | return flag.ErrHelp 40 | } else if fs.NArg() > 1 { 41 | return fmt.Errorf("too many arguments") 42 | } 43 | 44 | f, err := os.Open(fs.Arg(0)) 45 | if err != nil { 46 | return err 47 | } 48 | defer func() { _ = f.Close() }() 49 | 50 | // Read database header to determine page size. 51 | buf := make([]byte, 100) 52 | if _, err := io.ReadFull(f, buf); err != nil { 53 | return err 54 | } 55 | pageSize := int(binary.BigEndian.Uint16(buf[16:18])) 56 | if pageSize == 1 { 57 | pageSize = 65536 58 | } 59 | 60 | // Reseek to beginning and compute checksum. 61 | if _, err := f.Seek(0, io.SeekStart); err != nil { 62 | return err 63 | } 64 | chksum, err := ltx.ChecksumReader(f, pageSize) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | fmt.Println(chksum) 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /cmd/ltx/dump.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "os" 9 | "time" 10 | 11 | "github.com/superfly/ltx" 12 | ) 13 | 14 | // DumpCommand represents a command to print the contents of a single LTX file. 15 | type DumpCommand struct{} 16 | 17 | // NewDumpCommand returns a new instance of DumpCommand. 18 | func NewDumpCommand() *DumpCommand { 19 | return &DumpCommand{} 20 | } 21 | 22 | // Run executes the command. 23 | func (c *DumpCommand) Run(ctx context.Context, args []string) (ret error) { 24 | fs := flag.NewFlagSet("ltx-dump", flag.ContinueOnError) 25 | fs.Usage = func() { 26 | fmt.Println(` 27 | The dump command writes out all data for a single LTX file. 28 | 29 | Usage: 30 | 31 | ltx dump [arguments] PATH 32 | 33 | Arguments: 34 | `[1:]) 35 | fs.PrintDefaults() 36 | fmt.Println() 37 | } 38 | if err := fs.Parse(args); err != nil { 39 | return err 40 | } else if fs.NArg() == 0 { 41 | return fmt.Errorf("filename required") 42 | } else if fs.NArg() > 1 { 43 | return fmt.Errorf("too many arguments") 44 | } 45 | 46 | f, err := os.Open(fs.Arg(0)) 47 | if err != nil { 48 | return err 49 | } 50 | defer func() { _ = f.Close() }() 51 | 52 | dec := ltx.NewDecoder(f) 53 | 54 | // Read & print header information. 55 | err = dec.DecodeHeader() 56 | hdr := dec.Header() 57 | fmt.Printf("# HEADER\n") 58 | fmt.Printf("Version: %d\n", hdr.Version) 59 | fmt.Printf("Flags: 0x%08x\n", hdr.Flags) 60 | fmt.Printf("Page size: %d\n", hdr.PageSize) 61 | fmt.Printf("Commit: %d\n", hdr.Commit) 62 | fmt.Printf("Min TXID: %s (%d)\n", hdr.MinTXID.String(), hdr.MinTXID) 63 | fmt.Printf("Max TXID: %s (%d)\n", hdr.MaxTXID.String(), hdr.MaxTXID) 64 | fmt.Printf("Timestamp: %s (%d)\n", time.UnixMilli(int64(hdr.Timestamp)).UTC().Format(time.RFC3339Nano), hdr.Timestamp) 65 | fmt.Printf("Pre-apply: %s\n", hdr.PreApplyChecksum) 66 | fmt.Printf("WAL offset: %d\n", hdr.WALOffset) 67 | fmt.Printf("WAL size: %d\n", hdr.WALSize) 68 | fmt.Printf("WAL salt: %08x %08x\n", hdr.WALSalt1, hdr.WALSalt2) 69 | fmt.Printf("\n") 70 | if err != nil { 71 | return err 72 | } 73 | 74 | fmt.Printf("# PAGE DATA\n") 75 | for i := 0; ; i++ { 76 | var pageHeader ltx.PageHeader 77 | data := make([]byte, hdr.PageSize) 78 | if err := dec.DecodePage(&pageHeader, data); err == io.EOF { 79 | break 80 | } else if err != nil { 81 | return fmt.Errorf("decode page frame %d: %w", i, err) 82 | } 83 | 84 | fmt.Printf("Frame #%d: pgno=%d\n", i, pageHeader.Pgno) 85 | } 86 | fmt.Printf("\n") 87 | 88 | // Close & verify file, print trailer. 89 | err = dec.Close() 90 | trailer := dec.Trailer() 91 | 92 | fmt.Printf("# TRAILER\n") 93 | fmt.Printf("Post-apply: %s\n", trailer.PostApplyChecksum) 94 | fmt.Printf("File Checksum: %s\n", trailer.FileChecksum) 95 | fmt.Printf("\n") 96 | if err != nil { 97 | return err 98 | } 99 | 100 | return nil 101 | } 102 | -------------------------------------------------------------------------------- /cmd/ltx/encode_db.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/binary" 7 | "flag" 8 | "fmt" 9 | "io" 10 | "os" 11 | "time" 12 | 13 | "github.com/superfly/ltx" 14 | ) 15 | 16 | const ( 17 | SQLITE_DATABASE_HEADER_STRING = "SQLite format 3\x00" 18 | SQLITE_DATABASE_HEADER_SIZE = 100 19 | ) 20 | 21 | // EncodeDBCommand represents a command to encode an SQLite database file as a single LTX file. 22 | type EncodeDBCommand struct{} 23 | 24 | // NewEncodeDBCommand returns a new instance of EncodeDBCommand. 25 | func NewEncodeDBCommand() *EncodeDBCommand { 26 | return &EncodeDBCommand{} 27 | } 28 | 29 | // Run executes the command. 30 | func (c *EncodeDBCommand) Run(ctx context.Context, args []string) (ret error) { 31 | fs := flag.NewFlagSet("ltx-encode-db", flag.ContinueOnError) 32 | outPath := fs.String("o", "", "output path") 33 | compressed := fs.Bool("c", false, "compress database pages") 34 | fs.Usage = func() { 35 | fmt.Println(` 36 | The encode-db command encodes an SQLite database into an LTX file. 37 | 38 | Usage: 39 | 40 | ltx encode-db [arguments] PATH 41 | 42 | Arguments: 43 | `[1:]) 44 | fs.PrintDefaults() 45 | fmt.Println() 46 | } 47 | if err := fs.Parse(args); err != nil { 48 | return err 49 | } else if fs.NArg() == 0 { 50 | return fmt.Errorf("filename required") 51 | } else if fs.NArg() > 1 { 52 | return fmt.Errorf("too many arguments") 53 | } else if *outPath == "" { 54 | return fmt.Errorf("required: -o PATH") 55 | } 56 | 57 | db, err := os.Open(fs.Arg(0)) 58 | if err != nil { 59 | return fmt.Errorf("open DB file: %w", err) 60 | } 61 | defer func() { _ = db.Close() }() 62 | 63 | out, err := os.OpenFile(*outPath, os.O_CREATE|os.O_WRONLY, 0o644) 64 | if err != nil { 65 | return fmt.Errorf("open output file: %w", err) 66 | } 67 | defer func() { _ = out.Close() }() 68 | 69 | rd, hdr, err := c.readSQLiteDatabaseHeader(db) 70 | if err != nil { 71 | return fmt.Errorf("read database header: %w", err) 72 | } 73 | 74 | var flags uint32 75 | var postApplyChecksum ltx.Checksum 76 | if *compressed { 77 | flags |= ltx.HeaderFlagCompressLZ4 78 | } 79 | 80 | enc := ltx.NewEncoder(out) 81 | if err := enc.EncodeHeader(ltx.Header{ 82 | Version: 1, 83 | Flags: flags, 84 | PageSize: hdr.pageSize, 85 | Commit: hdr.pageN, 86 | MinTXID: ltx.TXID(1), 87 | MaxTXID: ltx.TXID(1), 88 | Timestamp: time.Now().UnixMilli(), 89 | }); err != nil { 90 | return fmt.Errorf("encode ltx header: %w", err) 91 | } 92 | 93 | buf := make([]byte, hdr.pageSize) 94 | for pgno := uint32(1); pgno <= hdr.pageN; pgno++ { 95 | if _, err := io.ReadFull(rd, buf); err != nil { 96 | return fmt.Errorf("read page %d: %w", pgno, err) 97 | } 98 | 99 | if pgno == ltx.LockPgno(hdr.pageSize) { 100 | continue 101 | } 102 | 103 | if err := enc.EncodePage(ltx.PageHeader{Pgno: pgno}, buf); err != nil { 104 | return fmt.Errorf("encode page %d: %w", pgno, err) 105 | } 106 | 107 | postApplyChecksum = ltx.ChecksumFlag | (postApplyChecksum ^ ltx.ChecksumPage(pgno, buf)) 108 | } 109 | 110 | enc.SetPostApplyChecksum(postApplyChecksum) 111 | if err := enc.Close(); err != nil { 112 | return fmt.Errorf("close ltx encoder: %w", err) 113 | } else if err := out.Sync(); err != nil { 114 | return fmt.Errorf("sync ltx file: %w", err) 115 | } else if err := out.Close(); err != nil { 116 | return fmt.Errorf("close ltx file: %w", err) 117 | } 118 | 119 | return nil 120 | } 121 | 122 | type sqliteDatabaseHeader struct { 123 | pageSize uint32 124 | pageN uint32 125 | } 126 | 127 | func (c *EncodeDBCommand) readSQLiteDatabaseHeader(rd io.Reader) (ord io.Reader, hdr sqliteDatabaseHeader, err error) { 128 | b := make([]byte, SQLITE_DATABASE_HEADER_SIZE) 129 | if _, err := io.ReadFull(rd, b); err == io.ErrUnexpectedEOF { 130 | return ord, hdr, fmt.Errorf("invalid database header") 131 | } else if err == io.EOF { 132 | return ord, hdr, fmt.Errorf("empty database") 133 | } else if err != nil { 134 | return ord, hdr, err 135 | } else if !bytes.Equal(b[:len(SQLITE_DATABASE_HEADER_STRING)], []byte(SQLITE_DATABASE_HEADER_STRING)) { 136 | return ord, hdr, fmt.Errorf("invalid database header") 137 | } 138 | 139 | hdr.pageSize = uint32(binary.BigEndian.Uint16(b[16:])) 140 | hdr.pageN = binary.BigEndian.Uint32(b[28:]) 141 | if hdr.pageSize == 1 { 142 | hdr.pageSize = 65536 143 | } 144 | 145 | ord = io.MultiReader(bytes.NewReader(b), rd) 146 | 147 | return ord, hdr, nil 148 | } 149 | -------------------------------------------------------------------------------- /cmd/ltx/list.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "os" 9 | "text/tabwriter" 10 | "time" 11 | 12 | "github.com/superfly/ltx" 13 | ) 14 | 15 | // ListCommand represents a command to print the header/trailer of one or more 16 | // LTX files in a table. 17 | type ListCommand struct{} 18 | 19 | // NewListCommand returns a new instance of ListCommand. 20 | func NewListCommand() *ListCommand { 21 | return &ListCommand{} 22 | } 23 | 24 | // Run executes the command. 25 | func (c *ListCommand) Run(ctx context.Context, args []string) (ret error) { 26 | fs := flag.NewFlagSet("ltx-list", flag.ContinueOnError) 27 | tsv := fs.Bool("tsv", false, "output as tab-separated values") 28 | fs.Usage = func() { 29 | fmt.Println(` 30 | The list command lists header & trailer information for a set of LTX files. 31 | 32 | Usage: 33 | 34 | ltx list [arguments] PATH [PATH...] 35 | 36 | Arguments: 37 | `[1:]) 38 | fs.PrintDefaults() 39 | fmt.Println() 40 | } 41 | if err := fs.Parse(args); err != nil { 42 | return err 43 | } else if fs.NArg() == 0 { 44 | return fmt.Errorf("at least one LTX file is required") 45 | } 46 | 47 | var w io.Writer = os.Stdout 48 | if !*tsv { 49 | tw := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) 50 | defer func() { _ = tw.Flush() }() 51 | w = tw 52 | } 53 | 54 | _, _ = fmt.Fprintln(w, "min_txid\tmax_txid\tcommit\tpages\tpreapply\tpostapply\ttimestamp\twal_offset\twal_size\twal_salt") 55 | for _, arg := range fs.Args() { 56 | if err := c.printFile(w, arg); err != nil { 57 | _, _ = fmt.Fprintf(os.Stderr, "%s: %s\n", arg, err) 58 | } 59 | } 60 | 61 | return nil 62 | } 63 | 64 | func (c *ListCommand) printFile(w io.Writer, filename string) error { 65 | f, err := os.Open(filename) 66 | if err != nil { 67 | return err 68 | } 69 | defer func() { _ = f.Close() }() 70 | 71 | dec := ltx.NewDecoder(f) 72 | if err := dec.Verify(); err != nil { 73 | return err 74 | } 75 | 76 | // Only show timestamp if it is actually set. 77 | timestamp := time.UnixMilli(dec.Header().Timestamp).UTC().Format(time.RFC3339) 78 | if dec.Header().Timestamp == 0 { 79 | timestamp = "" 80 | } 81 | 82 | _, _ = fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\t%s\t%s\t%d\t%d\t%08x %08x\n", 83 | dec.Header().MinTXID.String(), 84 | dec.Header().MaxTXID.String(), 85 | dec.Header().Commit, 86 | dec.PageN(), 87 | dec.Header().PreApplyChecksum, 88 | dec.Trailer().PostApplyChecksum, 89 | timestamp, 90 | dec.Header().WALOffset, 91 | dec.Header().WALSize, 92 | dec.Header().WALSalt1, dec.Header().WALSalt2, 93 | ) 94 | 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /cmd/ltx/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | // Build information. 12 | var ( 13 | Version = "" 14 | Commit = "" 15 | ) 16 | 17 | func main() { 18 | m := NewMain() 19 | if err := m.Run(context.Background(), os.Args[1:]); err == flag.ErrHelp { 20 | os.Exit(1) 21 | } else if err != nil { 22 | fmt.Println(err) 23 | os.Exit(1) 24 | } 25 | } 26 | 27 | // Main represents the main program execution. 28 | type Main struct{} 29 | 30 | // NewMain returns a new instance of Main. 31 | func NewMain() *Main { 32 | return &Main{} 33 | } 34 | 35 | // Run executes the program. 36 | func (m *Main) Run(ctx context.Context, args []string) (err error) { 37 | // Extract command name. 38 | var cmd string 39 | if len(args) > 0 { 40 | cmd, args = args[0], args[1:] 41 | } 42 | 43 | switch cmd { 44 | case "apply": 45 | return NewApplyCommand().Run(ctx, args) 46 | case "checksum": 47 | return NewChecksumCommand().Run(ctx, args) 48 | case "dump": 49 | return NewDumpCommand().Run(ctx, args) 50 | case "encode-db": 51 | return NewEncodeDBCommand().Run(ctx, args) 52 | case "list": 53 | return NewListCommand().Run(ctx, args) 54 | case "verify": 55 | return NewVerifyCommand().Run(ctx, args) 56 | case "version": 57 | if Version != "" { 58 | fmt.Printf("ltx %s, commit=%s\n", Version, Commit) 59 | } else if Commit != "" { 60 | fmt.Printf("ltx commit=%s\n", Commit) 61 | } else { 62 | fmt.Println("ltx development build") 63 | } 64 | return nil 65 | default: 66 | if cmd == "" || cmd == "help" || strings.HasPrefix(cmd, "-") { 67 | m.Usage() 68 | return flag.ErrHelp 69 | } 70 | return fmt.Errorf("ltx %s: unknown command", cmd) 71 | } 72 | } 73 | 74 | // Usage prints the help screen to STDOUT. 75 | func (m *Main) Usage() { 76 | fmt.Println(` 77 | ltx is a command-line tool for inspecting LTX files. 78 | 79 | Usage: 80 | 81 | ltx [arguments] 82 | 83 | The commands are: 84 | 85 | apply applies a set of LTX files to a database 86 | checksum computes the LTX checksum of a database file 87 | dump writes out metadata and page headers for a set of LTX files 88 | list lists header & trailer fields for LTX files in a table 89 | verify reads & verifies checksums of a set of LTX files 90 | version prints the version 91 | `[1:]) 92 | } 93 | -------------------------------------------------------------------------------- /cmd/ltx/verify.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/superfly/ltx" 10 | ) 11 | 12 | // VerifyCommand represents a command to verify the integrity of LTX files. 13 | type VerifyCommand struct{} 14 | 15 | // NewVerifyCommand returns a new instance of VerifyCommand. 16 | func NewVerifyCommand() *VerifyCommand { 17 | return &VerifyCommand{} 18 | } 19 | 20 | // Run executes the command. 21 | func (c *VerifyCommand) Run(ctx context.Context, args []string) (ret error) { 22 | fs := flag.NewFlagSet("ltx-verify", flag.ContinueOnError) 23 | fs.Usage = func() { 24 | fmt.Println(` 25 | The verify command reads one or more LTX files and verifies its integrity. 26 | 27 | Usage: 28 | 29 | ltx verify PATH [PATH...] 30 | 31 | `[1:], 32 | ) 33 | } 34 | if err := fs.Parse(args); err != nil { 35 | return err 36 | } else if fs.NArg() == 0 { 37 | return fmt.Errorf("at least one LTX file must be specified") 38 | } 39 | 40 | var okN, errorN int 41 | for _, filename := range fs.Args() { 42 | if err := c.verifyFile(ctx, filename); err != nil { 43 | errorN++ 44 | fmt.Printf("%s: %s\n", filename, err) 45 | continue 46 | } 47 | 48 | okN++ 49 | } 50 | 51 | if errorN != 0 { 52 | return fmt.Errorf("%d ok, %d invalid", okN, errorN) 53 | } 54 | 55 | fmt.Println("ok") 56 | return nil 57 | } 58 | 59 | func (c *VerifyCommand) verifyFile(ctx context.Context, filename string) error { 60 | f, err := os.Open(filename) 61 | if err != nil { 62 | return err 63 | } 64 | defer func() { _ = f.Close() }() 65 | 66 | return ltx.NewDecoder(f).Verify() 67 | } 68 | -------------------------------------------------------------------------------- /compactor.go: -------------------------------------------------------------------------------- 1 | package ltx 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "sort" 8 | ) 9 | 10 | // Compactor represents a compactor of LTX files. 11 | type Compactor struct { 12 | enc *Encoder 13 | inputs []*compactorInput 14 | 15 | // These flags will be set when encoding the header. 16 | HeaderFlags uint32 17 | 18 | // If true, the compactor will not validate that input files have contiguous 19 | // transaction IDs. This is false by default but can be enabled when 20 | // rebuilding snapshots with missing transactions. 21 | AllowNonContiguousTXIDs bool 22 | } 23 | 24 | // NewCompactor returns a new instance of Compactor with default settings. 25 | func NewCompactor(w io.Writer, rdrs []io.Reader) *Compactor { 26 | c := &Compactor{ 27 | enc: NewEncoder(w), 28 | } 29 | 30 | c.inputs = make([]*compactorInput, len(rdrs)) 31 | for i := range c.inputs { 32 | c.inputs[i] = &compactorInput{dec: NewDecoder(rdrs[i])} 33 | } 34 | return c 35 | } 36 | 37 | // Header returns the LTX header of the compacted file. Only valid after successful Compact(). 38 | func (c *Compactor) Header() Header { return c.enc.Header() } 39 | 40 | // Trailer returns the LTX trailer of the compacted file. Only valid after successful Compact(). 41 | func (c *Compactor) Trailer() Trailer { return c.enc.Trailer() } 42 | 43 | // Compact merges the input readers into a single LTX writer. 44 | func (c *Compactor) Compact(ctx context.Context) (retErr error) { 45 | if len(c.inputs) == 0 { 46 | return fmt.Errorf("at least one input reader required") 47 | } 48 | 49 | // Read headers from all inputs. 50 | for _, input := range c.inputs { 51 | if err := input.dec.DecodeHeader(); err != nil { 52 | return 53 | } 54 | } 55 | 56 | // Sort inputs by transaction ID. 57 | sort.Slice(c.inputs, func(i, j int) bool { 58 | return c.inputs[i].dec.Header().MinTXID < c.inputs[j].dec.Header().MaxTXID 59 | }) 60 | 61 | // Validate that reader page sizes match & TXIDs are contiguous. 62 | for i := 1; i < len(c.inputs); i++ { 63 | prevHdr := c.inputs[i-1].dec.Header() 64 | hdr := c.inputs[i].dec.Header() 65 | 66 | if prevHdr.PageSize != hdr.PageSize { 67 | return fmt.Errorf("input files have mismatched page sizes: %d != %d", prevHdr.PageSize, hdr.PageSize) 68 | } 69 | if !c.AllowNonContiguousTXIDs && prevHdr.MaxTXID+1 != hdr.MinTXID { 70 | return fmt.Errorf("non-contiguous transaction ids in input files: (%s,%s) -> (%s,%s)", 71 | prevHdr.MinTXID.String(), prevHdr.MaxTXID.String(), 72 | hdr.MinTXID.String(), hdr.MaxTXID.String(), 73 | ) 74 | } 75 | } 76 | 77 | // Fetch the first and last headers from the sorted readers. 78 | minHdr := c.inputs[0].dec.Header() 79 | maxHdr := c.inputs[len(c.inputs)-1].dec.Header() 80 | 81 | // Generate output header. Skip NodeID as it's not meaningful after compaction. 82 | if err := c.enc.EncodeHeader(Header{ 83 | Version: Version, 84 | Flags: c.HeaderFlags, 85 | PageSize: minHdr.PageSize, 86 | Commit: maxHdr.Commit, 87 | MinTXID: minHdr.MinTXID, 88 | MaxTXID: maxHdr.MaxTXID, 89 | Timestamp: maxHdr.Timestamp, 90 | PreApplyChecksum: minHdr.PreApplyChecksum, 91 | }); err != nil { 92 | return fmt.Errorf("write header: %w", err) 93 | } 94 | 95 | // Write page headers & data. 96 | if err := c.writePageBlock(ctx); err != nil { 97 | return err 98 | } 99 | 100 | // Close readers to ensure they're valid. 101 | for i, input := range c.inputs { 102 | if err := input.dec.Close(); err != nil { 103 | return fmt.Errorf("close reader %d: %w", i, err) 104 | } 105 | } 106 | 107 | // Close encoder. 108 | c.enc.SetPostApplyChecksum(c.inputs[len(c.inputs)-1].dec.Trailer().PostApplyChecksum) 109 | if err := c.enc.Close(); err != nil { 110 | return fmt.Errorf("close encoder: %w", err) 111 | } 112 | 113 | return nil 114 | } 115 | 116 | func (c *Compactor) writePageBlock(ctx context.Context) error { 117 | // Allocate buffers. 118 | for _, input := range c.inputs { 119 | input.buf.data = make([]byte, c.enc.Header().PageSize) 120 | } 121 | 122 | // Iterate over readers and merge together. 123 | for { 124 | // Read next page frame for each buffer. 125 | pgno, err := c.fillPageBuffers(ctx) 126 | if err != nil { 127 | return err 128 | } else if pgno == 0 { 129 | break // no more page frames, exit. 130 | } 131 | 132 | // Write page from latest input. 133 | if err := c.writePageBuffer(ctx, pgno); err != nil { 134 | return err 135 | } 136 | } 137 | 138 | return nil 139 | } 140 | 141 | // fillPageBuffers reads the next page frame into each input buffer. 142 | func (c *Compactor) fillPageBuffers(ctx context.Context) (pgno uint32, err error) { 143 | for i := range c.inputs { 144 | input := c.inputs[i] 145 | 146 | // Fill buffer if it is empty. 147 | if input.buf.hdr.IsZero() { 148 | if err := input.dec.DecodePage(&input.buf.hdr, input.buf.data); err == io.EOF { 149 | continue // end of page block 150 | } else if err != nil { 151 | return 0, fmt.Errorf("read page header %d: %w", i, err) 152 | } 153 | } 154 | 155 | // Find the lowest page number among the buffers. 156 | if pgno == 0 || input.buf.hdr.Pgno < pgno { 157 | pgno = input.buf.hdr.Pgno 158 | } 159 | } 160 | return pgno, nil 161 | } 162 | 163 | // writePageBuffer writes the buffer with a matching pgno from the latest input. 164 | func (c *Compactor) writePageBuffer(ctx context.Context, pgno uint32) error { 165 | commit := c.enc.Header().Commit 166 | 167 | var pageWritten bool 168 | for i := len(c.inputs) - 1; i >= 0; i-- { 169 | input := c.inputs[i] 170 | // Skip if buffer does have matching page number. 171 | if input.buf.hdr.Pgno != pgno { 172 | continue 173 | } 174 | 175 | // Clear buffer. 176 | hdr, data := input.buf.hdr, input.buf.data 177 | input.buf.hdr = PageHeader{} 178 | 179 | // If page number has not been written yet, copy from input file. 180 | if pageWritten { 181 | continue 182 | } else if pgno > commit { 183 | continue // out of range of final database size, skip 184 | } 185 | pageWritten = true 186 | 187 | if err := c.enc.EncodePage(hdr, data); err != nil { 188 | return fmt.Errorf("copy page %d header: %w", pgno, err) 189 | } 190 | } 191 | 192 | return nil 193 | } 194 | 195 | type compactorInput struct { 196 | dec *Decoder 197 | buf struct { 198 | hdr PageHeader 199 | data []byte 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /compactor_test.go: -------------------------------------------------------------------------------- 1 | package ltx_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "testing" 8 | 9 | "github.com/superfly/ltx" 10 | ) 11 | 12 | func TestCompactor_Compact(t *testing.T) { 13 | t.Run("SingleFilePageDataOnly", func(t *testing.T) { 14 | input := <x.FileSpec{ 15 | Header: ltx.Header{ 16 | Version: 1, 17 | PageSize: 512, 18 | Commit: 1, 19 | MinTXID: 1, 20 | MaxTXID: 1, 21 | Timestamp: 1000, 22 | PreApplyChecksum: 0, 23 | }, 24 | Pages: []ltx.PageSpec{ 25 | { 26 | Header: ltx.PageHeader{Pgno: 1}, 27 | Data: bytes.Repeat([]byte("1"), 512), 28 | }, 29 | }, 30 | Trailer: ltx.Trailer{ 31 | PostApplyChecksum: 0xeb1a999231044ddd, 32 | FileChecksum: 0x897cc5d024cd382a, 33 | }, 34 | } 35 | var buf0 bytes.Buffer 36 | writeFileSpec(t, &buf0, input) 37 | 38 | var output bytes.Buffer 39 | c := ltx.NewCompactor(&output, []io.Reader{&buf0}) 40 | if err := c.Compact(context.Background()); err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | spec := readFileSpec(t, &output, int64(output.Len())) 45 | assertFileSpecEqual(t, spec, input) 46 | // assertFileSpecChecksum(t, spec, input) // output should be exact copy 47 | 48 | // Ensure header/trailer available. 49 | if got, want := c.Header(), input.Header; got != want { 50 | t.Fatalf("Header()=%#v, want %#v", got, want) 51 | } 52 | if got, want := c.Trailer(), input.Trailer; got != want { 53 | t.Fatalf("Trailer()=%#v, want %#v", got, want) 54 | } 55 | }) 56 | 57 | t.Run("SnapshotPageDataOnly", func(t *testing.T) { 58 | spec, err := compactFileSpecs(t, 59 | <x.FileSpec{ 60 | Header: ltx.Header{ 61 | Version: 1, 62 | PageSize: 1024, 63 | Commit: 3, 64 | MinTXID: 1, 65 | MaxTXID: 1, 66 | Timestamp: 1000, 67 | }, 68 | Pages: []ltx.PageSpec{ 69 | {Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x81}, 1024)}, 70 | {Header: ltx.PageHeader{Pgno: 2}, Data: bytes.Repeat([]byte{0x82}, 1024)}, 71 | {Header: ltx.PageHeader{Pgno: 3}, Data: bytes.Repeat([]byte{0x83}, 1024)}, 72 | }, 73 | Trailer: ltx.Trailer{ 74 | PostApplyChecksum: 0x8a249272ad9f7dea, 75 | }, 76 | }, 77 | <x.FileSpec{ 78 | Header: ltx.Header{ 79 | Version: 1, 80 | PageSize: 1024, 81 | Commit: 3, 82 | MinTXID: 2, 83 | MaxTXID: 2, 84 | Timestamp: 2000, 85 | PreApplyChecksum: 0x8a249272ad9f7dea, 86 | }, 87 | Pages: []ltx.PageSpec{ 88 | {Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x91}, 1024)}, 89 | {Header: ltx.PageHeader{Pgno: 3}, Data: bytes.Repeat([]byte{0x93}, 1024)}, 90 | }, 91 | Trailer: ltx.Trailer{ 92 | PostApplyChecksum: 0x8a249272ad9f7dea, 93 | }, 94 | }, 95 | ) 96 | if err != nil { 97 | t.Fatal(err) 98 | } 99 | 100 | assertFileSpecEqual(t, spec, <x.FileSpec{ 101 | Header: ltx.Header{ 102 | Version: 1, 103 | PageSize: 1024, 104 | Commit: 3, 105 | MinTXID: 1, 106 | MaxTXID: 2, 107 | Timestamp: 2000, 108 | }, 109 | Pages: []ltx.PageSpec{ 110 | {Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x91}, 1024)}, 111 | {Header: ltx.PageHeader{Pgno: 2}, Data: bytes.Repeat([]byte{0x82}, 1024)}, 112 | {Header: ltx.PageHeader{Pgno: 3}, Data: bytes.Repeat([]byte{0x93}, 1024)}, 113 | }, 114 | Trailer: ltx.Trailer{ 115 | PostApplyChecksum: 0x8a249272ad9f7dea, 116 | FileChecksum: 0xcaf341fe1e6cddfb, 117 | }, 118 | }) 119 | }) 120 | t.Run("NonSnapshotPageDataOnly", func(t *testing.T) { 121 | spec, err := compactFileSpecs(t, 122 | <x.FileSpec{ 123 | Header: ltx.Header{ 124 | Version: 1, 125 | PageSize: 1024, 126 | Commit: 3, 127 | MinTXID: 2, 128 | MaxTXID: 3, 129 | Timestamp: 1000, 130 | PreApplyChecksum: ltx.ChecksumFlag | 2, 131 | }, 132 | Pages: []ltx.PageSpec{ 133 | {Header: ltx.PageHeader{Pgno: 3}, Data: bytes.Repeat([]byte{0x83}, 1024)}, 134 | }, 135 | Trailer: ltx.Trailer{ 136 | PostApplyChecksum: ltx.ChecksumFlag | 3, 137 | }, 138 | }, 139 | <x.FileSpec{ 140 | Header: ltx.Header{ 141 | Version: 1, 142 | PageSize: 1024, 143 | Commit: 3, 144 | MinTXID: 4, 145 | MaxTXID: 5, 146 | Timestamp: 2000, 147 | PreApplyChecksum: ltx.ChecksumFlag | 4, 148 | }, 149 | Pages: []ltx.PageSpec{ 150 | {Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x91}, 1024)}, 151 | }, 152 | Trailer: ltx.Trailer{ 153 | PostApplyChecksum: ltx.ChecksumFlag | 5, 154 | }, 155 | }, 156 | <x.FileSpec{ 157 | Header: ltx.Header{ 158 | Version: 1, 159 | PageSize: 1024, 160 | Commit: 5, 161 | MinTXID: 6, 162 | MaxTXID: 9, 163 | Timestamp: 3000, 164 | PreApplyChecksum: ltx.ChecksumFlag | 6, 165 | }, 166 | Pages: []ltx.PageSpec{ 167 | {Header: ltx.PageHeader{Pgno: 2}, Data: bytes.Repeat([]byte{0xa2}, 1024)}, 168 | {Header: ltx.PageHeader{Pgno: 3}, Data: bytes.Repeat([]byte{0xa3}, 1024)}, 169 | {Header: ltx.PageHeader{Pgno: 5}, Data: bytes.Repeat([]byte{0xa5}, 1024)}, 170 | }, 171 | Trailer: ltx.Trailer{ 172 | PostApplyChecksum: ltx.ChecksumFlag | 9, 173 | }, 174 | }, 175 | ) 176 | if err != nil { 177 | t.Fatal(err) 178 | } 179 | 180 | assertFileSpecEqual(t, spec, <x.FileSpec{ 181 | Header: ltx.Header{ 182 | Version: 1, 183 | PageSize: 1024, 184 | Commit: 5, 185 | MinTXID: 2, 186 | MaxTXID: 9, 187 | Timestamp: 3000, 188 | PreApplyChecksum: ltx.ChecksumFlag | 2, 189 | }, 190 | Pages: []ltx.PageSpec{ 191 | {Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x91}, 1024)}, 192 | {Header: ltx.PageHeader{Pgno: 2}, Data: bytes.Repeat([]byte{0xa2}, 1024)}, 193 | {Header: ltx.PageHeader{Pgno: 3}, Data: bytes.Repeat([]byte{0xa3}, 1024)}, 194 | {Header: ltx.PageHeader{Pgno: 5}, Data: bytes.Repeat([]byte{0xa5}, 1024)}, 195 | }, 196 | Trailer: ltx.Trailer{ 197 | PostApplyChecksum: ltx.ChecksumFlag | 9, 198 | FileChecksum: 0xead633959f3c67a8, 199 | }, 200 | }) 201 | }) 202 | 203 | t.Run("Shrinking", func(t *testing.T) { 204 | spec, err := compactFileSpecs(t, 205 | <x.FileSpec{ 206 | Header: ltx.Header{Version: 1, PageSize: 1024, Commit: 3, MinTXID: 2, MaxTXID: 3, Timestamp: 1000, PreApplyChecksum: ltx.ChecksumFlag | 2}, 207 | Pages: []ltx.PageSpec{ 208 | {Header: ltx.PageHeader{Pgno: 3}, Data: bytes.Repeat([]byte{0x83}, 1024)}, 209 | }, 210 | Trailer: ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag | 3}, 211 | }, 212 | <x.FileSpec{ 213 | Header: ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 4, MaxTXID: 5, Timestamp: 2000, PreApplyChecksum: ltx.ChecksumFlag | 4}, 214 | Pages: []ltx.PageSpec{ 215 | {Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x91}, 1024)}, 216 | }, 217 | Trailer: ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag | 5}, 218 | }, 219 | ) 220 | if err != nil { 221 | t.Fatal(err) 222 | } 223 | 224 | assertFileSpecEqual(t, spec, <x.FileSpec{ 225 | Header: ltx.Header{ 226 | Version: 1, 227 | PageSize: 1024, 228 | Commit: 2, 229 | MinTXID: 2, 230 | MaxTXID: 5, 231 | Timestamp: 2000, 232 | PreApplyChecksum: ltx.ChecksumFlag | 2, 233 | }, 234 | Pages: []ltx.PageSpec{ 235 | {Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x91}, 1024)}, 236 | }, 237 | Trailer: ltx.Trailer{ 238 | PostApplyChecksum: ltx.ChecksumFlag | 5, 239 | FileChecksum: 0xf688132c3904f118, 240 | }, 241 | }) 242 | }) 243 | 244 | t.Run("ErrInputReaderRequired", func(t *testing.T) { 245 | c := ltx.NewCompactor(&bytes.Buffer{}, nil) 246 | if err := c.Compact(context.Background()); err == nil || err.Error() != `at least one input reader required` { 247 | t.Fatalf("unexpected error: %s", err) 248 | } 249 | }) 250 | t.Run("ErrPageSizeMismatch", func(t *testing.T) { 251 | _, err := compactFileSpecs(t, 252 | <x.FileSpec{ 253 | Header: ltx.Header{Version: 1, PageSize: 512, Commit: 1, MinTXID: 1, MaxTXID: 1, Timestamp: 1000}, 254 | Pages: []ltx.PageSpec{{Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x81}, 512)}}, 255 | Trailer: ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag | 1}, 256 | }, 257 | <x.FileSpec{ 258 | Header: ltx.Header{Version: 1, PageSize: 1024, Commit: 1, MinTXID: 1, MaxTXID: 1, Timestamp: 1000}, 259 | Pages: []ltx.PageSpec{{Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x91}, 1024)}}, 260 | Trailer: ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag | 1}, 261 | }, 262 | ) 263 | if err == nil || err.Error() != `input files have mismatched page sizes: 512 != 1024` { 264 | t.Fatalf("unexpected error: %s", err) 265 | } 266 | }) 267 | t.Run("ErrNonContiguousTXID", func(t *testing.T) { 268 | _, err := compactFileSpecs(t, 269 | <x.FileSpec{ 270 | Header: ltx.Header{Version: 1, PageSize: 1024, Commit: 1, MinTXID: 1, MaxTXID: 2, Timestamp: 1000}, 271 | Pages: []ltx.PageSpec{{Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x81}, 1024)}}, 272 | Trailer: ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag | 1}, 273 | }, 274 | <x.FileSpec{ 275 | Header: ltx.Header{Version: 1, PageSize: 1024, Commit: 1, MinTXID: 4, MaxTXID: 4, Timestamp: 1000, PreApplyChecksum: ltx.ChecksumFlag | 2}, 276 | Pages: []ltx.PageSpec{{Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x91}, 1024)}}, 277 | Trailer: ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag | 1}, 278 | }, 279 | ) 280 | if err == nil || err.Error() != `non-contiguous transaction ids in input files: (0000000000000001,0000000000000002) -> (0000000000000004,0000000000000004)` { 281 | t.Fatalf("unexpected error: %s", err) 282 | } 283 | }) 284 | t.Run("AllowNonContiguousTXID", func(t *testing.T) { 285 | bufs := make([]bytes.Buffer, 2) 286 | writeFileSpec(t, &bufs[0], <x.FileSpec{ 287 | Header: ltx.Header{Version: 1, PageSize: 1024, Commit: 1, MinTXID: 1, MaxTXID: 2, Timestamp: 1000}, 288 | Pages: []ltx.PageSpec{{Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x81}, 1024)}}, 289 | Trailer: ltx.Trailer{PostApplyChecksum: 0xeb953fc47685d740}, 290 | }) 291 | 292 | writeFileSpec(t, &bufs[1], <x.FileSpec{ 293 | Header: ltx.Header{Version: 1, PageSize: 1024, Commit: 1, MinTXID: 4, MaxTXID: 4, Timestamp: 1000, PreApplyChecksum: ltx.ChecksumFlag | 2}, 294 | Pages: []ltx.PageSpec{{Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte{0x91}, 1024)}}, 295 | Trailer: ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag | 1}, 296 | }) 297 | 298 | // Compact files together. 299 | c := ltx.NewCompactor(io.Discard, []io.Reader{&bufs[0], &bufs[1]}) 300 | c.AllowNonContiguousTXIDs = true 301 | if err := c.Compact(context.Background()); err != nil { 302 | t.Fatalf("unexpected error: %s", err) 303 | } 304 | }) 305 | } 306 | -------------------------------------------------------------------------------- /decoder.go: -------------------------------------------------------------------------------- 1 | package ltx 2 | 3 | import ( 4 | "fmt" 5 | "hash" 6 | "hash/crc64" 7 | "io" 8 | 9 | "github.com/pierrec/lz4/v4" 10 | ) 11 | 12 | // Decoder represents a decoder of an LTX file. 13 | type Decoder struct { 14 | underlying io.Reader // main reader 15 | r io.Reader // current reader (either main or lz4) 16 | 17 | header Header 18 | trailer Trailer 19 | state string 20 | 21 | chksum Checksum 22 | hash hash.Hash64 23 | pageN int // pages read 24 | n int64 // bytes read 25 | } 26 | 27 | // NewDecoder returns a new instance of Decoder. 28 | func NewDecoder(r io.Reader) *Decoder { 29 | return &Decoder{ 30 | underlying: r, 31 | r: r, 32 | state: stateHeader, 33 | chksum: ChecksumFlag, 34 | hash: crc64.New(crc64.MakeTable(crc64.ISO)), 35 | } 36 | } 37 | 38 | // N returns the number of bytes read. 39 | func (dec *Decoder) N() int64 { return dec.n } 40 | 41 | // PageN returns the number of pages read. 42 | func (dec *Decoder) PageN() int { return dec.pageN } 43 | 44 | // Header returns a copy of the header. 45 | func (dec *Decoder) Header() Header { return dec.header } 46 | 47 | // Trailer returns a copy of the trailer. File checksum available after Close(). 48 | func (dec *Decoder) Trailer() Trailer { return dec.trailer } 49 | 50 | // PostApplyPos returns the replication position after underlying the LTX file is applied. 51 | // Only valid after successful Close(). 52 | func (dec *Decoder) PostApplyPos() Pos { 53 | return Pos{ 54 | TXID: dec.header.MaxTXID, 55 | PostApplyChecksum: dec.trailer.PostApplyChecksum, 56 | } 57 | } 58 | 59 | // Close verifies the reader is at the end of the file and that the checksum matches. 60 | func (dec *Decoder) Close() error { 61 | if dec.state == stateClosed { 62 | return nil // no-op 63 | } else if dec.state != stateClose { 64 | return fmt.Errorf("cannot close, expected %s", dec.state) 65 | } 66 | 67 | // Read trailer. 68 | b := make([]byte, TrailerSize) 69 | if _, err := io.ReadFull(dec.r, b); err != nil { 70 | return err 71 | } else if err := dec.trailer.UnmarshalBinary(b); err != nil { 72 | return fmt.Errorf("unmarshal trailer: %w", err) 73 | } 74 | 75 | // TODO: Ensure last read page is equal to the commit for snapshot LTX files 76 | 77 | dec.writeToHash(b[:TrailerChecksumOffset]) 78 | 79 | // Compare checksum with checksum in trailer. 80 | if chksum := ChecksumFlag | Checksum(dec.hash.Sum64()); chksum != dec.trailer.FileChecksum { 81 | return ErrChecksumMismatch 82 | } 83 | 84 | // Verify post-apply checksum for snapshot files. 85 | if dec.header.IsSnapshot() { 86 | if dec.trailer.PostApplyChecksum != dec.chksum { 87 | return fmt.Errorf("post-apply checksum in trailer (%s) does not match calculated checksum (%s)", dec.trailer.PostApplyChecksum, dec.chksum) 88 | } 89 | } 90 | 91 | // Update state to mark as closed. 92 | dec.state = stateClosed 93 | 94 | return nil 95 | } 96 | 97 | // DecodeHeader reads the LTX file header frame and stores it internally. 98 | // Call Header() to retrieve the header after this is successfully called. 99 | func (dec *Decoder) DecodeHeader() error { 100 | b := make([]byte, HeaderSize) 101 | if _, err := io.ReadFull(dec.r, b); err != nil { 102 | return err 103 | } else if err := dec.header.UnmarshalBinary(b); err != nil { 104 | return fmt.Errorf("unmarshal header: %w", err) 105 | } 106 | 107 | dec.writeToHash(b) 108 | dec.state = statePage 109 | 110 | if err := dec.header.Validate(); err != nil { 111 | return err 112 | } 113 | 114 | // Use LZ4 reader if compression is enabled. 115 | if dec.header.Flags&HeaderFlagCompressLZ4 != 0 { 116 | dec.r = lz4.NewReader(dec.underlying) 117 | } 118 | 119 | return nil 120 | } 121 | 122 | // DecodePage reads the next page header into hdr and associated page data. 123 | func (dec *Decoder) DecodePage(hdr *PageHeader, data []byte) error { 124 | if dec.state == stateClosed { 125 | return ErrDecoderClosed 126 | } else if dec.state == stateClose { 127 | return io.EOF 128 | } else if dec.state != statePage { 129 | return fmt.Errorf("cannot read page header, expected %s", dec.state) 130 | } else if uint32(len(data)) != dec.header.PageSize { 131 | return fmt.Errorf("invalid page buffer size: %d, expecting %d", len(data), dec.header.PageSize) 132 | } 133 | 134 | // Read and unmarshal page header. 135 | b := make([]byte, PageHeaderSize) 136 | if _, err := io.ReadFull(dec.r, b); err != nil { 137 | return err 138 | } else if err := hdr.UnmarshalBinary(b); err != nil { 139 | return fmt.Errorf("unmarshal: %w", err) 140 | } 141 | 142 | dec.writeToHash(b) 143 | 144 | // An empty page header indicates the end of the page block. 145 | if hdr.IsZero() { 146 | // Revert back to regular reader if we were using a compressed reader. 147 | // We need to read off the LZ4 trailer frame to ensure we hit EOF. 148 | if zr, ok := dec.r.(*lz4.Reader); ok { 149 | if _, err := io.ReadFull(zr, make([]byte, 1)); err != io.EOF { 150 | return fmt.Errorf("expected lz4 end frame") 151 | } 152 | dec.r = dec.underlying 153 | } 154 | 155 | dec.state = stateClose 156 | return io.EOF 157 | } 158 | 159 | if err := hdr.Validate(); err != nil { 160 | return err 161 | } 162 | 163 | // Read page data next. 164 | if _, err := io.ReadFull(dec.r, data); err != nil { 165 | return err 166 | } 167 | dec.writeToHash(data) 168 | dec.pageN++ 169 | 170 | // Calculate checksum while decoding snapshots. 171 | if dec.header.IsSnapshot() { 172 | if hdr.Pgno != LockPgno(dec.header.PageSize) { 173 | dec.chksum = ChecksumFlag | (dec.chksum ^ ChecksumPage(hdr.Pgno, data)) 174 | } 175 | } 176 | 177 | return nil 178 | } 179 | 180 | // Verify reads the entire file. Header & trailer can be accessed via methods 181 | // after the file is successfully verified. All other data is discarded. 182 | func (dec *Decoder) Verify() error { 183 | if err := dec.DecodeHeader(); err != nil { 184 | return fmt.Errorf("decode header: %w", err) 185 | } 186 | 187 | var pageHeader PageHeader 188 | data := make([]byte, dec.header.PageSize) 189 | for i := 0; ; i++ { 190 | if err := dec.DecodePage(&pageHeader, data); err == io.EOF { 191 | break 192 | } else if err != nil { 193 | return fmt.Errorf("decode page %d: %w", i, err) 194 | } 195 | } 196 | 197 | if err := dec.Close(); err != nil { 198 | return fmt.Errorf("close reader: %w", err) 199 | } 200 | return nil 201 | } 202 | 203 | // DecodeDatabaseTo decodes the LTX file as a SQLite database to w. 204 | // The LTX file MUST be a snapshot file. 205 | func (dec *Decoder) DecodeDatabaseTo(w io.Writer) error { 206 | if err := dec.DecodeHeader(); err != nil { 207 | return fmt.Errorf("decode header: %w", err) 208 | } 209 | 210 | hdr := dec.Header() 211 | lockPgno := hdr.LockPgno() 212 | if !dec.header.IsSnapshot() { 213 | return fmt.Errorf("cannot decode non-snapshot LTX file to SQLite database") 214 | } 215 | 216 | var pageHeader PageHeader 217 | data := make([]byte, dec.header.PageSize) 218 | for pgno := uint32(1); pgno <= hdr.Commit; pgno++ { 219 | if pgno == lockPgno { 220 | // Write empty page for lock page. 221 | for i := range data { 222 | data[i] = 0 223 | } 224 | } else { 225 | // Otherwise read the page from the LTX decoder. 226 | if err := dec.DecodePage(&pageHeader, data); err != nil { 227 | return fmt.Errorf("decode page %d: %w", pgno, err) 228 | } else if pageHeader.Pgno != pgno { 229 | return fmt.Errorf("unexpected pgno while decoding page: read %d, expected %d", pageHeader.Pgno, pgno) 230 | } 231 | } 232 | 233 | if _, err := w.Write(data); err != nil { 234 | return fmt.Errorf("write page %d: %w", pgno, err) 235 | } 236 | } 237 | 238 | // Issue one more final read and expect to see an EOF. This is required so 239 | // that the decoder can successfully close and validate. 240 | if err := dec.DecodePage(&pageHeader, data); err == nil { 241 | return fmt.Errorf("unexpected page %d after commit %d", pageHeader.Pgno, hdr.Commit) 242 | } else if err != io.EOF { 243 | return fmt.Errorf("unexpected error decoding after end of database: %w", err) 244 | } 245 | 246 | if err := dec.Close(); err != nil { 247 | return fmt.Errorf("close decoder: %w", err) 248 | } 249 | return nil 250 | } 251 | 252 | func (dec *Decoder) writeToHash(b []byte) { 253 | _, _ = dec.hash.Write(b) 254 | dec.n += int64(len(b)) 255 | } 256 | 257 | // DecodeHeader decodes the header from r. Returns the header & read bytes. 258 | func DecodeHeader(r io.Reader) (hdr Header, data []byte, err error) { 259 | data = make([]byte, HeaderSize) 260 | n, err := io.ReadFull(r, data) 261 | if err != nil { 262 | return hdr, data[:n], err 263 | } else if err := hdr.UnmarshalBinary(data); err != nil { 264 | return hdr, data[:n], err 265 | } 266 | return hdr, data, nil 267 | } 268 | -------------------------------------------------------------------------------- /decoder_test.go: -------------------------------------------------------------------------------- 1 | package ltx_test 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/superfly/ltx" 10 | ) 11 | 12 | func TestDecoder(t *testing.T) { 13 | testDecoder(t, "OK", 0) 14 | testDecoder(t, "CompressLZ4", ltx.HeaderFlagCompressLZ4) 15 | } 16 | 17 | func testDecoder(t *testing.T, name string, flags uint32) { 18 | t.Run(name, func(t *testing.T) { 19 | spec := <x.FileSpec{ 20 | Header: ltx.Header{ 21 | Version: 1, 22 | Flags: flags, 23 | PageSize: 1024, 24 | Commit: 2, 25 | MinTXID: 1, 26 | MaxTXID: 1, 27 | Timestamp: 1000, 28 | }, 29 | Pages: []ltx.PageSpec{ 30 | {Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte("2"), 1024)}, 31 | {Header: ltx.PageHeader{Pgno: 2}, Data: bytes.Repeat([]byte("3"), 1024)}, 32 | }, 33 | Trailer: ltx.Trailer{PostApplyChecksum: 0xe1899b6d587aaaaa}, 34 | } 35 | 36 | // Write spec to file. 37 | var buf bytes.Buffer 38 | writeFileSpec(t, &buf, spec) 39 | 40 | // Read and verify data matches spec. 41 | dec := ltx.NewDecoder(&buf) 42 | 43 | // Verify header. 44 | if err := dec.DecodeHeader(); err != nil { 45 | t.Fatal(err) 46 | } else if got, want := dec.Header(), spec.Header; !reflect.DeepEqual(got, want) { 47 | t.Fatalf("header mismatch:\ngot=%#v\nwant=%#v", got, want) 48 | } 49 | 50 | // Verify page headers. 51 | for i := range spec.Pages { 52 | var hdr ltx.PageHeader 53 | data := make([]byte, 1024) 54 | if err := dec.DecodePage(&hdr, data); err != nil { 55 | t.Fatal(err) 56 | } else if got, want := hdr, spec.Pages[i].Header; got != want { 57 | t.Fatalf("page hdr mismatch:\ngot=%#v\nwant=%#v", got, want) 58 | } else if got, want := data, spec.Pages[i].Data; !bytes.Equal(got, want) { 59 | t.Fatalf("page data mismatch:\ngot=%#v\nwant=%#v", got, want) 60 | } 61 | } 62 | 63 | if err := dec.DecodePage(<x.PageHeader{}, make([]byte, 1024)); err != io.EOF { 64 | t.Fatal("expected page header eof") 65 | } 66 | 67 | // Close reader to verify integrity. 68 | if err := dec.Close(); err != nil { 69 | t.Fatal(err) 70 | } 71 | 72 | if got, want := dec.Header().PreApplyPos(), (ltx.Pos{}); got != want { 73 | t.Fatalf("PreApplyPos=%s, want %s", got, want) 74 | } 75 | if got, want := dec.PostApplyPos(), (ltx.Pos{1, 0xe1899b6d587aaaaa}); got != want { 76 | t.Fatalf("PostApplyPos=%s, want %s", got, want) 77 | } 78 | }) 79 | } 80 | 81 | func TestDecoder_Decode_CommitZero(t *testing.T) { 82 | spec := <x.FileSpec{ 83 | Header: ltx.Header{ 84 | Version: 1, 85 | Flags: 0, 86 | PageSize: 1024, 87 | Commit: 0, 88 | MinTXID: 1, 89 | MaxTXID: 1, 90 | Timestamp: 1000, 91 | }, 92 | Trailer: ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag}, 93 | } 94 | 95 | // Write spec to file. 96 | var buf bytes.Buffer 97 | writeFileSpec(t, &buf, spec) 98 | 99 | // Read and verify data matches spec. 100 | dec := ltx.NewDecoder(&buf) 101 | 102 | // Verify header. 103 | if err := dec.DecodeHeader(); err != nil { 104 | t.Fatal(err) 105 | } else if got, want := dec.Header(), spec.Header; !reflect.DeepEqual(got, want) { 106 | t.Fatalf("header mismatch:\ngot=%#v\nwant=%#v", got, want) 107 | } 108 | 109 | if err := dec.DecodePage(<x.PageHeader{}, make([]byte, 1024)); err != io.EOF { 110 | t.Fatal("expected page header eof") 111 | } 112 | 113 | // Close reader to verify integrity. 114 | if err := dec.Close(); err != nil { 115 | t.Fatal(err) 116 | } 117 | 118 | if got, want := dec.Header().PreApplyPos(), (ltx.Pos{}); got != want { 119 | t.Fatalf("PreApplyPos=%s, want %s", got, want) 120 | } 121 | if got, want := dec.PostApplyPos(), (ltx.Pos{1, ltx.ChecksumFlag}); got != want { 122 | t.Fatalf("PostApplyPos=%s, want %s", got, want) 123 | } 124 | } 125 | 126 | func TestDecoder_DecodeDatabaseTo(t *testing.T) { 127 | t.Run("OK", func(t *testing.T) { 128 | spec := <x.FileSpec{ 129 | Header: ltx.Header{Version: 1, Flags: 0, PageSize: 512, Commit: 2, MinTXID: 1, MaxTXID: 2, Timestamp: 1000}, 130 | Pages: []ltx.PageSpec{ 131 | {Header: ltx.PageHeader{Pgno: 1}, Data: bytes.Repeat([]byte("2"), 512)}, 132 | {Header: ltx.PageHeader{Pgno: 2}, Data: bytes.Repeat([]byte("3"), 512)}, 133 | }, 134 | Trailer: ltx.Trailer{PostApplyChecksum: 0x8b87423eeeeeeeee}, 135 | } 136 | 137 | // Decode serialized LTX file. 138 | var buf bytes.Buffer 139 | writeFileSpec(t, &buf, spec) 140 | dec := ltx.NewDecoder(&buf) 141 | 142 | var out bytes.Buffer 143 | if err := dec.DecodeDatabaseTo(&out); err != nil { 144 | t.Fatal(err) 145 | } else if got, want := out.Bytes(), append(bytes.Repeat([]byte("2"), 512), bytes.Repeat([]byte("3"), 512)...); !bytes.Equal(got, want) { 146 | t.Fatal("output mismatch") 147 | } 148 | }) 149 | 150 | t.Run("WithLockPage", func(t *testing.T) { 151 | lockPgno := ltx.LockPgno(4096) 152 | commit := lockPgno + 10 153 | 154 | var want bytes.Buffer 155 | var buf bytes.Buffer 156 | enc := ltx.NewEncoder(&buf) 157 | if err := enc.EncodeHeader(ltx.Header{Version: 1, Flags: 0, PageSize: 4096, Commit: commit, MinTXID: 1, MaxTXID: 2, Timestamp: 1000}); err != nil { 158 | t.Fatal(err) 159 | } 160 | 161 | pageData := bytes.Repeat([]byte("x"), 4096) 162 | for pgno := uint32(1); pgno <= commit; pgno++ { 163 | if pgno == lockPgno { 164 | _, _ = want.Write(make([]byte, 4096)) 165 | continue 166 | } 167 | 168 | _, _ = want.Write(pageData) 169 | if err := enc.EncodePage(ltx.PageHeader{Pgno: pgno}, pageData); err != nil { 170 | t.Fatal(err) 171 | } 172 | } 173 | 174 | enc.SetPostApplyChecksum(0xc19b668c376662c7) 175 | if err := enc.Close(); err != nil { 176 | t.Fatal(err) 177 | } 178 | 179 | // Decode serialized LTX file. 180 | dec := ltx.NewDecoder(&buf) 181 | 182 | var out bytes.Buffer 183 | if err := dec.DecodeDatabaseTo(&out); err != nil { 184 | t.Fatal(err) 185 | } else if got, want := out.Bytes(), want.Bytes(); !bytes.Equal(got, want) { 186 | t.Fatal("output mismatch") 187 | } 188 | }) 189 | 190 | t.Run("ErrNonSnapshot", func(t *testing.T) { 191 | spec := <x.FileSpec{ 192 | Header: ltx.Header{Version: 1, Flags: 0, PageSize: 512, Commit: 2, MinTXID: 2, MaxTXID: 2, Timestamp: 1000, PreApplyChecksum: ltx.ChecksumFlag | 1}, 193 | Pages: []ltx.PageSpec{ 194 | {Header: ltx.PageHeader{Pgno: 2}, Data: bytes.Repeat([]byte("3"), 512)}, 195 | }, 196 | Trailer: ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag | 1}, 197 | } 198 | 199 | // Decode serialized LTX file. 200 | var buf bytes.Buffer 201 | writeFileSpec(t, &buf, spec) 202 | dec := ltx.NewDecoder(&buf) 203 | if err := dec.DecodeDatabaseTo(io.Discard); err == nil || err.Error() != `cannot decode non-snapshot LTX file to SQLite database` { 204 | t.Fatal(err) 205 | } 206 | }) 207 | } 208 | -------------------------------------------------------------------------------- /dist/ltx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/superfly/ltx/2c8411b9cc92ef74f5a7068c4844e4c2e7ccac57/dist/ltx -------------------------------------------------------------------------------- /encoder.go: -------------------------------------------------------------------------------- 1 | package ltx 2 | 3 | import ( 4 | "fmt" 5 | "hash" 6 | "hash/crc64" 7 | "io" 8 | 9 | "github.com/pierrec/lz4/v4" 10 | ) 11 | 12 | // Encoder implements a encoder for an LTX file. 13 | type Encoder struct { 14 | underlying io.Writer // main writer 15 | w io.Writer // current writer (main or lz4 writer) 16 | state string 17 | 18 | header Header 19 | trailer Trailer 20 | hash hash.Hash64 21 | n int64 // bytes written 22 | 23 | // Track how many of each write has occurred to move state. 24 | prevPgno uint32 25 | pagesWritten uint32 26 | } 27 | 28 | // NewEncoder returns a new instance of Encoder. 29 | func NewEncoder(w io.Writer) *Encoder { 30 | return &Encoder{ 31 | underlying: w, 32 | w: w, 33 | state: stateHeader, 34 | } 35 | } 36 | 37 | // N returns the number of bytes written. 38 | func (enc *Encoder) N() int64 { return enc.n } 39 | 40 | // Header returns a copy of the header. 41 | func (enc *Encoder) Header() Header { return enc.header } 42 | 43 | // Trailer returns a copy of the trailer. File checksum available after Close(). 44 | func (enc *Encoder) Trailer() Trailer { return enc.trailer } 45 | 46 | // PostApplyPos returns the replication position after underlying the LTX file is applied. 47 | // Only valid after successful Close(). 48 | func (enc *Encoder) PostApplyPos() Pos { 49 | return Pos{ 50 | TXID: enc.header.MaxTXID, 51 | PostApplyChecksum: enc.trailer.PostApplyChecksum, 52 | } 53 | } 54 | 55 | // SetPostApplyChecksum sets the post-apply checksum of the database. 56 | // Must call before Close(). 57 | func (enc *Encoder) SetPostApplyChecksum(chksum Checksum) { 58 | enc.trailer.PostApplyChecksum = chksum 59 | } 60 | 61 | // Close flushes the checksum to the header. 62 | func (enc *Encoder) Close() error { 63 | if enc.state == stateClosed { 64 | return nil // no-op 65 | } else if enc.state != statePage { 66 | return fmt.Errorf("cannot close, expected %s", enc.state) 67 | } 68 | 69 | // Marshal empty page header to mark end of page block. 70 | b0, err := (&PageHeader{}).MarshalBinary() 71 | if err != nil { 72 | return fmt.Errorf("marshal empty page header: %w", err) 73 | } else if _, err := enc.write(b0); err != nil { 74 | return fmt.Errorf("write empty page header: %w", err) 75 | } 76 | 77 | // Close the compressed writer, if in use. 78 | if zw, ok := enc.w.(*lz4.Writer); ok { 79 | if err := zw.Close(); err != nil { 80 | return fmt.Errorf("cannot close lz4 writer: %w", err) 81 | } 82 | } 83 | 84 | // Revert to original writer now that we've passed the compressed body. 85 | enc.w = enc.underlying 86 | 87 | // Marshal trailer to bytes. 88 | b1, err := enc.trailer.MarshalBinary() 89 | if err != nil { 90 | return fmt.Errorf("marshal trailer: %w", err) 91 | } 92 | enc.writeToHash(b1[:TrailerChecksumOffset]) 93 | enc.trailer.FileChecksum = ChecksumFlag | Checksum(enc.hash.Sum64()) 94 | 95 | // Validate trailer now that we have the file checksum. 96 | if err := enc.trailer.Validate(); err != nil { 97 | return fmt.Errorf("validate trailer: %w", err) 98 | } 99 | 100 | // If we are encoding a deletion LTX file then ensure that we have an empty checksum. 101 | if enc.header.Commit == 0 && enc.trailer.PostApplyChecksum != ChecksumFlag { 102 | return fmt.Errorf("post-apply checksum must be empty for zero-length database") 103 | } 104 | 105 | // Remarshal with correct checksum. 106 | b1, err = enc.trailer.MarshalBinary() 107 | if err != nil { 108 | return fmt.Errorf("marshal trailer: %w", err) 109 | } else if _, err := enc.w.Write(b1); err != nil { 110 | return fmt.Errorf("write trailer: %w", err) 111 | } 112 | enc.n += ChecksumSize 113 | 114 | enc.state = stateClosed 115 | 116 | return nil 117 | } 118 | 119 | // EncodeHeader writes hdr to the file's header block. 120 | func (enc *Encoder) EncodeHeader(hdr Header) error { 121 | if enc.state == stateClosed { 122 | return ErrEncoderClosed 123 | } else if enc.state != stateHeader { 124 | return fmt.Errorf("cannot encode header frame, expected %s", enc.state) 125 | } else if err := hdr.Validate(); err != nil { 126 | return err 127 | } 128 | 129 | enc.header = hdr 130 | 131 | // Initialize hash. 132 | enc.hash = crc64.New(crc64.MakeTable(crc64.ISO)) 133 | 134 | // Write header to underlying writer. 135 | b, err := enc.header.MarshalBinary() 136 | if err != nil { 137 | return fmt.Errorf("marshal header: %w", err) 138 | } else if _, err := enc.write(b); err != nil { 139 | return fmt.Errorf("write header: %w", err) 140 | } 141 | 142 | // Use a compressed writer for the body if LZ4 is enabled. 143 | if enc.header.Flags&HeaderFlagCompressLZ4 != 0 { 144 | zw := lz4.NewWriter(enc.underlying) 145 | if err := zw.Apply(lz4.BlockSizeOption(lz4.Block64Kb)); err != nil { // minimize memory allocation 146 | return fmt.Errorf("cannot set lz4 block size: %w", err) 147 | } 148 | if err := zw.Apply(lz4.CompressionLevelOption(lz4.Fast)); err != nil { 149 | return fmt.Errorf("cannot set lz4 compression level: %w", err) 150 | } 151 | enc.w = zw 152 | } 153 | 154 | // Move writer state to write page headers. 155 | enc.state = statePage // file must have at least one page 156 | 157 | return nil 158 | } 159 | 160 | // EncodePage writes hdr & data to the file's page block. 161 | func (enc *Encoder) EncodePage(hdr PageHeader, data []byte) (err error) { 162 | if enc.state == stateClosed { 163 | return ErrEncoderClosed 164 | } else if enc.state != statePage { 165 | return fmt.Errorf("cannot encode page header, expected %s", enc.state) 166 | } else if hdr.Pgno > enc.header.Commit { 167 | return fmt.Errorf("page number %d out-of-bounds for commit size %d", hdr.Pgno, enc.header.Commit) 168 | } else if err := hdr.Validate(); err != nil { 169 | return err 170 | } else if uint32(len(data)) != enc.header.PageSize { 171 | return fmt.Errorf("invalid page buffer size: %d, expecting %d", len(data), enc.header.PageSize) 172 | } 173 | 174 | lockPgno := LockPgno(enc.header.PageSize) 175 | if hdr.Pgno == lockPgno { 176 | return fmt.Errorf("cannot encode lock page: pgno=%d", hdr.Pgno) 177 | } 178 | 179 | // Snapshots must start with page 1 and include all pages up to the commit size. 180 | // Non-snapshot files can include any pages but they must be in order. 181 | if enc.header.IsSnapshot() { 182 | if enc.prevPgno == 0 && hdr.Pgno != 1 { 183 | return fmt.Errorf("snapshot transaction file must start with page number 1") 184 | } 185 | 186 | if enc.prevPgno == lockPgno-1 { 187 | if hdr.Pgno != enc.prevPgno+2 { // skip lock page 188 | return fmt.Errorf("nonsequential page numbers in snapshot transaction (skip lock page): %d,%d", enc.prevPgno, hdr.Pgno) 189 | } 190 | } else if enc.prevPgno != 0 && hdr.Pgno != enc.prevPgno+1 { 191 | return fmt.Errorf("nonsequential page numbers in snapshot transaction: %d,%d", enc.prevPgno, hdr.Pgno) 192 | } 193 | } else { 194 | if enc.prevPgno >= hdr.Pgno { 195 | return fmt.Errorf("out-of-order page numbers: %d,%d", enc.prevPgno, hdr.Pgno) 196 | } 197 | } 198 | 199 | // Encode & write header. 200 | b, err := hdr.MarshalBinary() 201 | if err != nil { 202 | return fmt.Errorf("marshal: %w", err) 203 | } else if _, err := enc.write(b); err != nil { 204 | return fmt.Errorf("write: %w", err) 205 | } 206 | 207 | // Write data to file. 208 | if _, err = enc.write(data); err != nil { 209 | return fmt.Errorf("write page data: %w", err) 210 | } 211 | 212 | enc.pagesWritten++ 213 | enc.prevPgno = hdr.Pgno 214 | return nil 215 | } 216 | 217 | // write to the current writer & add to the checksum. 218 | func (enc *Encoder) write(b []byte) (n int, err error) { 219 | n, err = enc.w.Write(b) 220 | enc.writeToHash(b[:n]) 221 | return n, err 222 | } 223 | 224 | func (enc *Encoder) writeToHash(b []byte) { 225 | _, _ = enc.hash.Write(b) 226 | enc.n += int64(len(b)) 227 | } 228 | -------------------------------------------------------------------------------- /encoder_test.go: -------------------------------------------------------------------------------- 1 | package ltx_test 2 | 3 | import ( 4 | "io" 5 | "math/rand" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/superfly/ltx" 10 | ) 11 | 12 | func TestEncoder(t *testing.T) { 13 | t.Run("OK", func(t *testing.T) { 14 | rnd := rand.New(rand.NewSource(0)) 15 | page0 := make([]byte, 4096) 16 | rnd.Read(page0) 17 | page1 := make([]byte, 4096) 18 | rnd.Read(page1) 19 | 20 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 21 | if err := enc.EncodeHeader(ltx.Header{ 22 | Version: 1, 23 | PageSize: 4096, 24 | Commit: 3, 25 | MinTXID: 5, 26 | MaxTXID: 6, 27 | Timestamp: 2000, 28 | PreApplyChecksum: ltx.ChecksumFlag | 5, 29 | }); err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | // Write pages. 34 | if err := enc.EncodePage(ltx.PageHeader{Pgno: 1}, page0); err != nil { 35 | t.Fatal(err) 36 | } 37 | 38 | if err := enc.EncodePage(ltx.PageHeader{Pgno: 2}, page1); err != nil { 39 | t.Fatal(err) 40 | } 41 | 42 | // Flush checksum to header. 43 | enc.SetPostApplyChecksum(ltx.ChecksumFlag | 6) 44 | if err := enc.Close(); err != nil { 45 | t.Fatal(err) 46 | } 47 | 48 | // Double close should be a no-op. 49 | if err := enc.Close(); err != nil { 50 | t.Fatal(err) 51 | } 52 | 53 | if got, want := enc.Header().PreApplyPos(), (ltx.Pos{4, ltx.ChecksumFlag | 5}); got != want { 54 | t.Fatalf("PreApplyPos=%s, want %s", got, want) 55 | } 56 | if got, want := enc.PostApplyPos(), (ltx.Pos{6, ltx.ChecksumFlag | 6}); got != want { 57 | t.Fatalf("PostApplyPos=%s, want %s", got, want) 58 | } 59 | }) 60 | 61 | // Ensure encoder can generate LTX files with a zero commit and no pages. 62 | t.Run("CommitZero", func(t *testing.T) { 63 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 64 | if err := enc.EncodeHeader(ltx.Header{ 65 | Version: 1, 66 | PageSize: 4096, 67 | Commit: 0, 68 | MinTXID: 5, 69 | MaxTXID: 6, 70 | Timestamp: 2000, 71 | PreApplyChecksum: ltx.ChecksumFlag | 5, 72 | }); err != nil { 73 | t.Fatal(err) 74 | } 75 | 76 | enc.SetPostApplyChecksum(ltx.ChecksumFlag) 77 | if err := enc.Close(); err != nil { 78 | t.Fatal(err) 79 | } 80 | 81 | if got, want := enc.Header().PreApplyPos(), (ltx.Pos{4, ltx.ChecksumFlag | 5}); got != want { 82 | t.Fatalf("PreApplyPos=%s, want %s", got, want) 83 | } 84 | if got, want := enc.PostApplyPos(), (ltx.Pos{6, ltx.ChecksumFlag}); got != want { 85 | t.Fatalf("PostApplyPos=%s, want %s", got, want) 86 | } 87 | }) 88 | 89 | // Ensure encoder has an empty post-apply checksum when encoding a deletion file. 90 | t.Run("ErrInvalidCommitZeroPostApplyChecksum", func(t *testing.T) { 91 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 92 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 4096, Commit: 0, MinTXID: 5, MaxTXID: 6, Timestamp: 2000, PreApplyChecksum: ltx.ChecksumFlag | 5}); err != nil { 93 | t.Fatal(err) 94 | } 95 | 96 | enc.SetPostApplyChecksum(ltx.ChecksumFlag | 1) 97 | if err := enc.Close(); err == nil || err.Error() != `post-apply checksum must be empty for zero-length database` { 98 | t.Fatalf("unexpected error: %s", err) 99 | } 100 | }) 101 | } 102 | 103 | func TestEncode_Close(t *testing.T) { 104 | t.Run("ErrInvalidState", func(t *testing.T) { 105 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 106 | if err := enc.Close(); err == nil || err.Error() != `cannot close, expected header` { 107 | t.Fatalf("unexpected error: %s", err) 108 | } 109 | }) 110 | 111 | t.Run("ErrClosed", func(t *testing.T) { 112 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 113 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 1024, Commit: 1, MinTXID: 1, MaxTXID: 1}); err != nil { 114 | t.Fatal(err) 115 | } else if err := enc.EncodePage(ltx.PageHeader{Pgno: 1}, make([]byte, 1024)); err != nil { 116 | t.Fatal(err) 117 | } 118 | 119 | enc.SetPostApplyChecksum(ltx.ChecksumFlag) 120 | if err := enc.Close(); err != nil { 121 | t.Fatal(err) 122 | } 123 | 124 | // Ensure all methods return an error after close. 125 | if err := enc.EncodeHeader(ltx.Header{}); err != ltx.ErrEncoderClosed { 126 | t.Fatal(err) 127 | } else if err := enc.EncodePage(ltx.PageHeader{}, nil); err != ltx.ErrEncoderClosed { 128 | t.Fatal(err) 129 | } 130 | }) 131 | } 132 | 133 | func TestEncode_EncodeHeader(t *testing.T) { 134 | t.Run("ErrInvalidState", func(t *testing.T) { 135 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 136 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 1024, Commit: 1, MinTXID: 1, MaxTXID: 1}); err != nil { 137 | t.Fatal(err) 138 | } 139 | if err := enc.EncodeHeader(ltx.Header{}); err == nil || err.Error() != `cannot encode header frame, expected page` { 140 | t.Fatal(err) 141 | } 142 | }) 143 | } 144 | 145 | func TestEncode_EncodePage(t *testing.T) { 146 | t.Run("ErrInvalidState", func(t *testing.T) { 147 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 148 | if err := enc.EncodePage(ltx.PageHeader{}, nil); err == nil || err.Error() != `cannot encode page header, expected header` { 149 | t.Fatal(err) 150 | } 151 | }) 152 | 153 | t.Run("ErrPageNumberRequired", func(t *testing.T) { 154 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 155 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 1024, Commit: 1, MinTXID: 1, MaxTXID: 1}); err != nil { 156 | t.Fatal(err) 157 | } else if err := enc.EncodePage(ltx.PageHeader{Pgno: 0}, nil); err == nil || err.Error() != `page number required` { 158 | t.Fatalf("unexpected error: %s", err) 159 | } 160 | }) 161 | 162 | t.Run("ErrPageNumberOutOfBounds", func(t *testing.T) { 163 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 164 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 1024, Commit: 4, MinTXID: 2, MaxTXID: 2, PreApplyChecksum: ltx.ChecksumFlag | 2}); err != nil { 165 | t.Fatal(err) 166 | } else if err := enc.EncodePage(ltx.PageHeader{Pgno: 5}, nil); err == nil || err.Error() != `page number 5 out-of-bounds for commit size 4` { 167 | t.Fatalf("unexpected error: %s", err) 168 | } 169 | }) 170 | 171 | t.Run("ErrSnapshotInitialPage", func(t *testing.T) { 172 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 173 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 1, MaxTXID: 2}); err != nil { 174 | t.Fatal(err) 175 | } else if err := enc.EncodePage(ltx.PageHeader{Pgno: 2}, make([]byte, 1024)); err == nil || err.Error() != `snapshot transaction file must start with page number 1` { 176 | t.Fatalf("unexpected error: %s", err) 177 | } 178 | }) 179 | 180 | t.Run("ErrSnapshotNonsequentialPages", func(t *testing.T) { 181 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 182 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 1024, Commit: 3, MinTXID: 1, MaxTXID: 1}); err != nil { 183 | t.Fatal(err) 184 | } 185 | if err := enc.EncodePage(ltx.PageHeader{Pgno: 1}, make([]byte, 1024)); err != nil { 186 | t.Fatal(err) 187 | } 188 | 189 | if err := enc.EncodePage(ltx.PageHeader{Pgno: 3}, make([]byte, 1024)); err == nil || err.Error() != `nonsequential page numbers in snapshot transaction: 1,3` { 190 | t.Fatalf("unexpected error: %s", err) 191 | } 192 | }) 193 | 194 | t.Run("ErrCannotEncodeLockPage", func(t *testing.T) { 195 | enc := ltx.NewEncoder(io.Discard) 196 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 4096, Commit: 262145, MinTXID: 1, MaxTXID: 1}); err != nil { 197 | t.Fatal(err) 198 | } 199 | 200 | pageBuf := make([]byte, 4096) 201 | for pgno := uint32(1); pgno <= 262144; pgno++ { 202 | if err := enc.EncodePage(ltx.PageHeader{Pgno: pgno}, pageBuf); err != nil { 203 | t.Fatal(err) 204 | } 205 | } 206 | 207 | // Try to encode lock page. 208 | if err := enc.EncodePage(ltx.PageHeader{Pgno: 262145}, pageBuf); err == nil || err.Error() != `cannot encode lock page: pgno=262145` { 209 | t.Fatalf("unexpected error: %s", err) 210 | } 211 | }) 212 | 213 | t.Run("ErrSnapshotNonsequentialPagesAfterLockPage", func(t *testing.T) { 214 | enc := ltx.NewEncoder(io.Discard) 215 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 4096, Commit: 262147, MinTXID: 1, MaxTXID: 1}); err != nil { 216 | t.Fatal(err) 217 | } 218 | 219 | pageBuf := make([]byte, 4096) 220 | for pgno := uint32(1); pgno <= 262144; pgno++ { 221 | if err := enc.EncodePage(ltx.PageHeader{Pgno: pgno}, pageBuf); err != nil { 222 | t.Fatal(err) 223 | } 224 | } 225 | 226 | // Try to encode lock page. 227 | if err := enc.EncodePage(ltx.PageHeader{Pgno: 262147}, pageBuf); err == nil || err.Error() != `nonsequential page numbers in snapshot transaction (skip lock page): 262144,262147` { 228 | t.Fatalf("unexpected error: %s", err) 229 | } 230 | }) 231 | 232 | t.Run("ErrOutOfOrderPages", func(t *testing.T) { 233 | enc := ltx.NewEncoder(createFile(t, filepath.Join(t.TempDir(), "ltx"))) 234 | if err := enc.EncodeHeader(ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 2, MaxTXID: 2, PreApplyChecksum: ltx.ChecksumFlag | 2}); err != nil { 235 | t.Fatal(err) 236 | } 237 | if err := enc.EncodePage(ltx.PageHeader{Pgno: 2}, make([]byte, 1024)); err != nil { 238 | t.Fatal(err) 239 | } 240 | if err := enc.EncodePage(ltx.PageHeader{Pgno: 1}, make([]byte, 1024)); err == nil || err.Error() != `out-of-order page numbers: 2,1` { 241 | t.Fatalf("unexpected error: %s", err) 242 | } 243 | }) 244 | } 245 | -------------------------------------------------------------------------------- /file_spec.go: -------------------------------------------------------------------------------- 1 | package ltx 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | ) 8 | 9 | // FileSpec is an in-memory representation of an LTX file. Typically used for testing. 10 | type FileSpec struct { 11 | Header Header 12 | Pages []PageSpec 13 | Trailer Trailer 14 | } 15 | 16 | // Write encodes a file spec to a file. 17 | func (s *FileSpec) WriteTo(dst io.Writer) (n int64, err error) { 18 | enc := NewEncoder(dst) 19 | if err := enc.EncodeHeader(s.Header); err != nil { 20 | return 0, fmt.Errorf("encode header: %s", err) 21 | } 22 | 23 | for i, page := range s.Pages { 24 | if err := enc.EncodePage(page.Header, page.Data); err != nil { 25 | return 0, fmt.Errorf("encode page[%d]: %s", i, err) 26 | } 27 | } 28 | 29 | enc.SetPostApplyChecksum(s.Trailer.PostApplyChecksum) 30 | 31 | if err := enc.Close(); err != nil { 32 | return 0, fmt.Errorf("close encoder: %s", err) 33 | } 34 | 35 | // Update checksums. 36 | s.Trailer = enc.Trailer() 37 | 38 | return enc.N(), nil 39 | } 40 | 41 | // ReadFromFile encodes a file spec to a file. Always return n of zero. 42 | func (s *FileSpec) ReadFrom(src io.Reader) (n int, err error) { 43 | dec := NewDecoder(src) 44 | 45 | // Read header frame and initialize spec slices to correct size. 46 | if err := dec.DecodeHeader(); err != nil { 47 | return 0, fmt.Errorf("read header: %s", err) 48 | } 49 | s.Header = dec.Header() 50 | 51 | // Read page frames. 52 | for { 53 | page := PageSpec{Data: make([]byte, s.Header.PageSize)} 54 | if err := dec.DecodePage(&page.Header, page.Data); err == io.EOF { 55 | break 56 | } else if err != nil { 57 | return 0, fmt.Errorf("read page header: %s", err) 58 | } 59 | 60 | s.Pages = append(s.Pages, page) 61 | } 62 | 63 | if err := dec.Close(); err != nil { 64 | return 0, fmt.Errorf("close reader: %s", err) 65 | } 66 | s.Trailer = dec.Trailer() 67 | 68 | return int(dec.N()), nil 69 | } 70 | 71 | // GoString returns the Go representation of s. 72 | func (s *FileSpec) GoString() string { 73 | var buf bytes.Buffer 74 | 75 | fmt.Fprintf(&buf, "{\n") 76 | fmt.Fprintf(&buf, "\tHeader: %#v,\n", s.Header) 77 | 78 | if s.Pages == nil { 79 | fmt.Fprintf(&buf, "\tPages: nil,\n") 80 | } else { 81 | fmt.Fprintf(&buf, "\tPages: []*PageSpec{,\n") 82 | for i := range s.Pages { 83 | fmt.Fprintf(&buf, "\t\t%#v,\n", &s.Pages[i]) 84 | } 85 | fmt.Fprintf(&buf, "\t},\n") 86 | } 87 | 88 | fmt.Fprintf(&buf, "\tTrailer: %#v,\n", s.Trailer) 89 | fmt.Fprintf(&buf, "}") 90 | 91 | return buf.String() 92 | } 93 | 94 | // PageSpec is an in-memory representation of an LTX page frame. Typically used for testing. 95 | type PageSpec struct { 96 | Header PageHeader 97 | Data []byte 98 | } 99 | 100 | // GoString returns the Go representation of s. 101 | func (s *PageSpec) GoString() string { 102 | var data string 103 | if len(s.Data) < 4 { 104 | data = fmt.Sprintf(`"%x"`, s.Data) 105 | } else { 106 | data = fmt.Sprintf(`"%x..."`, s.Data[:4]) 107 | } 108 | return fmt.Sprintf(`{Header:%#v, Data:[]byte(%s)}`, s.Header, data) 109 | } 110 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/superfly/ltx 2 | 3 | go 1.18 4 | 5 | require github.com/pierrec/lz4/v4 v4.1.17 // indirect 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 2 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 3 | -------------------------------------------------------------------------------- /ltx.go: -------------------------------------------------------------------------------- 1 | // Package ltx reads and writes Liteserver Transaction (LTX) files. 2 | package ltx 3 | 4 | import ( 5 | "bytes" 6 | "encoding/binary" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "regexp" 12 | "strconv" 13 | "time" 14 | ) 15 | 16 | const ( 17 | // Magic is the first 4 bytes of every LTX file. 18 | Magic = "LTX1" 19 | 20 | // Version is the current version of the LTX file format. 21 | Version = 1 22 | ) 23 | 24 | // Size constants. 25 | const ( 26 | HeaderSize = 100 27 | PageHeaderSize = 4 28 | TrailerSize = 16 29 | ) 30 | 31 | // RFC3339Milli is the standard time format for LTX timestamps. 32 | // It uses fixed-width millisecond resolution which makes it sortable. 33 | const RFC3339Milli = "2006-01-02T15:04:05.000Z07:00" 34 | 35 | // Checksum size & positions. 36 | const ( 37 | ChecksumSize = 8 38 | TrailerChecksumOffset = TrailerSize - ChecksumSize 39 | ) 40 | 41 | // Errors 42 | var ( 43 | ErrInvalidFile = errors.New("invalid LTX file") 44 | ErrDecoderClosed = errors.New("ltx decoder closed") 45 | ErrEncoderClosed = errors.New("ltx encoder closed") 46 | 47 | ErrNoChecksum = errors.New("no file checksum") 48 | ErrInvalidChecksumFormat = errors.New("invalid file checksum format") 49 | ErrChecksumMismatch = errors.New("file checksum mismatch") 50 | ) 51 | 52 | // ChecksumFlag is a flag on the checksum to ensure it is non-zero. 53 | const ChecksumFlag Checksum = 1 << 63 54 | 55 | // internal reader/writer states 56 | const ( 57 | stateHeader = "header" 58 | statePage = "page" 59 | stateClose = "close" 60 | stateClosed = "closed" 61 | ) 62 | 63 | // Pos represents the transactional position of a database. 64 | type Pos struct { 65 | TXID TXID 66 | PostApplyChecksum Checksum 67 | } 68 | 69 | // NewPos returns a new instance of Pos. 70 | func NewPos(txID TXID, postApplyChecksum Checksum) Pos { 71 | return Pos{ 72 | TXID: txID, 73 | PostApplyChecksum: postApplyChecksum, 74 | } 75 | } 76 | 77 | // ParsePos parses Pos from its string representation. 78 | func ParsePos(s string) (Pos, error) { 79 | if len(s) != 33 { 80 | return Pos{}, fmt.Errorf("invalid formatted position length: %q", s) 81 | } 82 | 83 | txid, err := ParseTXID(s[:16]) 84 | if err != nil { 85 | return Pos{}, err 86 | } 87 | 88 | checksum, err := ParseChecksum(s[17:]) 89 | if err != nil { 90 | return Pos{}, err 91 | } 92 | 93 | return Pos{ 94 | TXID: txid, 95 | PostApplyChecksum: checksum, 96 | }, nil 97 | } 98 | 99 | // String returns a string representation of the position. 100 | func (p Pos) String() string { 101 | return fmt.Sprintf("%s/%s", p.TXID, p.PostApplyChecksum) 102 | } 103 | 104 | // IsZero returns true if the position is empty. 105 | func (p Pos) IsZero() bool { 106 | return p == (Pos{}) 107 | } 108 | 109 | // PosMismatchError is returned when an LTX file is not contiguous with the current position. 110 | type PosMismatchError struct { 111 | Pos Pos `json:"pos"` 112 | } 113 | 114 | // NewPosMismatchError returns a new instance of PosMismatchError. 115 | func NewPosMismatchError(pos Pos) *PosMismatchError { 116 | return &PosMismatchError{Pos: pos} 117 | } 118 | 119 | // Error returns the string representation of the error. 120 | func (e *PosMismatchError) Error() string { 121 | return fmt.Sprintf("ltx position mismatch (%s)", e.Pos) 122 | } 123 | 124 | // TXID represents a transaction ID. 125 | type TXID uint64 126 | 127 | // ParseTXID parses a 16-character hex string into a transaction ID. 128 | func ParseTXID(s string) (TXID, error) { 129 | if len(s) != 16 { 130 | return 0, fmt.Errorf("invalid formatted transaction id length: %q", s) 131 | } 132 | v, err := strconv.ParseUint(s, 16, 64) 133 | if err != nil { 134 | return 0, fmt.Errorf("invalid transaction id format: %q", s) 135 | } 136 | return TXID(v), nil 137 | } 138 | 139 | // String returns id formatted as a fixed-width hex number. 140 | func (t TXID) String() string { 141 | return fmt.Sprintf("%016x", uint64(t)) 142 | } 143 | 144 | func (t TXID) MarshalJSON() ([]byte, error) { 145 | return []byte(`"` + t.String() + `"`), nil 146 | } 147 | 148 | func (t *TXID) UnmarshalJSON(data []byte) (err error) { 149 | var s *string 150 | if err := json.Unmarshal(data, &s); err != nil { 151 | return fmt.Errorf("cannot unmarshal TXID from JSON value") 152 | } 153 | 154 | // Set to zero if value is nil. 155 | if s == nil { 156 | *t = 0 157 | return nil 158 | } 159 | 160 | txID, err := ParseTXID(*s) 161 | if err != nil { 162 | return fmt.Errorf("cannot parse TXID from JSON string: %q", *s) 163 | } 164 | *t = TXID(txID) 165 | 166 | return nil 167 | } 168 | 169 | // Header flags. 170 | const ( 171 | HeaderFlagMask = uint32(0x00000001) 172 | 173 | HeaderFlagCompressLZ4 = uint32(0x00000001) 174 | ) 175 | 176 | // Header represents the header frame of an LTX file. 177 | type Header struct { 178 | Version int // based on magic 179 | Flags uint32 // reserved flags 180 | PageSize uint32 // page size, in bytes 181 | Commit uint32 // db size after transaction, in pages 182 | MinTXID TXID // minimum transaction ID 183 | MaxTXID TXID // maximum transaction ID 184 | Timestamp int64 // milliseconds since unix epoch 185 | PreApplyChecksum Checksum // rolling checksum of database before applying this LTX file 186 | WALOffset int64 // file offset from original WAL; zero if journal 187 | WALSize int64 // size of original WAL segment; zero if journal 188 | WALSalt1 uint32 // header salt-1 from original WAL; zero if journal or compaction 189 | WALSalt2 uint32 // header salt-2 from original WAL; zero if journal or compaction 190 | NodeID uint64 // node id where the LTX file was created, zero if unset 191 | } 192 | 193 | // IsSnapshot returns true if header represents a complete database snapshot. 194 | // This is true if the header includes the initial transaction. Snapshots must 195 | // include all pages in the database. 196 | func (h *Header) IsSnapshot() bool { 197 | return h.MinTXID == 1 198 | } 199 | 200 | // LockPgno returns the lock page number based on the header's page size. 201 | func (h *Header) LockPgno() uint32 { 202 | return LockPgno(h.PageSize) 203 | } 204 | 205 | // Validate returns an error if h is invalid. 206 | func (h *Header) Validate() error { 207 | if h.Version != Version { 208 | return fmt.Errorf("invalid version") 209 | } 210 | if !IsValidHeaderFlags(h.Flags) { 211 | return fmt.Errorf("invalid flags: 0x%08x", h.Flags) 212 | } 213 | if !IsValidPageSize(h.PageSize) { 214 | return fmt.Errorf("invalid page size: %d", h.PageSize) 215 | } 216 | if h.MinTXID == 0 { 217 | return fmt.Errorf("minimum transaction id required") 218 | } 219 | if h.MaxTXID == 0 { 220 | return fmt.Errorf("maximum transaction id required") 221 | } 222 | if h.MinTXID > h.MaxTXID { 223 | return fmt.Errorf("transaction ids out of order: (%d,%d)", h.MinTXID, h.MaxTXID) 224 | } 225 | 226 | if h.WALOffset < 0 { 227 | return fmt.Errorf("wal offset cannot be negative: %d", h.WALOffset) 228 | } 229 | if h.WALSize < 0 { 230 | return fmt.Errorf("wal size cannot be negative: %d", h.WALSize) 231 | } 232 | 233 | if h.WALSalt1 != 0 || h.WALSalt2 != 0 { 234 | if h.WALOffset == 0 { 235 | return fmt.Errorf("wal offset required if salt exists") 236 | } 237 | if h.WALSize == 0 { 238 | return fmt.Errorf("wal size required if salt exists") 239 | } 240 | } 241 | 242 | if h.WALOffset != 0 && h.WALSize == 0 { 243 | return fmt.Errorf("wal size required if wal offset exists") 244 | } 245 | if h.WALOffset == 0 && h.WALSize != 0 { 246 | return fmt.Errorf("wal offset required if wal size exists") 247 | } 248 | 249 | // Snapshots are LTX files which have a minimum TXID of 1. This means they 250 | // must have all database pages included in them and they have no previous checksum. 251 | if h.IsSnapshot() { 252 | if h.PreApplyChecksum != 0 { 253 | return fmt.Errorf("pre-apply checksum must be zero on snapshots") 254 | } 255 | } else { 256 | if h.PreApplyChecksum == 0 { 257 | return fmt.Errorf("pre-apply checksum required on non-snapshot files") 258 | } 259 | if h.PreApplyChecksum&ChecksumFlag == 0 { 260 | return fmt.Errorf("invalid pre-apply checksum format") 261 | } 262 | } 263 | 264 | return nil 265 | } 266 | 267 | // PreApplyPos returns the replication position before the LTX file is applies. 268 | func (h Header) PreApplyPos() Pos { 269 | return Pos{ 270 | TXID: h.MinTXID - 1, 271 | PostApplyChecksum: h.PreApplyChecksum, 272 | } 273 | } 274 | 275 | // MarshalBinary encodes h to a byte slice. 276 | func (h *Header) MarshalBinary() ([]byte, error) { 277 | b := make([]byte, HeaderSize) 278 | copy(b[0:4], Magic) 279 | binary.BigEndian.PutUint32(b[4:], h.Flags) 280 | binary.BigEndian.PutUint32(b[8:], h.PageSize) 281 | binary.BigEndian.PutUint32(b[12:], h.Commit) 282 | binary.BigEndian.PutUint64(b[16:], uint64(h.MinTXID)) 283 | binary.BigEndian.PutUint64(b[24:], uint64(h.MaxTXID)) 284 | binary.BigEndian.PutUint64(b[32:], uint64(h.Timestamp)) 285 | binary.BigEndian.PutUint64(b[40:], uint64(h.PreApplyChecksum)) 286 | binary.BigEndian.PutUint64(b[48:], uint64(h.WALOffset)) 287 | binary.BigEndian.PutUint64(b[56:], uint64(h.WALSize)) 288 | binary.BigEndian.PutUint32(b[64:], h.WALSalt1) 289 | binary.BigEndian.PutUint32(b[68:], h.WALSalt2) 290 | binary.BigEndian.PutUint64(b[72:], h.NodeID) 291 | return b, nil 292 | } 293 | 294 | // UnmarshalBinary decodes h from a byte slice. 295 | func (h *Header) UnmarshalBinary(b []byte) error { 296 | if len(b) < HeaderSize { 297 | return io.ErrShortBuffer 298 | } 299 | 300 | h.Flags = binary.BigEndian.Uint32(b[4:]) 301 | h.PageSize = binary.BigEndian.Uint32(b[8:]) 302 | h.Commit = binary.BigEndian.Uint32(b[12:]) 303 | h.MinTXID = TXID(binary.BigEndian.Uint64(b[16:])) 304 | h.MaxTXID = TXID(binary.BigEndian.Uint64(b[24:])) 305 | h.Timestamp = int64(binary.BigEndian.Uint64(b[32:])) 306 | h.PreApplyChecksum = Checksum(binary.BigEndian.Uint64(b[40:])) 307 | h.WALOffset = int64(binary.BigEndian.Uint64(b[48:])) 308 | h.WALSize = int64(binary.BigEndian.Uint64(b[56:])) 309 | h.WALSalt1 = binary.BigEndian.Uint32(b[64:]) 310 | h.WALSalt2 = binary.BigEndian.Uint32(b[68:]) 311 | h.NodeID = binary.BigEndian.Uint64(b[72:]) 312 | 313 | if string(b[0:4]) != Magic { 314 | return ErrInvalidFile 315 | } 316 | h.Version = Version 317 | 318 | return nil 319 | } 320 | 321 | // PeekHeader reads & unmarshals the header from r. 322 | // It returns a new io.Reader that prepends the header data back on. 323 | func PeekHeader(r io.Reader) (Header, io.Reader, error) { 324 | buf := make([]byte, HeaderSize) 325 | n, err := io.ReadFull(r, buf) 326 | r = io.MultiReader(bytes.NewReader(buf[:n]), r) 327 | if err != nil { 328 | return Header{}, r, err 329 | } 330 | 331 | var hdr Header 332 | err = hdr.UnmarshalBinary(buf) 333 | return hdr, r, err 334 | } 335 | 336 | // IsValidHeaderFlags returns true unless flags outside the valid mask are set. 337 | func IsValidHeaderFlags(flags uint32) bool { 338 | return flags == (flags & HeaderFlagMask) 339 | } 340 | 341 | // Trailer represents the ending frame of an LTX file. 342 | type Trailer struct { 343 | PostApplyChecksum Checksum // rolling checksum of database after this LTX file is applied 344 | FileChecksum Checksum // crc64 checksum of entire file 345 | } 346 | 347 | // Validate returns an error if t is invalid. 348 | func (t *Trailer) Validate() error { 349 | if t.PostApplyChecksum == 0 { 350 | return fmt.Errorf("post-apply checksum required") 351 | } else if t.PostApplyChecksum&ChecksumFlag == 0 { 352 | return fmt.Errorf("invalid post-apply checksum format") 353 | } 354 | 355 | if t.FileChecksum == 0 { 356 | return fmt.Errorf("file checksum required") 357 | } else if t.FileChecksum&ChecksumFlag == 0 { 358 | return fmt.Errorf("invalid file checksum format") 359 | } 360 | return nil 361 | } 362 | 363 | // MarshalBinary encodes h to a byte slice. 364 | func (t *Trailer) MarshalBinary() ([]byte, error) { 365 | b := make([]byte, TrailerSize) 366 | binary.BigEndian.PutUint64(b[0:], uint64(t.PostApplyChecksum)) 367 | binary.BigEndian.PutUint64(b[8:], uint64(t.FileChecksum)) 368 | return b, nil 369 | } 370 | 371 | // UnmarshalBinary decodes h from a byte slice. 372 | func (t *Trailer) UnmarshalBinary(b []byte) error { 373 | if len(b) < TrailerSize { 374 | return io.ErrShortBuffer 375 | } 376 | 377 | t.PostApplyChecksum = Checksum(binary.BigEndian.Uint64(b[0:])) 378 | t.FileChecksum = Checksum(binary.BigEndian.Uint64(b[8:])) 379 | return nil 380 | } 381 | 382 | // MaxPageSize is the maximum allowed page size for SQLite. 383 | const MaxPageSize = 65536 384 | 385 | // IsValidPageSize returns true if sz is between 512 and 64K and a power of two. 386 | func IsValidPageSize(sz uint32) bool { 387 | for i := uint32(512); i <= MaxPageSize; i *= 2 { 388 | if sz == i { 389 | return true 390 | } 391 | } 392 | return false 393 | } 394 | 395 | // PageHeader represents the header for a single page in an LTX file. 396 | type PageHeader struct { 397 | Pgno uint32 398 | } 399 | 400 | // IsZero returns true if the header is empty. 401 | func (h *PageHeader) IsZero() bool { 402 | return *h == (PageHeader{}) 403 | } 404 | 405 | // Validate returns an error if h is invalid. 406 | func (h *PageHeader) Validate() error { 407 | if h.Pgno == 0 { 408 | return fmt.Errorf("page number required") 409 | } 410 | return nil 411 | } 412 | 413 | // MarshalBinary encodes h to a byte slice. 414 | func (h *PageHeader) MarshalBinary() ([]byte, error) { 415 | b := make([]byte, PageHeaderSize) 416 | binary.BigEndian.PutUint32(b[0:], h.Pgno) 417 | return b, nil 418 | } 419 | 420 | // UnmarshalBinary decodes h from a byte slice. 421 | func (h *PageHeader) UnmarshalBinary(b []byte) error { 422 | if len(b) < PageHeaderSize { 423 | return io.ErrShortBuffer 424 | } 425 | 426 | h.Pgno = binary.BigEndian.Uint32(b[0:]) 427 | return nil 428 | } 429 | 430 | // ParseFilename parses a transaction range from an LTX file. 431 | func ParseFilename(name string) (minTXID, maxTXID TXID, err error) { 432 | a := filenameRegex.FindStringSubmatch(name) 433 | if a == nil { 434 | return 0, 0, fmt.Errorf("invalid ltx filename: %s", name) 435 | } 436 | 437 | min, _ := strconv.ParseUint(a[1], 16, 64) 438 | max, _ := strconv.ParseUint(a[2], 16, 64) 439 | return TXID(min), TXID(max), nil 440 | } 441 | 442 | // FormatTimestamp returns t with a fixed-width, millisecond-resolution UTC format. 443 | func FormatTimestamp(t time.Time) string { 444 | return t.UTC().Format(RFC3339Milli) 445 | } 446 | 447 | // ParseTimestamp parses a timestamp as RFC3339Milli (fixed-width) but will 448 | // fallback to RFC3339Nano if it fails. This is to support timestamps written 449 | // before the introduction of the standard time format. 450 | func ParseTimestamp(value string) (time.Time, error) { 451 | // Attempt standard format first. 452 | t, err := time.Parse(RFC3339Milli, value) 453 | if err == nil { 454 | return t, nil 455 | } 456 | 457 | // If the standard fails, fallback to stdlib format but truncate to milliseconds. 458 | t2, err2 := time.Parse(time.RFC3339Nano, value) 459 | if err2 != nil { 460 | return t, err // use original error on failure. 461 | } 462 | return t2.Truncate(time.Millisecond), nil 463 | } 464 | 465 | var filenameRegex = regexp.MustCompile(`^([0-9a-f]{16})-([0-9a-f]{16})\.ltx$`) 466 | 467 | // FormatFilename returns an LTX filename representing a range of transactions. 468 | func FormatFilename(minTXID, maxTXID TXID) string { 469 | return fmt.Sprintf("%s-%s.ltx", minTXID.String(), maxTXID.String()) 470 | } 471 | 472 | const PENDING_BYTE = 0x40000000 473 | 474 | // LockPgno returns the page number where the PENDING_BYTE exists. 475 | func LockPgno(pageSize uint32) uint32 { 476 | return uint32(PENDING_BYTE/int64(pageSize)) + 1 477 | } 478 | -------------------------------------------------------------------------------- /ltx_test.go: -------------------------------------------------------------------------------- 1 | package ltx_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "math" 10 | "math/rand" 11 | "os" 12 | "reflect" 13 | "testing" 14 | "time" 15 | 16 | "github.com/superfly/ltx" 17 | ) 18 | 19 | func TestNewPos(t *testing.T) { 20 | pos := ltx.NewPos(1000, 2000) 21 | if got, want := pos.TXID, ltx.TXID(1000); got != want { 22 | t.Fatalf("TXID=%s, want %s", got, want) 23 | } 24 | if got, want := pos.PostApplyChecksum, ltx.Checksum(2000); got != want { 25 | t.Fatalf("PostApplyChecksum=%v, want %v", got, want) 26 | } 27 | } 28 | 29 | func TestPos_String(t *testing.T) { 30 | pos := ltx.NewPos(1000, 2000) 31 | if got, want := pos.String(), "00000000000003e8/00000000000007d0"; got != want { 32 | t.Fatalf("Pos = %s, want = %s", got, want) 33 | } 34 | } 35 | 36 | func TestParsePos(t *testing.T) { 37 | t.Run("OK", func(t *testing.T) { 38 | if v, err := ltx.ParsePos("00000000000003e8/00000000000007d0"); err != nil { 39 | t.Fatal(err) 40 | } else if got, want := v, ltx.NewPos(1000, 2000); got != want { 41 | t.Fatalf("got=%d, want %d", got, want) 42 | } 43 | }) 44 | t.Run("ErrTooShort", func(t *testing.T) { 45 | if _, err := ltx.ParsePos("00000000000003e8"); err == nil || err.Error() != `invalid formatted position length: "00000000000003e8"` { 46 | t.Fatal(err) 47 | } 48 | }) 49 | } 50 | 51 | func TestHeader_Validate(t *testing.T) { 52 | t.Run("OK", func(t *testing.T) { 53 | hdr := ltx.Header{ 54 | Version: 1, 55 | PageSize: 1024, 56 | Commit: 2, 57 | MinTXID: 3, 58 | MaxTXID: 4, 59 | PreApplyChecksum: ltx.ChecksumFlag, 60 | WALSalt1: 5, 61 | WALSalt2: 6, 62 | WALOffset: 7, 63 | WALSize: 8, 64 | } 65 | if err := hdr.Validate(); err != nil { 66 | t.Fatal(err) 67 | } 68 | }) 69 | t.Run("CommitZero", func(t *testing.T) { 70 | hdr := ltx.Header{ 71 | Version: 1, 72 | PageSize: 1024, 73 | Commit: 0, 74 | MinTXID: 5, 75 | MaxTXID: 5, 76 | PreApplyChecksum: ltx.ChecksumFlag, 77 | } 78 | if err := hdr.Validate(); err != nil { 79 | t.Fatal(err) 80 | } 81 | }) 82 | t.Run("ErrVersion", func(t *testing.T) { 83 | hdr := ltx.Header{} 84 | if err := hdr.Validate(); err == nil || err.Error() != `invalid version` { 85 | t.Fatalf("unexpected error: %s", err) 86 | } 87 | }) 88 | t.Run("ErrFlags", func(t *testing.T) { 89 | hdr := ltx.Header{Version: 1, Flags: 2} 90 | if err := hdr.Validate(); err == nil || err.Error() != `invalid flags: 0x00000002` { 91 | t.Fatalf("unexpected error: %s", err) 92 | } 93 | }) 94 | t.Run("ErrInvalidPageSize", func(t *testing.T) { 95 | hdr := ltx.Header{Version: 1, PageSize: 1000} 96 | if err := hdr.Validate(); err == nil || err.Error() != `invalid page size: 1000` { 97 | t.Fatalf("unexpected error: %s", err) 98 | } 99 | }) 100 | t.Run("ErrMinTXIDRequired", func(t *testing.T) { 101 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 2} 102 | if err := hdr.Validate(); err == nil || err.Error() != `minimum transaction id required` { 103 | t.Fatalf("unexpected error: %s", err) 104 | } 105 | }) 106 | t.Run("ErrMaxTXIDRequired", func(t *testing.T) { 107 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 3} 108 | if err := hdr.Validate(); err == nil || err.Error() != `maximum transaction id required` { 109 | t.Fatalf("unexpected error: %s", err) 110 | } 111 | }) 112 | t.Run("ErrTXIDOutOfOrderRequired", func(t *testing.T) { 113 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 3, MaxTXID: 2} 114 | if err := hdr.Validate(); err == nil || err.Error() != `transaction ids out of order: (3,2)` { 115 | t.Fatalf("unexpected error: %s", err) 116 | } 117 | }) 118 | t.Run("ErrNegativeWALOffset", func(t *testing.T) { 119 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 1, MaxTXID: 1, WALOffset: -1000} 120 | if err := hdr.Validate(); err == nil || err.Error() != `wal offset cannot be negative: -1000` { 121 | t.Fatalf("unexpected error: %s", err) 122 | } 123 | }) 124 | t.Run("ErrNegativeWALSize", func(t *testing.T) { 125 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 1, MaxTXID: 1, WALOffset: 32, WALSize: -1000} 126 | if err := hdr.Validate(); err == nil || err.Error() != `wal size cannot be negative: -1000` { 127 | t.Fatalf("unexpected error: %s", err) 128 | } 129 | }) 130 | t.Run("ErrWALOffsetRequiredWithWALSalt", func(t *testing.T) { 131 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 1, MaxTXID: 1, WALSalt1: 100} 132 | if err := hdr.Validate(); err == nil || err.Error() != `wal offset required if salt exists` { 133 | t.Fatalf("unexpected error: %s", err) 134 | } 135 | }) 136 | t.Run("ErrWALSizeRequiredWithWALSalt", func(t *testing.T) { 137 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 1, MaxTXID: 1, WALSalt2: 100, WALOffset: 1000} 138 | if err := hdr.Validate(); err == nil || err.Error() != `wal size required if salt exists` { 139 | t.Fatalf("unexpected error: %s", err) 140 | } 141 | }) 142 | t.Run("ErrWALSizeRequiredWithWALOffset", func(t *testing.T) { 143 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 1, MaxTXID: 1, WALOffset: 1000} 144 | if err := hdr.Validate(); err == nil || err.Error() != `wal size required if wal offset exists` { 145 | t.Fatalf("unexpected error: %s", err) 146 | } 147 | }) 148 | t.Run("ErrWALOffsetRequiredWithWALSize", func(t *testing.T) { 149 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 2, MinTXID: 1, MaxTXID: 1, WALSize: 1000} 150 | if err := hdr.Validate(); err == nil || err.Error() != `wal offset required if wal size exists` { 151 | t.Fatalf("unexpected error: %s", err) 152 | } 153 | }) 154 | t.Run("ErrSnapshotPreApplyChecksumNotAllowed", func(t *testing.T) { 155 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 4, MinTXID: 1, MaxTXID: 3, PreApplyChecksum: 1} 156 | if err := hdr.Validate(); err == nil || err.Error() != `pre-apply checksum must be zero on snapshots` { 157 | t.Fatalf("unexpected error: %s", err) 158 | } 159 | }) 160 | t.Run("ErrNonSnapshotPreApplyChecksumRequired", func(t *testing.T) { 161 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 4, MinTXID: 2, MaxTXID: 3} 162 | if err := hdr.Validate(); err == nil || err.Error() != `pre-apply checksum required on non-snapshot files` { 163 | t.Fatalf("unexpected error: %s", err) 164 | } 165 | }) 166 | t.Run("ErrInvalidPreApplyChecksumFormat", func(t *testing.T) { 167 | hdr := ltx.Header{Version: 1, PageSize: 1024, Commit: 4, MinTXID: 2, MaxTXID: 3, PreApplyChecksum: 1} 168 | if err := hdr.Validate(); err == nil || err.Error() != `invalid pre-apply checksum format` { 169 | t.Fatalf("unexpected error: %s", err) 170 | } 171 | }) 172 | } 173 | 174 | func TestHeader_MarshalBinary(t *testing.T) { 175 | hdr := ltx.Header{ 176 | Version: ltx.Version, 177 | Flags: 0, 178 | PageSize: 1024, 179 | Commit: 1006, 180 | MinTXID: 1007, 181 | MaxTXID: 1008, 182 | Timestamp: 1009, 183 | PreApplyChecksum: 1011, 184 | WALSalt1: 1012, 185 | WALSalt2: 1013, 186 | WALOffset: 1014, 187 | WALSize: 1015, 188 | } 189 | 190 | var other ltx.Header 191 | if b, err := hdr.MarshalBinary(); err != nil { 192 | t.Fatal(err) 193 | } else if err := other.UnmarshalBinary(b); err != nil { 194 | t.Fatal(err) 195 | } else if !reflect.DeepEqual(hdr, other) { 196 | t.Fatalf("mismatch:\ngot=%#v\nwant=%#v", hdr, other) 197 | } 198 | } 199 | 200 | func TestHeader_UnmarshalBinary(t *testing.T) { 201 | t.Run("ErrShortBuffer", func(t *testing.T) { 202 | var hdr ltx.Header 203 | if err := hdr.UnmarshalBinary(make([]byte, 10)); err != io.ErrShortBuffer { 204 | t.Fatal(err) 205 | } 206 | }) 207 | t.Run("ErrInvalidFile", func(t *testing.T) { 208 | var hdr ltx.Header 209 | if err := hdr.UnmarshalBinary(make([]byte, ltx.HeaderSize)); err != ltx.ErrInvalidFile { 210 | t.Fatal(err) 211 | } 212 | }) 213 | } 214 | 215 | func TestPeekHeader(t *testing.T) { 216 | t.Run("OK", func(t *testing.T) { 217 | hdr := ltx.Header{ 218 | Version: ltx.Version, 219 | Flags: 0, 220 | PageSize: 1024, 221 | Commit: 1006, 222 | MinTXID: 1007, 223 | MaxTXID: 1008, 224 | Timestamp: 1009, 225 | PreApplyChecksum: 1011, 226 | WALSalt1: 1012, 227 | WALSalt2: 1013, 228 | WALOffset: 1014, 229 | WALSize: 1015, 230 | } 231 | b, err := hdr.MarshalBinary() 232 | if err != nil { 233 | t.Fatal(err) 234 | } 235 | 236 | var buf bytes.Buffer 237 | buf.Write(b) 238 | buf.Write([]byte("foobar")) 239 | 240 | // Read the header once. 241 | other, r, err := ltx.PeekHeader(&buf) 242 | if err != nil { 243 | t.Fatal(err) 244 | } else if !reflect.DeepEqual(hdr, other) { 245 | t.Fatalf("mismatch:\ngot=%#v\nwant=%#v", hdr, other) 246 | } 247 | 248 | // Read it again from the returned reader. 249 | if other, _, err = ltx.PeekHeader(r); err != nil { 250 | t.Fatal(err) 251 | } else if !reflect.DeepEqual(hdr, other) { 252 | t.Fatalf("mismatch:\ngot=%#v\nwant=%#v", hdr, other) 253 | } 254 | 255 | // Read the rest of the data. 256 | if trailing, err := io.ReadAll(r); err != nil { 257 | t.Fatal(err) 258 | } else if got, want := string(trailing), "foobar"; got != want { 259 | t.Fatalf("trailing=%s, want %s", got, want) 260 | } 261 | }) 262 | 263 | t.Run("EOF", func(t *testing.T) { 264 | if _, _, err := ltx.PeekHeader(bytes.NewReader(nil)); err != io.EOF { 265 | t.Fatal(err) 266 | } 267 | }) 268 | t.Run("ErrUnexpectedEOF", func(t *testing.T) { 269 | if _, _, err := ltx.PeekHeader(bytes.NewReader([]byte("foo"))); err != io.ErrUnexpectedEOF { 270 | t.Fatal(err) 271 | } 272 | }) 273 | } 274 | 275 | func TestPageHeader_Validate(t *testing.T) { 276 | t.Run("OK", func(t *testing.T) { 277 | hdr := ltx.PageHeader{Pgno: 1} 278 | if err := hdr.Validate(); err != nil { 279 | t.Fatal(err) 280 | } 281 | }) 282 | t.Run("ErrPgnoRequired", func(t *testing.T) { 283 | hdr := ltx.PageHeader{} 284 | if err := hdr.Validate(); err == nil || err.Error() != `page number required` { 285 | t.Fatalf("unexpected error: %s", err) 286 | } 287 | }) 288 | } 289 | 290 | func TestPageHeader_MarshalBinary(t *testing.T) { 291 | hdr := ltx.PageHeader{Pgno: 1000} 292 | 293 | var other ltx.PageHeader 294 | if b, err := hdr.MarshalBinary(); err != nil { 295 | t.Fatal(err) 296 | } else if err := other.UnmarshalBinary(b); err != nil { 297 | t.Fatal(err) 298 | } else if !reflect.DeepEqual(hdr, other) { 299 | t.Fatalf("mismatch:\ngot=%#v\nwant=%#v", hdr, other) 300 | } 301 | } 302 | 303 | func TestPageHeader_UnmarshalBinary(t *testing.T) { 304 | t.Run("ErrShortBuffer", func(t *testing.T) { 305 | var hdr ltx.PageHeader 306 | if err := hdr.UnmarshalBinary(make([]byte, 2)); err != io.ErrShortBuffer { 307 | t.Fatal(err) 308 | } 309 | }) 310 | } 311 | 312 | func TestTrailer_Validate(t *testing.T) { 313 | t.Run("OK", func(t *testing.T) { 314 | trailer := ltx.Trailer{ 315 | PostApplyChecksum: ltx.ChecksumFlag | 1, 316 | FileChecksum: ltx.ChecksumFlag | 2, 317 | } 318 | if err := trailer.Validate(); err != nil { 319 | t.Fatal(err) 320 | } 321 | }) 322 | t.Run("ErrPostApplyChecksumRequired", func(t *testing.T) { 323 | trailer := ltx.Trailer{} 324 | if err := trailer.Validate(); err == nil || err.Error() != `post-apply checksum required` { 325 | t.Fatalf("unexpected error: %s", err) 326 | } 327 | }) 328 | t.Run("ErrInvalidPostApplyChecksum", func(t *testing.T) { 329 | trailer := ltx.Trailer{PostApplyChecksum: 1} 330 | if err := trailer.Validate(); err == nil || err.Error() != `invalid post-apply checksum format` { 331 | t.Fatalf("unexpected error: %s", err) 332 | } 333 | }) 334 | t.Run("ErrFileChecksumRequired", func(t *testing.T) { 335 | trailer := ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag} 336 | if err := trailer.Validate(); err == nil || err.Error() != `file checksum required` { 337 | t.Fatalf("unexpected error: %s", err) 338 | } 339 | }) 340 | t.Run("ErrInvalidFileChecksum", func(t *testing.T) { 341 | trailer := ltx.Trailer{PostApplyChecksum: ltx.ChecksumFlag, FileChecksum: 1} 342 | if err := trailer.Validate(); err == nil || err.Error() != `invalid file checksum format` { 343 | t.Fatalf("unexpected error: %s", err) 344 | } 345 | }) 346 | } 347 | 348 | func TestIsValidHeaderFlags(t *testing.T) { 349 | if !ltx.IsValidHeaderFlags(0) { 350 | t.Fatal("expected valid") 351 | } else if ltx.IsValidHeaderFlags(100) { 352 | t.Fatal("expected invalid") 353 | } 354 | } 355 | 356 | func TestIsValidPageSize(t *testing.T) { 357 | t.Run("OK", func(t *testing.T) { 358 | for _, sz := range []uint32{512, 1024, 2048, 4096, 8192, 16384, 32768, 65536} { 359 | if !ltx.IsValidPageSize(sz) { 360 | t.Fatalf("expected page size of %d to be valid", sz) 361 | } 362 | } 363 | }) 364 | t.Run("TooSmall", func(t *testing.T) { 365 | if ltx.IsValidPageSize(256) { 366 | t.Fatal("expected invalid") 367 | } 368 | }) 369 | t.Run("TooLarge", func(t *testing.T) { 370 | if ltx.IsValidPageSize(131072) { 371 | t.Fatal("expected invalid") 372 | } 373 | }) 374 | t.Run("NotPowerOfTwo", func(t *testing.T) { 375 | if ltx.IsValidPageSize(1000) { 376 | t.Fatal("expected invalid") 377 | } 378 | }) 379 | } 380 | 381 | func TestParseFilename(t *testing.T) { 382 | t.Run("OK", func(t *testing.T) { 383 | if min, max, err := ltx.ParseFilename("0000000000000001-00000000000003e8.ltx"); err != nil { 384 | t.Fatal(err) 385 | } else if got, want := min, ltx.TXID(1); got != want { 386 | t.Fatalf("min=%d, want %d", got, want) 387 | } else if got, want := max, ltx.TXID(1000); got != want { 388 | t.Fatalf("max=%d, want %d", got, want) 389 | } 390 | }) 391 | 392 | t.Run("ErrInvalid", func(t *testing.T) { 393 | if _, _, err := ltx.ParseFilename("000000000000000z-00000000000003e8.ltx"); err == nil { 394 | t.Fatal("expected error") 395 | } 396 | if _, _, err := ltx.ParseFilename("0000000000000001.ltx"); err == nil { 397 | t.Fatal("expected error") 398 | } 399 | if _, _, err := ltx.ParseFilename("000000000000000z-00000000000003e8.zzz"); err == nil { 400 | t.Fatal("expected error") 401 | } 402 | }) 403 | } 404 | 405 | func TestChecksumReader(t *testing.T) { 406 | t.Run("OK", func(t *testing.T) { 407 | r := io.MultiReader( 408 | bytes.NewReader(bytes.Repeat([]byte("\x01"), 512)), 409 | bytes.NewReader(bytes.Repeat([]byte("\x02"), 512)), 410 | bytes.NewReader(bytes.Repeat([]byte("\x03"), 512)), 411 | ) 412 | if chksum, err := ltx.ChecksumReader(r, 512); err != nil { 413 | t.Fatal(err) 414 | } else if got, want := chksum, ltx.Checksum(0xefffffffecd99000); got != want { 415 | t.Fatalf("got=%x, want %x", got, want) 416 | } 417 | }) 418 | 419 | t.Run("ErrUnexpectedEOF", func(t *testing.T) { 420 | r := bytes.NewReader(bytes.Repeat([]byte("\x01"), 512)) 421 | if _, err := ltx.ChecksumReader(r, 1024); err != io.ErrUnexpectedEOF { 422 | t.Fatal(err) 423 | } 424 | }) 425 | } 426 | 427 | func TestTXID_MarshalJSON(t *testing.T) { 428 | t.Run("OK", func(t *testing.T) { 429 | txID := ltx.TXID(1000) 430 | if buf, err := json.Marshal(txID); err != nil { 431 | t.Fatal(err) 432 | } else if got, want := string(buf), `"00000000000003e8"`; got != want { 433 | t.Fatalf("got=%q, want %q", got, want) 434 | } 435 | }) 436 | t.Run("Map", func(t *testing.T) { 437 | m := map[string]ltx.TXID{"x": 1000, "y": 2000} 438 | if buf, err := json.Marshal(m); err != nil { 439 | t.Fatal(err) 440 | } else if got, want := string(buf), `{"x":"00000000000003e8","y":"00000000000007d0"}`; got != want { 441 | t.Fatalf("got=%q, want %q", got, want) 442 | } 443 | }) 444 | } 445 | 446 | func TestTXID_UnmarshalJSON(t *testing.T) { 447 | t.Run("OK", func(t *testing.T) { 448 | var txID ltx.TXID 449 | if err := json.Unmarshal([]byte(`"00000000000003e8"`), &txID); err != nil { 450 | t.Fatal(err) 451 | } else if got, want := txID, ltx.TXID(1000); got != want { 452 | t.Fatalf("got=%q, want %q", got, want) 453 | } 454 | }) 455 | t.Run("Null", func(t *testing.T) { 456 | var txID ltx.TXID 457 | if err := json.Unmarshal([]byte(`null`), &txID); err != nil { 458 | t.Fatal(err) 459 | } else if got, want := txID, ltx.TXID(0); got != want { 460 | t.Fatalf("got=%q, want %q", got, want) 461 | } 462 | }) 463 | t.Run("Map", func(t *testing.T) { 464 | var m map[string]ltx.TXID 465 | if err := json.Unmarshal([]byte(`{"x":"00000000000003e8","y":"00000000000007d0"}`), &m); err != nil { 466 | t.Fatal(err) 467 | } else if !reflect.DeepEqual(m, map[string]ltx.TXID{"x": 1000, "y": 2000}) { 468 | t.Fatalf("unexpected map: %#v", m) 469 | } 470 | }) 471 | t.Run("ErrInvalidType", func(t *testing.T) { 472 | var txID ltx.TXID 473 | if err := json.Unmarshal([]byte(`123`), &txID); err == nil || err.Error() != `cannot unmarshal TXID from JSON value` { 474 | t.Fatalf("unexpected error: %s", err) 475 | } 476 | }) 477 | t.Run("ErrStringFormat", func(t *testing.T) { 478 | var txID ltx.TXID 479 | if err := json.Unmarshal([]byte(`"xyz"`), &txID); err == nil || err.Error() != `cannot parse TXID from JSON string: "xyz"` { 480 | t.Fatalf("unexpected error: %s", err) 481 | } 482 | }) 483 | } 484 | 485 | func TestTXID_String(t *testing.T) { 486 | if got, want := ltx.TXID(0).String(), "0000000000000000"; got != want { 487 | t.Fatalf("got=%q, want %q", got, want) 488 | } 489 | if got, want := ltx.TXID(1000).String(), "00000000000003e8"; got != want { 490 | t.Fatalf("got=%q, want %q", got, want) 491 | } 492 | if got, want := ltx.TXID(math.MaxUint64).String(), "ffffffffffffffff"; got != want { 493 | t.Fatalf("got=%q, want %q", got, want) 494 | } 495 | } 496 | 497 | func TestParseTXID(t *testing.T) { 498 | t.Run("OK", func(t *testing.T) { 499 | if v, err := ltx.ParseTXID("0000000000000000"); err != nil { 500 | t.Fatal(err) 501 | } else if got, want := v, ltx.TXID(0); got != want { 502 | t.Fatalf("got=%d, want %d", got, want) 503 | } 504 | 505 | if v, err := ltx.ParseTXID("00000000000003e8"); err != nil { 506 | t.Fatal(err) 507 | } else if got, want := v, ltx.TXID(1000); got != want { 508 | t.Fatalf("got=%d, want %d", got, want) 509 | } 510 | 511 | if v, err := ltx.ParseTXID("ffffffffffffffff"); err != nil { 512 | t.Fatal(err) 513 | } else if got, want := v, ltx.TXID(math.MaxUint64); got != want { 514 | t.Fatalf("got=%d, want %d", got, want) 515 | } 516 | }) 517 | t.Run("ErrTooShort", func(t *testing.T) { 518 | if _, err := ltx.ParseTXID("000000000e38"); err == nil || err.Error() != `invalid formatted transaction id length: "000000000e38"` { 519 | t.Fatal(err) 520 | } 521 | }) 522 | t.Run("ErrTooLong", func(t *testing.T) { 523 | if _, err := ltx.ParseTXID("ffffffffffffffff0"); err == nil || err.Error() != `invalid formatted transaction id length: "ffffffffffffffff0"` { 524 | t.Fatal(err) 525 | } 526 | }) 527 | t.Run("ErrInvalidFormat", func(t *testing.T) { 528 | if _, err := ltx.ParseTXID("xxxxxxxxxxxxxxxx"); err == nil || err.Error() != `invalid transaction id format: "xxxxxxxxxxxxxxxx"` { 529 | t.Fatal(err) 530 | } 531 | }) 532 | } 533 | 534 | func TestChecksum_MarshalJSON(t *testing.T) { 535 | t.Run("OK", func(t *testing.T) { 536 | chksum := ltx.Checksum(1000) 537 | if buf, err := json.Marshal(chksum); err != nil { 538 | t.Fatal(err) 539 | } else if got, want := string(buf), `"00000000000003e8"`; got != want { 540 | t.Fatalf("got=%q, want %q", got, want) 541 | } 542 | }) 543 | t.Run("Map", func(t *testing.T) { 544 | m := map[string]ltx.Checksum{"x": 1000, "y": 2000} 545 | if buf, err := json.Marshal(m); err != nil { 546 | t.Fatal(err) 547 | } else if got, want := string(buf), `{"x":"00000000000003e8","y":"00000000000007d0"}`; got != want { 548 | t.Fatalf("got=%q, want %q", got, want) 549 | } 550 | }) 551 | } 552 | 553 | func TestChecksum_UnmarshalJSON(t *testing.T) { 554 | t.Run("OK", func(t *testing.T) { 555 | var chksum ltx.Checksum 556 | if err := json.Unmarshal([]byte(`"00000000000003e8"`), &chksum); err != nil { 557 | t.Fatal(err) 558 | } else if got, want := chksum, ltx.Checksum(1000); got != want { 559 | t.Fatalf("got=%q, want %q", got, want) 560 | } 561 | }) 562 | t.Run("Null", func(t *testing.T) { 563 | var chksum ltx.Checksum 564 | if err := json.Unmarshal([]byte(`null`), &chksum); err != nil { 565 | t.Fatal(err) 566 | } else if got, want := chksum, ltx.Checksum(0); got != want { 567 | t.Fatalf("got=%q, want %q", got, want) 568 | } 569 | }) 570 | t.Run("Map", func(t *testing.T) { 571 | var m map[string]ltx.Checksum 572 | if err := json.Unmarshal([]byte(`{"x":"00000000000003e8","y":"00000000000007d0"}`), &m); err != nil { 573 | t.Fatal(err) 574 | } else if !reflect.DeepEqual(m, map[string]ltx.Checksum{"x": 1000, "y": 2000}) { 575 | t.Fatalf("unexpected map: %#v", m) 576 | } 577 | }) 578 | t.Run("ErrInvalidType", func(t *testing.T) { 579 | var chksum ltx.Checksum 580 | if err := json.Unmarshal([]byte(`123`), &chksum); err == nil || err.Error() != `cannot unmarshal checksum from JSON value` { 581 | t.Fatalf("unexpected error: %s", err) 582 | } 583 | }) 584 | t.Run("ErrStringFormat", func(t *testing.T) { 585 | var chksum ltx.Checksum 586 | if err := json.Unmarshal([]byte(`"xyz"`), &chksum); err == nil || err.Error() != `cannot parse checksum from JSON string: "xyz"` { 587 | t.Fatalf("unexpected error: %s", err) 588 | } 589 | }) 590 | } 591 | 592 | func TestChecksum_String(t *testing.T) { 593 | if got, want := ltx.Checksum(0).String(), "0000000000000000"; got != want { 594 | t.Fatalf("got=%q, want %q", got, want) 595 | } 596 | if got, want := ltx.Checksum(1000).String(), "00000000000003e8"; got != want { 597 | t.Fatalf("got=%q, want %q", got, want) 598 | } 599 | if got, want := ltx.Checksum(math.MaxUint64).String(), "ffffffffffffffff"; got != want { 600 | t.Fatalf("got=%q, want %q", got, want) 601 | } 602 | } 603 | 604 | func TestParseChecksum(t *testing.T) { 605 | t.Run("OK", func(t *testing.T) { 606 | if v, err := ltx.ParseChecksum("0000000000000000"); err != nil { 607 | t.Fatal(err) 608 | } else if got, want := v, ltx.Checksum(0); got != want { 609 | t.Fatalf("got=%d, want %d", got, want) 610 | } 611 | 612 | if v, err := ltx.ParseChecksum("00000000000003e8"); err != nil { 613 | t.Fatal(err) 614 | } else if got, want := v, ltx.Checksum(1000); got != want { 615 | t.Fatalf("got=%d, want %d", got, want) 616 | } 617 | 618 | if v, err := ltx.ParseChecksum("ffffffffffffffff"); err != nil { 619 | t.Fatal(err) 620 | } else if got, want := v, ltx.Checksum(math.MaxUint64); got != want { 621 | t.Fatalf("got=%d, want %d", got, want) 622 | } 623 | }) 624 | t.Run("ErrTooShort", func(t *testing.T) { 625 | if _, err := ltx.ParseChecksum("000000000e38"); err == nil || err.Error() != `invalid formatted checksum length: "000000000e38"` { 626 | t.Fatal(err) 627 | } 628 | }) 629 | t.Run("ErrTooLong", func(t *testing.T) { 630 | if _, err := ltx.ParseChecksum("ffffffffffffffff0"); err == nil || err.Error() != `invalid formatted checksum length: "ffffffffffffffff0"` { 631 | t.Fatal(err) 632 | } 633 | }) 634 | t.Run("ErrInvalidFormat", func(t *testing.T) { 635 | if _, err := ltx.ParseChecksum("xxxxxxxxxxxxxxxx"); err == nil || err.Error() != `invalid checksum format: "xxxxxxxxxxxxxxxx"` { 636 | t.Fatal(err) 637 | } 638 | }) 639 | } 640 | 641 | func TestFormatTimestamp(t *testing.T) { 642 | for _, tt := range []struct { 643 | t time.Time 644 | want string 645 | }{ 646 | {time.Date(2000, 10, 20, 30, 40, 50, 0, time.UTC), "2000-10-21T06:40:50.000Z"}, 647 | {time.Date(2000, 10, 20, 30, 40, 50, 123000000, time.UTC), "2000-10-21T06:40:50.123Z"}, 648 | {time.Date(2000, 10, 20, 30, 40, 50, 120000000, time.UTC), "2000-10-21T06:40:50.120Z"}, 649 | {time.Date(2000, 10, 20, 30, 40, 50, 100000000, time.UTC), "2000-10-21T06:40:50.100Z"}, 650 | {time.Date(2000, 10, 20, 30, 40, 50, 100000, time.UTC), "2000-10-21T06:40:50.000Z"}, // submillisecond 651 | } { 652 | if got := ltx.FormatTimestamp(tt.t); got != tt.want { 653 | t.Fatalf("got=%s, want %s", got, tt.want) 654 | } 655 | } 656 | } 657 | 658 | func TestParseTimestamp(t *testing.T) { 659 | for _, tt := range []struct { 660 | str string 661 | want time.Time 662 | }{ 663 | {"2000-10-21T06:40:50.000Z", time.Date(2000, 10, 20, 30, 40, 50, 0, time.UTC)}, 664 | {"2000-10-21T06:40:50.123Z", time.Date(2000, 10, 20, 30, 40, 50, 123000000, time.UTC)}, 665 | {"2000-10-21T06:40:50.120Z", time.Date(2000, 10, 20, 30, 40, 50, 120000000, time.UTC)}, 666 | {"2000-10-21T06:40:50.100Z", time.Date(2000, 10, 20, 30, 40, 50, 100000000, time.UTC)}, 667 | {"2000-10-21T06:40:50Z", time.Date(2000, 10, 20, 30, 40, 50, 0, time.UTC)}, 668 | {"2000-10-21T06:40:50.000123Z", time.Date(2000, 10, 20, 30, 40, 50, 0, time.UTC)}, 669 | {"2000-10-21T06:40:50.000000123Z", time.Date(2000, 10, 20, 30, 40, 50, 0, time.UTC)}, 670 | } { 671 | if got, err := ltx.ParseTimestamp(tt.str); err != nil { 672 | t.Fatal(err) 673 | } else if !got.Equal(tt.want) { 674 | t.Fatalf("got=%s, want %s", got, tt.want) 675 | } 676 | } 677 | } 678 | 679 | func BenchmarkChecksumPage(b *testing.B) { 680 | for _, pageSize := range []int{512, 1024, 2048, 4096, 8192, 16384, 32768, 65536} { 681 | b.Run(fmt.Sprint(pageSize), func(b *testing.B) { 682 | benchmarkChecksumPage(b, pageSize) 683 | }) 684 | } 685 | } 686 | 687 | func benchmarkChecksumPage(b *testing.B, pageSize int) { 688 | data := make([]byte, pageSize) 689 | _, _ = rand.Read(data) 690 | b.ReportAllocs() 691 | b.SetBytes(int64(pageSize)) 692 | b.ResetTimer() 693 | 694 | for i := 0; i < b.N; i++ { 695 | ltx.ChecksumPage(uint32(i%math.MaxUint32), data) 696 | } 697 | } 698 | 699 | func BenchmarkChecksumPageWithHasher(b *testing.B) { 700 | for _, pageSize := range []int{512, 1024, 2048, 4096, 8192, 16384, 32768, 65536} { 701 | b.Run(fmt.Sprint(pageSize), func(b *testing.B) { 702 | benchmarkChecksumPageWithHasher(b, pageSize) 703 | }) 704 | } 705 | } 706 | 707 | func benchmarkChecksumPageWithHasher(b *testing.B, pageSize int) { 708 | data := make([]byte, pageSize) 709 | _, _ = rand.Read(data) 710 | b.ReportAllocs() 711 | b.SetBytes(int64(pageSize)) 712 | b.ResetTimer() 713 | 714 | h := ltx.NewHasher() 715 | for i := 0; i < b.N; i++ { 716 | ltx.ChecksumPageWithHasher(h, uint32(i%math.MaxUint32), data) 717 | } 718 | } 719 | 720 | // BenchmarkXOR simulates the sum of checksums for a 1GB database (assuming 4KB pages). 721 | func BenchmarkXOR(b *testing.B) { 722 | const pageSize = 4096 723 | const pageN = (1 << 30) / pageSize 724 | 725 | m := make(map[uint32]ltx.Checksum) 726 | page := make([]byte, pageSize) 727 | for pgno := uint32(1); pgno <= pageN; pgno++ { 728 | _, _ = rand.Read(page) 729 | m[pgno] = ltx.ChecksumPage(pgno, page) 730 | } 731 | b.SetBytes(int64(pageN * pageSize)) 732 | b.ResetTimer() 733 | 734 | for i := 0; i < b.N; i++ { 735 | var chksum ltx.Checksum 736 | for pgno := uint32(1); pgno <= pageN; pgno++ { 737 | chksum ^= m[pgno] 738 | } 739 | } 740 | } 741 | 742 | // createFile creates a file and returns the file handle. Closes on cleanup. 743 | func createFile(tb testing.TB, name string) *os.File { 744 | tb.Helper() 745 | f, err := os.Create(name) 746 | if err != nil { 747 | tb.Fatal(err) 748 | } 749 | tb.Cleanup(func() { _ = f.Close() }) 750 | return f 751 | } 752 | 753 | // writeFileSpec is a helper function for writing a spec to a file. 754 | func writeFileSpec(tb testing.TB, w io.Writer, spec *ltx.FileSpec) int64 { 755 | tb.Helper() 756 | n, err := spec.WriteTo(w) 757 | if err != nil { 758 | tb.Fatal(err) 759 | } 760 | return int64(n) 761 | } 762 | 763 | // readFileSpec is a helper function for reading a spec from a file. 764 | func readFileSpec(tb testing.TB, r io.Reader, size int64) *ltx.FileSpec { 765 | tb.Helper() 766 | var spec ltx.FileSpec 767 | if _, err := spec.ReadFrom(r); err != nil { 768 | tb.Fatal(err) 769 | } 770 | return &spec 771 | } 772 | 773 | // compactFileSpecs compacts a set of file specs to a new spec. 774 | func compactFileSpecs(tb testing.TB, inputs ...*ltx.FileSpec) (*ltx.FileSpec, error) { 775 | tb.Helper() 776 | 777 | // Write input specs to file. 778 | wtrs := make([]io.Writer, len(inputs)) 779 | rdrs := make([]io.Reader, len(inputs)) 780 | for i, input := range inputs { 781 | var buf bytes.Buffer 782 | wtrs[i], rdrs[i] = &buf, &buf 783 | writeFileSpec(tb, wtrs[i], input) 784 | } 785 | 786 | // Compact files together. 787 | var output bytes.Buffer 788 | c := ltx.NewCompactor(&output, rdrs) 789 | if err := c.Compact(context.Background()); err != nil { 790 | return nil, err 791 | } 792 | return readFileSpec(tb, &output, int64(output.Len())), nil 793 | } 794 | 795 | // assertFileSpecEqual checks x & y for equality. Fail on inequality. 796 | func assertFileSpecEqual(tb testing.TB, x, y *ltx.FileSpec) { 797 | tb.Helper() 798 | 799 | if got, want := x.Header, y.Header; got != want { 800 | tb.Fatalf("header mismatch:\ngot=%#v\nwant=%#v", got, want) 801 | } 802 | 803 | if got, want := len(x.Pages), len(y.Pages); got != want { 804 | tb.Fatalf("page count: %d, want %d", got, want) 805 | } 806 | for i := range x.Pages { 807 | if got, want := x.Pages[i].Header, y.Pages[i].Header; got != want { 808 | tb.Fatalf("page header mismatch: i=%d\ngot=%#v\nwant=%#v", i, got, want) 809 | } 810 | if got, want := x.Pages[i].Data, y.Pages[i].Data; !bytes.Equal(got, want) { 811 | tb.Fatalf("page data mismatch: i=%d\ngot=%#v\nwant=%#v", i, got, want) 812 | } 813 | } 814 | 815 | if got, want := x.Trailer, y.Trailer; got != want { 816 | tb.Fatalf("trailer mismatch:\ngot=%#v\nwant=%#v", got, want) 817 | } 818 | } 819 | --------------------------------------------------------------------------------