├── .gitignore ├── etc ├── litestream.service ├── gon.hcl ├── litestream.yml ├── build.ps1 ├── nfpm.yml └── litestream.wxs ├── internal ├── internal_unix.go ├── internal_windows.go └── internal.go ├── Dockerfile ├── cmd └── litestream │ ├── main_notwindows.go │ ├── version.go │ ├── databases.go │ ├── main_windows.go │ ├── snapshots.go │ ├── wal.go │ ├── generations.go │ ├── main_test.go │ ├── replicate.go │ └── restore.go ├── .github ├── CONTRIBUTING.md └── workflows │ ├── release.docker.yml │ ├── release.linux.yml │ └── commit.yml ├── Makefile ├── replica_client.go ├── go.mod ├── mock └── replica_client.go ├── README.md ├── replica_test.go ├── file ├── replica_client_test.go └── replica_client.go ├── LICENSE ├── gcs └── replica_client.go ├── sftp └── replica_client.go ├── litestream.go ├── abs └── replica_client.go ├── db_test.go └── litestream_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /dist 3 | -------------------------------------------------------------------------------- /etc/litestream.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Litestream 3 | 4 | [Service] 5 | Restart=always 6 | ExecStart=/usr/bin/litestream replicate 7 | 8 | [Install] 9 | WantedBy=multi-user.target 10 | -------------------------------------------------------------------------------- /etc/gon.hcl: -------------------------------------------------------------------------------- 1 | source = ["./dist/litestream"] 2 | bundle_id = "com.middlemost.litestream" 3 | 4 | apple_id { 5 | username = "benbjohnson@yahoo.com" 6 | password = "@env:AC_PASSWORD" 7 | } 8 | 9 | sign { 10 | application_identity = "Developer ID Application: Middlemost Systems, LLC" 11 | } 12 | 13 | zip { 14 | output_path = "dist/litestream.zip" 15 | } 16 | -------------------------------------------------------------------------------- /etc/litestream.yml: -------------------------------------------------------------------------------- 1 | # AWS credentials 2 | # access-key-id: AKIAxxxxxxxxxxxxxxxx 3 | # secret-access-key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxxxxxxx 4 | 5 | # dbs: 6 | # - path: /path/to/primary/db # Database to replicate from 7 | # replicas: 8 | # - path: /path/to/replica # File-based replication 9 | # - url: s3://my.bucket.com/db # S3-based replication 10 | 11 | -------------------------------------------------------------------------------- /internal/internal_unix.go: -------------------------------------------------------------------------------- 1 | // +build aix darwin dragonfly freebsd linux netbsd openbsd solaris 2 | 3 | package internal 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | ) 9 | 10 | // Fileinfo returns syscall fields from a FileInfo object. 11 | func Fileinfo(fi os.FileInfo) (uid, gid int) { 12 | if fi == nil { 13 | return -1, -1 14 | } 15 | stat := fi.Sys().(*syscall.Stat_t) 16 | return int(stat.Uid), int(stat.Gid) 17 | } 18 | 19 | func fixRootDirectory(p string) string { 20 | return p 21 | } 22 | -------------------------------------------------------------------------------- /etc/build.ps1: -------------------------------------------------------------------------------- 1 | [CmdletBinding()] 2 | Param ( 3 | [Parameter(Mandatory = $true)] 4 | [String] $Version 5 | ) 6 | $ErrorActionPreference = "Stop" 7 | 8 | # Update working directory. 9 | Push-Location $PSScriptRoot 10 | Trap { 11 | Pop-Location 12 | } 13 | 14 | Invoke-Expression "candle.exe -nologo -arch x64 -ext WixUtilExtension -out litestream.wixobj -dVersion=`"$Version`" litestream.wxs" 15 | Invoke-Expression "light.exe -nologo -spdb -ext WixUtilExtension -out `"litestream-${Version}.msi`" litestream.wixobj" 16 | 17 | Pop-Location 18 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20.1 as builder 2 | 3 | WORKDIR /src/litestream 4 | COPY . . 5 | 6 | ARG LITESTREAM_VERSION=latest 7 | 8 | RUN --mount=type=cache,target=/root/.cache/go-build \ 9 | --mount=type=cache,target=/go/pkg \ 10 | go build -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}' -extldflags '-static'" -tags osusergo,netgo,sqlite_omit_load_extension -o /usr/local/bin/litestream ./cmd/litestream 11 | 12 | 13 | FROM alpine:3.17.2 14 | COPY --from=builder /usr/local/bin/litestream /usr/local/bin/litestream 15 | ENTRYPOINT ["/usr/local/bin/litestream"] 16 | CMD [] 17 | -------------------------------------------------------------------------------- /cmd/litestream/main_notwindows.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package main 4 | 5 | import ( 6 | "context" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | ) 11 | 12 | const defaultConfigPath = "/etc/litestream.yml" 13 | 14 | func isWindowsService() (bool, error) { 15 | return false, nil 16 | } 17 | 18 | func runWindowsService(ctx context.Context) error { 19 | panic("cannot run windows service as unix process") 20 | } 21 | 22 | func signalChan() <-chan os.Signal { 23 | ch := make(chan os.Signal, 2) 24 | signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) 25 | return ch 26 | } 27 | -------------------------------------------------------------------------------- /internal/internal_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package internal 4 | 5 | import ( 6 | "os" 7 | ) 8 | 9 | // Fileinfo returns syscall fields from a FileInfo object. 10 | func Fileinfo(fi os.FileInfo) (uid, gid int) { 11 | return -1, -1 12 | } 13 | 14 | // fixRootDirectory is copied from the standard library for use with mkdirAll() 15 | func fixRootDirectory(p string) string { 16 | if len(p) == len(`\\?\c:`) { 17 | if os.IsPathSeparator(p[0]) && os.IsPathSeparator(p[1]) && p[2] == '?' && os.IsPathSeparator(p[3]) && p[5] == ':' { 18 | return p + `\` 19 | } 20 | } 21 | return p 22 | } 23 | -------------------------------------------------------------------------------- /etc/nfpm.yml: -------------------------------------------------------------------------------- 1 | name: litestream 2 | arch: "${GOARCH}" 3 | platform: "${GOOS}" 4 | version: "${LITESTREAM_VERSION}" 5 | section: "default" 6 | priority: "extra" 7 | maintainer: "Ben Johnson " 8 | description: Litestream is a tool for real-time replication of SQLite databases. 9 | homepage: "https://github.com/benbjohnson/litestream" 10 | license: "Apache 2" 11 | contents: 12 | - src: ./litestream 13 | dst: /usr/bin/litestream 14 | - src: ./litestream.yml 15 | dst: /etc/litestream.yml 16 | type: config 17 | - src: ./litestream.service 18 | dst: /usr/lib/systemd/system/litestream.service 19 | type: config 20 | -------------------------------------------------------------------------------- /cmd/litestream/version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | ) 8 | 9 | // VersionCommand represents a command to print the current version. 10 | type VersionCommand struct{} 11 | 12 | // Run executes the command. 13 | func (c *VersionCommand) Run(ctx context.Context, args []string) (err error) { 14 | fs := flag.NewFlagSet("litestream-version", flag.ContinueOnError) 15 | fs.Usage = c.Usage 16 | if err := fs.Parse(args); err != nil { 17 | return err 18 | } 19 | 20 | fmt.Println(Version) 21 | 22 | return nil 23 | } 24 | 25 | // Usage prints the help screen to STDOUT. 26 | func (c *VersionCommand) Usage() { 27 | fmt.Println(` 28 | Prints the version. 29 | 30 | Usage: 31 | 32 | litestream version 33 | `[1:]) 34 | } 35 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contribution Policy 2 | 3 | Initially, Litestream was closed to outside contributions. The goal was to 4 | reduce burnout by limiting the maintenance overhead of reviewing and validating 5 | third-party code. However, this policy is overly broad and has prevented small, 6 | easily testable patches from being contributed. 7 | 8 | Litestream is now open to code contributions for bug fixes only. Features carry 9 | a long-term maintenance burden so they will not be accepted at this time. 10 | Please [submit an issue][new-issue] if you have a feature you'd like to 11 | request. 12 | 13 | If you find mistakes in the documentation, please submit a fix to the 14 | [documentation repository][docs]. 15 | 16 | [new-issue]: https://github.com/benbjohnson/litestream/issues/new 17 | [docs]: https://github.com/benbjohnson/litestream.io 18 | 19 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: 2 | 3 | docker: 4 | docker build -t litestream . 5 | 6 | dist-linux: 7 | mkdir -p dist 8 | cp etc/litestream.yml dist/litestream.yml 9 | docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e GOOS=linux -e GOARCH=amd64 golang:1.16 go build -v -ldflags "-s -w" -o dist/litestream ./cmd/litestream 10 | tar -cz -f dist/litestream-linux-amd64.tar.gz -C dist litestream 11 | 12 | dist-linux-arm: 13 | docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e CGO_ENABLED=1 -e CC=arm-linux-gnueabihf-gcc -e GOOS=linux -e GOARCH=arm golang-xc:1.16 go build -v -o dist/litestream-linux-arm ./cmd/litestream 14 | 15 | dist-linux-arm64: 16 | docker run --rm -v "${PWD}":/usr/src/litestream -w /usr/src/litestream -e CGO_ENABLED=1 -e CC=aarch64-linux-gnu-gcc -e GOOS=linux -e GOARCH=arm64 golang-xc:1.16 go build -v -o dist/litestream-linux-arm64 ./cmd/litestream 17 | 18 | dist-macos: 19 | ifndef LITESTREAM_VERSION 20 | $(error LITESTREAM_VERSION is undefined) 21 | endif 22 | mkdir -p dist 23 | go build -v -ldflags "-s -w -X 'main.Version=${LITESTREAM_VERSION}'" -o dist/litestream ./cmd/litestream 24 | gon etc/gon.hcl 25 | mv dist/litestream.zip dist/litestream-${LITESTREAM_VERSION}-darwin-amd64.zip 26 | openssl dgst -sha256 dist/litestream-${LITESTREAM_VERSION}-darwin-amd64.zip 27 | 28 | clean: 29 | rm -rf dist 30 | 31 | .PHONY: default dist-linux dist-macos clean 32 | -------------------------------------------------------------------------------- /.github/workflows/release.docker.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: 4 | - published 5 | pull_request: 6 | types: 7 | - opened 8 | - synchronize 9 | - reopened 10 | branches-ignore: 11 | - "dependabot/**" 12 | 13 | name: Release (Docker) 14 | jobs: 15 | docker: 16 | runs-on: ubuntu-latest 17 | env: 18 | PLATFORMS: "linux/amd64,linux/arm64" 19 | VERSION: "${{ github.event_name == 'release' && github.event.release.name || github.sha }}" 20 | 21 | steps: 22 | - uses: actions/checkout@v2 23 | - uses: docker/setup-qemu-action@v1 24 | - uses: docker/setup-buildx-action@v1 25 | 26 | - uses: docker/login-action@v1 27 | with: 28 | username: benbjohnson 29 | password: ${{ secrets.DOCKERHUB_TOKEN }} 30 | 31 | - id: meta 32 | uses: docker/metadata-action@v3 33 | with: 34 | images: litestream/litestream 35 | tags: | 36 | type=ref,event=branch 37 | type=ref,event=pr 38 | type=sha 39 | type=sha,format=long 40 | type=semver,pattern={{version}} 41 | type=semver,pattern={{major}}.{{minor}} 42 | 43 | - uses: docker/build-push-action@v2 44 | with: 45 | context: . 46 | push: true 47 | platforms: ${{ env.PLATFORMS }} 48 | tags: ${{ steps.meta.outputs.tags }} 49 | labels: ${{ steps.meta.outputs.labels }} 50 | build-args: | 51 | LITESTREAM_VERSION=${{ env.VERSION }} -------------------------------------------------------------------------------- /cmd/litestream/databases.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "strings" 9 | "text/tabwriter" 10 | ) 11 | 12 | // DatabasesCommand is a command for listing managed databases. 13 | type DatabasesCommand struct{} 14 | 15 | // Run executes the command. 16 | func (c *DatabasesCommand) Run(ctx context.Context, args []string) (err error) { 17 | fs := flag.NewFlagSet("litestream-databases", flag.ContinueOnError) 18 | configPath, noExpandEnv := registerConfigFlag(fs) 19 | fs.Usage = c.Usage 20 | if err := fs.Parse(args); err != nil { 21 | return err 22 | } else if fs.NArg() != 0 { 23 | return fmt.Errorf("too many arguments") 24 | } 25 | 26 | // Load configuration. 27 | if *configPath == "" { 28 | *configPath = DefaultConfigPath() 29 | } 30 | config, err := ReadConfigFile(*configPath, !*noExpandEnv) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | // List all databases. 36 | w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) 37 | defer w.Flush() 38 | 39 | fmt.Fprintln(w, "path\treplicas") 40 | for _, dbConfig := range config.DBs { 41 | db, err := NewDBFromConfig(dbConfig) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | var replicaNames []string 47 | for _, r := range db.Replicas { 48 | replicaNames = append(replicaNames, r.Name()) 49 | } 50 | 51 | fmt.Fprintf(w, "%s\t%s\n", 52 | db.Path(), 53 | strings.Join(replicaNames, ","), 54 | ) 55 | } 56 | 57 | return nil 58 | } 59 | 60 | // Usage prints the help screen to STDOUT. 61 | func (c *DatabasesCommand) Usage() { 62 | fmt.Printf(` 63 | The databases command lists all databases in the configuration file. 64 | 65 | Usage: 66 | 67 | litestream databases [arguments] 68 | 69 | Arguments: 70 | 71 | -config PATH 72 | Specifies the configuration file. 73 | Defaults to %s 74 | 75 | -no-expand-env 76 | Disables environment variable expansion in configuration file. 77 | 78 | `[1:], 79 | DefaultConfigPath(), 80 | ) 81 | } 82 | -------------------------------------------------------------------------------- /replica_client.go: -------------------------------------------------------------------------------- 1 | package litestream 2 | 3 | import ( 4 | "context" 5 | "io" 6 | ) 7 | 8 | // ReplicaClient represents client to connect to a Replica. 9 | type ReplicaClient interface { 10 | // Returns the type of client. 11 | Type() string 12 | 13 | // Returns a list of available generations. 14 | Generations(ctx context.Context) ([]string, error) 15 | 16 | // Deletes all snapshots & WAL segments within a generation. 17 | DeleteGeneration(ctx context.Context, generation string) error 18 | 19 | // Returns an iterator of all snapshots within a generation on the replica. 20 | Snapshots(ctx context.Context, generation string) (SnapshotIterator, error) 21 | 22 | // Writes LZ4 compressed snapshot data to the replica at a given index 23 | // within a generation. Returns metadata for the snapshot. 24 | WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (SnapshotInfo, error) 25 | 26 | // Deletes a snapshot with the given generation & index. 27 | DeleteSnapshot(ctx context.Context, generation string, index int) error 28 | 29 | // Returns a reader that contains LZ4 compressed snapshot data for a 30 | // given index within a generation. Returns an os.ErrNotFound error if 31 | // the snapshot does not exist. 32 | SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) 33 | 34 | // Returns an iterator of all WAL segments within a generation on the replica. 35 | WALSegments(ctx context.Context, generation string) (WALSegmentIterator, error) 36 | 37 | // Writes an LZ4 compressed WAL segment at a given position. 38 | // Returns metadata for the written segment. 39 | WriteWALSegment(ctx context.Context, pos Pos, r io.Reader) (WALSegmentInfo, error) 40 | 41 | // Deletes one or more WAL segments at the given positions. 42 | DeleteWALSegments(ctx context.Context, a []Pos) error 43 | 44 | // Returns a reader that contains an LZ4 compressed WAL segment at a given 45 | // index/offset within a generation. Returns an os.ErrNotFound error if the 46 | // WAL segment does not exist. 47 | WALSegmentReader(ctx context.Context, pos Pos) (io.ReadCloser, error) 48 | } 49 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/benbjohnson/litestream 2 | 3 | go 1.19 4 | 5 | require ( 6 | cloud.google.com/go/storage v1.31.0 7 | filippo.io/age v1.1.1 8 | github.com/Azure/azure-storage-blob-go v0.15.0 9 | github.com/aws/aws-sdk-go v1.44.318 10 | github.com/mattn/go-shellwords v1.0.12 11 | github.com/mattn/go-sqlite3 v1.14.17 12 | github.com/pierrec/lz4/v4 v4.1.18 13 | github.com/pkg/sftp v1.13.5 14 | github.com/prometheus/client_golang v1.16.0 15 | golang.org/x/crypto v0.12.0 16 | golang.org/x/sync v0.3.0 17 | golang.org/x/sys v0.11.0 18 | google.golang.org/api v0.135.0 19 | gopkg.in/yaml.v2 v2.4.0 20 | ) 21 | 22 | require ( 23 | cloud.google.com/go v0.110.7 // indirect 24 | cloud.google.com/go/compute v1.23.0 // indirect 25 | cloud.google.com/go/compute/metadata v0.2.3 // indirect 26 | cloud.google.com/go/iam v1.1.1 // indirect 27 | github.com/Azure/azure-pipeline-go v0.2.3 // indirect 28 | github.com/beorn7/perks v1.0.1 // indirect 29 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 30 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 31 | github.com/golang/protobuf v1.5.3 // indirect 32 | github.com/google/go-cmp v0.5.9 // indirect 33 | github.com/google/s2a-go v0.1.4 // indirect 34 | github.com/google/uuid v1.3.0 // indirect 35 | github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect 36 | github.com/googleapis/gax-go/v2 v2.12.0 // indirect 37 | github.com/jmespath/go-jmespath v0.4.0 // indirect 38 | github.com/kr/fs v0.1.0 // indirect 39 | github.com/mattn/go-ieproxy v0.0.11 // indirect 40 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 41 | github.com/prometheus/client_model v0.4.0 // indirect 42 | github.com/prometheus/common v0.44.0 // indirect 43 | github.com/prometheus/procfs v0.11.1 // indirect 44 | go.opencensus.io v0.24.0 // indirect 45 | golang.org/x/net v0.14.0 // indirect 46 | golang.org/x/oauth2 v0.11.0 // indirect 47 | golang.org/x/text v0.12.0 // indirect 48 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect 49 | google.golang.org/appengine v1.6.7 // indirect 50 | google.golang.org/genproto v0.0.0-20230807174057-1744710a1577 // indirect 51 | google.golang.org/genproto/googleapis/api v0.0.0-20230807174057-1744710a1577 // indirect 52 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect 53 | google.golang.org/grpc v1.57.0 // indirect 54 | google.golang.org/protobuf v1.31.0 // indirect 55 | ) 56 | -------------------------------------------------------------------------------- /etc/litestream.wxs: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 21 | 29 | 30 | 31 | 32 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 51 | 52 | 61 | 67 | 68 | 69 | 70 | 77 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /mock/replica_client.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/benbjohnson/litestream" 8 | ) 9 | 10 | var _ litestream.ReplicaClient = (*ReplicaClient)(nil) 11 | 12 | type ReplicaClient struct { 13 | GenerationsFunc func(ctx context.Context) ([]string, error) 14 | DeleteGenerationFunc func(ctx context.Context, generation string) error 15 | SnapshotsFunc func(ctx context.Context, generation string) (litestream.SnapshotIterator, error) 16 | WriteSnapshotFunc func(ctx context.Context, generation string, index int, r io.Reader) (litestream.SnapshotInfo, error) 17 | DeleteSnapshotFunc func(ctx context.Context, generation string, index int) error 18 | SnapshotReaderFunc func(ctx context.Context, generation string, index int) (io.ReadCloser, error) 19 | WALSegmentsFunc func(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) 20 | WriteWALSegmentFunc func(ctx context.Context, pos litestream.Pos, r io.Reader) (litestream.WALSegmentInfo, error) 21 | DeleteWALSegmentsFunc func(ctx context.Context, a []litestream.Pos) error 22 | WALSegmentReaderFunc func(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) 23 | } 24 | 25 | func (c *ReplicaClient) Type() string { return "mock" } 26 | 27 | func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) { 28 | return c.GenerationsFunc(ctx) 29 | } 30 | 31 | func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error { 32 | return c.DeleteGenerationFunc(ctx, generation) 33 | } 34 | 35 | func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) { 36 | return c.SnapshotsFunc(ctx, generation) 37 | } 38 | 39 | func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, r io.Reader) (litestream.SnapshotInfo, error) { 40 | return c.WriteSnapshotFunc(ctx, generation, index, r) 41 | } 42 | 43 | func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error { 44 | return c.DeleteSnapshotFunc(ctx, generation, index) 45 | } 46 | 47 | func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) { 48 | return c.SnapshotReaderFunc(ctx, generation, index) 49 | } 50 | 51 | func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) { 52 | return c.WALSegmentsFunc(ctx, generation) 53 | } 54 | 55 | func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, r io.Reader) (litestream.WALSegmentInfo, error) { 56 | return c.WriteWALSegmentFunc(ctx, pos, r) 57 | } 58 | 59 | func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error { 60 | return c.DeleteWALSegmentsFunc(ctx, a) 61 | } 62 | 63 | func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) { 64 | return c.WALSegmentReaderFunc(ctx, pos) 65 | } 66 | -------------------------------------------------------------------------------- /.github/workflows/release.linux.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: 4 | - created 5 | 6 | name: release (linux) 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | include: 13 | - arch: amd64 14 | cc: gcc 15 | - arch: arm64 16 | cc: aarch64-linux-gnu-gcc 17 | - arch: arm 18 | arm: 6 19 | cc: arm-linux-gnueabi-gcc 20 | - arch: arm 21 | arm: 7 22 | cc: arm-linux-gnueabihf-gcc 23 | 24 | env: 25 | GOOS: linux 26 | GOARCH: ${{ matrix.arch }} 27 | GOARM: ${{ matrix.arm }} 28 | CC: ${{ matrix.cc }} 29 | 30 | steps: 31 | - uses: actions/checkout@v2 32 | - uses: actions/setup-go@v2 33 | with: 34 | go-version: '1.20' 35 | 36 | - id: release 37 | uses: bruceadams/get-release@v1.2.2 38 | env: 39 | GITHUB_TOKEN: ${{ github.token }} 40 | 41 | - name: Install cross-compilers 42 | run: | 43 | sudo apt-get update 44 | sudo apt-get install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf gcc-arm-linux-gnueabi 45 | 46 | - name: Install nfpm 47 | run: | 48 | wget https://github.com/goreleaser/nfpm/releases/download/v2.2.3/nfpm_2.2.3_Linux_x86_64.tar.gz 49 | tar zxvf nfpm_2.2.3_Linux_x86_64.tar.gz 50 | 51 | - name: Build litestream 52 | run: | 53 | rm -rf dist 54 | mkdir -p dist 55 | cp etc/litestream.yml etc/litestream.service dist 56 | cat etc/nfpm.yml | LITESTREAM_VERSION=${{ steps.release.outputs.tag_name }} envsubst > dist/nfpm.yml 57 | CGO_ENABLED=1 go build -ldflags "-s -w -extldflags "-static" -X 'main.Version=${{ steps.release.outputs.tag_name }}'" -tags osusergo,netgo,sqlite_omit_load_extension -o dist/litestream ./cmd/litestream 58 | 59 | cd dist 60 | tar -czvf litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.tar.gz litestream 61 | ../nfpm pkg --config nfpm.yml --packager deb --target litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.deb 62 | 63 | - name: Upload release tarball 64 | uses: actions/upload-release-asset@v1.0.2 65 | env: 66 | GITHUB_TOKEN: ${{ github.token }} 67 | with: 68 | upload_url: ${{ steps.release.outputs.upload_url }} 69 | asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.tar.gz 70 | asset_name: litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.tar.gz 71 | asset_content_type: application/gzip 72 | 73 | - name: Upload debian package 74 | uses: actions/upload-release-asset@v1.0.2 75 | env: 76 | GITHUB_TOKEN: ${{ github.token }} 77 | with: 78 | upload_url: ${{ steps.release.outputs.upload_url }} 79 | asset_path: ./dist/litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.deb 80 | asset_name: litestream-${{ steps.release.outputs.tag_name }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}.deb 81 | asset_content_type: application/octet-stream 82 | -------------------------------------------------------------------------------- /cmd/litestream/main_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package main 4 | 5 | import ( 6 | "context" 7 | "io" 8 | "log" 9 | "os" 10 | "os/signal" 11 | 12 | "golang.org/x/sys/windows" 13 | "golang.org/x/sys/windows/svc" 14 | "golang.org/x/sys/windows/svc/eventlog" 15 | ) 16 | 17 | const defaultConfigPath = `C:\Litestream\litestream.yml` 18 | 19 | // serviceName is the Windows Service name. 20 | const serviceName = "Litestream" 21 | 22 | // isWindowsService returns true if currently executing within a Windows service. 23 | func isWindowsService() (bool, error) { 24 | return svc.IsWindowsService() 25 | } 26 | 27 | func runWindowsService(ctx context.Context) error { 28 | // Attempt to install new log service. This will fail if already installed. 29 | // We don't log the error because we don't have anywhere to log until we open the log. 30 | _ = eventlog.InstallAsEventCreate(serviceName, eventlog.Error|eventlog.Warning|eventlog.Info) 31 | 32 | elog, err := eventlog.Open(serviceName) 33 | if err != nil { 34 | return err 35 | } 36 | defer elog.Close() 37 | 38 | // Set eventlog as log writer while running. 39 | log.SetOutput((*eventlogWriter)(elog)) 40 | defer log.SetOutput(os.Stderr) 41 | 42 | log.Print("Litestream service starting") 43 | 44 | if err := svc.Run(serviceName, &windowsService{ctx: ctx}); err != nil { 45 | return errStop 46 | } 47 | 48 | log.Print("Litestream service stopped") 49 | return nil 50 | } 51 | 52 | // windowsService is an interface adapter for svc.Handler. 53 | type windowsService struct { 54 | ctx context.Context 55 | } 56 | 57 | func (s *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, statusCh chan<- svc.Status) (svcSpecificEC bool, exitCode uint32) { 58 | var err error 59 | 60 | // Notify Windows that the service is starting up. 61 | statusCh <- svc.Status{State: svc.StartPending} 62 | 63 | // Instantiate replication command and load configuration. 64 | c := NewReplicateCommand() 65 | if c.Config, err = ReadConfigFile(DefaultConfigPath(), true); err != nil { 66 | log.Printf("cannot load configuration: %s", err) 67 | return true, 1 68 | } 69 | 70 | // Execute replication command. 71 | if err := c.Run(s.ctx); err != nil { 72 | log.Printf("cannot replicate: %s", err) 73 | statusCh <- svc.Status{State: svc.StopPending} 74 | return true, 2 75 | } 76 | 77 | // Notify Windows that the service is now running. 78 | statusCh <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop} 79 | 80 | for { 81 | select { 82 | case req := <-r: 83 | switch req.Cmd { 84 | case svc.Stop: 85 | c.Close() 86 | statusCh <- svc.Status{State: svc.StopPending} 87 | return false, windows.NO_ERROR 88 | case svc.Interrogate: 89 | statusCh <- req.CurrentStatus 90 | default: 91 | log.Printf("Litestream service received unexpected change request cmd: %d", req.Cmd) 92 | } 93 | } 94 | } 95 | } 96 | 97 | // Ensure implementation implements io.Writer interface. 98 | var _ io.Writer = (*eventlogWriter)(nil) 99 | 100 | // eventlogWriter is an adapter for using eventlog.Log as an io.Writer. 101 | type eventlogWriter eventlog.Log 102 | 103 | func (w *eventlogWriter) Write(p []byte) (n int, err error) { 104 | elog := (*eventlog.Log)(w) 105 | return 0, elog.Info(1, string(p)) 106 | } 107 | 108 | func signalChan() <-chan os.Signal { 109 | ch := make(chan os.Signal, 1) 110 | signal.Notify(ch, os.Interrupt) 111 | return ch 112 | } 113 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Litestream 2 | ![GitHub release (latest by date)](https://img.shields.io/github/v/release/benbjohnson/litestream) 3 | ![Status](https://img.shields.io/badge/status-beta-blue) 4 | ![GitHub](https://img.shields.io/github/license/benbjohnson/litestream) 5 | [![Docker Pulls](https://img.shields.io/docker/pulls/litestream/litestream.svg?maxAge=604800)](https://hub.docker.com/r/litestream/litestream/) 6 | ![test](https://github.com/benbjohnson/litestream/workflows/test/badge.svg) 7 | ========== 8 | 9 | Litestream is a standalone disaster recovery tool for SQLite. It runs as a 10 | background process and safely replicates changes incrementally to another file 11 | or S3. Litestream only communicates with SQLite through the SQLite API so it 12 | will not corrupt your database. 13 | 14 | If you need support or have ideas for improving Litestream, please join the 15 | [Litestream Slack][slack] or visit the [GitHub Discussions](https://github.com/benbjohnson/litestream/discussions). 16 | Please visit the [Litestream web site](https://litestream.io) for installation 17 | instructions and documentation. 18 | 19 | If you find this project interesting, please consider starring the project on 20 | GitHub. 21 | 22 | [slack]: https://join.slack.com/t/litestream/shared_invite/zt-n0j4s3ci-lx1JziR3bV6L2NMF723H3Q 23 | 24 | 25 | ## Acknowledgements 26 | 27 | While the Litestream project does not accept external code patches, many 28 | of the most valuable contributions are in the forms of testing, feedback, and 29 | documentation. These help harden software and streamline usage for other users. 30 | 31 | I want to give special thanks to individuals who invest much of their time and 32 | energy into the project to help make it better: 33 | 34 | - Thanks to [Cory LaNou](https://twitter.com/corylanou) for giving early feedback and testing when Litestream was still pre-release. 35 | - Thanks to [Michael Lynch](https://github.com/mtlynch) for digging into issues and contributing to the documentation. 36 | - Thanks to [Kurt Mackey](https://twitter.com/mrkurt) for feedback and testing. 37 | - Thanks to [Sam Weston](https://twitter.com/cablespaghetti) for figuring out how to run Litestream on Kubernetes and writing up the docs for it. 38 | - Thanks to [Rafael](https://github.com/netstx) & [Jungle Boogie](https://github.com/jungle-boogie) for helping to get OpenBSD release builds working. 39 | - Thanks to [Simon Gottschlag](https://github.com/simongottschlag), [Marin](https://github.com/supermarin),[Victor Björklund](https://github.com/victorbjorklund), [Jonathan Beri](https://twitter.com/beriberikix) [Yuri](https://github.com/yurivish), [Nathan Probst](https://github.com/nprbst), [Yann Coleu](https://github.com/yanc0), and [Nicholas Grilly](https://twitter.com/ngrilly) for frequent feedback, testing, & support. 40 | 41 | Huge thanks to fly.io for their support and for contributing credits for testing and development! 42 | 43 | 44 | ## Contribution Policy 45 | 46 | Initially, Litestream was closed to outside contributions. The goal was to 47 | reduce burnout by limiting the maintenance overhead of reviewing and validating 48 | third-party code. However, this policy is overly broad and has prevented small, 49 | easily testable patches from being contributed. 50 | 51 | Litestream is now open to code contributions for bug fixes only. Features carry 52 | a long-term maintenance burden so they will not be accepted at this time. 53 | Please [submit an issue][new-issue] if you have a feature you'd like to 54 | request. 55 | 56 | If you find mistakes in the documentation, please submit a fix to the 57 | [documentation repository][docs]. 58 | 59 | [new-issue]: https://github.com/benbjohnson/litestream/issues/new 60 | [docs]: https://github.com/benbjohnson/litestream.io 61 | 62 | -------------------------------------------------------------------------------- /cmd/litestream/snapshots.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "text/tabwriter" 10 | "time" 11 | 12 | "github.com/benbjohnson/litestream" 13 | ) 14 | 15 | // SnapshotsCommand represents a command to list snapshots for a command. 16 | type SnapshotsCommand struct{} 17 | 18 | // Run executes the command. 19 | func (c *SnapshotsCommand) Run(ctx context.Context, args []string) (err error) { 20 | fs := flag.NewFlagSet("litestream-snapshots", flag.ContinueOnError) 21 | configPath, noExpandEnv := registerConfigFlag(fs) 22 | replicaName := fs.String("replica", "", "replica name") 23 | fs.Usage = c.Usage 24 | if err := fs.Parse(args); err != nil { 25 | return err 26 | } else if fs.NArg() == 0 || fs.Arg(0) == "" { 27 | return fmt.Errorf("database path required") 28 | } else if fs.NArg() > 1 { 29 | return fmt.Errorf("too many arguments") 30 | } 31 | 32 | var db *litestream.DB 33 | var r *litestream.Replica 34 | if isURL(fs.Arg(0)) { 35 | if *configPath != "" { 36 | return fmt.Errorf("cannot specify a replica URL and the -config flag") 37 | } 38 | if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil { 39 | return err 40 | } 41 | } else { 42 | if *configPath == "" { 43 | *configPath = DefaultConfigPath() 44 | } 45 | 46 | // Load configuration. 47 | config, err := ReadConfigFile(*configPath, !*noExpandEnv) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | // Lookup database from configuration file by path. 53 | if path, err := expand(fs.Arg(0)); err != nil { 54 | return err 55 | } else if dbc := config.DBConfig(path); dbc == nil { 56 | return fmt.Errorf("database not found in config: %s", path) 57 | } else if db, err = NewDBFromConfig(dbc); err != nil { 58 | return err 59 | } 60 | 61 | // Filter by replica, if specified. 62 | if *replicaName != "" { 63 | if r = db.Replica(*replicaName); r == nil { 64 | return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path()) 65 | } 66 | } 67 | } 68 | 69 | // Find snapshots by db or replica. 70 | var replicas []*litestream.Replica 71 | if r != nil { 72 | replicas = []*litestream.Replica{r} 73 | } else { 74 | replicas = db.Replicas 75 | } 76 | 77 | // List all snapshots. 78 | w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) 79 | defer w.Flush() 80 | 81 | fmt.Fprintln(w, "replica\tgeneration\tindex\tsize\tcreated") 82 | for _, r := range replicas { 83 | infos, err := r.Snapshots(ctx) 84 | if err != nil { 85 | log.Printf("cannot determine snapshots: %s", err) 86 | continue 87 | } 88 | for _, info := range infos { 89 | fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%s\n", 90 | r.Name(), 91 | info.Generation, 92 | info.Index, 93 | info.Size, 94 | info.CreatedAt.Format(time.RFC3339), 95 | ) 96 | } 97 | } 98 | 99 | return nil 100 | } 101 | 102 | // Usage prints the help screen to STDOUT. 103 | func (c *SnapshotsCommand) Usage() { 104 | fmt.Printf(` 105 | The snapshots command lists all snapshots available for a database or replica. 106 | 107 | Usage: 108 | 109 | litestream snapshots [arguments] DB_PATH 110 | 111 | litestream snapshots [arguments] REPLICA_URL 112 | 113 | Arguments: 114 | 115 | -config PATH 116 | Specifies the configuration file. 117 | Defaults to %s 118 | 119 | -no-expand-env 120 | Disables environment variable expansion in configuration file. 121 | 122 | -replica NAME 123 | Optional, filter by a specific replica. 124 | 125 | Examples: 126 | 127 | # List all snapshots for a database. 128 | $ litestream snapshots /path/to/db 129 | 130 | # List all snapshots on S3. 131 | $ litestream snapshots -replica s3 /path/to/db 132 | 133 | # List all snapshots by replica URL. 134 | $ litestream snapshots s3://mybkt/db 135 | 136 | `[1:], 137 | DefaultConfigPath(), 138 | ) 139 | } 140 | -------------------------------------------------------------------------------- /internal/internal.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "syscall" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/promauto" 10 | ) 11 | 12 | // ReadCloser wraps a reader to also attach a separate closer. 13 | type ReadCloser struct { 14 | r io.Reader 15 | c io.Closer 16 | } 17 | 18 | // NewReadCloser returns a new instance of ReadCloser. 19 | func NewReadCloser(r io.Reader, c io.Closer) *ReadCloser { 20 | return &ReadCloser{r, c} 21 | } 22 | 23 | // Read reads bytes into the underlying reader. 24 | func (r *ReadCloser) Read(p []byte) (n int, err error) { 25 | return r.r.Read(p) 26 | } 27 | 28 | // Close closes the reader (if implementing io.ReadCloser) and the Closer. 29 | func (r *ReadCloser) Close() error { 30 | if rc, ok := r.r.(io.Closer); ok { 31 | if err := rc.Close(); err != nil { 32 | r.c.Close() 33 | return err 34 | } 35 | } 36 | return r.c.Close() 37 | } 38 | 39 | // ReadCounter wraps an io.Reader and counts the total number of bytes read. 40 | type ReadCounter struct { 41 | r io.Reader 42 | n int64 43 | } 44 | 45 | // NewReadCounter returns a new instance of ReadCounter that wraps r. 46 | func NewReadCounter(r io.Reader) *ReadCounter { 47 | return &ReadCounter{r: r} 48 | } 49 | 50 | // Read reads from the underlying reader into p and adds the bytes read to the counter. 51 | func (r *ReadCounter) Read(p []byte) (int, error) { 52 | n, err := r.r.Read(p) 53 | r.n += int64(n) 54 | return n, err 55 | } 56 | 57 | // N returns the total number of bytes read. 58 | func (r *ReadCounter) N() int64 { return r.n } 59 | 60 | // CreateFile creates the file and matches the mode & uid/gid of fi. 61 | func CreateFile(filename string, fi os.FileInfo) (*os.File, error) { 62 | mode := os.FileMode(0600) 63 | if fi != nil { 64 | mode = fi.Mode() 65 | } 66 | 67 | f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | uid, gid := Fileinfo(fi) 73 | _ = f.Chown(uid, gid) 74 | return f, nil 75 | } 76 | 77 | // MkdirAll is a copy of os.MkdirAll() except that it attempts to set the 78 | // mode/uid/gid to match fi for each created directory. 79 | func MkdirAll(path string, fi os.FileInfo) error { 80 | uid, gid := Fileinfo(fi) 81 | 82 | // Fast path: if we can tell whether path is a directory or file, stop with success or error. 83 | dir, err := os.Stat(path) 84 | if err == nil { 85 | if dir.IsDir() { 86 | return nil 87 | } 88 | return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR} 89 | } 90 | 91 | // Slow path: make sure parent exists and then call Mkdir for path. 92 | i := len(path) 93 | for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. 94 | i-- 95 | } 96 | 97 | j := i 98 | for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. 99 | j-- 100 | } 101 | 102 | if j > 1 { 103 | // Create parent. 104 | err = MkdirAll(fixRootDirectory(path[:j-1]), fi) 105 | if err != nil { 106 | return err 107 | } 108 | } 109 | 110 | // Parent now exists; invoke Mkdir and use its result. 111 | mode := os.FileMode(0700) 112 | if fi != nil { 113 | mode = fi.Mode() 114 | } 115 | err = os.Mkdir(path, mode) 116 | if err != nil { 117 | // Handle arguments like "foo/." by 118 | // double-checking that directory doesn't exist. 119 | dir, err1 := os.Lstat(path) 120 | if err1 == nil && dir.IsDir() { 121 | _ = os.Chown(path, uid, gid) 122 | return nil 123 | } 124 | return err 125 | } 126 | _ = os.Chown(path, uid, gid) 127 | return nil 128 | } 129 | 130 | // Shared replica metrics. 131 | var ( 132 | OperationTotalCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{ 133 | Name: "litestream_replica_operation_total", 134 | Help: "The number of replica operations performed", 135 | }, []string{"replica_type", "operation"}) 136 | 137 | OperationBytesCounterVec = promauto.NewCounterVec(prometheus.CounterOpts{ 138 | Name: "litestream_replica_operation_bytes", 139 | Help: "The number of bytes used by replica operations", 140 | }, []string{"replica_type", "operation"}) 141 | ) 142 | -------------------------------------------------------------------------------- /.github/workflows/commit.yml: -------------------------------------------------------------------------------- 1 | on: push 2 | 3 | jobs: 4 | build: 5 | name: Build & Unit Test 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v2 9 | - uses: actions/setup-go@v2 10 | with: 11 | go-version: '1.20' 12 | - uses: actions/cache@v2 13 | with: 14 | path: ~/go/pkg/mod 15 | key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }} 16 | restore-keys: ${{ inputs.os }}-go- 17 | 18 | - run: go env 19 | 20 | - run: go install ./cmd/litestream 21 | 22 | - run: go test -v ./... 23 | 24 | # - name: Build integration test 25 | # run: go test -c ./integration 26 | # 27 | # - uses: actions/upload-artifact@v2 28 | # with: 29 | # name: integration.test 30 | # path: integration.test 31 | # if-no-files-found: error 32 | 33 | # long-running-test: 34 | # name: Run Long Running Unit Test 35 | # runs-on: ubuntu-latest 36 | # steps: 37 | # - uses: actions/checkout@v2 38 | # - uses: actions/setup-go@v2 39 | # with: 40 | # go-version: '1.20' 41 | # - uses: actions/cache@v2 42 | # with: 43 | # path: ~/go/pkg/mod 44 | # key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }} 45 | # restore-keys: ${{ inputs.os }}-go- 46 | # 47 | # - run: go install ./cmd/litestream 48 | # - run: go test -v -run=TestCmd_Replicate_LongRunning ./integration -long-running-duration 1m 49 | 50 | # s3-integration-test: 51 | # name: Run S3 Integration Tests 52 | # runs-on: ubuntu-latest 53 | # needs: build 54 | # steps: 55 | # - uses: actions/download-artifact@v2 56 | # with: 57 | # name: integration.test 58 | # - run: chmod +x integration.test 59 | # 60 | # - run: ./integration.test -test.v -test.run=TestReplicaClient -replica-type s3 61 | # env: 62 | # LITESTREAM_S3_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }} 63 | # LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }} 64 | # LITESTREAM_S3_REGION: us-east-1 65 | # LITESTREAM_S3_BUCKET: integration.litestream.io 66 | 67 | # gcp-integration-test: 68 | # name: Run GCP Integration Tests 69 | # runs-on: ubuntu-latest 70 | # needs: build 71 | # steps: 72 | # - name: Extract GCP credentials 73 | # run: 'echo "$GOOGLE_APPLICATION_CREDENTIALS" > /opt/gcp.json' 74 | # shell: bash 75 | # env: 76 | # GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}} 77 | # 78 | # - uses: actions/download-artifact@v2 79 | # with: 80 | # name: integration.test 81 | # - run: chmod +x integration.test 82 | # 83 | # - run: ./integration.test -test.v -test.run=TestReplicaClient -replica-type gcs 84 | # env: 85 | # GOOGLE_APPLICATION_CREDENTIALS: /opt/gcp.json 86 | # LITESTREAM_GCS_BUCKET: integration.litestream.io 87 | 88 | # abs-integration-test: 89 | # name: Run Azure Blob Store Integration Tests 90 | # runs-on: ubuntu-latest 91 | # needs: build 92 | # steps: 93 | # - uses: actions/download-artifact@v2 94 | # with: 95 | # name: integration.test 96 | # - run: chmod +x integration.test 97 | # 98 | # - run: ./integration.test -test.v -test.run=TestReplicaClient -replica-type abs 99 | # env: 100 | # LITESTREAM_ABS_ACCOUNT_NAME: ${{ secrets.LITESTREAM_ABS_ACCOUNT_NAME }} 101 | # LITESTREAM_ABS_ACCOUNT_KEY: ${{ secrets.LITESTREAM_ABS_ACCOUNT_KEY }} 102 | # LITESTREAM_ABS_BUCKET: integration 103 | 104 | # sftp-integration-test: 105 | # name: Run SFTP Integration Tests 106 | # runs-on: ubuntu-latest 107 | # needs: build 108 | # steps: 109 | # - name: Extract SSH key 110 | # run: 'echo "$LITESTREAM_SFTP_KEY" > /opt/id_ed25519' 111 | # shell: bash 112 | # env: 113 | # LITESTREAM_SFTP_KEY: ${{secrets.LITESTREAM_SFTP_KEY}} 114 | # 115 | # - name: Run sftp tests 116 | # run: go test -v -run=TestReplicaClient ./integration -replica-type sftp 117 | # env: 118 | # LITESTREAM_SFTP_HOST: ${{ secrets.LITESTREAM_SFTP_HOST }} 119 | # LITESTREAM_SFTP_USER: ${{ secrets.LITESTREAM_SFTP_USER }} 120 | # LITESTREAM_SFTP_KEY_PATH: /opt/id_ed25519 121 | # LITESTREAM_SFTP_PATH: ${{ secrets.LITESTREAM_SFTP_PATH }} 122 | -------------------------------------------------------------------------------- /cmd/litestream/wal.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "text/tabwriter" 10 | "time" 11 | 12 | "github.com/benbjohnson/litestream" 13 | ) 14 | 15 | // WALCommand represents a command to list WAL files for a database. 16 | type WALCommand struct{} 17 | 18 | // Run executes the command. 19 | func (c *WALCommand) Run(ctx context.Context, args []string) (err error) { 20 | fs := flag.NewFlagSet("litestream-wal", flag.ContinueOnError) 21 | configPath, noExpandEnv := registerConfigFlag(fs) 22 | replicaName := fs.String("replica", "", "replica name") 23 | generation := fs.String("generation", "", "generation name") 24 | fs.Usage = c.Usage 25 | if err := fs.Parse(args); err != nil { 26 | return err 27 | } else if fs.NArg() == 0 || fs.Arg(0) == "" { 28 | return fmt.Errorf("database path required") 29 | } else if fs.NArg() > 1 { 30 | return fmt.Errorf("too many arguments") 31 | } 32 | 33 | var db *litestream.DB 34 | var r *litestream.Replica 35 | if isURL(fs.Arg(0)) { 36 | if *configPath != "" { 37 | return fmt.Errorf("cannot specify a replica URL and the -config flag") 38 | } 39 | if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil { 40 | return err 41 | } 42 | } else { 43 | if *configPath == "" { 44 | *configPath = DefaultConfigPath() 45 | } 46 | 47 | // Load configuration. 48 | config, err := ReadConfigFile(*configPath, !*noExpandEnv) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | // Lookup database from configuration file by path. 54 | if path, err := expand(fs.Arg(0)); err != nil { 55 | return err 56 | } else if dbc := config.DBConfig(path); dbc == nil { 57 | return fmt.Errorf("database not found in config: %s", path) 58 | } else if db, err = NewDBFromConfig(dbc); err != nil { 59 | return err 60 | } 61 | 62 | // Filter by replica, if specified. 63 | if *replicaName != "" { 64 | if r = db.Replica(*replicaName); r == nil { 65 | return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path()) 66 | } 67 | } 68 | } 69 | 70 | // Find WAL files by db or replica. 71 | var replicas []*litestream.Replica 72 | if r != nil { 73 | replicas = []*litestream.Replica{r} 74 | } else { 75 | replicas = db.Replicas 76 | } 77 | 78 | // List all WAL files. 79 | w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) 80 | defer w.Flush() 81 | 82 | fmt.Fprintln(w, "replica\tgeneration\tindex\toffset\tsize\tcreated") 83 | for _, r := range replicas { 84 | var generations []string 85 | if *generation != "" { 86 | generations = []string{*generation} 87 | } else { 88 | if generations, err = r.Client.Generations(ctx); err != nil { 89 | log.Printf("%s: cannot determine generations: %s", r.Name(), err) 90 | continue 91 | } 92 | } 93 | 94 | for _, generation := range generations { 95 | if err := func() error { 96 | itr, err := r.Client.WALSegments(ctx, generation) 97 | if err != nil { 98 | return err 99 | } 100 | defer itr.Close() 101 | 102 | for itr.Next() { 103 | info := itr.WALSegment() 104 | 105 | fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%d\t%s\n", 106 | r.Name(), 107 | info.Generation, 108 | info.Index, 109 | info.Offset, 110 | info.Size, 111 | info.CreatedAt.Format(time.RFC3339), 112 | ) 113 | } 114 | return itr.Close() 115 | }(); err != nil { 116 | log.Printf("%s: cannot fetch wal segments: %s", r.Name(), err) 117 | continue 118 | } 119 | } 120 | } 121 | 122 | return nil 123 | } 124 | 125 | // Usage prints the help screen to STDOUT. 126 | func (c *WALCommand) Usage() { 127 | fmt.Printf(` 128 | The wal command lists all wal segments available for a database. 129 | 130 | Usage: 131 | 132 | litestream wal [arguments] DB_PATH 133 | 134 | litestream wal [arguments] REPLICA_URL 135 | 136 | Arguments: 137 | 138 | -config PATH 139 | Specifies the configuration file. 140 | Defaults to %s 141 | 142 | -no-expand-env 143 | Disables environment variable expansion in configuration file. 144 | 145 | -replica NAME 146 | Optional, filter by a specific replica. 147 | 148 | -generation NAME 149 | Optional, filter by a specific generation. 150 | 151 | Examples: 152 | 153 | # List all WAL segments for a database. 154 | $ litestream wal /path/to/db 155 | 156 | # List all WAL segments on S3 for a specific generation. 157 | $ litestream wal -replica s3 -generation xxxxxxxx /path/to/db 158 | 159 | # List all WAL segments for replica URL. 160 | $ litestream wal s3://mybkt/db 161 | 162 | `[1:], 163 | DefaultConfigPath(), 164 | ) 165 | } 166 | -------------------------------------------------------------------------------- /replica_test.go: -------------------------------------------------------------------------------- 1 | package litestream_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "os" 8 | "testing" 9 | 10 | "github.com/benbjohnson/litestream" 11 | "github.com/benbjohnson/litestream/file" 12 | "github.com/benbjohnson/litestream/mock" 13 | "github.com/pierrec/lz4/v4" 14 | ) 15 | 16 | func TestReplica_Name(t *testing.T) { 17 | t.Run("WithName", func(t *testing.T) { 18 | if got, want := litestream.NewReplica(nil, "NAME").Name(), "NAME"; got != want { 19 | t.Fatalf("Name()=%v, want %v", got, want) 20 | } 21 | }) 22 | t.Run("WithoutName", func(t *testing.T) { 23 | r := litestream.NewReplica(nil, "") 24 | r.Client = &mock.ReplicaClient{} 25 | if got, want := r.Name(), "mock"; got != want { 26 | t.Fatalf("Name()=%v, want %v", got, want) 27 | } 28 | }) 29 | } 30 | 31 | func TestReplica_Sync(t *testing.T) { 32 | db, sqldb := MustOpenDBs(t) 33 | defer MustCloseDBs(t, db, sqldb) 34 | 35 | // Execute a query to force a write to the WAL. 36 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 37 | t.Fatal(err) 38 | } 39 | 40 | // Issue initial database sync to setup generation. 41 | if err := db.Sync(context.Background()); err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | // Fetch current database position. 46 | dpos, err := db.Pos() 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | 51 | c := file.NewReplicaClient(t.TempDir()) 52 | r := litestream.NewReplica(db, "") 53 | c.Replica, r.Client = r, c 54 | 55 | if err := r.Sync(context.Background()); err != nil { 56 | t.Fatal(err) 57 | } 58 | 59 | // Verify client generation matches database. 60 | generations, err := c.Generations(context.Background()) 61 | if err != nil { 62 | t.Fatal(err) 63 | } else if got, want := len(generations), 1; got != want { 64 | t.Fatalf("len(generations)=%v, want %v", got, want) 65 | } else if got, want := generations[0], dpos.Generation; got != want { 66 | t.Fatalf("generations[0]=%v, want %v", got, want) 67 | } 68 | 69 | // Verify WAL matches replica WAL. 70 | if b0, err := os.ReadFile(db.Path() + "-wal"); err != nil { 71 | t.Fatal(err) 72 | } else if r, err := c.WALSegmentReader(context.Background(), litestream.Pos{Generation: generations[0], Index: 0, Offset: 0}); err != nil { 73 | t.Fatal(err) 74 | } else if b1, err := io.ReadAll(lz4.NewReader(r)); err != nil { 75 | t.Fatal(err) 76 | } else if err := r.Close(); err != nil { 77 | t.Fatal(err) 78 | } else if !bytes.Equal(b0, b1) { 79 | t.Fatalf("wal mismatch: len(%d), len(%d)", len(b0), len(b1)) 80 | } 81 | } 82 | 83 | func TestReplica_Snapshot(t *testing.T) { 84 | db, sqldb := MustOpenDBs(t) 85 | defer MustCloseDBs(t, db, sqldb) 86 | 87 | c := file.NewReplicaClient(t.TempDir()) 88 | r := litestream.NewReplica(db, "") 89 | r.Client = c 90 | 91 | // Execute a query to force a write to the WAL. 92 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 93 | t.Fatal(err) 94 | } else if err := db.Sync(context.Background()); err != nil { 95 | t.Fatal(err) 96 | } else if err := r.Sync(context.Background()); err != nil { 97 | t.Fatal(err) 98 | } 99 | 100 | // Fetch current database position & snapshot. 101 | pos0, err := db.Pos() 102 | if err != nil { 103 | t.Fatal(err) 104 | } else if info, err := r.Snapshot(context.Background()); err != nil { 105 | t.Fatal(err) 106 | } else if got, want := info.Pos(), pos0.Truncate(); got != want { 107 | t.Fatalf("pos=%s, want %s", got, want) 108 | } 109 | 110 | // Sync database and then replica. 111 | if err := db.Sync(context.Background()); err != nil { 112 | t.Fatal(err) 113 | } else if err := r.Sync(context.Background()); err != nil { 114 | t.Fatal(err) 115 | } 116 | 117 | // Execute a query to force a write to the WAL & truncate to start new index. 118 | if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz');`); err != nil { 119 | t.Fatal(err) 120 | } else if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil { 121 | t.Fatal(err) 122 | } 123 | 124 | // Fetch current database position & snapshot. 125 | pos1, err := db.Pos() 126 | if err != nil { 127 | t.Fatal(err) 128 | } else if info, err := r.Snapshot(context.Background()); err != nil { 129 | t.Fatal(err) 130 | } else if got, want := info.Pos(), pos1.Truncate(); got != want { 131 | t.Fatalf("pos=%v, want %v", got, want) 132 | } 133 | 134 | // Verify two snapshots exist. 135 | if infos, err := r.Snapshots(context.Background()); err != nil { 136 | t.Fatal(err) 137 | } else if got, want := len(infos), 2; got != want { 138 | t.Fatalf("len=%v, want %v", got, want) 139 | } else if got, want := infos[0].Pos(), pos0.Truncate(); got != want { 140 | t.Fatalf("info[0]=%s, want %s", got, want) 141 | } else if got, want := infos[1].Pos(), pos1.Truncate(); got != want { 142 | t.Fatalf("info[1]=%s, want %s", got, want) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /cmd/litestream/generations.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "text/tabwriter" 10 | "time" 11 | 12 | "github.com/benbjohnson/litestream" 13 | ) 14 | 15 | // GenerationsCommand represents a command to list all generations for a database. 16 | type GenerationsCommand struct{} 17 | 18 | // Run executes the command. 19 | func (c *GenerationsCommand) Run(ctx context.Context, args []string) (err error) { 20 | fs := flag.NewFlagSet("litestream-generations", flag.ContinueOnError) 21 | configPath, noExpandEnv := registerConfigFlag(fs) 22 | replicaName := fs.String("replica", "", "replica name") 23 | fs.Usage = c.Usage 24 | if err := fs.Parse(args); err != nil { 25 | return err 26 | } else if fs.NArg() == 0 || fs.Arg(0) == "" { 27 | return fmt.Errorf("database path or replica URL required") 28 | } else if fs.NArg() > 1 { 29 | return fmt.Errorf("too many arguments") 30 | } 31 | 32 | var db *litestream.DB 33 | var r *litestream.Replica 34 | dbUpdatedAt := time.Now() 35 | if isURL(fs.Arg(0)) { 36 | if *configPath != "" { 37 | return fmt.Errorf("cannot specify a replica URL and the -config flag") 38 | } 39 | if r, err = NewReplicaFromConfig(&ReplicaConfig{URL: fs.Arg(0)}, nil); err != nil { 40 | return err 41 | } 42 | } else { 43 | if *configPath == "" { 44 | *configPath = DefaultConfigPath() 45 | } 46 | 47 | // Load configuration. 48 | config, err := ReadConfigFile(*configPath, !*noExpandEnv) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | // Lookup database from configuration file by path. 54 | if path, err := expand(fs.Arg(0)); err != nil { 55 | return err 56 | } else if dbc := config.DBConfig(path); dbc == nil { 57 | return fmt.Errorf("database not found in config: %s", path) 58 | } else if db, err = NewDBFromConfig(dbc); err != nil { 59 | return err 60 | } 61 | 62 | // Filter by replica, if specified. 63 | if *replicaName != "" { 64 | if r = db.Replica(*replicaName); r == nil { 65 | return fmt.Errorf("replica %q not found for database %q", *replicaName, db.Path()) 66 | } 67 | } 68 | 69 | // Determine last time database or WAL was updated. 70 | if dbUpdatedAt, err = db.UpdatedAt(); err != nil { 71 | return err 72 | } 73 | } 74 | 75 | var replicas []*litestream.Replica 76 | if r != nil { 77 | replicas = []*litestream.Replica{r} 78 | } else { 79 | replicas = db.Replicas 80 | } 81 | 82 | // List each generation. 83 | w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) 84 | defer w.Flush() 85 | 86 | fmt.Fprintln(w, "name\tgeneration\tlag\tstart\tend") 87 | for _, r := range replicas { 88 | generations, err := r.Client.Generations(ctx) 89 | if err != nil { 90 | log.Printf("%s: cannot list generations: %s", r.Name(), err) 91 | continue 92 | } 93 | 94 | // Iterate over each generation for the replica. 95 | for _, generation := range generations { 96 | createdAt, updatedAt, err := r.GenerationTimeBounds(ctx, generation) 97 | if err != nil { 98 | log.Printf("%s: cannot determine generation time bounds: %s", r.Name(), err) 99 | continue 100 | } 101 | 102 | fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", 103 | r.Name(), 104 | generation, 105 | truncateDuration(dbUpdatedAt.Sub(updatedAt)).String(), 106 | createdAt.Format(time.RFC3339), 107 | updatedAt.Format(time.RFC3339), 108 | ) 109 | } 110 | } 111 | 112 | return nil 113 | } 114 | 115 | // Usage prints the help message to STDOUT. 116 | func (c *GenerationsCommand) Usage() { 117 | fmt.Printf(` 118 | The generations command lists all generations for a database or replica. It also 119 | lists stats about their lag behind the primary database and the time range they 120 | cover. 121 | 122 | Usage: 123 | 124 | litestream generations [arguments] DB_PATH 125 | 126 | litestream generations [arguments] REPLICA_URL 127 | 128 | Arguments: 129 | 130 | -config PATH 131 | Specifies the configuration file. 132 | Defaults to %s 133 | 134 | -no-expand-env 135 | Disables environment variable expansion in configuration file. 136 | 137 | -replica NAME 138 | Optional, filters by replica. 139 | 140 | `[1:], 141 | DefaultConfigPath(), 142 | ) 143 | } 144 | 145 | func truncateDuration(d time.Duration) time.Duration { 146 | if d < 0 { 147 | if d < -10*time.Second { 148 | return d.Truncate(time.Second) 149 | } else if d < -time.Second { 150 | return d.Truncate(time.Second / 10) 151 | } else if d < -time.Millisecond { 152 | return d.Truncate(time.Millisecond) 153 | } else if d < -time.Microsecond { 154 | return d.Truncate(time.Microsecond) 155 | } 156 | return d 157 | } 158 | 159 | if d > 10*time.Second { 160 | return d.Truncate(time.Second) 161 | } else if d > time.Second { 162 | return d.Truncate(time.Second / 10) 163 | } else if d > time.Millisecond { 164 | return d.Truncate(time.Millisecond) 165 | } else if d > time.Microsecond { 166 | return d.Truncate(time.Microsecond) 167 | } 168 | return d 169 | } 170 | -------------------------------------------------------------------------------- /cmd/litestream/main_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | main "github.com/benbjohnson/litestream/cmd/litestream" 10 | "github.com/benbjohnson/litestream/file" 11 | "github.com/benbjohnson/litestream/gcs" 12 | "github.com/benbjohnson/litestream/s3" 13 | ) 14 | 15 | func TestReadConfigFile(t *testing.T) { 16 | // Ensure global AWS settings are propagated down to replica configurations. 17 | t.Run("PropagateGlobalSettings", func(t *testing.T) { 18 | filename := filepath.Join(t.TempDir(), "litestream.yml") 19 | if err := ioutil.WriteFile(filename, []byte(` 20 | access-key-id: XXX 21 | secret-access-key: YYY 22 | 23 | dbs: 24 | - path: /path/to/db 25 | replicas: 26 | - url: s3://foo/bar 27 | `[1:]), 0666); err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | config, err := main.ReadConfigFile(filename, true) 32 | if err != nil { 33 | t.Fatal(err) 34 | } else if got, want := config.AccessKeyID, `XXX`; got != want { 35 | t.Fatalf("AccessKeyID=%v, want %v", got, want) 36 | } else if got, want := config.SecretAccessKey, `YYY`; got != want { 37 | t.Fatalf("SecretAccessKey=%v, want %v", got, want) 38 | } else if got, want := config.DBs[0].Replicas[0].AccessKeyID, `XXX`; got != want { 39 | t.Fatalf("Replica.AccessKeyID=%v, want %v", got, want) 40 | } else if got, want := config.DBs[0].Replicas[0].SecretAccessKey, `YYY`; got != want { 41 | t.Fatalf("Replica.SecretAccessKey=%v, want %v", got, want) 42 | } 43 | }) 44 | 45 | // Ensure environment variables are expanded. 46 | t.Run("ExpandEnv", func(t *testing.T) { 47 | os.Setenv("LITESTREAM_TEST_0129380", "/path/to/db") 48 | os.Setenv("LITESTREAM_TEST_1872363", "s3://foo/bar") 49 | 50 | filename := filepath.Join(t.TempDir(), "litestream.yml") 51 | if err := ioutil.WriteFile(filename, []byte(` 52 | dbs: 53 | - path: $LITESTREAM_TEST_0129380 54 | replicas: 55 | - url: ${LITESTREAM_TEST_1872363} 56 | - url: ${LITESTREAM_TEST_NO_SUCH_ENV} 57 | `[1:]), 0666); err != nil { 58 | t.Fatal(err) 59 | } 60 | 61 | config, err := main.ReadConfigFile(filename, true) 62 | if err != nil { 63 | t.Fatal(err) 64 | } else if got, want := config.DBs[0].Path, `/path/to/db`; got != want { 65 | t.Fatalf("DB.Path=%v, want %v", got, want) 66 | } else if got, want := config.DBs[0].Replicas[0].URL, `s3://foo/bar`; got != want { 67 | t.Fatalf("Replica[0].URL=%v, want %v", got, want) 68 | } else if got, want := config.DBs[0].Replicas[1].URL, ``; got != want { 69 | t.Fatalf("Replica[1].URL=%v, want %v", got, want) 70 | } 71 | }) 72 | 73 | // Ensure environment variables are not expanded. 74 | t.Run("NoExpandEnv", func(t *testing.T) { 75 | os.Setenv("LITESTREAM_TEST_9847533", "s3://foo/bar") 76 | 77 | filename := filepath.Join(t.TempDir(), "litestream.yml") 78 | if err := ioutil.WriteFile(filename, []byte(` 79 | dbs: 80 | - path: /path/to/db 81 | replicas: 82 | - url: ${LITESTREAM_TEST_9847533} 83 | `[1:]), 0666); err != nil { 84 | t.Fatal(err) 85 | } 86 | 87 | config, err := main.ReadConfigFile(filename, false) 88 | if err != nil { 89 | t.Fatal(err) 90 | } else if got, want := config.DBs[0].Replicas[0].URL, `${LITESTREAM_TEST_9847533}`; got != want { 91 | t.Fatalf("Replica.URL=%v, want %v", got, want) 92 | } 93 | }) 94 | } 95 | 96 | func TestNewFileReplicaFromConfig(t *testing.T) { 97 | r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{Path: "/foo"}, nil) 98 | if err != nil { 99 | t.Fatal(err) 100 | } else if client, ok := r.Client.(*file.ReplicaClient); !ok { 101 | t.Fatal("unexpected replica type") 102 | } else if got, want := client.Path(), "/foo"; got != want { 103 | t.Fatalf("Path=%s, want %s", got, want) 104 | } 105 | } 106 | 107 | func TestNewS3ReplicaFromConfig(t *testing.T) { 108 | t.Run("URL", func(t *testing.T) { 109 | r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo/bar"}, nil) 110 | if err != nil { 111 | t.Fatal(err) 112 | } else if client, ok := r.Client.(*s3.ReplicaClient); !ok { 113 | t.Fatal("unexpected replica type") 114 | } else if got, want := client.Bucket, "foo"; got != want { 115 | t.Fatalf("Bucket=%s, want %s", got, want) 116 | } else if got, want := client.Path, "bar"; got != want { 117 | t.Fatalf("Path=%s, want %s", got, want) 118 | } else if got, want := client.Region, ""; got != want { 119 | t.Fatalf("Region=%s, want %s", got, want) 120 | } else if got, want := client.Endpoint, ""; got != want { 121 | t.Fatalf("Endpoint=%s, want %s", got, want) 122 | } else if got, want := client.ForcePathStyle, false; got != want { 123 | t.Fatalf("ForcePathStyle=%v, want %v", got, want) 124 | } 125 | }) 126 | 127 | t.Run("MinIO", func(t *testing.T) { 128 | r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.localhost:9000/bar"}, nil) 129 | if err != nil { 130 | t.Fatal(err) 131 | } else if client, ok := r.Client.(*s3.ReplicaClient); !ok { 132 | t.Fatal("unexpected replica type") 133 | } else if got, want := client.Bucket, "foo"; got != want { 134 | t.Fatalf("Bucket=%s, want %s", got, want) 135 | } else if got, want := client.Path, "bar"; got != want { 136 | t.Fatalf("Path=%s, want %s", got, want) 137 | } else if got, want := client.Region, "us-east-1"; got != want { 138 | t.Fatalf("Region=%s, want %s", got, want) 139 | } else if got, want := client.Endpoint, "http://localhost:9000"; got != want { 140 | t.Fatalf("Endpoint=%s, want %s", got, want) 141 | } else if got, want := client.ForcePathStyle, true; got != want { 142 | t.Fatalf("ForcePathStyle=%v, want %v", got, want) 143 | } 144 | }) 145 | 146 | t.Run("Backblaze", func(t *testing.T) { 147 | r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "s3://foo.s3.us-west-000.backblazeb2.com/bar"}, nil) 148 | if err != nil { 149 | t.Fatal(err) 150 | } else if client, ok := r.Client.(*s3.ReplicaClient); !ok { 151 | t.Fatal("unexpected replica type") 152 | } else if got, want := client.Bucket, "foo"; got != want { 153 | t.Fatalf("Bucket=%s, want %s", got, want) 154 | } else if got, want := client.Path, "bar"; got != want { 155 | t.Fatalf("Path=%s, want %s", got, want) 156 | } else if got, want := client.Region, "us-west-000"; got != want { 157 | t.Fatalf("Region=%s, want %s", got, want) 158 | } else if got, want := client.Endpoint, "https://s3.us-west-000.backblazeb2.com"; got != want { 159 | t.Fatalf("Endpoint=%s, want %s", got, want) 160 | } else if got, want := client.ForcePathStyle, true; got != want { 161 | t.Fatalf("ForcePathStyle=%v, want %v", got, want) 162 | } 163 | }) 164 | } 165 | 166 | func TestNewGCSReplicaFromConfig(t *testing.T) { 167 | r, err := main.NewReplicaFromConfig(&main.ReplicaConfig{URL: "gcs://foo/bar"}, nil) 168 | if err != nil { 169 | t.Fatal(err) 170 | } else if client, ok := r.Client.(*gcs.ReplicaClient); !ok { 171 | t.Fatal("unexpected replica type") 172 | } else if got, want := client.Bucket, "foo"; got != want { 173 | t.Fatalf("Bucket=%s, want %s", got, want) 174 | } else if got, want := client.Path, "bar"; got != want { 175 | t.Fatalf("Path=%s, want %s", got, want) 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /cmd/litestream/replicate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "net" 9 | "net/http" 10 | _ "net/http/pprof" 11 | "os" 12 | "os/exec" 13 | 14 | "github.com/benbjohnson/litestream" 15 | "github.com/benbjohnson/litestream/abs" 16 | "github.com/benbjohnson/litestream/file" 17 | "github.com/benbjohnson/litestream/gcs" 18 | "github.com/benbjohnson/litestream/s3" 19 | "github.com/benbjohnson/litestream/sftp" 20 | "github.com/mattn/go-shellwords" 21 | "github.com/prometheus/client_golang/prometheus/promhttp" 22 | ) 23 | 24 | // ReplicateCommand represents a command that continuously replicates SQLite databases. 25 | type ReplicateCommand struct { 26 | cmd *exec.Cmd // subcommand 27 | execCh chan error // subcommand error channel 28 | 29 | Config Config 30 | 31 | // List of managed databases specified in the config. 32 | DBs []*litestream.DB 33 | } 34 | 35 | func NewReplicateCommand() *ReplicateCommand { 36 | return &ReplicateCommand{ 37 | execCh: make(chan error), 38 | } 39 | } 40 | 41 | // ParseFlags parses the CLI flags and loads the configuration file. 42 | func (c *ReplicateCommand) ParseFlags(ctx context.Context, args []string) (err error) { 43 | fs := flag.NewFlagSet("litestream-replicate", flag.ContinueOnError) 44 | execFlag := fs.String("exec", "", "execute subcommand") 45 | tracePath := fs.String("trace", "", "trace path") 46 | configPath, noExpandEnv := registerConfigFlag(fs) 47 | fs.Usage = c.Usage 48 | if err := fs.Parse(args); err != nil { 49 | return err 50 | } 51 | 52 | // Load configuration or use CLI args to build db/replica. 53 | if fs.NArg() == 1 { 54 | return fmt.Errorf("must specify at least one replica URL for %s", fs.Arg(0)) 55 | } else if fs.NArg() > 1 { 56 | if *configPath != "" { 57 | return fmt.Errorf("cannot specify a replica URL and the -config flag") 58 | } 59 | 60 | dbConfig := &DBConfig{Path: fs.Arg(0)} 61 | for _, u := range fs.Args()[1:] { 62 | syncInterval := litestream.DefaultSyncInterval 63 | dbConfig.Replicas = append(dbConfig.Replicas, &ReplicaConfig{ 64 | URL: u, 65 | SyncInterval: &syncInterval, 66 | }) 67 | } 68 | c.Config.DBs = []*DBConfig{dbConfig} 69 | } else { 70 | if *configPath == "" { 71 | *configPath = DefaultConfigPath() 72 | } 73 | if c.Config, err = ReadConfigFile(*configPath, !*noExpandEnv); err != nil { 74 | return err 75 | } 76 | } 77 | 78 | // Override config exec command, if specified. 79 | if *execFlag != "" { 80 | c.Config.Exec = *execFlag 81 | } 82 | 83 | // Enable trace logging. 84 | if *tracePath != "" { 85 | f, err := os.Create(*tracePath) 86 | if err != nil { 87 | return err 88 | } 89 | defer f.Close() 90 | litestream.Tracef = log.New(f, "", log.LstdFlags|log.Lmicroseconds|log.LUTC|log.Lshortfile).Printf 91 | } 92 | 93 | return nil 94 | } 95 | 96 | // Run loads all databases specified in the configuration. 97 | func (c *ReplicateCommand) Run() (err error) { 98 | // Display version information. 99 | log.Printf("litestream %s", Version) 100 | 101 | // Setup databases. 102 | if len(c.Config.DBs) == 0 { 103 | log.Println("no databases specified in configuration") 104 | } 105 | 106 | for _, dbConfig := range c.Config.DBs { 107 | db, err := NewDBFromConfig(dbConfig) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | // Open database & attach to program. 113 | if err := db.Open(); err != nil { 114 | return err 115 | } 116 | c.DBs = append(c.DBs, db) 117 | } 118 | 119 | // Notify user that initialization is done. 120 | for _, db := range c.DBs { 121 | log.Printf("initialized db: %s", db.Path()) 122 | for _, r := range db.Replicas { 123 | switch client := r.Client.(type) { 124 | case *file.ReplicaClient: 125 | log.Printf("replicating to: name=%q type=%q path=%q", r.Name(), client.Type(), client.Path()) 126 | case *s3.ReplicaClient: 127 | log.Printf("replicating to: name=%q type=%q bucket=%q path=%q region=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Region, client.Endpoint, r.SyncInterval) 128 | case *gcs.ReplicaClient: 129 | log.Printf("replicating to: name=%q type=%q bucket=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, r.SyncInterval) 130 | case *abs.ReplicaClient: 131 | log.Printf("replicating to: name=%q type=%q bucket=%q path=%q endpoint=%q sync-interval=%s", r.Name(), client.Type(), client.Bucket, client.Path, client.Endpoint, r.SyncInterval) 132 | case *sftp.ReplicaClient: 133 | log.Printf("replicating to: name=%q type=%q host=%q user=%q path=%q sync-interval=%s", r.Name(), client.Type(), client.Host, client.User, client.Path, r.SyncInterval) 134 | default: 135 | log.Printf("replicating to: name=%q type=%q", r.Name(), client.Type()) 136 | } 137 | } 138 | } 139 | 140 | // Serve metrics over HTTP if enabled. 141 | if c.Config.Addr != "" { 142 | hostport := c.Config.Addr 143 | if host, port, _ := net.SplitHostPort(c.Config.Addr); port == "" { 144 | return fmt.Errorf("must specify port for bind address: %q", c.Config.Addr) 145 | } else if host == "" { 146 | hostport = net.JoinHostPort("localhost", port) 147 | } 148 | 149 | log.Printf("serving metrics on http://%s/metrics", hostport) 150 | go func() { 151 | http.Handle("/metrics", promhttp.Handler()) 152 | if err := http.ListenAndServe(c.Config.Addr, nil); err != nil { 153 | log.Printf("cannot start metrics server: %s", err) 154 | } 155 | }() 156 | } 157 | 158 | // Parse exec commands args & start subprocess. 159 | if c.Config.Exec != "" { 160 | execArgs, err := shellwords.Parse(c.Config.Exec) 161 | if err != nil { 162 | return fmt.Errorf("cannot parse exec command: %w", err) 163 | } 164 | 165 | c.cmd = exec.Command(execArgs[0], execArgs[1:]...) 166 | c.cmd.Env = os.Environ() 167 | c.cmd.Stdout = os.Stdout 168 | c.cmd.Stderr = os.Stderr 169 | if err := c.cmd.Start(); err != nil { 170 | return fmt.Errorf("cannot start exec command: %w", err) 171 | } 172 | go func() { c.execCh <- c.cmd.Wait() }() 173 | } 174 | 175 | return nil 176 | } 177 | 178 | // Close closes all open databases. 179 | func (c *ReplicateCommand) Close() (err error) { 180 | for _, db := range c.DBs { 181 | if e := db.Close(); e != nil { 182 | log.Printf("error closing db: path=%s err=%s", db.Path(), e) 183 | if err == nil { 184 | err = e 185 | } 186 | } 187 | } 188 | return err 189 | } 190 | 191 | // Usage prints the help screen to STDOUT. 192 | func (c *ReplicateCommand) Usage() { 193 | fmt.Printf(` 194 | The replicate command starts a server to monitor & replicate databases. 195 | You can specify your database & replicas in a configuration file or you can 196 | replicate a single database file by specifying its path and its replicas in the 197 | command line arguments. 198 | 199 | Usage: 200 | 201 | litestream replicate [arguments] 202 | 203 | litestream replicate [arguments] DB_PATH REPLICA_URL [REPLICA_URL...] 204 | 205 | Arguments: 206 | 207 | -config PATH 208 | Specifies the configuration file. 209 | Defaults to %s 210 | 211 | -exec CMD 212 | Executes a subcommand. Litestream will exit when the child 213 | process exits. Useful for simple process management. 214 | 215 | -no-expand-env 216 | Disables environment variable expansion in configuration file. 217 | 218 | -trace PATH 219 | Write verbose trace logging to PATH. 220 | 221 | `[1:], DefaultConfigPath()) 222 | } 223 | -------------------------------------------------------------------------------- /cmd/litestream/restore.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | "strconv" 11 | "time" 12 | 13 | "github.com/benbjohnson/litestream" 14 | ) 15 | 16 | // RestoreCommand represents a command to restore a database from a backup. 17 | type RestoreCommand struct{} 18 | 19 | // Run executes the command. 20 | func (c *RestoreCommand) Run(ctx context.Context, args []string) (err error) { 21 | opt := litestream.NewRestoreOptions() 22 | opt.Verbose = true 23 | 24 | fs := flag.NewFlagSet("litestream-restore", flag.ContinueOnError) 25 | configPath, noExpandEnv := registerConfigFlag(fs) 26 | fs.StringVar(&opt.OutputPath, "o", "", "output path") 27 | fs.StringVar(&opt.ReplicaName, "replica", "", "replica name") 28 | fs.StringVar(&opt.Generation, "generation", "", "generation name") 29 | fs.Var((*indexVar)(&opt.Index), "index", "wal index") 30 | fs.IntVar(&opt.Parallelism, "parallelism", opt.Parallelism, "parallelism") 31 | ifDBNotExists := fs.Bool("if-db-not-exists", false, "") 32 | ifReplicaExists := fs.Bool("if-replica-exists", false, "") 33 | timestampStr := fs.String("timestamp", "", "timestamp") 34 | verbose := fs.Bool("v", false, "verbose output") 35 | fs.Usage = c.Usage 36 | if err := fs.Parse(args); err != nil { 37 | return err 38 | } else if fs.NArg() == 0 || fs.Arg(0) == "" { 39 | return fmt.Errorf("database path or replica URL required") 40 | } else if fs.NArg() > 1 { 41 | return fmt.Errorf("too many arguments") 42 | } 43 | 44 | // Parse timestamp, if specified. 45 | if *timestampStr != "" { 46 | if opt.Timestamp, err = time.Parse(time.RFC3339, *timestampStr); err != nil { 47 | return errors.New("invalid -timestamp, must specify in ISO 8601 format (e.g. 2000-01-01T00:00:00Z)") 48 | } 49 | } 50 | 51 | // Instantiate logger if verbose output is enabled. 52 | if *verbose { 53 | opt.Logger = log.New(os.Stderr, "", log.LstdFlags|log.Lmicroseconds) 54 | } 55 | 56 | // Determine replica & generation to restore from. 57 | var r *litestream.Replica 58 | if isURL(fs.Arg(0)) { 59 | if *configPath != "" { 60 | return fmt.Errorf("cannot specify a replica URL and the -config flag") 61 | } 62 | if r, err = c.loadFromURL(ctx, fs.Arg(0), *ifDBNotExists, &opt); err == errSkipDBExists { 63 | fmt.Println("database already exists, skipping") 64 | return nil 65 | } else if err != nil { 66 | return err 67 | } 68 | } else { 69 | if *configPath == "" { 70 | *configPath = DefaultConfigPath() 71 | } 72 | if r, err = c.loadFromConfig(ctx, fs.Arg(0), *configPath, !*noExpandEnv, *ifDBNotExists, &opt); err == errSkipDBExists { 73 | fmt.Println("database already exists, skipping") 74 | return nil 75 | } else if err != nil { 76 | return err 77 | } 78 | } 79 | 80 | // Return an error if no matching targets found. 81 | // If optional flag set, return success. Useful for automated recovery. 82 | if opt.Generation == "" { 83 | if *ifReplicaExists { 84 | fmt.Println("no matching backups found") 85 | return nil 86 | } 87 | return fmt.Errorf("no matching backups found") 88 | } 89 | 90 | return r.Restore(ctx, opt) 91 | } 92 | 93 | // loadFromURL creates a replica & updates the restore options from a replica URL. 94 | func (c *RestoreCommand) loadFromURL(ctx context.Context, replicaURL string, ifDBNotExists bool, opt *litestream.RestoreOptions) (*litestream.Replica, error) { 95 | if opt.OutputPath == "" { 96 | return nil, fmt.Errorf("output path required") 97 | } 98 | 99 | // Exit successfully if the output file already exists. 100 | if _, err := os.Stat(opt.OutputPath); !os.IsNotExist(err) && ifDBNotExists { 101 | return nil, errSkipDBExists 102 | } 103 | 104 | syncInterval := litestream.DefaultSyncInterval 105 | r, err := NewReplicaFromConfig(&ReplicaConfig{ 106 | URL: replicaURL, 107 | SyncInterval: &syncInterval, 108 | }, nil) 109 | if err != nil { 110 | return nil, err 111 | } 112 | opt.Generation, _, err = r.CalcRestoreTarget(ctx, *opt) 113 | return r, err 114 | } 115 | 116 | // loadFromConfig returns a replica & updates the restore options from a DB reference. 117 | func (c *RestoreCommand) loadFromConfig(ctx context.Context, dbPath, configPath string, expandEnv, ifDBNotExists bool, opt *litestream.RestoreOptions) (*litestream.Replica, error) { 118 | // Load configuration. 119 | config, err := ReadConfigFile(configPath, expandEnv) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | // Lookup database from configuration file by path. 125 | if dbPath, err = expand(dbPath); err != nil { 126 | return nil, err 127 | } 128 | dbConfig := config.DBConfig(dbPath) 129 | if dbConfig == nil { 130 | return nil, fmt.Errorf("database not found in config: %s", dbPath) 131 | } 132 | db, err := NewDBFromConfig(dbConfig) 133 | if err != nil { 134 | return nil, err 135 | } 136 | 137 | // Restore into original database path if not specified. 138 | if opt.OutputPath == "" { 139 | opt.OutputPath = dbPath 140 | } 141 | 142 | // Exit successfully if the output file already exists. 143 | if _, err := os.Stat(opt.OutputPath); !os.IsNotExist(err) && ifDBNotExists { 144 | return nil, errSkipDBExists 145 | } 146 | 147 | // Determine the appropriate replica & generation to restore from, 148 | r, generation, err := db.CalcRestoreTarget(ctx, *opt) 149 | if err != nil { 150 | return nil, err 151 | } 152 | opt.Generation = generation 153 | 154 | return r, nil 155 | } 156 | 157 | // Usage prints the help screen to STDOUT. 158 | func (c *RestoreCommand) Usage() { 159 | fmt.Printf(` 160 | The restore command recovers a database from a previous snapshot and WAL. 161 | 162 | Usage: 163 | 164 | litestream restore [arguments] DB_PATH 165 | 166 | litestream restore [arguments] REPLICA_URL 167 | 168 | Arguments: 169 | 170 | -config PATH 171 | Specifies the configuration file. 172 | Defaults to %s 173 | 174 | -no-expand-env 175 | Disables environment variable expansion in configuration file. 176 | 177 | -replica NAME 178 | Restore from a specific replica. 179 | Defaults to replica with latest data. 180 | 181 | -generation NAME 182 | Restore from a specific generation. 183 | Defaults to generation with latest data. 184 | 185 | -index NUM 186 | Restore up to a specific hex-encoded WAL index (inclusive). 187 | Defaults to use the highest available index. 188 | 189 | -timestamp TIMESTAMP 190 | Restore to a specific point-in-time. 191 | Defaults to use the latest available backup. 192 | 193 | -o PATH 194 | Output path of the restored database. 195 | Defaults to original DB path. 196 | 197 | -if-db-not-exists 198 | Returns exit code of 0 if the database already exists. 199 | 200 | -if-replica-exists 201 | Returns exit code of 0 if no backups found. 202 | 203 | -parallelism NUM 204 | Determines the number of WAL files downloaded in parallel. 205 | Defaults to `+strconv.Itoa(litestream.DefaultRestoreParallelism)+`. 206 | 207 | -v 208 | Verbose output. 209 | 210 | 211 | Examples: 212 | 213 | # Restore latest replica for database to original location. 214 | $ litestream restore /path/to/db 215 | 216 | # Restore replica for database to a given point in time. 217 | $ litestream restore -timestamp 2020-01-01T00:00:00Z /path/to/db 218 | 219 | # Restore latest replica for database to new /tmp directory 220 | $ litestream restore -o /tmp/db /path/to/db 221 | 222 | # Restore database from latest generation on S3. 223 | $ litestream restore -replica s3 /path/to/db 224 | 225 | # Restore database from specific generation on S3. 226 | $ litestream restore -replica s3 -generation xxxxxxxx /path/to/db 227 | 228 | `[1:], 229 | DefaultConfigPath(), 230 | ) 231 | } 232 | 233 | var errSkipDBExists = errors.New("database already exists, skipping") 234 | -------------------------------------------------------------------------------- /file/replica_client_test.go: -------------------------------------------------------------------------------- 1 | package file_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/benbjohnson/litestream/file" 7 | ) 8 | 9 | func TestReplicaClient_Path(t *testing.T) { 10 | c := file.NewReplicaClient("/foo/bar") 11 | if got, want := c.Path(), "/foo/bar"; got != want { 12 | t.Fatalf("Path()=%v, want %v", got, want) 13 | } 14 | } 15 | 16 | func TestReplicaClient_Type(t *testing.T) { 17 | if got, want := file.NewReplicaClient("").Type(), "file"; got != want { 18 | t.Fatalf("Type()=%v, want %v", got, want) 19 | } 20 | } 21 | 22 | func TestReplicaClient_GenerationsDir(t *testing.T) { 23 | t.Run("OK", func(t *testing.T) { 24 | if got, err := file.NewReplicaClient("/foo").GenerationsDir(); err != nil { 25 | t.Fatal(err) 26 | } else if want := "/foo/generations"; got != want { 27 | t.Fatalf("GenerationsDir()=%v, want %v", got, want) 28 | } 29 | }) 30 | t.Run("ErrNoPath", func(t *testing.T) { 31 | if _, err := file.NewReplicaClient("").GenerationsDir(); err == nil || err.Error() != `file replica path required` { 32 | t.Fatalf("unexpected error: %v", err) 33 | } 34 | }) 35 | } 36 | 37 | func TestReplicaClient_GenerationDir(t *testing.T) { 38 | t.Run("OK", func(t *testing.T) { 39 | if got, err := file.NewReplicaClient("/foo").GenerationDir("0123456701234567"); err != nil { 40 | t.Fatal(err) 41 | } else if want := "/foo/generations/0123456701234567"; got != want { 42 | t.Fatalf("GenerationDir()=%v, want %v", got, want) 43 | } 44 | }) 45 | t.Run("ErrNoPath", func(t *testing.T) { 46 | if _, err := file.NewReplicaClient("").GenerationDir("0123456701234567"); err == nil || err.Error() != `file replica path required` { 47 | t.Fatalf("expected error: %v", err) 48 | } 49 | }) 50 | t.Run("ErrNoGeneration", func(t *testing.T) { 51 | if _, err := file.NewReplicaClient("/foo").GenerationDir(""); err == nil || err.Error() != `generation required` { 52 | t.Fatalf("expected error: %v", err) 53 | } 54 | }) 55 | } 56 | 57 | func TestReplicaClient_SnapshotsDir(t *testing.T) { 58 | t.Run("OK", func(t *testing.T) { 59 | if got, err := file.NewReplicaClient("/foo").SnapshotsDir("0123456701234567"); err != nil { 60 | t.Fatal(err) 61 | } else if want := "/foo/generations/0123456701234567/snapshots"; got != want { 62 | t.Fatalf("SnapshotsDir()=%v, want %v", got, want) 63 | } 64 | }) 65 | t.Run("ErrNoPath", func(t *testing.T) { 66 | if _, err := file.NewReplicaClient("").SnapshotsDir("0123456701234567"); err == nil || err.Error() != `file replica path required` { 67 | t.Fatalf("unexpected error: %v", err) 68 | } 69 | }) 70 | t.Run("ErrNoGeneration", func(t *testing.T) { 71 | if _, err := file.NewReplicaClient("/foo").SnapshotsDir(""); err == nil || err.Error() != `generation required` { 72 | t.Fatalf("unexpected error: %v", err) 73 | } 74 | }) 75 | } 76 | 77 | func TestReplicaClient_SnapshotPath(t *testing.T) { 78 | t.Run("OK", func(t *testing.T) { 79 | if got, err := file.NewReplicaClient("/foo").SnapshotPath("0123456701234567", 1000); err != nil { 80 | t.Fatal(err) 81 | } else if want := "/foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want { 82 | t.Fatalf("SnapshotPath()=%v, want %v", got, want) 83 | } 84 | }) 85 | t.Run("ErrNoPath", func(t *testing.T) { 86 | if _, err := file.NewReplicaClient("").SnapshotPath("0123456701234567", 1000); err == nil || err.Error() != `file replica path required` { 87 | t.Fatalf("unexpected error: %v", err) 88 | } 89 | }) 90 | t.Run("ErrNoGeneration", func(t *testing.T) { 91 | if _, err := file.NewReplicaClient("/foo").SnapshotPath("", 1000); err == nil || err.Error() != `generation required` { 92 | t.Fatalf("unexpected error: %v", err) 93 | } 94 | }) 95 | } 96 | 97 | func TestReplicaClient_WALDir(t *testing.T) { 98 | t.Run("OK", func(t *testing.T) { 99 | if got, err := file.NewReplicaClient("/foo").WALDir("0123456701234567"); err != nil { 100 | t.Fatal(err) 101 | } else if want := "/foo/generations/0123456701234567/wal"; got != want { 102 | t.Fatalf("WALDir()=%v, want %v", got, want) 103 | } 104 | }) 105 | t.Run("ErrNoPath", func(t *testing.T) { 106 | if _, err := file.NewReplicaClient("").WALDir("0123456701234567"); err == nil || err.Error() != `file replica path required` { 107 | t.Fatalf("unexpected error: %v", err) 108 | } 109 | }) 110 | t.Run("ErrNoGeneration", func(t *testing.T) { 111 | if _, err := file.NewReplicaClient("/foo").WALDir(""); err == nil || err.Error() != `generation required` { 112 | t.Fatalf("unexpected error: %v", err) 113 | } 114 | }) 115 | } 116 | 117 | func TestReplicaClient_WALSegmentPath(t *testing.T) { 118 | t.Run("OK", func(t *testing.T) { 119 | if got, err := file.NewReplicaClient("/foo").WALSegmentPath("0123456701234567", 1000, 1001); err != nil { 120 | t.Fatal(err) 121 | } else if want := "/foo/generations/0123456701234567/wal/000003e8_000003e9.wal.lz4"; got != want { 122 | t.Fatalf("WALPath()=%v, want %v", got, want) 123 | } 124 | }) 125 | t.Run("ErrNoPath", func(t *testing.T) { 126 | if _, err := file.NewReplicaClient("").WALSegmentPath("0123456701234567", 1000, 0); err == nil || err.Error() != `file replica path required` { 127 | t.Fatalf("unexpected error: %v", err) 128 | } 129 | }) 130 | t.Run("ErrNoGeneration", func(t *testing.T) { 131 | if _, err := file.NewReplicaClient("/foo").WALSegmentPath("", 1000, 0); err == nil || err.Error() != `generation required` { 132 | t.Fatalf("unexpected error: %v", err) 133 | } 134 | }) 135 | } 136 | 137 | /* 138 | func TestReplica_Sync(t *testing.T) { 139 | // Ensure replica can successfully sync after DB has sync'd. 140 | t.Run("InitialSync", func(t *testing.T) { 141 | db, sqldb := MustOpenDBs(t) 142 | defer MustCloseDBs(t, db, sqldb) 143 | 144 | r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir())) 145 | r.MonitorEnabled = false 146 | db.Replicas = []*litestream.Replica{r} 147 | 148 | // Sync database & then sync replica. 149 | if err := db.Sync(context.Background()); err != nil { 150 | t.Fatal(err) 151 | } else if err := r.Sync(context.Background()); err != nil { 152 | t.Fatal(err) 153 | } 154 | 155 | // Ensure posistions match. 156 | if want, err := db.Pos(); err != nil { 157 | t.Fatal(err) 158 | } else if got, err := r.Pos(context.Background()); err != nil { 159 | t.Fatal(err) 160 | } else if got != want { 161 | t.Fatalf("Pos()=%v, want %v", got, want) 162 | } 163 | }) 164 | 165 | // Ensure replica can successfully sync multiple times. 166 | t.Run("MultiSync", func(t *testing.T) { 167 | db, sqldb := MustOpenDBs(t) 168 | defer MustCloseDBs(t, db, sqldb) 169 | 170 | r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir())) 171 | r.MonitorEnabled = false 172 | db.Replicas = []*litestream.Replica{r} 173 | 174 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 175 | t.Fatal(err) 176 | } 177 | 178 | // Write to the database multiple times and sync after each write. 179 | for i, n := 0, db.MinCheckpointPageN*2; i < n; i++ { 180 | if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz')`); err != nil { 181 | t.Fatal(err) 182 | } 183 | 184 | // Sync periodically. 185 | if i%100 == 0 || i == n-1 { 186 | if err := db.Sync(context.Background()); err != nil { 187 | t.Fatal(err) 188 | } else if err := r.Sync(context.Background()); err != nil { 189 | t.Fatal(err) 190 | } 191 | } 192 | } 193 | 194 | // Ensure posistions match. 195 | pos, err := db.Pos() 196 | if err != nil { 197 | t.Fatal(err) 198 | } else if got, want := pos.Index, 2; got != want { 199 | t.Fatalf("Index=%v, want %v", got, want) 200 | } 201 | 202 | if want, err := r.Pos(context.Background()); err != nil { 203 | t.Fatal(err) 204 | } else if got := pos; got != want { 205 | t.Fatalf("Pos()=%v, want %v", got, want) 206 | } 207 | }) 208 | 209 | // Ensure replica returns an error if there is no generation available from the DB. 210 | t.Run("ErrNoGeneration", func(t *testing.T) { 211 | db, sqldb := MustOpenDBs(t) 212 | defer MustCloseDBs(t, db, sqldb) 213 | 214 | r := litestream.NewReplica(db, "", file.NewReplicaClient(t.TempDir())) 215 | r.MonitorEnabled = false 216 | db.Replicas = []*litestream.Replica{r} 217 | 218 | if err := r.Sync(context.Background()); err == nil || err.Error() != `no generation, waiting for data` { 219 | t.Fatal(err) 220 | } 221 | }) 222 | } 223 | */ 224 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /file/replica_client.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "sort" 11 | 12 | "github.com/benbjohnson/litestream" 13 | "github.com/benbjohnson/litestream/internal" 14 | ) 15 | 16 | // ReplicaClientType is the client type for this package. 17 | const ReplicaClientType = "file" 18 | 19 | var _ litestream.ReplicaClient = (*ReplicaClient)(nil) 20 | 21 | // ReplicaClient is a client for writing snapshots & WAL segments to disk. 22 | type ReplicaClient struct { 23 | path string // destination path 24 | 25 | Replica *litestream.Replica 26 | } 27 | 28 | // NewReplicaClient returns a new instance of ReplicaClient. 29 | func NewReplicaClient(path string) *ReplicaClient { 30 | return &ReplicaClient{ 31 | path: path, 32 | } 33 | } 34 | 35 | // db returns the database, if available. 36 | func (c *ReplicaClient) db() *litestream.DB { 37 | if c.Replica == nil { 38 | return nil 39 | } 40 | return c.Replica.DB() 41 | } 42 | 43 | // Type returns "file" as the client type. 44 | func (c *ReplicaClient) Type() string { 45 | return ReplicaClientType 46 | } 47 | 48 | // Path returns the destination path to replicate the database to. 49 | func (c *ReplicaClient) Path() string { 50 | return c.path 51 | } 52 | 53 | // GenerationsDir returns the path to a generation root directory. 54 | func (c *ReplicaClient) GenerationsDir() (string, error) { 55 | if c.path == "" { 56 | return "", fmt.Errorf("file replica path required") 57 | } 58 | return filepath.Join(c.path, "generations"), nil 59 | } 60 | 61 | // GenerationDir returns the path to a generation's root directory. 62 | func (c *ReplicaClient) GenerationDir(generation string) (string, error) { 63 | dir, err := c.GenerationsDir() 64 | if err != nil { 65 | return "", err 66 | } else if generation == "" { 67 | return "", fmt.Errorf("generation required") 68 | } 69 | return filepath.Join(dir, generation), nil 70 | } 71 | 72 | // SnapshotsDir returns the path to a generation's snapshot directory. 73 | func (c *ReplicaClient) SnapshotsDir(generation string) (string, error) { 74 | dir, err := c.GenerationDir(generation) 75 | if err != nil { 76 | return "", err 77 | } 78 | return filepath.Join(dir, "snapshots"), nil 79 | } 80 | 81 | // SnapshotPath returns the path to an uncompressed snapshot file. 82 | func (c *ReplicaClient) SnapshotPath(generation string, index int) (string, error) { 83 | dir, err := c.SnapshotsDir(generation) 84 | if err != nil { 85 | return "", err 86 | } 87 | return filepath.Join(dir, litestream.FormatSnapshotPath(index)), nil 88 | } 89 | 90 | // WALDir returns the path to a generation's WAL directory 91 | func (c *ReplicaClient) WALDir(generation string) (string, error) { 92 | dir, err := c.GenerationDir(generation) 93 | if err != nil { 94 | return "", err 95 | } 96 | return filepath.Join(dir, "wal"), nil 97 | } 98 | 99 | // WALSegmentPath returns the path to a WAL segment file. 100 | func (c *ReplicaClient) WALSegmentPath(generation string, index int, offset int64) (string, error) { 101 | dir, err := c.WALDir(generation) 102 | if err != nil { 103 | return "", err 104 | } 105 | return filepath.Join(dir, litestream.FormatWALSegmentPath(index, offset)), nil 106 | } 107 | 108 | // Generations returns a list of available generation names. 109 | func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) { 110 | root, err := c.GenerationsDir() 111 | if err != nil { 112 | return nil, fmt.Errorf("cannot determine generations path: %w", err) 113 | } 114 | 115 | fis, err := ioutil.ReadDir(root) 116 | if os.IsNotExist(err) { 117 | return nil, nil 118 | } else if err != nil { 119 | return nil, err 120 | } 121 | 122 | var generations []string 123 | for _, fi := range fis { 124 | if !litestream.IsGenerationName(fi.Name()) { 125 | continue 126 | } else if !fi.IsDir() { 127 | continue 128 | } 129 | generations = append(generations, fi.Name()) 130 | } 131 | return generations, nil 132 | } 133 | 134 | // DeleteGeneration deletes all snapshots & WAL segments within a generation. 135 | func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error { 136 | dir, err := c.GenerationDir(generation) 137 | if err != nil { 138 | return fmt.Errorf("cannot determine generation path: %w", err) 139 | } 140 | 141 | if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { 142 | return err 143 | } 144 | return nil 145 | } 146 | 147 | // Snapshots returns an iterator over all available snapshots for a generation. 148 | func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) { 149 | dir, err := c.SnapshotsDir(generation) 150 | if err != nil { 151 | return nil, fmt.Errorf("cannot determine snapshots path: %w", err) 152 | } 153 | 154 | f, err := os.Open(dir) 155 | if os.IsNotExist(err) { 156 | return litestream.NewSnapshotInfoSliceIterator(nil), nil 157 | } else if err != nil { 158 | return nil, err 159 | } 160 | defer f.Close() 161 | 162 | fis, err := f.Readdir(-1) 163 | if err != nil { 164 | return nil, err 165 | } 166 | 167 | // Iterate over every file and convert to metadata. 168 | infos := make([]litestream.SnapshotInfo, 0, len(fis)) 169 | for _, fi := range fis { 170 | // Parse index from filename. 171 | index, err := litestream.ParseSnapshotPath(fi.Name()) 172 | if err != nil { 173 | continue 174 | } 175 | 176 | infos = append(infos, litestream.SnapshotInfo{ 177 | Generation: generation, 178 | Index: index, 179 | Size: fi.Size(), 180 | CreatedAt: fi.ModTime().UTC(), 181 | }) 182 | } 183 | 184 | sort.Sort(litestream.SnapshotInfoSlice(infos)) 185 | 186 | return litestream.NewSnapshotInfoSliceIterator(infos), nil 187 | } 188 | 189 | // WriteSnapshot writes LZ4 compressed data from rd into a file on disk. 190 | func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) { 191 | filename, err := c.SnapshotPath(generation, index) 192 | if err != nil { 193 | return info, fmt.Errorf("cannot determine snapshot path: %w", err) 194 | } 195 | 196 | var fileInfo, dirInfo os.FileInfo 197 | if db := c.db(); db != nil { 198 | fileInfo, dirInfo = db.FileInfo(), db.DirInfo() 199 | } 200 | 201 | // Ensure parent directory exists. 202 | if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil { 203 | return info, err 204 | } 205 | 206 | // Write snapshot to temporary file next to destination path. 207 | f, err := internal.CreateFile(filename+".tmp", fileInfo) 208 | if err != nil { 209 | return info, err 210 | } 211 | defer f.Close() 212 | 213 | if _, err := io.Copy(f, rd); err != nil { 214 | return info, err 215 | } else if err := f.Sync(); err != nil { 216 | return info, err 217 | } else if err := f.Close(); err != nil { 218 | return info, err 219 | } 220 | 221 | // Build metadata. 222 | fi, err := os.Stat(filename + ".tmp") 223 | if err != nil { 224 | return info, err 225 | } 226 | info = litestream.SnapshotInfo{ 227 | Generation: generation, 228 | Index: index, 229 | Size: fi.Size(), 230 | CreatedAt: fi.ModTime().UTC(), 231 | } 232 | 233 | // Move snapshot to final path when it has been fully written & synced to disk. 234 | if err := os.Rename(filename+".tmp", filename); err != nil { 235 | return info, err 236 | } 237 | 238 | return info, nil 239 | } 240 | 241 | // SnapshotReader returns a reader for snapshot data at the given generation/index. 242 | // Returns os.ErrNotExist if no matching index is found. 243 | func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) { 244 | filename, err := c.SnapshotPath(generation, index) 245 | if err != nil { 246 | return nil, fmt.Errorf("cannot determine snapshot path: %w", err) 247 | } 248 | return os.Open(filename) 249 | } 250 | 251 | // DeleteSnapshot deletes a snapshot with the given generation & index. 252 | func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error { 253 | filename, err := c.SnapshotPath(generation, index) 254 | if err != nil { 255 | return fmt.Errorf("cannot determine snapshot path: %w", err) 256 | } 257 | if err := os.Remove(filename); err != nil && !os.IsNotExist(err) { 258 | return err 259 | } 260 | return nil 261 | } 262 | 263 | // WALSegments returns an iterator over all available WAL files for a generation. 264 | func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) { 265 | dir, err := c.WALDir(generation) 266 | if err != nil { 267 | return nil, fmt.Errorf("cannot determine wal path: %w", err) 268 | } 269 | 270 | f, err := os.Open(dir) 271 | if os.IsNotExist(err) { 272 | return litestream.NewWALSegmentInfoSliceIterator(nil), nil 273 | } else if err != nil { 274 | return nil, err 275 | } 276 | defer f.Close() 277 | 278 | fis, err := f.Readdir(-1) 279 | if err != nil { 280 | return nil, err 281 | } 282 | 283 | // Iterate over every file and convert to metadata. 284 | infos := make([]litestream.WALSegmentInfo, 0, len(fis)) 285 | for _, fi := range fis { 286 | // Parse index from filename. 287 | index, offset, err := litestream.ParseWALSegmentPath(fi.Name()) 288 | if err != nil { 289 | continue 290 | } 291 | 292 | infos = append(infos, litestream.WALSegmentInfo{ 293 | Generation: generation, 294 | Index: index, 295 | Offset: offset, 296 | Size: fi.Size(), 297 | CreatedAt: fi.ModTime().UTC(), 298 | }) 299 | } 300 | 301 | sort.Sort(litestream.WALSegmentInfoSlice(infos)) 302 | 303 | return litestream.NewWALSegmentInfoSliceIterator(infos), nil 304 | } 305 | 306 | // WriteWALSegment writes LZ4 compressed data from rd into a file on disk. 307 | func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) { 308 | filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset) 309 | if err != nil { 310 | return info, fmt.Errorf("cannot determine wal segment path: %w", err) 311 | } 312 | 313 | var fileInfo, dirInfo os.FileInfo 314 | if db := c.db(); db != nil { 315 | fileInfo, dirInfo = db.FileInfo(), db.DirInfo() 316 | } 317 | 318 | // Ensure parent directory exists. 319 | if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil { 320 | return info, err 321 | } 322 | 323 | // Write WAL segment to temporary file next to destination path. 324 | f, err := internal.CreateFile(filename+".tmp", fileInfo) 325 | if err != nil { 326 | return info, err 327 | } 328 | defer f.Close() 329 | 330 | if _, err := io.Copy(f, rd); err != nil { 331 | return info, err 332 | } else if err := f.Sync(); err != nil { 333 | return info, err 334 | } else if err := f.Close(); err != nil { 335 | return info, err 336 | } 337 | 338 | // Build metadata. 339 | fi, err := os.Stat(filename + ".tmp") 340 | if err != nil { 341 | return info, err 342 | } 343 | info = litestream.WALSegmentInfo{ 344 | Generation: pos.Generation, 345 | Index: pos.Index, 346 | Offset: pos.Offset, 347 | Size: fi.Size(), 348 | CreatedAt: fi.ModTime().UTC(), 349 | } 350 | 351 | // Move WAL segment to final path when it has been written & synced to disk. 352 | if err := os.Rename(filename+".tmp", filename); err != nil { 353 | return info, err 354 | } 355 | 356 | return info, nil 357 | } 358 | 359 | // WALSegmentReader returns a reader for a section of WAL data at the given position. 360 | // Returns os.ErrNotExist if no matching index/offset is found. 361 | func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) { 362 | filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset) 363 | if err != nil { 364 | return nil, fmt.Errorf("cannot determine wal segment path: %w", err) 365 | } 366 | return os.Open(filename) 367 | } 368 | 369 | // DeleteWALSegments deletes WAL segments at the given positions. 370 | func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error { 371 | for _, pos := range a { 372 | filename, err := c.WALSegmentPath(pos.Generation, pos.Index, pos.Offset) 373 | if err != nil { 374 | return fmt.Errorf("cannot determine wal segment path: %w", err) 375 | } 376 | if err := os.Remove(filename); err != nil && !os.IsNotExist(err) { 377 | return err 378 | } 379 | } 380 | return nil 381 | } 382 | -------------------------------------------------------------------------------- /gcs/replica_client.go: -------------------------------------------------------------------------------- 1 | package gcs 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | "cloud.google.com/go/storage" 14 | "github.com/benbjohnson/litestream" 15 | "github.com/benbjohnson/litestream/internal" 16 | "google.golang.org/api/iterator" 17 | ) 18 | 19 | // ReplicaClientType is the client type for this package. 20 | const ReplicaClientType = "gcs" 21 | 22 | var _ litestream.ReplicaClient = (*ReplicaClient)(nil) 23 | 24 | // ReplicaClient is a client for writing snapshots & WAL segments to disk. 25 | type ReplicaClient struct { 26 | mu sync.Mutex 27 | client *storage.Client // gcs client 28 | bkt *storage.BucketHandle // gcs bucket handle 29 | 30 | // GCS bucket information 31 | Bucket string 32 | Path string 33 | } 34 | 35 | // NewReplicaClient returns a new instance of ReplicaClient. 36 | func NewReplicaClient() *ReplicaClient { 37 | return &ReplicaClient{} 38 | } 39 | 40 | // Type returns "gcs" as the client type. 41 | func (c *ReplicaClient) Type() string { 42 | return ReplicaClientType 43 | } 44 | 45 | // Init initializes the connection to GCS. No-op if already initialized. 46 | func (c *ReplicaClient) Init(ctx context.Context) (err error) { 47 | c.mu.Lock() 48 | defer c.mu.Unlock() 49 | 50 | if c.client != nil { 51 | return nil 52 | } 53 | 54 | if c.client, err = storage.NewClient(ctx); err != nil { 55 | return err 56 | } 57 | c.bkt = c.client.Bucket(c.Bucket) 58 | 59 | return nil 60 | } 61 | 62 | // Generations returns a list of available generation names. 63 | func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) { 64 | if err := c.Init(ctx); err != nil { 65 | return nil, err 66 | } 67 | 68 | // Construct query to only pull generation directory names. 69 | query := &storage.Query{ 70 | Delimiter: "/", 71 | Prefix: litestream.GenerationsPath(c.Path) + "/", 72 | } 73 | 74 | // Loop over results and only build list of generation-formatted names. 75 | it := c.bkt.Objects(ctx, query) 76 | var generations []string 77 | for { 78 | attrs, err := it.Next() 79 | if err == iterator.Done { 80 | break 81 | } else if err != nil { 82 | return nil, err 83 | } 84 | 85 | name := path.Base(strings.TrimSuffix(attrs.Prefix, "/")) 86 | if !litestream.IsGenerationName(name) { 87 | continue 88 | } 89 | generations = append(generations, name) 90 | } 91 | 92 | return generations, nil 93 | } 94 | 95 | // DeleteGeneration deletes all snapshots & WAL segments within a generation. 96 | func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error { 97 | if err := c.Init(ctx); err != nil { 98 | return err 99 | } 100 | 101 | dir, err := litestream.GenerationPath(c.Path, generation) 102 | if err != nil { 103 | return fmt.Errorf("cannot determine generation path: %w", err) 104 | } 105 | 106 | // Iterate over every object in generation and delete it. 107 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc() 108 | for it := c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"}); ; { 109 | attrs, err := it.Next() 110 | if err == iterator.Done { 111 | break 112 | } else if err != nil { 113 | return err 114 | } 115 | 116 | if err := c.bkt.Object(attrs.Name).Delete(ctx); isNotExists(err) { 117 | continue 118 | } else if err != nil { 119 | return fmt.Errorf("cannot delete object %q: %w", attrs.Name, err) 120 | } 121 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc() 122 | } 123 | 124 | // log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation) 125 | 126 | return nil 127 | } 128 | 129 | // Snapshots returns an iterator over all available snapshots for a generation. 130 | func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) { 131 | if err := c.Init(ctx); err != nil { 132 | return nil, err 133 | } 134 | dir, err := litestream.SnapshotsPath(c.Path, generation) 135 | if err != nil { 136 | return nil, fmt.Errorf("cannot determine snapshots path: %w", err) 137 | } 138 | return newSnapshotIterator(generation, c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"})), nil 139 | } 140 | 141 | // WriteSnapshot writes LZ4 compressed data from rd to the object storage. 142 | func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) { 143 | if err := c.Init(ctx); err != nil { 144 | return info, err 145 | } 146 | 147 | key, err := litestream.SnapshotPath(c.Path, generation, index) 148 | if err != nil { 149 | return info, fmt.Errorf("cannot determine snapshot path: %w", err) 150 | } 151 | startTime := time.Now() 152 | 153 | w := c.bkt.Object(key).NewWriter(ctx) 154 | defer w.Close() 155 | 156 | n, err := io.Copy(w, rd) 157 | if err != nil { 158 | return info, err 159 | } else if err := w.Close(); err != nil { 160 | return info, err 161 | } 162 | 163 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc() 164 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n)) 165 | 166 | // log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond)) 167 | 168 | return litestream.SnapshotInfo{ 169 | Generation: generation, 170 | Index: index, 171 | Size: n, 172 | CreatedAt: startTime.UTC(), 173 | }, nil 174 | } 175 | 176 | // SnapshotReader returns a reader for snapshot data at the given generation/index. 177 | func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) { 178 | if err := c.Init(ctx); err != nil { 179 | return nil, err 180 | } 181 | 182 | key, err := litestream.SnapshotPath(c.Path, generation, index) 183 | if err != nil { 184 | return nil, fmt.Errorf("cannot determine snapshot path: %w", err) 185 | } 186 | 187 | r, err := c.bkt.Object(key).NewReader(ctx) 188 | if isNotExists(err) { 189 | return nil, os.ErrNotExist 190 | } else if err != nil { 191 | return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err) 192 | } 193 | 194 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc() 195 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(r.Attrs.Size)) 196 | 197 | return r, nil 198 | } 199 | 200 | // DeleteSnapshot deletes a snapshot with the given generation & index. 201 | func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error { 202 | if err := c.Init(ctx); err != nil { 203 | return err 204 | } 205 | 206 | key, err := litestream.SnapshotPath(c.Path, generation, index) 207 | if err != nil { 208 | return fmt.Errorf("cannot determine snapshot path: %w", err) 209 | } 210 | 211 | if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) { 212 | return fmt.Errorf("cannot delete snapshot %q: %w", key, err) 213 | } 214 | 215 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc() 216 | return nil 217 | } 218 | 219 | // WALSegments returns an iterator over all available WAL files for a generation. 220 | func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) { 221 | if err := c.Init(ctx); err != nil { 222 | return nil, err 223 | } 224 | dir, err := litestream.WALPath(c.Path, generation) 225 | if err != nil { 226 | return nil, fmt.Errorf("cannot determine wal path: %w", err) 227 | } 228 | return newWALSegmentIterator(generation, c.bkt.Objects(ctx, &storage.Query{Prefix: dir + "/"})), nil 229 | } 230 | 231 | // WriteWALSegment writes LZ4 compressed data from rd into a file on disk. 232 | func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) { 233 | if err := c.Init(ctx); err != nil { 234 | return info, err 235 | } 236 | 237 | key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) 238 | if err != nil { 239 | return info, fmt.Errorf("cannot determine wal segment path: %w", err) 240 | } 241 | startTime := time.Now() 242 | 243 | w := c.bkt.Object(key).NewWriter(ctx) 244 | defer w.Close() 245 | 246 | n, err := io.Copy(w, rd) 247 | if err != nil { 248 | return info, err 249 | } else if err := w.Close(); err != nil { 250 | return info, err 251 | } 252 | 253 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc() 254 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n)) 255 | 256 | return litestream.WALSegmentInfo{ 257 | Generation: pos.Generation, 258 | Index: pos.Index, 259 | Offset: pos.Offset, 260 | Size: n, 261 | CreatedAt: startTime.UTC(), 262 | }, nil 263 | } 264 | 265 | // WALSegmentReader returns a reader for a section of WAL data at the given index. 266 | // Returns os.ErrNotExist if no matching index/offset is found. 267 | func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) { 268 | if err := c.Init(ctx); err != nil { 269 | return nil, err 270 | } 271 | 272 | key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) 273 | if err != nil { 274 | return nil, fmt.Errorf("cannot determine wal segment path: %w", err) 275 | } 276 | 277 | r, err := c.bkt.Object(key).NewReader(ctx) 278 | if isNotExists(err) { 279 | return nil, os.ErrNotExist 280 | } else if err != nil { 281 | return nil, err 282 | } 283 | 284 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc() 285 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(r.Attrs.Size)) 286 | 287 | return r, nil 288 | } 289 | 290 | // DeleteWALSegments deletes WAL segments with at the given positions. 291 | func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error { 292 | if err := c.Init(ctx); err != nil { 293 | return err 294 | } 295 | 296 | for _, pos := range a { 297 | key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) 298 | if err != nil { 299 | return fmt.Errorf("cannot determine wal segment path: %w", err) 300 | } 301 | 302 | if err := c.bkt.Object(key).Delete(ctx); err != nil && !isNotExists(err) { 303 | return fmt.Errorf("cannot delete wal segment %q: %w", key, err) 304 | } 305 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc() 306 | } 307 | 308 | return nil 309 | } 310 | 311 | type snapshotIterator struct { 312 | generation string 313 | 314 | it *storage.ObjectIterator 315 | info litestream.SnapshotInfo 316 | err error 317 | } 318 | 319 | func newSnapshotIterator(generation string, it *storage.ObjectIterator) *snapshotIterator { 320 | return &snapshotIterator{ 321 | generation: generation, 322 | it: it, 323 | } 324 | } 325 | 326 | func (itr *snapshotIterator) Close() (err error) { 327 | return itr.err 328 | } 329 | 330 | func (itr *snapshotIterator) Next() bool { 331 | // Exit if an error has already occurred. 332 | if itr.err != nil { 333 | return false 334 | } 335 | 336 | for { 337 | // Fetch next object. 338 | attrs, err := itr.it.Next() 339 | if err == iterator.Done { 340 | return false 341 | } else if err != nil { 342 | itr.err = err 343 | return false 344 | } 345 | 346 | // Parse index, otherwise skip to the next object. 347 | index, err := litestream.ParseSnapshotPath(path.Base(attrs.Name)) 348 | if err != nil { 349 | continue 350 | } 351 | 352 | // Store current snapshot and return. 353 | itr.info = litestream.SnapshotInfo{ 354 | Generation: itr.generation, 355 | Index: index, 356 | Size: attrs.Size, 357 | CreatedAt: attrs.Created.UTC(), 358 | } 359 | return true 360 | } 361 | } 362 | 363 | func (itr *snapshotIterator) Err() error { return itr.err } 364 | 365 | func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo { return itr.info } 366 | 367 | type walSegmentIterator struct { 368 | generation string 369 | 370 | it *storage.ObjectIterator 371 | info litestream.WALSegmentInfo 372 | err error 373 | } 374 | 375 | func newWALSegmentIterator(generation string, it *storage.ObjectIterator) *walSegmentIterator { 376 | return &walSegmentIterator{ 377 | generation: generation, 378 | it: it, 379 | } 380 | } 381 | 382 | func (itr *walSegmentIterator) Close() (err error) { 383 | return itr.err 384 | } 385 | 386 | func (itr *walSegmentIterator) Next() bool { 387 | // Exit if an error has already occurred. 388 | if itr.err != nil { 389 | return false 390 | } 391 | 392 | for { 393 | // Fetch next object. 394 | attrs, err := itr.it.Next() 395 | if err == iterator.Done { 396 | return false 397 | } else if err != nil { 398 | itr.err = err 399 | return false 400 | } 401 | 402 | // Parse index & offset, otherwise skip to the next object. 403 | index, offset, err := litestream.ParseWALSegmentPath(path.Base(attrs.Name)) 404 | if err != nil { 405 | continue 406 | } 407 | 408 | // Store current snapshot and return. 409 | itr.info = litestream.WALSegmentInfo{ 410 | Generation: itr.generation, 411 | Index: index, 412 | Offset: offset, 413 | Size: attrs.Size, 414 | CreatedAt: attrs.Created.UTC(), 415 | } 416 | return true 417 | } 418 | } 419 | 420 | func (itr *walSegmentIterator) Err() error { return itr.err } 421 | 422 | func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo { 423 | return itr.info 424 | } 425 | 426 | func isNotExists(err error) bool { 427 | return err == storage.ErrObjectNotExist 428 | } 429 | -------------------------------------------------------------------------------- /sftp/replica_client.go: -------------------------------------------------------------------------------- 1 | package sftp 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net" 9 | "os" 10 | "path" 11 | "sort" 12 | "sync" 13 | "time" 14 | 15 | "github.com/benbjohnson/litestream" 16 | "github.com/benbjohnson/litestream/internal" 17 | "github.com/pkg/sftp" 18 | "golang.org/x/crypto/ssh" 19 | ) 20 | 21 | // ReplicaClientType is the client type for this package. 22 | const ReplicaClientType = "sftp" 23 | 24 | // Default settings for replica client. 25 | const ( 26 | DefaultDialTimeout = 30 * time.Second 27 | ) 28 | 29 | var _ litestream.ReplicaClient = (*ReplicaClient)(nil) 30 | 31 | // ReplicaClient is a client for writing snapshots & WAL segments to disk. 32 | type ReplicaClient struct { 33 | mu sync.Mutex 34 | sshClient *ssh.Client 35 | sftpClient *sftp.Client 36 | 37 | // SFTP connection info 38 | Host string 39 | User string 40 | Password string 41 | Path string 42 | KeyPath string 43 | DialTimeout time.Duration 44 | } 45 | 46 | // NewReplicaClient returns a new instance of ReplicaClient. 47 | func NewReplicaClient() *ReplicaClient { 48 | return &ReplicaClient{ 49 | DialTimeout: DefaultDialTimeout, 50 | } 51 | } 52 | 53 | // Type returns "gcs" as the client type. 54 | func (c *ReplicaClient) Type() string { 55 | return ReplicaClientType 56 | } 57 | 58 | // Init initializes the connection to GCS. No-op if already initialized. 59 | func (c *ReplicaClient) Init(ctx context.Context) (_ *sftp.Client, err error) { 60 | c.mu.Lock() 61 | defer c.mu.Unlock() 62 | 63 | if c.sftpClient != nil { 64 | return c.sftpClient, nil 65 | } 66 | 67 | if c.User == "" { 68 | return nil, fmt.Errorf("sftp user required") 69 | } 70 | 71 | // Build SSH configuration & auth methods 72 | config := &ssh.ClientConfig{ 73 | User: c.User, 74 | HostKeyCallback: ssh.InsecureIgnoreHostKey(), 75 | BannerCallback: ssh.BannerDisplayStderr(), 76 | } 77 | if c.Password != "" { 78 | config.Auth = append(config.Auth, ssh.Password(c.Password)) 79 | } 80 | 81 | if c.KeyPath != "" { 82 | buf, err := os.ReadFile(c.KeyPath) 83 | if err != nil { 84 | return nil, fmt.Errorf("cannot read sftp key path: %w", err) 85 | } 86 | 87 | signer, err := ssh.ParsePrivateKey(buf) 88 | if err != nil { 89 | return nil, fmt.Errorf("cannot parse sftp key path: %w", err) 90 | } 91 | config.Auth = append(config.Auth, ssh.PublicKeys(signer)) 92 | } 93 | 94 | // Append standard port, if necessary. 95 | host := c.Host 96 | if _, _, err := net.SplitHostPort(c.Host); err != nil { 97 | host = net.JoinHostPort(c.Host, "22") 98 | } 99 | 100 | // Connect via SSH. 101 | if c.sshClient, err = ssh.Dial("tcp", host, config); err != nil { 102 | return nil, err 103 | } 104 | 105 | // Wrap connection with an SFTP client. 106 | if c.sftpClient, err = sftp.NewClient(c.sshClient); err != nil { 107 | c.sshClient.Close() 108 | c.sshClient = nil 109 | return nil, err 110 | } 111 | 112 | return c.sftpClient, nil 113 | } 114 | 115 | // Generations returns a list of available generation names. 116 | func (c *ReplicaClient) Generations(ctx context.Context) (_ []string, err error) { 117 | defer func() { c.resetOnConnError(err) }() 118 | 119 | sftpClient, err := c.Init(ctx) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | fis, err := sftpClient.ReadDir(litestream.GenerationsPath(c.Path)) 125 | if os.IsNotExist(err) { 126 | return nil, nil 127 | } else if err != nil { 128 | return nil, err 129 | } 130 | 131 | var generations []string 132 | for _, fi := range fis { 133 | if !fi.IsDir() { 134 | continue 135 | } 136 | 137 | name := path.Base(fi.Name()) 138 | if !litestream.IsGenerationName(name) { 139 | continue 140 | } 141 | generations = append(generations, name) 142 | } 143 | 144 | sort.Strings(generations) 145 | 146 | return generations, nil 147 | } 148 | 149 | // DeleteGeneration deletes all snapshots & WAL segments within a generation. 150 | func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) (err error) { 151 | defer func() { c.resetOnConnError(err) }() 152 | 153 | sftpClient, err := c.Init(ctx) 154 | if err != nil { 155 | return err 156 | } 157 | 158 | dir, err := litestream.GenerationPath(c.Path, generation) 159 | if err != nil { 160 | return fmt.Errorf("cannot determine generation path: %w", err) 161 | } 162 | 163 | var dirs []string 164 | walker := sftpClient.Walk(dir) 165 | for walker.Step() { 166 | if err := walker.Err(); err != nil { 167 | return fmt.Errorf("cannot walk path %q: %w", walker.Path(), err) 168 | } 169 | if walker.Stat().IsDir() { 170 | dirs = append(dirs, walker.Path()) 171 | continue 172 | } 173 | 174 | if err := sftpClient.Remove(walker.Path()); err != nil && !os.IsNotExist(err) { 175 | return fmt.Errorf("cannot delete file %q: %w", walker.Path(), err) 176 | } 177 | 178 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc() 179 | } 180 | 181 | // Remove directories in reverse order after they have been emptied. 182 | for i := len(dirs) - 1; i >= 0; i-- { 183 | filename := dirs[i] 184 | if err := sftpClient.RemoveDirectory(filename); err != nil && !os.IsNotExist(err) { 185 | return fmt.Errorf("cannot delete directory %q: %w", filename, err) 186 | } 187 | } 188 | 189 | // log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation) 190 | 191 | return nil 192 | } 193 | 194 | // Snapshots returns an iterator over all available snapshots for a generation. 195 | func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (_ litestream.SnapshotIterator, err error) { 196 | defer func() { c.resetOnConnError(err) }() 197 | 198 | sftpClient, err := c.Init(ctx) 199 | if err != nil { 200 | return nil, err 201 | } 202 | 203 | dir, err := litestream.SnapshotsPath(c.Path, generation) 204 | if err != nil { 205 | return nil, fmt.Errorf("cannot determine snapshots path: %w", err) 206 | } 207 | 208 | fis, err := sftpClient.ReadDir(dir) 209 | if os.IsNotExist(err) { 210 | return litestream.NewSnapshotInfoSliceIterator(nil), nil 211 | } else if err != nil { 212 | return nil, err 213 | } 214 | 215 | // Iterate over every file and convert to metadata. 216 | infos := make([]litestream.SnapshotInfo, 0, len(fis)) 217 | for _, fi := range fis { 218 | // Parse index from filename. 219 | index, err := litestream.ParseSnapshotPath(path.Base(fi.Name())) 220 | if err != nil { 221 | continue 222 | } 223 | 224 | infos = append(infos, litestream.SnapshotInfo{ 225 | Generation: generation, 226 | Index: index, 227 | Size: fi.Size(), 228 | CreatedAt: fi.ModTime().UTC(), 229 | }) 230 | } 231 | 232 | sort.Sort(litestream.SnapshotInfoSlice(infos)) 233 | 234 | return litestream.NewSnapshotInfoSliceIterator(infos), nil 235 | } 236 | 237 | // WriteSnapshot writes LZ4 compressed data from rd to the object storage. 238 | func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) { 239 | defer func() { c.resetOnConnError(err) }() 240 | 241 | sftpClient, err := c.Init(ctx) 242 | if err != nil { 243 | return info, err 244 | } 245 | 246 | filename, err := litestream.SnapshotPath(c.Path, generation, index) 247 | if err != nil { 248 | return info, fmt.Errorf("cannot determine snapshot path: %w", err) 249 | } 250 | startTime := time.Now() 251 | 252 | if err := sftpClient.MkdirAll(path.Dir(filename)); err != nil { 253 | return info, fmt.Errorf("cannot make parent wal segment directory %q: %w", path.Dir(filename), err) 254 | } 255 | 256 | f, err := sftpClient.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC) 257 | if err != nil { 258 | return info, fmt.Errorf("cannot open snapshot file for writing: %w", err) 259 | } 260 | defer f.Close() 261 | 262 | n, err := io.Copy(f, rd) 263 | if err != nil { 264 | return info, err 265 | } else if err := f.Close(); err != nil { 266 | return info, err 267 | } 268 | 269 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc() 270 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n)) 271 | 272 | // log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond)) 273 | 274 | return litestream.SnapshotInfo{ 275 | Generation: generation, 276 | Index: index, 277 | Size: n, 278 | CreatedAt: startTime.UTC(), 279 | }, nil 280 | } 281 | 282 | // SnapshotReader returns a reader for snapshot data at the given generation/index. 283 | func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (_ io.ReadCloser, err error) { 284 | defer func() { c.resetOnConnError(err) }() 285 | 286 | sftpClient, err := c.Init(ctx) 287 | if err != nil { 288 | return nil, err 289 | } 290 | 291 | filename, err := litestream.SnapshotPath(c.Path, generation, index) 292 | if err != nil { 293 | return nil, fmt.Errorf("cannot determine snapshot path: %w", err) 294 | } 295 | 296 | f, err := sftpClient.Open(filename) 297 | if err != nil { 298 | return nil, err 299 | } 300 | 301 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc() 302 | 303 | return f, nil 304 | } 305 | 306 | // DeleteSnapshot deletes a snapshot with the given generation & index. 307 | func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) (err error) { 308 | defer func() { c.resetOnConnError(err) }() 309 | 310 | sftpClient, err := c.Init(ctx) 311 | if err != nil { 312 | return err 313 | } 314 | 315 | filename, err := litestream.SnapshotPath(c.Path, generation, index) 316 | if err != nil { 317 | return fmt.Errorf("cannot determine snapshot path: %w", err) 318 | } 319 | 320 | if err := sftpClient.Remove(filename); err != nil && !os.IsNotExist(err) { 321 | return fmt.Errorf("cannot delete snapshot %q: %w", filename, err) 322 | } 323 | 324 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc() 325 | return nil 326 | } 327 | 328 | // WALSegments returns an iterator over all available WAL files for a generation. 329 | func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (_ litestream.WALSegmentIterator, err error) { 330 | defer func() { c.resetOnConnError(err) }() 331 | 332 | sftpClient, err := c.Init(ctx) 333 | if err != nil { 334 | return nil, err 335 | } 336 | 337 | dir, err := litestream.WALPath(c.Path, generation) 338 | if err != nil { 339 | return nil, fmt.Errorf("cannot determine wal path: %w", err) 340 | } 341 | 342 | fis, err := sftpClient.ReadDir(dir) 343 | if os.IsNotExist(err) { 344 | return litestream.NewWALSegmentInfoSliceIterator(nil), nil 345 | } else if err != nil { 346 | return nil, err 347 | } 348 | 349 | // Iterate over every file and convert to metadata. 350 | infos := make([]litestream.WALSegmentInfo, 0, len(fis)) 351 | for _, fi := range fis { 352 | index, offset, err := litestream.ParseWALSegmentPath(path.Base(fi.Name())) 353 | if err != nil { 354 | continue 355 | } 356 | 357 | infos = append(infos, litestream.WALSegmentInfo{ 358 | Generation: generation, 359 | Index: index, 360 | Offset: offset, 361 | Size: fi.Size(), 362 | CreatedAt: fi.ModTime().UTC(), 363 | }) 364 | } 365 | 366 | sort.Sort(litestream.WALSegmentInfoSlice(infos)) 367 | 368 | return litestream.NewWALSegmentInfoSliceIterator(infos), nil 369 | } 370 | 371 | // WriteWALSegment writes LZ4 compressed data from rd into a file on disk. 372 | func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) { 373 | defer func() { c.resetOnConnError(err) }() 374 | 375 | sftpClient, err := c.Init(ctx) 376 | if err != nil { 377 | return info, err 378 | } 379 | 380 | filename, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) 381 | if err != nil { 382 | return info, fmt.Errorf("cannot determine wal segment path: %w", err) 383 | } 384 | startTime := time.Now() 385 | 386 | if err := sftpClient.MkdirAll(path.Dir(filename)); err != nil { 387 | return info, fmt.Errorf("cannot make parent snapshot directory %q: %w", path.Dir(filename), err) 388 | } 389 | 390 | f, err := sftpClient.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC) 391 | if err != nil { 392 | return info, fmt.Errorf("cannot open snapshot file for writing: %w", err) 393 | } 394 | defer f.Close() 395 | 396 | n, err := io.Copy(f, rd) 397 | if err != nil { 398 | return info, err 399 | } else if err := f.Close(); err != nil { 400 | return info, err 401 | } 402 | 403 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc() 404 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n)) 405 | 406 | return litestream.WALSegmentInfo{ 407 | Generation: pos.Generation, 408 | Index: pos.Index, 409 | Offset: pos.Offset, 410 | Size: n, 411 | CreatedAt: startTime.UTC(), 412 | }, nil 413 | } 414 | 415 | // WALSegmentReader returns a reader for a section of WAL data at the given index. 416 | // Returns os.ErrNotExist if no matching index/offset is found. 417 | func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (_ io.ReadCloser, err error) { 418 | defer func() { c.resetOnConnError(err) }() 419 | 420 | sftpClient, err := c.Init(ctx) 421 | if err != nil { 422 | return nil, err 423 | } 424 | 425 | filename, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) 426 | if err != nil { 427 | return nil, fmt.Errorf("cannot determine wal segment path: %w", err) 428 | } 429 | 430 | f, err := sftpClient.Open(filename) 431 | if err != nil { 432 | return nil, err 433 | } 434 | 435 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc() 436 | 437 | return f, nil 438 | } 439 | 440 | // DeleteWALSegments deletes WAL segments with at the given positions. 441 | func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) (err error) { 442 | defer func() { c.resetOnConnError(err) }() 443 | 444 | sftpClient, err := c.Init(ctx) 445 | if err != nil { 446 | return err 447 | } 448 | 449 | for _, pos := range a { 450 | filename, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) 451 | if err != nil { 452 | return fmt.Errorf("cannot determine wal segment path: %w", err) 453 | } 454 | 455 | if err := sftpClient.Remove(filename); err != nil && !os.IsNotExist(err) { 456 | return fmt.Errorf("cannot delete wal segment %q: %w", filename, err) 457 | } 458 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc() 459 | } 460 | 461 | return nil 462 | } 463 | 464 | // Cleanup deletes path & generations directories after empty. 465 | func (c *ReplicaClient) Cleanup(ctx context.Context) (err error) { 466 | defer func() { c.resetOnConnError(err) }() 467 | 468 | sftpClient, err := c.Init(ctx) 469 | if err != nil { 470 | return err 471 | } 472 | 473 | if err := sftpClient.RemoveDirectory(litestream.GenerationsPath(c.Path)); err != nil && !os.IsNotExist(err) { 474 | return fmt.Errorf("cannot delete generations path: %w", err) 475 | } else if err := sftpClient.RemoveDirectory(c.Path); err != nil && !os.IsNotExist(err) { 476 | return fmt.Errorf("cannot delete path: %w", err) 477 | } 478 | return nil 479 | } 480 | 481 | // resetOnConnError closes & clears the client if a connection error occurs. 482 | func (c *ReplicaClient) resetOnConnError(err error) { 483 | if !errors.Is(err, sftp.ErrSSHFxConnectionLost) { 484 | return 485 | } 486 | 487 | if c.sftpClient != nil { 488 | c.sftpClient.Close() 489 | c.sftpClient = nil 490 | } 491 | if c.sshClient != nil { 492 | c.sshClient.Close() 493 | c.sshClient = nil 494 | } 495 | } 496 | -------------------------------------------------------------------------------- /litestream.go: -------------------------------------------------------------------------------- 1 | package litestream 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/binary" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "os" 10 | "path" 11 | "path/filepath" 12 | "regexp" 13 | "strconv" 14 | "strings" 15 | "time" 16 | 17 | "github.com/mattn/go-sqlite3" 18 | ) 19 | 20 | // Naming constants. 21 | const ( 22 | MetaDirSuffix = "-litestream" 23 | 24 | WALDirName = "wal" 25 | WALExt = ".wal" 26 | WALSegmentExt = ".wal.lz4" 27 | SnapshotExt = ".snapshot.lz4" 28 | 29 | GenerationNameLen = 16 30 | ) 31 | 32 | // SQLite checkpoint modes. 33 | const ( 34 | CheckpointModePassive = "PASSIVE" 35 | CheckpointModeFull = "FULL" 36 | CheckpointModeRestart = "RESTART" 37 | CheckpointModeTruncate = "TRUNCATE" 38 | ) 39 | 40 | // Litestream errors. 41 | var ( 42 | ErrNoGeneration = errors.New("no generation available") 43 | ErrNoSnapshots = errors.New("no snapshots available") 44 | ErrChecksumMismatch = errors.New("invalid replica, checksum mismatch") 45 | ) 46 | 47 | var ( 48 | // LogWriter is the destination writer for all logging. 49 | LogWriter = os.Stdout 50 | 51 | // LogFlags are the flags passed to log.New(). 52 | LogFlags = 0 53 | ) 54 | 55 | func init() { 56 | sql.Register("litestream-sqlite3", &sqlite3.SQLiteDriver{ 57 | ConnectHook: func(conn *sqlite3.SQLiteConn) error { 58 | if err := conn.SetFileControlInt("main", sqlite3.SQLITE_FCNTL_PERSIST_WAL, 1); err != nil { 59 | return fmt.Errorf("cannot set file control: %w", err) 60 | } 61 | return nil 62 | }, 63 | }) 64 | } 65 | 66 | // SnapshotIterator represents an iterator over a collection of snapshot metadata. 67 | type SnapshotIterator interface { 68 | io.Closer 69 | 70 | // Prepares the the next snapshot for reading with the Snapshot() method. 71 | // Returns true if another snapshot is available. Returns false if no more 72 | // snapshots are available or if an error occured. 73 | Next() bool 74 | 75 | // Returns an error that occurred during iteration. 76 | Err() error 77 | 78 | // Returns metadata for the currently positioned snapshot. 79 | Snapshot() SnapshotInfo 80 | } 81 | 82 | // SliceSnapshotIterator returns all snapshots from an iterator as a slice. 83 | func SliceSnapshotIterator(itr SnapshotIterator) ([]SnapshotInfo, error) { 84 | var a []SnapshotInfo 85 | for itr.Next() { 86 | a = append(a, itr.Snapshot()) 87 | } 88 | return a, itr.Close() 89 | } 90 | 91 | var _ SnapshotIterator = (*SnapshotInfoSliceIterator)(nil) 92 | 93 | // SnapshotInfoSliceIterator represents an iterator for iterating over a slice of snapshots. 94 | type SnapshotInfoSliceIterator struct { 95 | init bool 96 | a []SnapshotInfo 97 | } 98 | 99 | // NewSnapshotInfoSliceIterator returns a new instance of SnapshotInfoSliceIterator. 100 | func NewSnapshotInfoSliceIterator(a []SnapshotInfo) *SnapshotInfoSliceIterator { 101 | return &SnapshotInfoSliceIterator{a: a} 102 | } 103 | 104 | // Close always returns nil. 105 | func (itr *SnapshotInfoSliceIterator) Close() error { return nil } 106 | 107 | // Next moves to the next snapshot. Returns true if another snapshot is available. 108 | func (itr *SnapshotInfoSliceIterator) Next() bool { 109 | if !itr.init { 110 | itr.init = true 111 | return len(itr.a) > 0 112 | } 113 | itr.a = itr.a[1:] 114 | return len(itr.a) > 0 115 | } 116 | 117 | // Err always returns nil. 118 | func (itr *SnapshotInfoSliceIterator) Err() error { return nil } 119 | 120 | // Snapshot returns the metadata from the currently positioned snapshot. 121 | func (itr *SnapshotInfoSliceIterator) Snapshot() SnapshotInfo { 122 | if len(itr.a) == 0 { 123 | return SnapshotInfo{} 124 | } 125 | return itr.a[0] 126 | } 127 | 128 | // WALSegmentIterator represents an iterator over a collection of WAL segments. 129 | type WALSegmentIterator interface { 130 | io.Closer 131 | 132 | // Prepares the the next WAL for reading with the WAL() method. 133 | // Returns true if another WAL is available. Returns false if no more 134 | // WAL files are available or if an error occured. 135 | Next() bool 136 | 137 | // Returns an error that occurred during iteration. 138 | Err() error 139 | 140 | // Returns metadata for the currently positioned WAL segment file. 141 | WALSegment() WALSegmentInfo 142 | } 143 | 144 | // SliceWALSegmentIterator returns all WAL segment files from an iterator as a slice. 145 | func SliceWALSegmentIterator(itr WALSegmentIterator) ([]WALSegmentInfo, error) { 146 | var a []WALSegmentInfo 147 | for itr.Next() { 148 | a = append(a, itr.WALSegment()) 149 | } 150 | return a, itr.Close() 151 | } 152 | 153 | var _ WALSegmentIterator = (*WALSegmentInfoSliceIterator)(nil) 154 | 155 | // WALSegmentInfoSliceIterator represents an iterator for iterating over a slice of wal segments. 156 | type WALSegmentInfoSliceIterator struct { 157 | init bool 158 | a []WALSegmentInfo 159 | } 160 | 161 | // NewWALSegmentInfoSliceIterator returns a new instance of WALSegmentInfoSliceIterator. 162 | func NewWALSegmentInfoSliceIterator(a []WALSegmentInfo) *WALSegmentInfoSliceIterator { 163 | return &WALSegmentInfoSliceIterator{a: a} 164 | } 165 | 166 | // Close always returns nil. 167 | func (itr *WALSegmentInfoSliceIterator) Close() error { return nil } 168 | 169 | // Next moves to the next wal segment. Returns true if another segment is available. 170 | func (itr *WALSegmentInfoSliceIterator) Next() bool { 171 | if !itr.init { 172 | itr.init = true 173 | return len(itr.a) > 0 174 | } 175 | itr.a = itr.a[1:] 176 | return len(itr.a) > 0 177 | } 178 | 179 | // Err always returns nil. 180 | func (itr *WALSegmentInfoSliceIterator) Err() error { return nil } 181 | 182 | // WALSegment returns the metadata from the currently positioned wal segment. 183 | func (itr *WALSegmentInfoSliceIterator) WALSegment() WALSegmentInfo { 184 | if len(itr.a) == 0 { 185 | return WALSegmentInfo{} 186 | } 187 | return itr.a[0] 188 | } 189 | 190 | // SnapshotInfo represents file information about a snapshot. 191 | type SnapshotInfo struct { 192 | Generation string 193 | Index int 194 | Size int64 195 | CreatedAt time.Time 196 | } 197 | 198 | // Pos returns the WAL position when the snapshot was made. 199 | func (info *SnapshotInfo) Pos() Pos { 200 | return Pos{Generation: info.Generation, Index: info.Index} 201 | } 202 | 203 | // SnapshotInfoSlice represents a slice of snapshot metadata. 204 | type SnapshotInfoSlice []SnapshotInfo 205 | 206 | func (a SnapshotInfoSlice) Len() int { return len(a) } 207 | 208 | func (a SnapshotInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 209 | 210 | func (a SnapshotInfoSlice) Less(i, j int) bool { 211 | if a[i].Generation != a[j].Generation { 212 | return a[i].Generation < a[j].Generation 213 | } 214 | return a[i].Index < a[j].Index 215 | } 216 | 217 | // FilterSnapshotsAfter returns all snapshots that were created on or after t. 218 | func FilterSnapshotsAfter(a []SnapshotInfo, t time.Time) []SnapshotInfo { 219 | other := make([]SnapshotInfo, 0, len(a)) 220 | for _, snapshot := range a { 221 | if !snapshot.CreatedAt.Before(t) { 222 | other = append(other, snapshot) 223 | } 224 | } 225 | return other 226 | } 227 | 228 | // FindMinSnapshotByGeneration finds the snapshot with the lowest index in a generation. 229 | func FindMinSnapshotByGeneration(a []SnapshotInfo, generation string) *SnapshotInfo { 230 | var min *SnapshotInfo 231 | for i := range a { 232 | snapshot := &a[i] 233 | 234 | if snapshot.Generation != generation { 235 | continue 236 | } else if min == nil || snapshot.Index < min.Index { 237 | min = snapshot 238 | } 239 | } 240 | return min 241 | } 242 | 243 | // WALInfo represents file information about a WAL file. 244 | type WALInfo struct { 245 | Generation string 246 | Index int 247 | CreatedAt time.Time 248 | } 249 | 250 | // WALInfoSlice represents a slice of WAL metadata. 251 | type WALInfoSlice []WALInfo 252 | 253 | func (a WALInfoSlice) Len() int { return len(a) } 254 | 255 | func (a WALInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 256 | 257 | func (a WALInfoSlice) Less(i, j int) bool { 258 | if a[i].Generation != a[j].Generation { 259 | return a[i].Generation < a[j].Generation 260 | } 261 | return a[i].Index < a[j].Index 262 | } 263 | 264 | // WALSegmentInfo represents file information about a WAL segment file. 265 | type WALSegmentInfo struct { 266 | Generation string 267 | Index int 268 | Offset int64 269 | Size int64 270 | CreatedAt time.Time 271 | } 272 | 273 | // Pos returns the WAL position when the segment was made. 274 | func (info *WALSegmentInfo) Pos() Pos { 275 | return Pos{Generation: info.Generation, Index: info.Index, Offset: info.Offset} 276 | } 277 | 278 | // WALSegmentInfoSlice represents a slice of WAL segment metadata. 279 | type WALSegmentInfoSlice []WALSegmentInfo 280 | 281 | func (a WALSegmentInfoSlice) Len() int { return len(a) } 282 | 283 | func (a WALSegmentInfoSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 284 | 285 | func (a WALSegmentInfoSlice) Less(i, j int) bool { 286 | if a[i].Generation != a[j].Generation { 287 | return a[i].Generation < a[j].Generation 288 | } else if a[i].Index != a[j].Index { 289 | return a[i].Index < a[j].Index 290 | } 291 | return a[i].Offset < a[j].Offset 292 | } 293 | 294 | // Pos is a position in the WAL for a generation. 295 | type Pos struct { 296 | Generation string // generation name 297 | Index int // wal file index 298 | Offset int64 // offset within wal file 299 | } 300 | 301 | // String returns a string representation. 302 | func (p Pos) String() string { 303 | if p.IsZero() { 304 | return "" 305 | } 306 | return fmt.Sprintf("%s/%08x:%d", p.Generation, p.Index, p.Offset) 307 | } 308 | 309 | // IsZero returns true if p is the zero value. 310 | func (p Pos) IsZero() bool { 311 | return p == (Pos{}) 312 | } 313 | 314 | // Truncate returns p with the offset truncated to zero. 315 | func (p Pos) Truncate() Pos { 316 | return Pos{Generation: p.Generation, Index: p.Index} 317 | } 318 | 319 | // Checksum computes a running SQLite checksum over a byte slice. 320 | func Checksum(bo binary.ByteOrder, s0, s1 uint32, b []byte) (uint32, uint32) { 321 | assert(len(b)%8 == 0, "misaligned checksum byte slice") 322 | 323 | // Iterate over 8-byte units and compute checksum. 324 | for i := 0; i < len(b); i += 8 { 325 | s0 += bo.Uint32(b[i:]) + s1 326 | s1 += bo.Uint32(b[i+4:]) + s0 327 | } 328 | return s0, s1 329 | } 330 | 331 | const ( 332 | // WALHeaderSize is the size of the WAL header, in bytes. 333 | WALHeaderSize = 32 334 | 335 | // WALFrameHeaderSize is the size of the WAL frame header, in bytes. 336 | WALFrameHeaderSize = 24 337 | ) 338 | 339 | // calcWALSize returns the size of the WAL, in bytes, for a given number of pages. 340 | func calcWALSize(pageSize int, n int) int64 { 341 | return int64(WALHeaderSize + ((WALFrameHeaderSize + pageSize) * n)) 342 | } 343 | 344 | // rollback rolls back tx. Ignores already-rolled-back errors. 345 | func rollback(tx *sql.Tx) error { 346 | if err := tx.Rollback(); err != nil && !strings.Contains(err.Error(), `transaction has already been committed or rolled back`) { 347 | return err 348 | } 349 | return nil 350 | } 351 | 352 | // readWALHeader returns the header read from a WAL file. 353 | func readWALHeader(filename string) ([]byte, error) { 354 | f, err := os.Open(filename) 355 | if err != nil { 356 | return nil, err 357 | } 358 | defer f.Close() 359 | 360 | buf := make([]byte, WALHeaderSize) 361 | n, err := io.ReadFull(f, buf) 362 | return buf[:n], err 363 | } 364 | 365 | // readWALFileAt reads a slice from a file. Do not use this with database files 366 | // as it causes problems with non-OFD locks. 367 | func readWALFileAt(filename string, offset, n int64) ([]byte, error) { 368 | f, err := os.Open(filename) 369 | if err != nil { 370 | return nil, err 371 | } 372 | defer f.Close() 373 | 374 | buf := make([]byte, n) 375 | if n, err := f.ReadAt(buf, offset); err != nil { 376 | return buf[:n], err 377 | } else if n < len(buf) { 378 | return buf[:n], io.ErrUnexpectedEOF 379 | } 380 | return buf, nil 381 | } 382 | 383 | // removeTmpFiles recursively finds and removes .tmp files. 384 | func removeTmpFiles(root string) error { 385 | return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { 386 | if err != nil { 387 | return nil // skip errored files 388 | } else if info.IsDir() { 389 | return nil // skip directories 390 | } else if !strings.HasSuffix(path, ".tmp") { 391 | return nil // skip non-temp files 392 | } 393 | return os.Remove(path) 394 | }) 395 | } 396 | 397 | // IsGenerationName returns true if s is the correct length and is only lowercase hex characters. 398 | func IsGenerationName(s string) bool { 399 | if len(s) != GenerationNameLen { 400 | return false 401 | } 402 | for _, ch := range s { 403 | if !isHexChar(ch) { 404 | return false 405 | } 406 | } 407 | return true 408 | } 409 | 410 | // GenerationsPath returns the path to a generation root directory. 411 | func GenerationsPath(root string) string { 412 | return path.Join(root, "generations") 413 | } 414 | 415 | // GenerationPath returns the path to a generation's root directory. 416 | func GenerationPath(root, generation string) (string, error) { 417 | dir := GenerationsPath(root) 418 | if generation == "" { 419 | return "", fmt.Errorf("generation required") 420 | } 421 | return path.Join(dir, generation), nil 422 | } 423 | 424 | // SnapshotsPath returns the path to a generation's snapshot directory. 425 | func SnapshotsPath(root, generation string) (string, error) { 426 | dir, err := GenerationPath(root, generation) 427 | if err != nil { 428 | return "", err 429 | } 430 | return path.Join(dir, "snapshots"), nil 431 | } 432 | 433 | // SnapshotPath returns the path to an uncompressed snapshot file. 434 | func SnapshotPath(root, generation string, index int) (string, error) { 435 | dir, err := SnapshotsPath(root, generation) 436 | if err != nil { 437 | return "", err 438 | } 439 | return path.Join(dir, FormatSnapshotPath(index)), nil 440 | } 441 | 442 | // WALPath returns the path to a generation's WAL directory 443 | func WALPath(root, generation string) (string, error) { 444 | dir, err := GenerationPath(root, generation) 445 | if err != nil { 446 | return "", err 447 | } 448 | return path.Join(dir, "wal"), nil 449 | } 450 | 451 | // WALSegmentPath returns the path to a WAL segment file. 452 | func WALSegmentPath(root, generation string, index int, offset int64) (string, error) { 453 | dir, err := WALPath(root, generation) 454 | if err != nil { 455 | return "", err 456 | } 457 | return path.Join(dir, FormatWALSegmentPath(index, offset)), nil 458 | } 459 | 460 | // IsSnapshotPath returns true if s is a path to a snapshot file. 461 | func IsSnapshotPath(s string) bool { 462 | return snapshotPathRegex.MatchString(s) 463 | } 464 | 465 | // ParseSnapshotPath returns the index for the snapshot. 466 | // Returns an error if the path is not a valid snapshot path. 467 | func ParseSnapshotPath(s string) (index int, err error) { 468 | s = filepath.Base(s) 469 | 470 | a := snapshotPathRegex.FindStringSubmatch(s) 471 | if a == nil { 472 | return 0, fmt.Errorf("invalid snapshot path: %s", s) 473 | } 474 | 475 | i64, _ := strconv.ParseUint(a[1], 16, 64) 476 | return int(i64), nil 477 | } 478 | 479 | // FormatSnapshotPath formats a snapshot filename with a given index. 480 | func FormatSnapshotPath(index int) string { 481 | assert(index >= 0, "snapshot index must be non-negative") 482 | return fmt.Sprintf("%08x%s", index, SnapshotExt) 483 | } 484 | 485 | var snapshotPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.snapshot\.lz4$`) 486 | 487 | // IsWALPath returns true if s is a path to a WAL file. 488 | func IsWALPath(s string) bool { 489 | return walPathRegex.MatchString(s) 490 | } 491 | 492 | // ParseWALPath returns the index for the WAL file. 493 | // Returns an error if the path is not a valid WAL path. 494 | func ParseWALPath(s string) (index int, err error) { 495 | s = filepath.Base(s) 496 | 497 | a := walPathRegex.FindStringSubmatch(s) 498 | if a == nil { 499 | return 0, fmt.Errorf("invalid wal path: %s", s) 500 | } 501 | 502 | i64, _ := strconv.ParseUint(a[1], 16, 64) 503 | return int(i64), nil 504 | } 505 | 506 | // FormatWALPath formats a WAL filename with a given index. 507 | func FormatWALPath(index int) string { 508 | assert(index >= 0, "wal index must be non-negative") 509 | return fmt.Sprintf("%08x%s", index, WALExt) 510 | } 511 | 512 | var walPathRegex = regexp.MustCompile(`^([0-9a-f]{8})\.wal$`) 513 | 514 | // ParseWALSegmentPath returns the index & offset for the WAL segment file. 515 | // Returns an error if the path is not a valid wal segment path. 516 | func ParseWALSegmentPath(s string) (index int, offset int64, err error) { 517 | s = filepath.Base(s) 518 | 519 | a := walSegmentPathRegex.FindStringSubmatch(s) 520 | if a == nil { 521 | return 0, 0, fmt.Errorf("invalid wal segment path: %s", s) 522 | } 523 | 524 | i64, _ := strconv.ParseUint(a[1], 16, 64) 525 | off64, _ := strconv.ParseUint(a[2], 16, 64) 526 | return int(i64), int64(off64), nil 527 | } 528 | 529 | // FormatWALSegmentPath formats a WAL segment filename with a given index & offset. 530 | func FormatWALSegmentPath(index int, offset int64) string { 531 | assert(index >= 0, "wal index must be non-negative") 532 | assert(offset >= 0, "wal offset must be non-negative") 533 | return fmt.Sprintf("%08x_%08x%s", index, offset, WALSegmentExt) 534 | } 535 | 536 | var walSegmentPathRegex = regexp.MustCompile(`^([0-9a-f]{8})(?:_([0-9a-f]{8}))\.wal\.lz4$`) 537 | 538 | // isHexChar returns true if ch is a lowercase hex character. 539 | func isHexChar(ch rune) bool { 540 | return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') 541 | } 542 | 543 | // Tracef is used for low-level tracing. 544 | var Tracef = func(format string, a ...interface{}) {} 545 | 546 | func assert(condition bool, message string) { 547 | if !condition { 548 | panic("assertion failed: " + message) 549 | } 550 | } 551 | -------------------------------------------------------------------------------- /abs/replica_client.go: -------------------------------------------------------------------------------- 1 | package abs 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/url" 8 | "os" 9 | "path" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | "github.com/Azure/azure-storage-blob-go/azblob" 15 | "github.com/benbjohnson/litestream" 16 | "github.com/benbjohnson/litestream/internal" 17 | "golang.org/x/sync/errgroup" 18 | ) 19 | 20 | // ReplicaClientType is the client type for this package. 21 | const ReplicaClientType = "abs" 22 | 23 | var _ litestream.ReplicaClient = (*ReplicaClient)(nil) 24 | 25 | // ReplicaClient is a client for writing snapshots & WAL segments to disk. 26 | type ReplicaClient struct { 27 | mu sync.Mutex 28 | containerURL *azblob.ContainerURL 29 | 30 | // Azure credentials 31 | AccountName string 32 | AccountKey string 33 | Endpoint string 34 | 35 | // Azure Blob Storage container information 36 | Bucket string 37 | Path string 38 | } 39 | 40 | // NewReplicaClient returns a new instance of ReplicaClient. 41 | func NewReplicaClient() *ReplicaClient { 42 | return &ReplicaClient{} 43 | } 44 | 45 | // Type returns "abs" as the client type. 46 | func (c *ReplicaClient) Type() string { 47 | return ReplicaClientType 48 | } 49 | 50 | // Init initializes the connection to Azure. No-op if already initialized. 51 | func (c *ReplicaClient) Init(ctx context.Context) (err error) { 52 | c.mu.Lock() 53 | defer c.mu.Unlock() 54 | 55 | if c.containerURL != nil { 56 | return nil 57 | } 58 | 59 | // Read account key from environment, if available. 60 | accountKey := c.AccountKey 61 | if accountKey == "" { 62 | accountKey = os.Getenv("LITESTREAM_AZURE_ACCOUNT_KEY") 63 | } 64 | 65 | // Authenticate to ACS. 66 | credential, err := azblob.NewSharedKeyCredential(c.AccountName, accountKey) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | // Construct & parse endpoint unless already set. 72 | endpoint := c.Endpoint 73 | if endpoint == "" { 74 | endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", c.AccountName) 75 | } 76 | endpointURL, err := url.Parse(endpoint) 77 | if err != nil { 78 | return fmt.Errorf("cannot parse azure endpoint: %w", err) 79 | } 80 | 81 | // Build pipeline and reference to container. 82 | pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{ 83 | Retry: azblob.RetryOptions{ 84 | TryTimeout: 24 * time.Hour, 85 | }, 86 | }) 87 | containerURL := azblob.NewServiceURL(*endpointURL, pipeline).NewContainerURL(c.Bucket) 88 | c.containerURL = &containerURL 89 | 90 | return nil 91 | } 92 | 93 | // Generations returns a list of available generation names. 94 | func (c *ReplicaClient) Generations(ctx context.Context) ([]string, error) { 95 | if err := c.Init(ctx); err != nil { 96 | return nil, err 97 | } 98 | 99 | var generations []string 100 | var marker azblob.Marker 101 | for marker.NotDone() { 102 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc() 103 | 104 | resp, err := c.containerURL.ListBlobsHierarchySegment(ctx, marker, "/", azblob.ListBlobsSegmentOptions{ 105 | Prefix: litestream.GenerationsPath(c.Path) + "/", 106 | }) 107 | if err != nil { 108 | return nil, err 109 | } 110 | marker = resp.NextMarker 111 | 112 | for _, prefix := range resp.Segment.BlobPrefixes { 113 | name := path.Base(strings.TrimSuffix(prefix.Name, "/")) 114 | if !litestream.IsGenerationName(name) { 115 | continue 116 | } 117 | generations = append(generations, name) 118 | } 119 | } 120 | 121 | return generations, nil 122 | } 123 | 124 | // DeleteGeneration deletes all snapshots & WAL segments within a generation. 125 | func (c *ReplicaClient) DeleteGeneration(ctx context.Context, generation string) error { 126 | if err := c.Init(ctx); err != nil { 127 | return err 128 | } 129 | 130 | dir, err := litestream.GenerationPath(c.Path, generation) 131 | if err != nil { 132 | return fmt.Errorf("cannot determine generation path: %w", err) 133 | } 134 | 135 | var marker azblob.Marker 136 | for marker.NotDone() { 137 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc() 138 | 139 | resp, err := c.containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: dir + "/"}) 140 | if err != nil { 141 | return err 142 | } 143 | marker = resp.NextMarker 144 | 145 | for _, item := range resp.Segment.BlobItems { 146 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc() 147 | 148 | blobURL := c.containerURL.NewBlobURL(item.Name) 149 | if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) { 150 | continue 151 | } else if err != nil { 152 | return err 153 | } 154 | } 155 | } 156 | 157 | // log.Printf("%s(%s): retainer: deleting generation: %s", r.db.Path(), r.Name(), generation) 158 | 159 | return nil 160 | } 161 | 162 | // Snapshots returns an iterator over all available snapshots for a generation. 163 | func (c *ReplicaClient) Snapshots(ctx context.Context, generation string) (litestream.SnapshotIterator, error) { 164 | if err := c.Init(ctx); err != nil { 165 | return nil, err 166 | } 167 | return newSnapshotIterator(ctx, generation, c), nil 168 | } 169 | 170 | // WriteSnapshot writes LZ4 compressed data from rd to the object storage. 171 | func (c *ReplicaClient) WriteSnapshot(ctx context.Context, generation string, index int, rd io.Reader) (info litestream.SnapshotInfo, err error) { 172 | if err := c.Init(ctx); err != nil { 173 | return info, err 174 | } 175 | 176 | key, err := litestream.SnapshotPath(c.Path, generation, index) 177 | if err != nil { 178 | return info, fmt.Errorf("cannot determine snapshot path: %w", err) 179 | } 180 | startTime := time.Now() 181 | 182 | rc := internal.NewReadCounter(rd) 183 | 184 | blobURL := c.containerURL.NewBlockBlobURL(key) 185 | if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{ 186 | BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"}, 187 | BlobAccessTier: azblob.DefaultAccessTier, 188 | }); err != nil { 189 | return info, err 190 | } 191 | 192 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc() 193 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N())) 194 | 195 | // log.Printf("%s(%s): snapshot: creating %s/%08x t=%s", r.db.Path(), r.Name(), generation, index, time.Since(startTime).Truncate(time.Millisecond)) 196 | 197 | return litestream.SnapshotInfo{ 198 | Generation: generation, 199 | Index: index, 200 | Size: rc.N(), 201 | CreatedAt: startTime.UTC(), 202 | }, nil 203 | } 204 | 205 | // SnapshotReader returns a reader for snapshot data at the given generation/index. 206 | func (c *ReplicaClient) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) { 207 | if err := c.Init(ctx); err != nil { 208 | return nil, err 209 | } 210 | 211 | key, err := litestream.SnapshotPath(c.Path, generation, index) 212 | if err != nil { 213 | return nil, fmt.Errorf("cannot determine snapshot path: %w", err) 214 | } 215 | 216 | blobURL := c.containerURL.NewBlobURL(key) 217 | resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) 218 | if isNotExists(err) { 219 | return nil, os.ErrNotExist 220 | } else if err != nil { 221 | return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err) 222 | } 223 | 224 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc() 225 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength())) 226 | 227 | return resp.Body(azblob.RetryReaderOptions{}), nil 228 | } 229 | 230 | // DeleteSnapshot deletes a snapshot with the given generation & index. 231 | func (c *ReplicaClient) DeleteSnapshot(ctx context.Context, generation string, index int) error { 232 | if err := c.Init(ctx); err != nil { 233 | return err 234 | } 235 | 236 | key, err := litestream.SnapshotPath(c.Path, generation, index) 237 | if err != nil { 238 | return fmt.Errorf("cannot determine snapshot path: %w", err) 239 | } 240 | 241 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc() 242 | 243 | blobURL := c.containerURL.NewBlobURL(key) 244 | if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) { 245 | return nil 246 | } else if err != nil { 247 | return fmt.Errorf("cannot delete snapshot %q: %w", key, err) 248 | } 249 | return nil 250 | } 251 | 252 | // WALSegments returns an iterator over all available WAL files for a generation. 253 | func (c *ReplicaClient) WALSegments(ctx context.Context, generation string) (litestream.WALSegmentIterator, error) { 254 | if err := c.Init(ctx); err != nil { 255 | return nil, err 256 | } 257 | return newWALSegmentIterator(ctx, generation, c), nil 258 | } 259 | 260 | // WriteWALSegment writes LZ4 compressed data from rd into a file on disk. 261 | func (c *ReplicaClient) WriteWALSegment(ctx context.Context, pos litestream.Pos, rd io.Reader) (info litestream.WALSegmentInfo, err error) { 262 | if err := c.Init(ctx); err != nil { 263 | return info, err 264 | } 265 | 266 | key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) 267 | if err != nil { 268 | return info, fmt.Errorf("cannot determine wal segment path: %w", err) 269 | } 270 | startTime := time.Now() 271 | 272 | rc := internal.NewReadCounter(rd) 273 | 274 | blobURL := c.containerURL.NewBlockBlobURL(key) 275 | if _, err := azblob.UploadStreamToBlockBlob(ctx, rc, blobURL, azblob.UploadStreamToBlockBlobOptions{ 276 | BlobHTTPHeaders: azblob.BlobHTTPHeaders{ContentType: "application/octet-stream"}, 277 | BlobAccessTier: azblob.DefaultAccessTier, 278 | }); err != nil { 279 | return info, err 280 | } 281 | 282 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc() 283 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(rc.N())) 284 | 285 | return litestream.WALSegmentInfo{ 286 | Generation: pos.Generation, 287 | Index: pos.Index, 288 | Offset: pos.Offset, 289 | Size: rc.N(), 290 | CreatedAt: startTime.UTC(), 291 | }, nil 292 | } 293 | 294 | // WALSegmentReader returns a reader for a section of WAL data at the given index. 295 | // Returns os.ErrNotExist if no matching index/offset is found. 296 | func (c *ReplicaClient) WALSegmentReader(ctx context.Context, pos litestream.Pos) (io.ReadCloser, error) { 297 | if err := c.Init(ctx); err != nil { 298 | return nil, err 299 | } 300 | 301 | key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) 302 | if err != nil { 303 | return nil, fmt.Errorf("cannot determine wal segment path: %w", err) 304 | } 305 | 306 | blobURL := c.containerURL.NewBlobURL(key) 307 | resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) 308 | if isNotExists(err) { 309 | return nil, os.ErrNotExist 310 | } else if err != nil { 311 | return nil, fmt.Errorf("cannot start new reader for %q: %w", key, err) 312 | } 313 | 314 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc() 315 | internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(resp.ContentLength())) 316 | 317 | return resp.Body(azblob.RetryReaderOptions{}), nil 318 | } 319 | 320 | // DeleteWALSegments deletes WAL segments with at the given positions. 321 | func (c *ReplicaClient) DeleteWALSegments(ctx context.Context, a []litestream.Pos) error { 322 | if err := c.Init(ctx); err != nil { 323 | return err 324 | } 325 | 326 | for _, pos := range a { 327 | key, err := litestream.WALSegmentPath(c.Path, pos.Generation, pos.Index, pos.Offset) 328 | if err != nil { 329 | return fmt.Errorf("cannot determine wal segment path: %w", err) 330 | } 331 | 332 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "DELETE").Inc() 333 | 334 | blobURL := c.containerURL.NewBlobURL(key) 335 | if _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}); isNotExists(err) { 336 | continue 337 | } else if err != nil { 338 | return fmt.Errorf("cannot delete wal segment %q: %w", key, err) 339 | } 340 | } 341 | 342 | return nil 343 | } 344 | 345 | type snapshotIterator struct { 346 | client *ReplicaClient 347 | generation string 348 | 349 | ch chan litestream.SnapshotInfo 350 | g errgroup.Group 351 | ctx context.Context 352 | cancel func() 353 | 354 | info litestream.SnapshotInfo 355 | err error 356 | } 357 | 358 | func newSnapshotIterator(ctx context.Context, generation string, client *ReplicaClient) *snapshotIterator { 359 | itr := &snapshotIterator{ 360 | client: client, 361 | generation: generation, 362 | ch: make(chan litestream.SnapshotInfo), 363 | } 364 | 365 | itr.ctx, itr.cancel = context.WithCancel(ctx) 366 | itr.g.Go(itr.fetch) 367 | 368 | return itr 369 | } 370 | 371 | // fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel. 372 | func (itr *snapshotIterator) fetch() error { 373 | defer close(itr.ch) 374 | 375 | dir, err := litestream.SnapshotsPath(itr.client.Path, itr.generation) 376 | if err != nil { 377 | return fmt.Errorf("cannot determine snapshots path: %w", err) 378 | } 379 | 380 | var marker azblob.Marker 381 | for marker.NotDone() { 382 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc() 383 | 384 | resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: dir + "/"}) 385 | if err != nil { 386 | return err 387 | } 388 | marker = resp.NextMarker 389 | 390 | for _, item := range resp.Segment.BlobItems { 391 | key := path.Base(item.Name) 392 | index, err := litestream.ParseSnapshotPath(key) 393 | if err != nil { 394 | continue 395 | } 396 | 397 | info := litestream.SnapshotInfo{ 398 | Generation: itr.generation, 399 | Index: index, 400 | Size: *item.Properties.ContentLength, 401 | CreatedAt: item.Properties.CreationTime.UTC(), 402 | } 403 | 404 | select { 405 | case <-itr.ctx.Done(): 406 | case itr.ch <- info: 407 | } 408 | } 409 | } 410 | return nil 411 | } 412 | 413 | func (itr *snapshotIterator) Close() (err error) { 414 | err = itr.err 415 | 416 | // Cancel context and wait for error group to finish. 417 | itr.cancel() 418 | if e := itr.g.Wait(); e != nil && err == nil { 419 | err = e 420 | } 421 | 422 | return err 423 | } 424 | 425 | func (itr *snapshotIterator) Next() bool { 426 | // Exit if an error has already occurred. 427 | if itr.err != nil { 428 | return false 429 | } 430 | 431 | // Return false if context was canceled or if there are no more snapshots. 432 | // Otherwise fetch the next snapshot and store it on the iterator. 433 | select { 434 | case <-itr.ctx.Done(): 435 | return false 436 | case info, ok := <-itr.ch: 437 | if !ok { 438 | return false 439 | } 440 | itr.info = info 441 | return true 442 | } 443 | } 444 | 445 | func (itr *snapshotIterator) Err() error { return itr.err } 446 | 447 | func (itr *snapshotIterator) Snapshot() litestream.SnapshotInfo { 448 | return itr.info 449 | } 450 | 451 | type walSegmentIterator struct { 452 | client *ReplicaClient 453 | generation string 454 | 455 | ch chan litestream.WALSegmentInfo 456 | g errgroup.Group 457 | ctx context.Context 458 | cancel func() 459 | 460 | info litestream.WALSegmentInfo 461 | err error 462 | } 463 | 464 | func newWALSegmentIterator(ctx context.Context, generation string, client *ReplicaClient) *walSegmentIterator { 465 | itr := &walSegmentIterator{ 466 | client: client, 467 | generation: generation, 468 | ch: make(chan litestream.WALSegmentInfo), 469 | } 470 | 471 | itr.ctx, itr.cancel = context.WithCancel(ctx) 472 | itr.g.Go(itr.fetch) 473 | 474 | return itr 475 | } 476 | 477 | // fetch runs in a separate goroutine to fetch pages of objects and stream them to a channel. 478 | func (itr *walSegmentIterator) fetch() error { 479 | defer close(itr.ch) 480 | 481 | dir, err := litestream.WALPath(itr.client.Path, itr.generation) 482 | if err != nil { 483 | return fmt.Errorf("cannot determine wal path: %w", err) 484 | } 485 | 486 | var marker azblob.Marker 487 | for marker.NotDone() { 488 | internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "LIST").Inc() 489 | 490 | resp, err := itr.client.containerURL.ListBlobsFlatSegment(itr.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: dir + "/"}) 491 | if err != nil { 492 | return err 493 | } 494 | marker = resp.NextMarker 495 | 496 | for _, item := range resp.Segment.BlobItems { 497 | key := path.Base(item.Name) 498 | index, offset, err := litestream.ParseWALSegmentPath(key) 499 | if err != nil { 500 | continue 501 | } 502 | 503 | info := litestream.WALSegmentInfo{ 504 | Generation: itr.generation, 505 | Index: index, 506 | Offset: offset, 507 | Size: *item.Properties.ContentLength, 508 | CreatedAt: item.Properties.CreationTime.UTC(), 509 | } 510 | 511 | select { 512 | case <-itr.ctx.Done(): 513 | case itr.ch <- info: 514 | } 515 | } 516 | } 517 | return nil 518 | } 519 | 520 | func (itr *walSegmentIterator) Close() (err error) { 521 | err = itr.err 522 | 523 | // Cancel context and wait for error group to finish. 524 | itr.cancel() 525 | if e := itr.g.Wait(); e != nil && err == nil { 526 | err = e 527 | } 528 | 529 | return err 530 | } 531 | 532 | func (itr *walSegmentIterator) Next() bool { 533 | // Exit if an error has already occurred. 534 | if itr.err != nil { 535 | return false 536 | } 537 | 538 | // Return false if context was canceled or if there are no more segments. 539 | // Otherwise fetch the next segment and store it on the iterator. 540 | select { 541 | case <-itr.ctx.Done(): 542 | return false 543 | case info, ok := <-itr.ch: 544 | if !ok { 545 | return false 546 | } 547 | itr.info = info 548 | return true 549 | } 550 | } 551 | 552 | func (itr *walSegmentIterator) Err() error { return itr.err } 553 | 554 | func (itr *walSegmentIterator) WALSegment() litestream.WALSegmentInfo { 555 | return itr.info 556 | } 557 | 558 | func isNotExists(err error) bool { 559 | switch err := err.(type) { 560 | case azblob.StorageError: 561 | return err.ServiceCode() == azblob.ServiceCodeBlobNotFound 562 | default: 563 | return false 564 | } 565 | } 566 | -------------------------------------------------------------------------------- /db_test.go: -------------------------------------------------------------------------------- 1 | package litestream_test 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "testing" 11 | "time" 12 | 13 | "github.com/benbjohnson/litestream" 14 | ) 15 | 16 | func TestDB_Path(t *testing.T) { 17 | db := litestream.NewDB("/tmp/db") 18 | if got, want := db.Path(), `/tmp/db`; got != want { 19 | t.Fatalf("Path()=%v, want %v", got, want) 20 | } 21 | } 22 | 23 | func TestDB_WALPath(t *testing.T) { 24 | db := litestream.NewDB("/tmp/db") 25 | if got, want := db.WALPath(), `/tmp/db-wal`; got != want { 26 | t.Fatalf("WALPath()=%v, want %v", got, want) 27 | } 28 | } 29 | 30 | func TestDB_MetaPath(t *testing.T) { 31 | t.Run("Absolute", func(t *testing.T) { 32 | db := litestream.NewDB("/tmp/db") 33 | if got, want := db.MetaPath(), `/tmp/.db-litestream`; got != want { 34 | t.Fatalf("MetaPath()=%v, want %v", got, want) 35 | } 36 | }) 37 | t.Run("Relative", func(t *testing.T) { 38 | db := litestream.NewDB("db") 39 | if got, want := db.MetaPath(), `.db-litestream`; got != want { 40 | t.Fatalf("MetaPath()=%v, want %v", got, want) 41 | } 42 | }) 43 | } 44 | 45 | func TestDB_GenerationNamePath(t *testing.T) { 46 | db := litestream.NewDB("/tmp/db") 47 | if got, want := db.GenerationNamePath(), `/tmp/.db-litestream/generation`; got != want { 48 | t.Fatalf("GenerationNamePath()=%v, want %v", got, want) 49 | } 50 | } 51 | 52 | func TestDB_GenerationPath(t *testing.T) { 53 | db := litestream.NewDB("/tmp/db") 54 | if got, want := db.GenerationPath("xxxx"), `/tmp/.db-litestream/generations/xxxx`; got != want { 55 | t.Fatalf("GenerationPath()=%v, want %v", got, want) 56 | } 57 | } 58 | 59 | func TestDB_ShadowWALDir(t *testing.T) { 60 | db := litestream.NewDB("/tmp/db") 61 | if got, want := db.ShadowWALDir("xxxx"), `/tmp/.db-litestream/generations/xxxx/wal`; got != want { 62 | t.Fatalf("ShadowWALDir()=%v, want %v", got, want) 63 | } 64 | } 65 | 66 | func TestDB_ShadowWALPath(t *testing.T) { 67 | db := litestream.NewDB("/tmp/db") 68 | if got, want := db.ShadowWALPath("xxxx", 1000), `/tmp/.db-litestream/generations/xxxx/wal/000003e8.wal`; got != want { 69 | t.Fatalf("ShadowWALPath()=%v, want %v", got, want) 70 | } 71 | } 72 | 73 | // Ensure we can check the last modified time of the real database and its WAL. 74 | func TestDB_UpdatedAt(t *testing.T) { 75 | t.Run("ErrNotExist", func(t *testing.T) { 76 | db := MustOpenDB(t) 77 | defer MustCloseDB(t, db) 78 | if _, err := db.UpdatedAt(); !os.IsNotExist(err) { 79 | t.Fatalf("unexpected error: %#v", err) 80 | } 81 | }) 82 | 83 | t.Run("DB", func(t *testing.T) { 84 | db, sqldb := MustOpenDBs(t) 85 | defer MustCloseDBs(t, db, sqldb) 86 | 87 | if t0, err := db.UpdatedAt(); err != nil { 88 | t.Fatal(err) 89 | } else if time.Since(t0) > 10*time.Second { 90 | t.Fatalf("unexpected updated at time: %s", t0) 91 | } 92 | }) 93 | 94 | t.Run("WAL", func(t *testing.T) { 95 | db, sqldb := MustOpenDBs(t) 96 | defer MustCloseDBs(t, db, sqldb) 97 | 98 | t0, err := db.UpdatedAt() 99 | if err != nil { 100 | t.Fatal(err) 101 | } 102 | 103 | sleepTime := 100 * time.Millisecond 104 | if os.Getenv("CI") != "" { 105 | sleepTime = 1 * time.Second 106 | } 107 | time.Sleep(sleepTime) 108 | 109 | if _, err := sqldb.Exec(`CREATE TABLE t (id INT);`); err != nil { 110 | t.Fatal(err) 111 | } 112 | 113 | if t1, err := db.UpdatedAt(); err != nil { 114 | t.Fatal(err) 115 | } else if !t1.After(t0) { 116 | t.Fatalf("expected newer updated at time: %s > %s", t1, t0) 117 | } 118 | }) 119 | } 120 | 121 | // Ensure we can compute a checksum on the real database. 122 | func TestDB_CRC64(t *testing.T) { 123 | t.Run("ErrNotExist", func(t *testing.T) { 124 | db := MustOpenDB(t) 125 | defer MustCloseDB(t, db) 126 | if _, _, err := db.CRC64(context.Background()); !os.IsNotExist(err) { 127 | t.Fatalf("unexpected error: %#v", err) 128 | } 129 | }) 130 | 131 | t.Run("DB", func(t *testing.T) { 132 | db, sqldb := MustOpenDBs(t) 133 | defer MustCloseDBs(t, db, sqldb) 134 | 135 | if err := db.Sync(context.Background()); err != nil { 136 | t.Fatal(err) 137 | } 138 | 139 | chksum0, _, err := db.CRC64(context.Background()) 140 | if err != nil { 141 | t.Fatal(err) 142 | } 143 | 144 | // Issue change that is applied to the WAL. Checksum should not change. 145 | if _, err := sqldb.Exec(`CREATE TABLE t (id INT);`); err != nil { 146 | t.Fatal(err) 147 | } else if chksum1, _, err := db.CRC64(context.Background()); err != nil { 148 | t.Fatal(err) 149 | } else if chksum0 == chksum1 { 150 | t.Fatal("expected different checksum event after WAL change") 151 | } 152 | 153 | // Checkpoint change into database. Checksum should change. 154 | if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil { 155 | t.Fatal(err) 156 | } 157 | 158 | if chksum2, _, err := db.CRC64(context.Background()); err != nil { 159 | t.Fatal(err) 160 | } else if chksum0 == chksum2 { 161 | t.Fatal("expected different checksums after checkpoint") 162 | } 163 | }) 164 | } 165 | 166 | // Ensure we can sync the real WAL to the shadow WAL. 167 | func TestDB_Sync(t *testing.T) { 168 | // Ensure sync is skipped if no database exists. 169 | t.Run("NoDB", func(t *testing.T) { 170 | db := MustOpenDB(t) 171 | defer MustCloseDB(t, db) 172 | if err := db.Sync(context.Background()); err != nil { 173 | t.Fatal(err) 174 | } 175 | }) 176 | 177 | // Ensure sync can successfully run on the initial sync. 178 | t.Run("Initial", func(t *testing.T) { 179 | db, sqldb := MustOpenDBs(t) 180 | defer MustCloseDBs(t, db, sqldb) 181 | 182 | if err := db.Sync(context.Background()); err != nil { 183 | t.Fatal(err) 184 | } 185 | 186 | // Verify page size if now available. 187 | if db.PageSize() == 0 { 188 | t.Fatal("expected page size after initial sync") 189 | } 190 | 191 | // Obtain real WAL size. 192 | fi, err := os.Stat(db.WALPath()) 193 | if err != nil { 194 | t.Fatal(err) 195 | } 196 | 197 | // Ensure position now available. 198 | if pos, err := db.Pos(); err != nil { 199 | t.Fatal(err) 200 | } else if pos.Generation == "" { 201 | t.Fatal("expected generation") 202 | } else if got, want := pos.Index, 0; got != want { 203 | t.Fatalf("pos.Index=%v, want %v", got, want) 204 | } else if got, want := pos.Offset, fi.Size(); got != want { 205 | t.Fatalf("pos.Offset=%v, want %v", got, want) 206 | } 207 | }) 208 | 209 | // Ensure DB can keep in sync across multiple Sync() invocations. 210 | t.Run("MultiSync", func(t *testing.T) { 211 | db, sqldb := MustOpenDBs(t) 212 | defer MustCloseDBs(t, db, sqldb) 213 | 214 | // Execute a query to force a write to the WAL. 215 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 216 | t.Fatal(err) 217 | } 218 | 219 | // Perform initial sync & grab initial position. 220 | if err := db.Sync(context.Background()); err != nil { 221 | t.Fatal(err) 222 | } 223 | 224 | pos0, err := db.Pos() 225 | if err != nil { 226 | t.Fatal(err) 227 | } 228 | 229 | // Insert into table. 230 | if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz');`); err != nil { 231 | t.Fatal(err) 232 | } 233 | 234 | // Sync to ensure position moves forward one page. 235 | if err := db.Sync(context.Background()); err != nil { 236 | t.Fatal(err) 237 | } else if pos1, err := db.Pos(); err != nil { 238 | t.Fatal(err) 239 | } else if pos0.Generation != pos1.Generation { 240 | t.Fatal("expected the same generation") 241 | } else if got, want := pos1.Index, pos0.Index; got != want { 242 | t.Fatalf("Index=%v, want %v", got, want) 243 | } else if got, want := pos1.Offset, pos0.Offset+4096+litestream.WALFrameHeaderSize; got != want { 244 | t.Fatalf("Offset=%v, want %v", got, want) 245 | } 246 | }) 247 | 248 | // Ensure a WAL file is created if one does not already exist. 249 | t.Run("NoWAL", func(t *testing.T) { 250 | db, sqldb := MustOpenDBs(t) 251 | defer MustCloseDBs(t, db, sqldb) 252 | 253 | // Issue initial sync and truncate WAL. 254 | if err := db.Sync(context.Background()); err != nil { 255 | t.Fatal(err) 256 | } 257 | 258 | // Obtain initial position. 259 | pos0, err := db.Pos() 260 | if err != nil { 261 | t.Fatal(err) 262 | } 263 | 264 | // Checkpoint & fully close which should close WAL file. 265 | if err := db.Checkpoint(context.Background(), litestream.CheckpointModeTruncate); err != nil { 266 | t.Fatal(err) 267 | } else if err := db.Close(); err != nil { 268 | t.Fatal(err) 269 | } else if err := sqldb.Close(); err != nil { 270 | t.Fatal(err) 271 | } 272 | 273 | // Remove WAL file. 274 | if err := os.Remove(db.WALPath()); err != nil { 275 | t.Fatal(err) 276 | } 277 | 278 | // Reopen the managed database. 279 | db = MustOpenDBAt(t, db.Path()) 280 | defer MustCloseDB(t, db) 281 | 282 | // Re-sync and ensure new generation has been created. 283 | if err := db.Sync(context.Background()); err != nil { 284 | t.Fatal(err) 285 | } 286 | 287 | // Obtain initial position. 288 | if pos1, err := db.Pos(); err != nil { 289 | t.Fatal(err) 290 | } else if pos0.Generation == pos1.Generation { 291 | t.Fatal("expected new generation after truncation") 292 | } 293 | }) 294 | 295 | // Ensure DB can start new generation if it detects it cannot verify last position. 296 | t.Run("OverwritePrevPosition", func(t *testing.T) { 297 | db, sqldb := MustOpenDBs(t) 298 | defer MustCloseDBs(t, db, sqldb) 299 | 300 | // Execute a query to force a write to the WAL. 301 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 302 | t.Fatal(err) 303 | } 304 | 305 | // Issue initial sync and truncate WAL. 306 | if err := db.Sync(context.Background()); err != nil { 307 | t.Fatal(err) 308 | } 309 | 310 | // Obtain initial position. 311 | pos0, err := db.Pos() 312 | if err != nil { 313 | t.Fatal(err) 314 | } 315 | 316 | // Fully close which should close WAL file. 317 | if err := db.Close(); err != nil { 318 | t.Fatal(err) 319 | } else if err := sqldb.Close(); err != nil { 320 | t.Fatal(err) 321 | } 322 | 323 | // Verify WAL does not exist. 324 | if _, err := os.Stat(db.WALPath()); !os.IsNotExist(err) { 325 | t.Fatal(err) 326 | } 327 | 328 | // Insert into table multiple times to move past old offset 329 | sqldb = MustOpenSQLDB(t, db.Path()) 330 | defer MustCloseSQLDB(t, sqldb) 331 | for i := 0; i < 100; i++ { 332 | if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz');`); err != nil { 333 | t.Fatal(err) 334 | } 335 | } 336 | 337 | // Reopen the managed database. 338 | db = MustOpenDBAt(t, db.Path()) 339 | defer MustCloseDB(t, db) 340 | 341 | // Re-sync and ensure new generation has been created. 342 | if err := db.Sync(context.Background()); err != nil { 343 | t.Fatal(err) 344 | } 345 | 346 | // Obtain initial position. 347 | if pos1, err := db.Pos(); err != nil { 348 | t.Fatal(err) 349 | } else if pos0.Generation == pos1.Generation { 350 | t.Fatal("expected new generation after truncation") 351 | } 352 | }) 353 | 354 | // Ensure DB can handle a mismatched header-only and start new generation. 355 | t.Run("WALHeaderMismatch", func(t *testing.T) { 356 | db, sqldb := MustOpenDBs(t) 357 | defer MustCloseDBs(t, db, sqldb) 358 | 359 | // Execute a query to force a write to the WAL and then sync. 360 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 361 | t.Fatal(err) 362 | } else if err := db.Sync(context.Background()); err != nil { 363 | t.Fatal(err) 364 | } 365 | 366 | // Grab initial position & close. 367 | pos0, err := db.Pos() 368 | if err != nil { 369 | t.Fatal(err) 370 | } else if err := db.Close(); err != nil { 371 | t.Fatal(err) 372 | } 373 | 374 | // Read existing file, update header checksum, and write back only header 375 | // to simulate a header with a mismatched checksum. 376 | shadowWALPath := db.ShadowWALPath(pos0.Generation, pos0.Index) 377 | if buf, err := ioutil.ReadFile(shadowWALPath); err != nil { 378 | t.Fatal(err) 379 | } else if err := ioutil.WriteFile(shadowWALPath, append(buf[:litestream.WALHeaderSize-8], 0, 0, 0, 0, 0, 0, 0, 0), 0600); err != nil { 380 | t.Fatal(err) 381 | } 382 | 383 | // Reopen managed database & ensure sync will still work. 384 | db = MustOpenDBAt(t, db.Path()) 385 | defer MustCloseDB(t, db) 386 | if err := db.Sync(context.Background()); err != nil { 387 | t.Fatal(err) 388 | } 389 | 390 | // Verify a new generation was started. 391 | if pos1, err := db.Pos(); err != nil { 392 | t.Fatal(err) 393 | } else if pos0.Generation == pos1.Generation { 394 | t.Fatal("expected new generation") 395 | } 396 | }) 397 | 398 | // Ensure DB can handle partial shadow WAL header write. 399 | t.Run("PartialShadowWALHeader", func(t *testing.T) { 400 | db, sqldb := MustOpenDBs(t) 401 | defer MustCloseDBs(t, db, sqldb) 402 | 403 | // Execute a query to force a write to the WAL and then sync. 404 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 405 | t.Fatal(err) 406 | } else if err := db.Sync(context.Background()); err != nil { 407 | t.Fatal(err) 408 | } 409 | 410 | pos0, err := db.Pos() 411 | if err != nil { 412 | t.Fatal(err) 413 | } 414 | 415 | // Close & truncate shadow WAL to simulate a partial header write. 416 | if err := db.Close(); err != nil { 417 | t.Fatal(err) 418 | } else if err := os.Truncate(db.ShadowWALPath(pos0.Generation, pos0.Index), litestream.WALHeaderSize-1); err != nil { 419 | t.Fatal(err) 420 | } 421 | 422 | // Reopen managed database & ensure sync will still work. 423 | db = MustOpenDBAt(t, db.Path()) 424 | defer MustCloseDB(t, db) 425 | if err := db.Sync(context.Background()); err != nil { 426 | t.Fatal(err) 427 | } 428 | 429 | // Verify a new generation was started. 430 | if pos1, err := db.Pos(); err != nil { 431 | t.Fatal(err) 432 | } else if pos0.Generation == pos1.Generation { 433 | t.Fatal("expected new generation") 434 | } 435 | }) 436 | 437 | // Ensure DB can handle partial shadow WAL writes. 438 | t.Run("PartialShadowWALFrame", func(t *testing.T) { 439 | db, sqldb := MustOpenDBs(t) 440 | defer MustCloseDBs(t, db, sqldb) 441 | 442 | // Execute a query to force a write to the WAL and then sync. 443 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 444 | t.Fatal(err) 445 | } else if err := db.Sync(context.Background()); err != nil { 446 | t.Fatal(err) 447 | } 448 | 449 | pos0, err := db.Pos() 450 | if err != nil { 451 | t.Fatal(err) 452 | } 453 | 454 | // Obtain current shadow WAL size. 455 | fi, err := os.Stat(db.ShadowWALPath(pos0.Generation, pos0.Index)) 456 | if err != nil { 457 | t.Fatal(err) 458 | } 459 | 460 | // Close & truncate shadow WAL to simulate a partial frame write. 461 | if err := db.Close(); err != nil { 462 | t.Fatal(err) 463 | } else if err := os.Truncate(db.ShadowWALPath(pos0.Generation, pos0.Index), fi.Size()-1); err != nil { 464 | t.Fatal(err) 465 | } 466 | 467 | // Reopen managed database & ensure sync will still work. 468 | db = MustOpenDBAt(t, db.Path()) 469 | defer MustCloseDB(t, db) 470 | if err := db.Sync(context.Background()); err != nil { 471 | t.Fatal(err) 472 | } 473 | 474 | // Verify same generation is kept. 475 | if pos1, err := db.Pos(); err != nil { 476 | t.Fatal(err) 477 | } else if got, want := pos1, pos0; got != want { 478 | t.Fatalf("Pos()=%s want %s", got, want) 479 | } 480 | 481 | // Ensure shadow WAL has recovered. 482 | if fi0, err := os.Stat(db.ShadowWALPath(pos0.Generation, pos0.Index)); err != nil { 483 | t.Fatal(err) 484 | } else if got, want := fi0.Size(), fi.Size(); got != want { 485 | t.Fatalf("Size()=%v, want %v", got, want) 486 | } 487 | }) 488 | 489 | // Ensure DB can handle a generation directory with a missing shadow WAL. 490 | t.Run("NoShadowWAL", func(t *testing.T) { 491 | db, sqldb := MustOpenDBs(t) 492 | defer MustCloseDBs(t, db, sqldb) 493 | 494 | // Execute a query to force a write to the WAL and then sync. 495 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 496 | t.Fatal(err) 497 | } else if err := db.Sync(context.Background()); err != nil { 498 | t.Fatal(err) 499 | } 500 | 501 | pos0, err := db.Pos() 502 | if err != nil { 503 | t.Fatal(err) 504 | } 505 | 506 | // Close & delete shadow WAL to simulate dir created but not WAL. 507 | if err := db.Close(); err != nil { 508 | t.Fatal(err) 509 | } else if err := os.Remove(db.ShadowWALPath(pos0.Generation, pos0.Index)); err != nil { 510 | t.Fatal(err) 511 | } 512 | 513 | // Reopen managed database & ensure sync will still work. 514 | db = MustOpenDBAt(t, db.Path()) 515 | defer MustCloseDB(t, db) 516 | if err := db.Sync(context.Background()); err != nil { 517 | t.Fatal(err) 518 | } 519 | 520 | // Verify new generation created but index/offset the same. 521 | if pos1, err := db.Pos(); err != nil { 522 | t.Fatal(err) 523 | } else if pos0.Generation == pos1.Generation { 524 | t.Fatal("expected new generation") 525 | } else if got, want := pos1.Index, pos0.Index; got != want { 526 | t.Fatalf("Index=%v want %v", got, want) 527 | } else if got, want := pos1.Offset, pos0.Offset; got != want { 528 | t.Fatalf("Offset=%v want %v", got, want) 529 | } 530 | }) 531 | 532 | // Ensure DB checkpoints after minimum number of pages. 533 | t.Run("MinCheckpointPageN", func(t *testing.T) { 534 | db, sqldb := MustOpenDBs(t) 535 | defer MustCloseDBs(t, db, sqldb) 536 | 537 | // Execute a query to force a write to the WAL and then sync. 538 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 539 | t.Fatal(err) 540 | } else if err := db.Sync(context.Background()); err != nil { 541 | t.Fatal(err) 542 | } 543 | 544 | // Write at least minimum number of pages to trigger rollover. 545 | for i := 0; i < db.MinCheckpointPageN; i++ { 546 | if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz');`); err != nil { 547 | t.Fatal(err) 548 | } 549 | } 550 | 551 | // Sync to shadow WAL. 552 | if err := db.Sync(context.Background()); err != nil { 553 | t.Fatal(err) 554 | } 555 | 556 | // Ensure position is now on the second index. 557 | if pos, err := db.Pos(); err != nil { 558 | t.Fatal(err) 559 | } else if got, want := pos.Index, 1; got != want { 560 | t.Fatalf("Index=%v, want %v", got, want) 561 | } 562 | }) 563 | 564 | // Ensure DB checkpoints after interval. 565 | t.Run("CheckpointInterval", func(t *testing.T) { 566 | db, sqldb := MustOpenDBs(t) 567 | defer MustCloseDBs(t, db, sqldb) 568 | 569 | // Execute a query to force a write to the WAL and then sync. 570 | if _, err := sqldb.Exec(`CREATE TABLE foo (bar TEXT);`); err != nil { 571 | t.Fatal(err) 572 | } else if err := db.Sync(context.Background()); err != nil { 573 | t.Fatal(err) 574 | } 575 | 576 | // Reduce checkpoint interval to ensure a rollover is triggered. 577 | db.CheckpointInterval = 1 * time.Nanosecond 578 | 579 | // Write to WAL & sync. 580 | if _, err := sqldb.Exec(`INSERT INTO foo (bar) VALUES ('baz');`); err != nil { 581 | t.Fatal(err) 582 | } else if err := db.Sync(context.Background()); err != nil { 583 | t.Fatal(err) 584 | } 585 | 586 | // Ensure position is now on the second index. 587 | if pos, err := db.Pos(); err != nil { 588 | t.Fatal(err) 589 | } else if got, want := pos.Index, 1; got != want { 590 | t.Fatalf("Index=%v, want %v", got, want) 591 | } 592 | }) 593 | } 594 | 595 | // MustOpenDBs returns a new instance of a DB & associated SQL DB. 596 | func MustOpenDBs(tb testing.TB) (*litestream.DB, *sql.DB) { 597 | tb.Helper() 598 | db := MustOpenDB(tb) 599 | return db, MustOpenSQLDB(tb, db.Path()) 600 | } 601 | 602 | // MustCloseDBs closes db & sqldb and removes the parent directory. 603 | func MustCloseDBs(tb testing.TB, db *litestream.DB, sqldb *sql.DB) { 604 | tb.Helper() 605 | MustCloseDB(tb, db) 606 | MustCloseSQLDB(tb, sqldb) 607 | } 608 | 609 | // MustOpenDB returns a new instance of a DB. 610 | func MustOpenDB(tb testing.TB) *litestream.DB { 611 | dir := tb.TempDir() 612 | return MustOpenDBAt(tb, filepath.Join(dir, "db")) 613 | } 614 | 615 | // MustOpenDBAt returns a new instance of a DB for a given path. 616 | func MustOpenDBAt(tb testing.TB, path string) *litestream.DB { 617 | tb.Helper() 618 | db := litestream.NewDB(path) 619 | db.MonitorInterval = 0 // disable background goroutine 620 | if err := db.Open(); err != nil { 621 | tb.Fatal(err) 622 | } 623 | return db 624 | } 625 | 626 | // MustCloseDB closes db and removes its parent directory. 627 | func MustCloseDB(tb testing.TB, db *litestream.DB) { 628 | tb.Helper() 629 | if err := db.Close(); err != nil && !strings.Contains(err.Error(), `database is closed`) { 630 | tb.Fatal(err) 631 | } else if err := os.RemoveAll(filepath.Dir(db.Path())); err != nil { 632 | tb.Fatal(err) 633 | } 634 | } 635 | 636 | // MustOpenSQLDB returns a database/sql DB. 637 | func MustOpenSQLDB(tb testing.TB, path string) *sql.DB { 638 | tb.Helper() 639 | d, err := sql.Open("sqlite3", path) 640 | if err != nil { 641 | tb.Fatal(err) 642 | } else if _, err := d.Exec(`PRAGMA journal_mode = wal;`); err != nil { 643 | tb.Fatal(err) 644 | } 645 | return d 646 | } 647 | 648 | // MustCloseSQLDB closes a database/sql DB. 649 | func MustCloseSQLDB(tb testing.TB, d *sql.DB) { 650 | tb.Helper() 651 | if err := d.Close(); err != nil { 652 | tb.Fatal(err) 653 | } 654 | } 655 | -------------------------------------------------------------------------------- /litestream_test.go: -------------------------------------------------------------------------------- 1 | package litestream_test 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/hex" 6 | "testing" 7 | 8 | "github.com/benbjohnson/litestream" 9 | _ "github.com/mattn/go-sqlite3" 10 | ) 11 | 12 | func TestChecksum(t *testing.T) { 13 | // Ensure a WAL header, frame header, & frame data can be checksummed in one pass. 14 | t.Run("OnePass", func(t *testing.T) { 15 | input, err := hex.DecodeString("377f0682002de218000010000000000052382eac857b1a4e00000002000000020d000000080fe0000ffc0ff80ff40ff00fec0fe80fe40fe000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000208020902070209020602090205020902040209020302090202020902010209") 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | 20 | s0, s1 := litestream.Checksum(binary.LittleEndian, 0, 0, input) 21 | if got, want := [2]uint32{s0, s1}, [2]uint32{0xdc2f3e84, 0x540488d3}; got != want { 22 | t.Fatalf("Checksum()=%x, want %x", got, want) 23 | } 24 | }) 25 | 26 | // Ensure we get the same result as OnePass even if we split up into multiple calls. 27 | t.Run("Incremental", func(t *testing.T) { 28 | // Compute checksum for beginning of WAL header. 29 | s0, s1 := litestream.Checksum(binary.LittleEndian, 0, 0, MustDecodeHexString("377f0682002de218000010000000000052382eac857b1a4e")) 30 | if got, want := [2]uint32{s0, s1}, [2]uint32{0x81153b65, 0x87178e8f}; got != want { 31 | t.Fatalf("Checksum()=%x, want %x", got, want) 32 | } 33 | 34 | // Continue checksum with WAL frame header & frame contents. 35 | s0a, s1a := litestream.Checksum(binary.LittleEndian, s0, s1, MustDecodeHexString("0000000200000002")) 36 | s0b, s1b := litestream.Checksum(binary.LittleEndian, s0a, s1a, MustDecodeHexString(`0d000000080fe0000ffc0ff80ff40ff00fec0fe80fe40fe000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000208020902070209020602090205020902040209020302090202020902010209`)) 37 | if got, want := [2]uint32{s0b, s1b}, [2]uint32{0xdc2f3e84, 0x540488d3}; got != want { 38 | t.Fatalf("Checksum()=%x, want %x", got, want) 39 | } 40 | }) 41 | } 42 | 43 | func TestGenerationsPath(t *testing.T) { 44 | t.Run("OK", func(t *testing.T) { 45 | if got, want := litestream.GenerationsPath("foo"), "foo/generations"; got != want { 46 | t.Fatalf("GenerationsPath()=%v, want %v", got, want) 47 | } 48 | }) 49 | t.Run("NoPath", func(t *testing.T) { 50 | if got, want := litestream.GenerationsPath(""), "generations"; got != want { 51 | t.Fatalf("GenerationsPath()=%v, want %v", got, want) 52 | } 53 | }) 54 | } 55 | 56 | func TestGenerationPath(t *testing.T) { 57 | t.Run("OK", func(t *testing.T) { 58 | if got, err := litestream.GenerationPath("foo", "0123456701234567"); err != nil { 59 | t.Fatal(err) 60 | } else if want := "foo/generations/0123456701234567"; got != want { 61 | t.Fatalf("GenerationPath()=%v, want %v", got, want) 62 | } 63 | }) 64 | t.Run("ErrNoGeneration", func(t *testing.T) { 65 | if _, err := litestream.GenerationPath("foo", ""); err == nil || err.Error() != `generation required` { 66 | t.Fatalf("expected error: %v", err) 67 | } 68 | }) 69 | } 70 | 71 | func TestSnapshotsPath(t *testing.T) { 72 | t.Run("OK", func(t *testing.T) { 73 | if got, err := litestream.SnapshotsPath("foo", "0123456701234567"); err != nil { 74 | t.Fatal(err) 75 | } else if want := "foo/generations/0123456701234567/snapshots"; got != want { 76 | t.Fatalf("SnapshotsPath()=%v, want %v", got, want) 77 | } 78 | }) 79 | t.Run("ErrNoGeneration", func(t *testing.T) { 80 | if _, err := litestream.SnapshotsPath("foo", ""); err == nil || err.Error() != `generation required` { 81 | t.Fatalf("unexpected error: %v", err) 82 | } 83 | }) 84 | } 85 | 86 | func TestSnapshotPath(t *testing.T) { 87 | t.Run("OK", func(t *testing.T) { 88 | if got, err := litestream.SnapshotPath("foo", "0123456701234567", 1000); err != nil { 89 | t.Fatal(err) 90 | } else if want := "foo/generations/0123456701234567/snapshots/000003e8.snapshot.lz4"; got != want { 91 | t.Fatalf("SnapshotPath()=%v, want %v", got, want) 92 | } 93 | }) 94 | t.Run("ErrNoGeneration", func(t *testing.T) { 95 | if _, err := litestream.SnapshotPath("foo", "", 1000); err == nil || err.Error() != `generation required` { 96 | t.Fatalf("unexpected error: %v", err) 97 | } 98 | }) 99 | } 100 | 101 | func TestWALPath(t *testing.T) { 102 | t.Run("OK", func(t *testing.T) { 103 | if got, err := litestream.WALPath("foo", "0123456701234567"); err != nil { 104 | t.Fatal(err) 105 | } else if want := "foo/generations/0123456701234567/wal"; got != want { 106 | t.Fatalf("WALPath()=%v, want %v", got, want) 107 | } 108 | }) 109 | t.Run("ErrNoGeneration", func(t *testing.T) { 110 | if _, err := litestream.WALPath("foo", ""); err == nil || err.Error() != `generation required` { 111 | t.Fatalf("unexpected error: %v", err) 112 | } 113 | }) 114 | } 115 | 116 | func TestWALSegmentPath(t *testing.T) { 117 | t.Run("OK", func(t *testing.T) { 118 | if got, err := litestream.WALSegmentPath("foo", "0123456701234567", 1000, 1001); err != nil { 119 | t.Fatal(err) 120 | } else if want := "foo/generations/0123456701234567/wal/000003e8_000003e9.wal.lz4"; got != want { 121 | t.Fatalf("WALPath()=%v, want %v", got, want) 122 | } 123 | }) 124 | t.Run("ErrNoGeneration", func(t *testing.T) { 125 | if _, err := litestream.WALSegmentPath("foo", "", 1000, 0); err == nil || err.Error() != `generation required` { 126 | t.Fatalf("unexpected error: %v", err) 127 | } 128 | }) 129 | } 130 | 131 | func TestFindMinSnapshotByGeneration(t *testing.T) { 132 | infos := []litestream.SnapshotInfo{ 133 | {Generation: "29cf4bced74e92ab", Index: 0}, 134 | {Generation: "5dfeb4aa03232553", Index: 24}, 135 | } 136 | if got, want := litestream.FindMinSnapshotByGeneration(infos, "29cf4bced74e92ab"), &infos[0]; got != want { 137 | t.Fatalf("info=%#v, want %#v", got, want) 138 | } 139 | } 140 | 141 | func MustDecodeHexString(s string) []byte { 142 | b, err := hex.DecodeString(s) 143 | if err != nil { 144 | panic(err) 145 | } 146 | return b 147 | } 148 | --------------------------------------------------------------------------------