├── src ├── log │ ├── mod.rs │ └── log.rs ├── base │ ├── numerical.rs │ ├── mod.rs │ ├── string.rs │ ├── generation.rs │ ├── compress.rs │ └── file.rs ├── cmd │ ├── mod.rs │ ├── command.rs │ ├── replicate.rs │ └── restore.rs ├── database │ ├── mod.rs │ └── database.rs ├── sync │ ├── mod.rs │ ├── shadow_wal_reader.rs │ ├── restore.rs │ └── replicate.rs ├── error │ ├── mod.rs │ ├── backtrace.rs │ ├── error_code.rs │ ├── error_into.rs │ └── error.rs ├── storage │ ├── mod.rs │ ├── operator.rs │ └── storage_client.rs ├── sqlite │ ├── mod.rs │ ├── wal_frame.rs │ ├── wal_header.rs │ └── common.rs ├── main.rs └── config │ ├── mod.rs │ ├── arg.rs │ ├── storage_params.rs │ └── config.rs ├── rust-toolchain.toml ├── clippy.toml ├── tests ├── config │ ├── fs_template.toml │ ├── ftp_template.toml │ └── s3_template.toml └── integration_test.py ├── rustfmt.toml ├── .github ├── services │ ├── ftp │ │ └── action.yml │ └── s3 │ │ └── minio │ │ └── action.yml ├── workflows │ ├── unit_test.yml │ ├── fs_integration_test.yml │ ├── ftp_integration_test.yml │ └── s3_integration_test.yml └── actions │ └── setup │ └── action.yml ├── fixtures ├── s3 │ └── docker-compose-minio.yml └── ftp │ └── docker-compose-vsftpd.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── etc └── sample.toml ├── config.md ├── README.md └── LICENSE /src/log/mod.rs: -------------------------------------------------------------------------------- 1 | #[allow(clippy::module_inception)] 2 | mod log; 3 | 4 | pub use log::init_log; 5 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.88" 3 | components = ["rustfmt", "clippy", "rust-analyzer"] 4 | -------------------------------------------------------------------------------- /src/base/numerical.rs: -------------------------------------------------------------------------------- 1 | pub fn is_power_of_two(num: u64) -> bool { 2 | num != 0 && (num & (num - 1)) == 0 3 | } 4 | -------------------------------------------------------------------------------- /src/cmd/mod.rs: -------------------------------------------------------------------------------- 1 | mod command; 2 | mod replicate; 3 | mod restore; 4 | 5 | pub use command::command; 6 | pub use replicate::Replicate; 7 | pub use restore::Restore; 8 | -------------------------------------------------------------------------------- /src/database/mod.rs: -------------------------------------------------------------------------------- 1 | #[allow(clippy::module_inception)] 2 | mod database; 3 | 4 | pub use database::DatabaseInfo; 5 | pub use database::DbCommand; 6 | pub use database::WalGenerationPos; 7 | pub use database::run_database; 8 | -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | disallowed-methods = [] 2 | 3 | disallowed-types = [] 4 | 5 | disallowed-macros = [] 6 | 7 | avoid-breaking-exported-api = true 8 | too-many-arguments-threshold = 10 9 | upper-case-acronyms-aggressive = false 10 | -------------------------------------------------------------------------------- /src/sync/mod.rs: -------------------------------------------------------------------------------- 1 | mod replicate; 2 | mod restore; 3 | mod shadow_wal_reader; 4 | 5 | pub use replicate::Replicate; 6 | pub use replicate::ReplicateCommand; 7 | pub use restore::run_restore; 8 | pub(crate) use shadow_wal_reader::ShadowWalReader; 9 | -------------------------------------------------------------------------------- /src/error/mod.rs: -------------------------------------------------------------------------------- 1 | mod backtrace; 2 | #[allow(clippy::module_inception)] 3 | mod error; 4 | mod error_code; 5 | mod error_into; 6 | 7 | pub(crate) use backtrace::capture; 8 | pub use error::Error; 9 | pub(crate) use error::ErrorCodeBacktrace; 10 | pub use error::Result; 11 | -------------------------------------------------------------------------------- /tests/config/fs_template.toml: -------------------------------------------------------------------------------- 1 | [log] 2 | level = "Debug" 3 | dir = "{root}" 4 | 5 | [[database]] 6 | db = "{root}/test.db" 7 | 8 | # sample of file system replicate config 9 | [[database.replicate]] 10 | name = "name of fs" 11 | params.type = "Fs" 12 | params.root = "{root}/replited" 13 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | version = "Two" 3 | reorder_imports = true 4 | imports_granularity = "Item" 5 | group_imports = "StdExternalCrate" 6 | where_single_line = true 7 | trailing_comma = "Vertical" 8 | overflow_delimited_expr = true 9 | format_code_in_doc_comments = true 10 | normalize_comments = true 11 | -------------------------------------------------------------------------------- /.github/services/ftp/action.yml: -------------------------------------------------------------------------------- 1 | name: vsftpd 2 | description: 'integration test for vsftpd' 3 | 4 | runs: 5 | using: "composite" 6 | steps: 7 | - name: Setup vsftpd Server 8 | shell: bash 9 | working-directory: fixtures/ftp 10 | run: docker compose -f docker-compose-vsftpd.yml up -d --wait 11 | -------------------------------------------------------------------------------- /fixtures/s3/docker-compose-minio.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | minio: 5 | image: quay.io/minio/minio:RELEASE.2024-01-18T22-51-28Z 6 | ports: 7 | - 9000:9000 8 | command: server /data 9 | environment: 10 | MINIO_ROOT_USER: "minioadmin" 11 | MINIO_ROOT_PASSWORD: "minioadmin" 12 | -------------------------------------------------------------------------------- /src/storage/mod.rs: -------------------------------------------------------------------------------- 1 | mod operator; 2 | mod storage_client; 3 | 4 | pub(crate) use operator::init_operator; 5 | pub use storage_client::RestoreInfo; 6 | pub use storage_client::RestoreWalSegments; 7 | pub use storage_client::SnapshotInfo; 8 | pub use storage_client::StorageClient; 9 | pub use storage_client::WalSegmentInfo; 10 | -------------------------------------------------------------------------------- /tests/config/ftp_template.toml: -------------------------------------------------------------------------------- 1 | [log] 2 | level = "Debug" 3 | dir = "{root}" 4 | 5 | [[database]] 6 | db = "{root}/test.db" 7 | 8 | # sample of file system replicate config 9 | [[database.replicate]] 10 | name = "ftp replicate" 11 | params.type = "Ftp" 12 | params.endpoint = "ftp://127.0.0.1:2121" 13 | params.root = "/" 14 | params.username = "admin" 15 | params.password = "admin" 16 | -------------------------------------------------------------------------------- /tests/config/s3_template.toml: -------------------------------------------------------------------------------- 1 | [log] 2 | level = "Debug" 3 | dir = "{root}" 4 | 5 | [[database]] 6 | db = "{root}/test.db" 7 | 8 | [[database.replicate]] 9 | name = "name of s3" 10 | params.type = "S3" 11 | params.endpoint = "http://127.0.0.1:9000" 12 | params.bucket = "test" 13 | params.region = "" 14 | params.root = "" 15 | params.access_key_id = "minioadmin" 16 | params.secret_access_key = "minioadmin" 17 | -------------------------------------------------------------------------------- /fixtures/ftp/docker-compose-vsftpd.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | vsftpd: 4 | image: fauria/vsftpd 5 | ports: 6 | - "2121:21" 7 | - "20000-22000:20000-22000" 8 | environment: 9 | FTP_USER: admin 10 | FTP_PASS: admin 11 | PASV_ADDRESS: 127.0.0.1 12 | PASV_MIN_PORT: 20000 13 | PASV_MAX_PORT: 22000 14 | volumes: 15 | - vsftpd-data:/home/vsftpd 16 | 17 | volumes: 18 | vsftpd-data: 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | # Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | # MSVC Windows builds of rustc generate these, which store debugging information 14 | *.pdb 15 | 16 | .test/ 17 | test.db 18 | -------------------------------------------------------------------------------- /src/sqlite/mod.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | mod wal_frame; 3 | mod wal_header; 4 | 5 | pub use common::CheckpointMode; 6 | pub use common::WAL_FRAME_HEADER_SIZE; 7 | pub use common::WAL_HEADER_BIG_ENDIAN_MAGIC; 8 | pub use common::WAL_HEADER_LITTLE_ENDIAN_MAGIC; 9 | pub use common::WAL_HEADER_SIZE; 10 | pub use common::align_frame; 11 | pub use common::checksum; 12 | pub(crate) use common::from_be_bytes_at; 13 | pub use common::read_last_checksum; 14 | pub use wal_frame::WALFrame; 15 | pub use wal_header::WALHeader; 16 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | #![allow(incomplete_features)] 3 | 4 | mod base; 5 | mod cmd; 6 | mod config; 7 | mod database; 8 | mod error; 9 | mod log; 10 | mod sqlite; 11 | mod storage; 12 | mod sync; 13 | 14 | use clap::Parser; 15 | use config::Arg; 16 | 17 | use crate::cmd::command; 18 | 19 | #[tokio::main] 20 | async fn main() -> anyhow::Result<()> { 21 | let arg = Arg::parse(); 22 | println!("arg: {:?}\n", arg); 23 | 24 | let mut cmd = command(arg)?; 25 | 26 | cmd.run().await?; 27 | 28 | Ok(()) 29 | } 30 | -------------------------------------------------------------------------------- /.github/services/s3/minio/action.yml: -------------------------------------------------------------------------------- 1 | name: minio_s3 2 | description: 'integration test for minio' 3 | 4 | runs: 5 | using: "composite" 6 | steps: 7 | - name: Setup MinIO Server 8 | shell: bash 9 | working-directory: fixtures/s3 10 | run: docker compose -f docker-compose-minio.yml up -d --wait 11 | - name: Setup test bucket 12 | shell: bash 13 | env: 14 | AWS_ACCESS_KEY_ID: "minioadmin" 15 | AWS_SECRET_ACCESS_KEY: "minioadmin" 16 | AWS_EC2_METADATA_DISABLED: "true" 17 | run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test 18 | -------------------------------------------------------------------------------- /.github/workflows/unit_test.yml: -------------------------------------------------------------------------------- 1 | name: Unit Test 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | plan: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: 'Checkout codes' 13 | uses: actions/checkout@v4 14 | with: 15 | # fetch depth set to 0 to make sure we have correct diff result. 16 | fetch-depth: 0 17 | 18 | - name: Setup Rust toolchain 19 | uses: ./.github/actions/setup 20 | 21 | - name: 'Run Unit tests' 22 | shell: bash 23 | run: | 24 | cargo test 25 | -------------------------------------------------------------------------------- /src/config/mod.rs: -------------------------------------------------------------------------------- 1 | mod arg; 2 | #[allow(clippy::module_inception)] 3 | mod config; 4 | mod storage_params; 5 | 6 | pub use arg::Arg; 7 | pub use arg::ArgCommand; 8 | pub use arg::RestoreOptions; 9 | pub use config::Config; 10 | pub use config::DbConfig; 11 | pub use config::LogConfig; 12 | pub use config::LogLevel; 13 | pub use config::StorageConfig; 14 | pub use storage_params::StorageAzblobConfig; 15 | pub use storage_params::StorageFsConfig; 16 | pub use storage_params::StorageFtpConfig; 17 | pub use storage_params::StorageGcsConfig; 18 | pub use storage_params::StorageParams; 19 | pub use storage_params::StorageS3Config; 20 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/) 6 | and this project adheres to [Semantic Versioning](https://semver.org/). 7 | 8 | 9 | 10 | ## [v0.1.0] - 2024-10-15 11 | 12 | ### Feature 13 | * feat: add replicate/restore sub commands. 14 | * feat: add fs/ftp/azure blob/gcs/s3 backend support. 15 | ### Docs 16 | * docs: add config.md about config fotmat. 17 | ### CI 18 | * ci: add integration test of ftp/s3/fs. 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/cmd/command.rs: -------------------------------------------------------------------------------- 1 | use super::Replicate; 2 | use super::Restore; 3 | use crate::config::Arg; 4 | use crate::config::ArgCommand; 5 | use crate::error::Result; 6 | 7 | pub const REPLICATE_CMD: &str = "replicate"; 8 | pub const RESTORE_CMD: &str = "restore"; 9 | 10 | #[async_trait::async_trait] 11 | pub trait Command { 12 | async fn run(&mut self) -> Result<()>; 13 | } 14 | 15 | pub fn command(arg: Arg) -> Result> { 16 | match &arg.cmd { 17 | ArgCommand::Replicate => Ok(Replicate::try_create(&arg.config)?), 18 | ArgCommand::Restore(options) => Ok(Restore::try_create(&arg.config, options.clone())?), 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /.github/workflows/fs_integration_test.yml: -------------------------------------------------------------------------------- 1 | name: Fs integration test 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | plan: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: 'Checkout codes' 13 | uses: actions/checkout@v4 14 | with: 15 | # fetch depth set to 0 to make sure we have correct diff result. 16 | fetch-depth: 0 17 | 18 | - name: Setup Rust toolchain 19 | uses: ./.github/actions/setup 20 | 21 | - name: 'Run integration test for fs' 22 | shell: bash 23 | run: | 24 | cargo build --release 25 | ls 26 | pwd 27 | python3 tests/integration_test.py 12000 fs ./target/release/replited 28 | -------------------------------------------------------------------------------- /.github/workflows/ftp_integration_test.yml: -------------------------------------------------------------------------------- 1 | name: Ftp integration test 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | plan: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: 'Checkout codes' 13 | uses: actions/checkout@v4 14 | with: 15 | # fetch depth set to 0 to make sure we have correct diff result. 16 | fetch-depth: 0 17 | 18 | - name: Setup Rust toolchain 19 | uses: ./.github/actions/setup 20 | 21 | - name: Setup Ftp Server 22 | uses: ./.github/services/ftp 23 | 24 | - name: 'Run integration test for ftp' 25 | shell: bash 26 | run: | 27 | cargo build 28 | ls 29 | pwd 30 | python3 tests/integration_test.py 12000 ftp ./target/debug/replited 31 | -------------------------------------------------------------------------------- /.github/workflows/s3_integration_test.yml: -------------------------------------------------------------------------------- 1 | name: S3 integration test 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | plan: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: 'Checkout codes' 13 | uses: actions/checkout@v4 14 | with: 15 | # fetch depth set to 0 to make sure we have correct diff result. 16 | fetch-depth: 0 17 | 18 | - name: Setup Rust toolchain 19 | uses: ./.github/actions/setup 20 | 21 | - name: Setup Minio 22 | uses: ./.github/services/s3/minio 23 | 24 | - name: 'Run integration test for s3' 25 | shell: bash 26 | run: | 27 | cargo build --release 28 | ls 29 | pwd 30 | python3 tests/integration_test.py 12000 s3 ./target/release/replited 31 | -------------------------------------------------------------------------------- /src/base/mod.rs: -------------------------------------------------------------------------------- 1 | mod compress; 2 | mod file; 3 | mod generation; 4 | mod numerical; 5 | mod string; 6 | 7 | pub use compress::compress_buffer; 8 | pub use compress::compress_file; 9 | pub use compress::decompressed_data; 10 | pub use file::generation_dir; 11 | pub use file::generation_file_path; 12 | pub use file::local_generations_dir; 13 | pub use file::parent_dir; 14 | pub use file::parse_snapshot_path; 15 | pub use file::parse_wal_path; 16 | pub use file::parse_wal_segment_path; 17 | pub use file::path_base; 18 | pub use file::remote_generations_dir; 19 | pub use file::shadow_wal_dir; 20 | pub use file::shadow_wal_file; 21 | pub use file::snapshot_file; 22 | pub use file::snapshots_dir; 23 | pub use file::walsegment_file; 24 | pub use file::walsegments_dir; 25 | pub use generation::Generation; 26 | pub use numerical::is_power_of_two; 27 | pub use string::mask_string; 28 | -------------------------------------------------------------------------------- /.github/actions/setup/action.yml: -------------------------------------------------------------------------------- 1 | name: Setup Rust Builder 2 | description: 'Prepare Rust Build Environment' 3 | 4 | runs: 5 | using: "composite" 6 | steps: 7 | - name: Setup rust related environment variables 8 | shell: bash 9 | run: | 10 | # Disable full debug symbol generation to speed up CI build and keep memory down 11 | # "1" means line tables only, which is useful for panic tracebacks. 12 | # About `force-frame-pointers`, here's the discussion history: https://github.com/apache/opendal/issues/3756 13 | echo "RUSTFLAGS=-C force-frame-pointers=yes -C debuginfo=1" >> $GITHUB_ENV 14 | # Enable backtraces 15 | echo "RUST_BACKTRACE=1" >> $GITHUB_ENV 16 | # Enable logging 17 | echo "RUST_LOG=opendal=trace" >> $GITHUB_ENV 18 | # Enable sparse index 19 | echo "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse" >> $GITHUB_ENV 20 | # Make sure rust has been setup 21 | cargo version 22 | -------------------------------------------------------------------------------- /src/cmd/replicate.rs: -------------------------------------------------------------------------------- 1 | use super::command::Command; 2 | use crate::config::Config; 3 | use crate::database::run_database; 4 | use crate::error::Result; 5 | use crate::log::init_log; 6 | 7 | pub struct Replicate { 8 | config: Config, 9 | } 10 | 11 | impl Replicate { 12 | pub fn try_create(config: &str) -> Result> { 13 | let config = Config::load(config)?; 14 | let log_config = config.log.clone(); 15 | 16 | init_log(log_config)?; 17 | Ok(Box::new(Replicate { config })) 18 | } 19 | } 20 | 21 | #[async_trait::async_trait] 22 | impl Command for Replicate { 23 | async fn run(&mut self) -> Result<()> { 24 | let mut handles = vec![]; 25 | for database in &self.config.database { 26 | let datatase = database.clone(); 27 | let handle = tokio::spawn(async move { 28 | let _ = run_database(datatase).await; 29 | }); 30 | 31 | handles.push(handle); 32 | } 33 | 34 | for h in handles { 35 | h.await.unwrap(); 36 | } 37 | Ok(()) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/base/string.rs: -------------------------------------------------------------------------------- 1 | /// Mask a string by "******", but keep `unmask_len` of suffix. 2 | #[inline] 3 | pub fn mask_string(s: &str, unmask_len: usize) -> String { 4 | if s.len() <= unmask_len { 5 | s.to_string() 6 | } else { 7 | let mut ret = "******".to_string(); 8 | ret.push_str(&s[(s.len() - unmask_len)..]); 9 | ret 10 | } 11 | } 12 | 13 | pub fn u8_array_as_hex(arr: &[u8]) -> String { 14 | let hex_str: String = arr 15 | .iter() 16 | .map(|byte| format!("0x{:02X}", byte)) 17 | .collect::>() 18 | .join(" "); 19 | 20 | hex_str 21 | } 22 | 23 | pub fn format_integer_with_leading_zeros(num: u32) -> String { 24 | format!("{:08X}", num) 25 | } 26 | 27 | #[cfg(test)] 28 | mod tests { 29 | use super::format_integer_with_leading_zeros; 30 | use crate::error::Result; 31 | 32 | #[test] 33 | fn test_format_integer_with_leading_zeros() -> Result<()> { 34 | let num = 0xab12; 35 | let hex = format_integer_with_leading_zeros(num); 36 | assert_eq!(&hex, "0000AB12"); 37 | Ok(()) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/base/generation.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Display; 2 | use std::fmt::Formatter; 3 | 4 | use uuid::NoContext; 5 | use uuid::Uuid; 6 | use uuid::timestamp; 7 | 8 | use crate::error::Result; 9 | 10 | #[derive(Eq, PartialEq, PartialOrd, Debug, Clone, Default)] 11 | pub struct Generation { 12 | uuid: Uuid, 13 | generation: String, 14 | } 15 | 16 | impl Generation { 17 | pub fn new() -> Self { 18 | let timestamp = timestamp::Timestamp::now(NoContext); 19 | Self::from_uuid(Uuid::new_v7(timestamp)) 20 | } 21 | 22 | fn from_uuid(uuid: Uuid) -> Self { 23 | let generation = uuid.simple().to_string(); 24 | Generation { uuid, generation } 25 | } 26 | 27 | pub fn try_create(generation: &str) -> Result { 28 | Ok(Self::from_uuid(Uuid::parse_str(generation)?)) 29 | } 30 | 31 | pub fn as_str(&self) -> &str { 32 | &self.generation 33 | } 34 | 35 | pub fn is_empty(&self) -> bool { 36 | self.uuid.is_nil() 37 | } 38 | } 39 | 40 | impl Display for Generation { 41 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 42 | write!(f, "{:?}", self.generation) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/cmd/restore.rs: -------------------------------------------------------------------------------- 1 | use super::command::Command; 2 | use crate::config::Config; 3 | use crate::config::RestoreOptions; 4 | use crate::error::Result; 5 | use crate::log::init_log; 6 | use crate::sync::run_restore; 7 | 8 | pub struct Restore { 9 | config: Config, 10 | options: RestoreOptions, 11 | } 12 | 13 | impl Restore { 14 | pub fn try_create(config: &str, options: RestoreOptions) -> Result> { 15 | let config = Config::load(config)?; 16 | let log_config = config.log.clone(); 17 | 18 | init_log(log_config)?; 19 | Ok(Box::new(Restore { config, options })) 20 | } 21 | } 22 | 23 | #[async_trait::async_trait] 24 | impl Command for Restore { 25 | async fn run(&mut self) -> Result<()> { 26 | self.options.validate()?; 27 | 28 | for config in &self.config.database { 29 | if config.db == self.options.db { 30 | let ret = run_restore(config, &self.options).await; 31 | println!("restore result: {:?}", ret); 32 | return Ok(()); 33 | } 34 | } 35 | 36 | println!("cannot find db {} in config file", self.options.db); 37 | 38 | Ok(()) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/log/log.rs: -------------------------------------------------------------------------------- 1 | use std::num::NonZeroUsize; 2 | 3 | use logforth::append::file::FileBuilder; 4 | use logforth::layout::TextLayout; 5 | use logforth::record::LevelFilter; 6 | 7 | use crate::config::LogConfig; 8 | use crate::config::LogLevel; 9 | use crate::error::Error; 10 | use crate::error::Result; 11 | 12 | pub fn init_log(log_config: LogConfig) -> Result<()> { 13 | let level: LevelFilter = match log_config.level { 14 | LogLevel::Error => LevelFilter::Error, 15 | LogLevel::Warn => LevelFilter::Warn, 16 | LogLevel::Info => LevelFilter::Info, 17 | LogLevel::Debug => LevelFilter::Debug, 18 | LogLevel::Trace => LevelFilter::Trace, 19 | LogLevel::Off => LevelFilter::Off, 20 | }; 21 | 22 | let file = FileBuilder::new(log_config.dir, "replited") 23 | .rollover_size(NonZeroUsize::new(1024 * 4096).unwrap()) // bytes 24 | .max_log_files(NonZeroUsize::new(9).unwrap()) 25 | .filename_suffix("log") 26 | .layout(TextLayout::default().no_color()) 27 | .build() 28 | .map_err(Error::from_std_error)?; 29 | 30 | logforth::starter_log::builder() 31 | .dispatch(|d| d.filter(level).append(file)) 32 | .apply(); 33 | 34 | Ok(()) 35 | } 36 | -------------------------------------------------------------------------------- /src/sqlite/wal_frame.rs: -------------------------------------------------------------------------------- 1 | use std::io::Read; 2 | 3 | use super::WAL_FRAME_HEADER_SIZE; 4 | use super::from_be_bytes_at; 5 | use crate::error::Result; 6 | 7 | #[derive(Clone, Debug, PartialEq)] 8 | pub struct WALFrame { 9 | pub data: Vec, 10 | pub page_num: u32, 11 | pub db_size: u32, 12 | pub salt1: u32, 13 | pub salt2: u32, 14 | pub checksum1: u32, 15 | pub checksum2: u32, 16 | } 17 | 18 | impl WALFrame { 19 | pub fn read(reader: &mut R, page_size: u64) -> Result { 20 | let mut data: Vec = vec![0u8; (WAL_FRAME_HEADER_SIZE + page_size) as usize]; 21 | reader.read_exact(&mut data)?; 22 | 23 | let page_num = from_be_bytes_at(&data, 0)?; 24 | let db_size = from_be_bytes_at(&data, 4)?; 25 | 26 | let checksum1 = from_be_bytes_at(&data, 16)?; 27 | let checksum2 = from_be_bytes_at(&data, 20)?; 28 | 29 | let salt1 = from_be_bytes_at(&data, 8)?; 30 | let salt2 = from_be_bytes_at(&data, 12)?; 31 | 32 | Ok(WALFrame { 33 | data, 34 | page_num, 35 | db_size, 36 | salt1, 37 | salt2, 38 | checksum1, 39 | checksum2, 40 | }) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "replited" 3 | version = "0.1.0" 4 | edition = "2024" 5 | rust-version = "1.88" 6 | description = "Replicate SQLite Daemon" 7 | 8 | [dependencies] 9 | anyhow = { version = "1.0.65" } 10 | async-trait = { version = "0.1.81" } 11 | backtrace = "0.3.73" 12 | chrono = { version = "0.4.31", features = ["serde"] } 13 | clap = { version = "4.4.2", features = ["derive"] } 14 | opendal = { version = "0.50.0", features = [ 15 | "layers-fastrace", 16 | "layers-async-backtrace", 17 | "services-azblob", 18 | "services-fs", 19 | "services-ftp", 20 | "services-gcs", 21 | "services-s3", 22 | ] } 23 | log = "0.4.17" 24 | logforth = { version = "0.28.1", features = ["starter-log"] } 25 | lz4 = "1.26.0" 26 | parking_lot = "0.12.1" 27 | paste = "1.0.9" 28 | regex = { version = "1.10.6" } 29 | reqwest = { version = "0.12", default-features = false, features = [ 30 | "json", 31 | "http2", 32 | "rustls-tls", 33 | "rustls-tls-native-roots", 34 | ] } 35 | reqwest-hickory-resolver = "0.1" 36 | rusqlite = { version = "0.32.1" } 37 | serde = { version = "1.0.164", features = ["derive", "rc"] } 38 | tempfile = "3.13.0" 39 | thiserror = { version = "1" } 40 | toml = "0.8.14" 41 | tokio = { version = "1.35.0", features = ["full"] } 42 | uuid = { version = "1.10.0", features = ["v7"] } 43 | 44 | [profile.release] 45 | lto = true 46 | codegen-units = 1 47 | 48 | [[bin]] 49 | name = "replited" 50 | path = "src/main.rs" 51 | doctest = false 52 | test = true 53 | -------------------------------------------------------------------------------- /etc/sample.toml: -------------------------------------------------------------------------------- 1 | [log] 2 | level = "Debug" 3 | dir = "/var/log/replited/" 4 | 5 | [[database]] 6 | db = "/var/sqlite/test.db" 7 | 8 | # each database has at least one replicate backend, 9 | # each replication config in one db MUST has different name 10 | 11 | # sample of azure blob replicate config 12 | [[database.replicate]] 13 | name = "name of azure blob" 14 | params.type = "Azb" 15 | params.endpoint = "xxx" 16 | params.container = "xxx" 17 | params.root = "" 18 | params.account_name = "xxx" 19 | params.account_key = "xxx" 20 | 21 | # sample of file system replicate config 22 | [[database.replicate]] 23 | name = "name of fs" 24 | params.type = "Fs" 25 | params.root = "/var/replited" 26 | 27 | # sample of ftp replicate config 28 | [[database.replicate]] 29 | name = "name of ftp" 30 | params.type = "Ftp" 31 | params.endpoint = "https://storage.googleapis.com" 32 | params.root = "/var/replited" 33 | params.username = "username" 34 | params.password = "password" 35 | 36 | # sample of gcs replicate config 37 | [[database.replicate]] 38 | name = "name of gcs" 39 | params.type = "Gcs" 40 | params.endpoint = "https://storage.googleapis.com" 41 | params.bucket = "sqlite" 42 | params.root = "" 43 | params.credential = "xxx" 44 | 45 | # sample of s3 replicate config 46 | [[database.replicate]] 47 | name = "name of s3" 48 | params.type = "S3" 49 | params.endpoint = "http://127.0.0.1:9900" 50 | params.bucket = "sqlite" 51 | params.region = "" 52 | params.root = "" 53 | params.access_key_id = "minioadmin" 54 | params.secret_access_key = "minioadmin" 55 | 56 | 57 | # can add more than one database config 58 | #[[database]] 59 | #db = "/var/sqlite/test2.db" 60 | 61 | #[[database.replicate]] 62 | -------------------------------------------------------------------------------- /src/config/arg.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use clap::Subcommand; 3 | 4 | use crate::error::Error; 5 | use crate::error::Result; 6 | 7 | #[derive(Parser, Debug)] 8 | #[command(author="replited", version, about="Replicate sqlite to everywhere", long_about = None)] 9 | pub struct Arg { 10 | #[arg(short, long, default_value = "/etc/replited.toml")] 11 | pub config: String, 12 | 13 | #[command(subcommand)] 14 | pub cmd: ArgCommand, 15 | } 16 | 17 | #[derive(Subcommand, Clone, Debug)] 18 | pub enum ArgCommand { 19 | Replicate, 20 | 21 | Restore(RestoreOptions), 22 | } 23 | 24 | #[derive(Parser, Debug, Clone)] 25 | pub struct RestoreOptions { 26 | // restore db path in config file 27 | #[arg(short, long, default_value = "")] 28 | pub db: String, 29 | 30 | // restore db output path 31 | #[arg(long, default_value = "")] 32 | pub output: String, 33 | // restore db generation string. 34 | // when empty, use the most recent generation from replicates. 35 | //#[arg(short, long, default_value = "")] 36 | // pub generation: String, 37 | 38 | // if overwrite existing db in the same path 39 | //#[arg(long, default_value_t = false)] 40 | // pub overwrite: bool, 41 | } 42 | 43 | impl RestoreOptions { 44 | pub fn validate(&self) -> Result<()> { 45 | if self.db.is_empty() { 46 | println!("restore MUST Specify db path in config"); 47 | return Err(Error::InvalidArg("arg MUST Specify db path in config")); 48 | } 49 | 50 | if self.output.is_empty() { 51 | println!("restore MUST Specify db output path"); 52 | return Err(Error::InvalidArg("arg MUST Specify db output pathg")); 53 | } 54 | 55 | Ok(()) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/base/compress.rs: -------------------------------------------------------------------------------- 1 | use std::fs::OpenOptions; 2 | use std::io::Read; 3 | use std::io::Write; 4 | 5 | use lz4::Decoder; 6 | use lz4::EncoderBuilder; 7 | 8 | use crate::error::Result; 9 | 10 | const COMPRESS_BUFFER_SIZE: usize = 102400; 11 | 12 | pub fn compress_buffer(data: &[u8]) -> Result> { 13 | let mut buffer = Vec::with_capacity(data.len()); 14 | let mut encoder = EncoderBuilder::new().build(&mut buffer)?; 15 | 16 | encoder.write_all(data)?; 17 | let (compressed_data, result) = encoder.finish(); 18 | result?; 19 | 20 | Ok(compressed_data.to_owned()) 21 | } 22 | 23 | pub fn compress_file(file_name: &str) -> Result> { 24 | // Open db file descriptor 25 | let mut reader = OpenOptions::new().read(true).open(file_name)?; 26 | let bytes = reader.metadata()?.len() as usize; 27 | let mut buffer = Vec::with_capacity(bytes); 28 | let mut encoder = EncoderBuilder::new().build(&mut buffer)?; 29 | 30 | let mut temp_buffer = vec![0; COMPRESS_BUFFER_SIZE]; 31 | 32 | loop { 33 | let bytes_read = reader.read(&mut temp_buffer)?; 34 | if bytes_read == 0 { 35 | break; // EOF 36 | } 37 | encoder.write_all(&temp_buffer[..bytes_read])?; 38 | } 39 | let (compressed_data, result) = encoder.finish(); 40 | result?; 41 | 42 | Ok(compressed_data.to_owned()) 43 | } 44 | 45 | pub fn decompressed_data(compressed_data: Vec) -> Result> { 46 | let compressed_data = compressed_data.as_slice(); 47 | let mut decoder = Decoder::new(compressed_data)?; 48 | let mut decompressed_data = Vec::new(); 49 | let mut buffer = vec![0; COMPRESS_BUFFER_SIZE]; 50 | 51 | loop { 52 | let bytes_read = decoder.read(&mut buffer)?; 53 | if bytes_read == 0 { 54 | break; // EOF 55 | } 56 | decompressed_data.extend_from_slice(&buffer[..bytes_read]); 57 | } 58 | 59 | Ok(decompressed_data) 60 | } 61 | -------------------------------------------------------------------------------- /src/error/backtrace.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::sync::atomic::AtomicUsize; 3 | use std::sync::atomic::Ordering; 4 | 5 | use crate::error::ErrorCodeBacktrace; 6 | 7 | // 0: not specified 1: disable 2: enable 8 | pub static USER_SET_ENABLE_BACKTRACE: AtomicUsize = AtomicUsize::new(0); 9 | 10 | pub fn set_backtrace(switch: bool) { 11 | if switch { 12 | USER_SET_ENABLE_BACKTRACE.store(2, Ordering::Relaxed); 13 | } else { 14 | USER_SET_ENABLE_BACKTRACE.store(1, Ordering::Relaxed); 15 | } 16 | } 17 | 18 | fn enable_rust_backtrace() -> bool { 19 | match USER_SET_ENABLE_BACKTRACE.load(Ordering::Relaxed) { 20 | 0 => {} 21 | 1 => return false, 22 | _ => return true, 23 | } 24 | 25 | let enabled = match std::env::var("RUST_LIB_BACKTRACE") { 26 | Ok(s) => s != "0", 27 | Err(_) => match std::env::var("RUST_BACKTRACE") { 28 | Ok(s) => s != "0", 29 | Err(_) => false, 30 | }, 31 | }; 32 | 33 | USER_SET_ENABLE_BACKTRACE.store(enabled as usize + 1, Ordering::Relaxed); 34 | enabled 35 | } 36 | 37 | enum BacktraceStyle { 38 | Symbols, 39 | Address, 40 | } 41 | 42 | fn backtrace_style() -> BacktraceStyle { 43 | static ENABLED: AtomicUsize = AtomicUsize::new(0); 44 | match ENABLED.load(Ordering::Relaxed) { 45 | 1 => return BacktraceStyle::Address, 46 | 2 => return BacktraceStyle::Symbols, 47 | _ => {} 48 | } 49 | 50 | let backtrace_style = match std::env::var("BACKTRACE_STYLE") { 51 | Ok(style) if style.eq_ignore_ascii_case("ADDRESS") => 1, 52 | _ => 2, 53 | }; 54 | 55 | ENABLED.store(backtrace_style, Ordering::Relaxed); 56 | match backtrace_style { 57 | 1 => BacktraceStyle::Address, 58 | _ => BacktraceStyle::Symbols, 59 | } 60 | } 61 | 62 | pub fn capture() -> Option { 63 | match enable_rust_backtrace() { 64 | false => None, 65 | true => match backtrace_style() { 66 | BacktraceStyle::Symbols => Some(ErrorCodeBacktrace::Symbols(Arc::new( 67 | backtrace::Backtrace::new(), 68 | ))), 69 | // TODO: get offset address(https://github.com/rust-lang/backtrace-rs/issues/434) 70 | BacktraceStyle::Address => Some(ErrorCodeBacktrace::Address(Arc::new( 71 | backtrace::Backtrace::new_unresolved(), 72 | ))), 73 | }, 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/error/error_code.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_snake_case)] 2 | 3 | use super::backtrace::capture; 4 | use crate::error::Error; 5 | 6 | macro_rules! build_error { 7 | ($($(#[$meta:meta])* $body:ident($code:expr)),*$(,)*) => { 8 | impl Error { 9 | $( 10 | paste::item! { 11 | $( 12 | #[$meta] 13 | )* 14 | pub const [< $body:snake:upper >]: u32 = $code; 15 | } 16 | $( 17 | #[$meta] 18 | )* 19 | pub fn $body(display_text: impl Into) -> Error { 20 | Error::create( 21 | $code, 22 | stringify!($body), 23 | display_text.into(), 24 | String::new(), 25 | None, 26 | capture(), 27 | ) 28 | } 29 | )* 30 | } 31 | } 32 | } 33 | 34 | build_error! { 35 | Ok(0), 36 | 37 | // config file error 38 | EmptyConfigFile(1), 39 | InvalidConfig(2), 40 | InvalidArg(3), 41 | ReadConfigFail(4), 42 | ParseConfigFail(5), 43 | 44 | // logger error 45 | InitLoggerError(10), 46 | /// Internal means this is the internal error that no action 47 | /// can be taken by neither developers or users. 48 | /// In most of the time, they are code bugs. 49 | /// 50 | /// If there is an error that are unexpected and no other actions 51 | /// to taken, please use this error code. 52 | /// 53 | /// # Notes 54 | /// 55 | /// This error should never be used to for error checking. An error 56 | /// that returns as internal error could be assigned a separate error 57 | /// code at anytime. 58 | Internal(11), 59 | 60 | // storage error 61 | StorageNotFound(51), 62 | StoragePermissionDenied(52), 63 | StorageOther(53), 64 | InvalidPath(54), 65 | 66 | // database error 67 | SpawnDatabaseTaskError(80), 68 | OverwriteDbError(81), 69 | NoGenerationError(82), 70 | WalReaderOffsetTooHighError(83), 71 | InvalidWalSegmentError(84), 72 | MismatchWalHeaderError(85), 73 | 74 | // 3rd crate error 75 | TokioError(100), 76 | OpenDalError(101), 77 | UUIDError(102), 78 | 79 | // sqlite error 80 | SqliteError(120), 81 | SqliteWalError(121), 82 | SqliteInvalidWalHeaderError(122), 83 | SqliteInvalidWalFrameHeaderError(123), 84 | SqliteInvalidWalFrameError(124), 85 | NoSnapshotError(125), 86 | NoWalsegmentError(126), 87 | BadShadowWalError(127), 88 | 89 | // other error 90 | PanicError(140), 91 | UnexpectedEofError(141), 92 | } 93 | -------------------------------------------------------------------------------- /src/sqlite/wal_header.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::ErrorKind; 3 | use std::io::Read; 4 | 5 | use super::from_be_bytes_at; 6 | use crate::base::is_power_of_two; 7 | use crate::error::Error; 8 | use crate::error::Result; 9 | use crate::sqlite::WAL_HEADER_BIG_ENDIAN_MAGIC; 10 | use crate::sqlite::WAL_HEADER_LITTLE_ENDIAN_MAGIC; 11 | use crate::sqlite::WAL_HEADER_SIZE; 12 | use crate::sqlite::checksum; 13 | 14 | #[derive(Clone, Debug, PartialEq)] 15 | pub struct WALHeader { 16 | pub data: Vec, 17 | pub salt1: u32, 18 | pub salt2: u32, 19 | pub page_size: u64, 20 | pub is_big_endian: bool, 21 | } 22 | 23 | impl WALHeader { 24 | // see: https://www.sqlite.org/fileformat2.html#walformat 25 | pub fn read_from(reader: &mut R) -> Result { 26 | let mut data: Vec = vec![0u8; WAL_HEADER_SIZE as usize]; 27 | if let Err(e) = reader.read_exact(&mut data) { 28 | if e.kind() == ErrorKind::UnexpectedEof { 29 | return Err(Error::SqliteInvalidWalHeaderError( 30 | "Invalid WAL frame header", 31 | )); 32 | } 33 | 34 | return Err(e.into()); 35 | } 36 | 37 | let magic: &[u8] = &data[0..4]; 38 | // check magic 39 | let is_big_endian = if magic == WAL_HEADER_BIG_ENDIAN_MAGIC { 40 | true 41 | } else if magic == WAL_HEADER_LITTLE_ENDIAN_MAGIC { 42 | false 43 | } else { 44 | return Err(Error::SqliteInvalidWalHeaderError( 45 | "Unknown WAL file header magic", 46 | )); 47 | }; 48 | 49 | // check page size 50 | let page_size = from_be_bytes_at(&data, 8)? as u64; 51 | if !is_power_of_two(page_size) || page_size < 1024 { 52 | return Err(Error::SqliteInvalidWalHeaderError("Invalid page size")); 53 | } 54 | 55 | // checksum 56 | let (s1, s2) = checksum(&data[0..24], 0, 0, is_big_endian); 57 | let checksum1 = from_be_bytes_at(&data, 24)?; 58 | let checksum2 = from_be_bytes_at(&data, 28)?; 59 | if checksum1 != s1 || checksum2 != s2 { 60 | return Err(Error::SqliteInvalidWalHeaderError( 61 | "Invalid wal header checksum", 62 | )); 63 | } 64 | 65 | let salt1 = from_be_bytes_at(&data, 16)?; 66 | let salt2 = from_be_bytes_at(&data, 20)?; 67 | 68 | Ok(WALHeader { 69 | data, 70 | salt1, 71 | salt2, 72 | page_size, 73 | is_big_endian, 74 | }) 75 | } 76 | 77 | pub fn read(file_path: &str) -> Result { 78 | let mut file = File::open(file_path)?; 79 | 80 | Self::read_from(&mut file) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/sync/shadow_wal_reader.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::fs::OpenOptions; 3 | use std::io; 4 | use std::io::Read; 5 | use std::io::Seek; 6 | use std::io::SeekFrom; 7 | use std::os::unix::fs::MetadataExt; 8 | 9 | use crate::base::shadow_wal_file; 10 | use crate::database::DatabaseInfo; 11 | use crate::database::WalGenerationPos; 12 | use crate::error::Error; 13 | use crate::error::Result; 14 | use crate::sqlite::align_frame; 15 | 16 | pub struct ShadowWalReader { 17 | pub position: WalGenerationPos, 18 | pub file: File, 19 | pub left: u64, 20 | } 21 | 22 | impl ShadowWalReader { 23 | // ShadowWALReader opens a reader for a shadow WAL file at a given position. 24 | // If the reader is at the end of the file, it attempts to return the next file. 25 | // 26 | // The caller should check Pos() & Size() on the returned reader to check offset. 27 | pub fn try_create(pos: WalGenerationPos, info: &DatabaseInfo) -> Result { 28 | let reader = ShadowWalReader::new(pos.clone(), info)?; 29 | if reader.left > 0 { 30 | return Ok(reader); 31 | } 32 | 33 | // no data, try next 34 | let mut pos = pos; 35 | pos.index += 1; 36 | pos.offset = 0; 37 | 38 | match ShadowWalReader::new(pos, info) { 39 | Err(e) => { 40 | if e.code() == Error::STORAGE_NOT_FOUND { 41 | return Err(Error::from_error_code( 42 | Error::UNEXPECTED_EOF_ERROR, 43 | "no wal shadow file".to_string(), 44 | )); 45 | } 46 | Err(e) 47 | } 48 | Ok(reader) => Ok(reader), 49 | } 50 | } 51 | 52 | fn new(pos: WalGenerationPos, info: &DatabaseInfo) -> Result { 53 | let file_name = shadow_wal_file(&info.meta_dir, pos.generation.as_str(), pos.index); 54 | let mut file = OpenOptions::new().read(true).open(file_name)?; 55 | let size = align_frame(info.page_size, file.metadata()?.size()); 56 | 57 | if pos.offset > size { 58 | return Err(Error::WalReaderOffsetTooHighError(format!( 59 | "wal reader offset {} > file size {}", 60 | pos.offset, size 61 | ))); 62 | } 63 | 64 | // Move file handle to offset position. 65 | file.seek(SeekFrom::Start(pos.offset))?; 66 | let left = size - pos.offset; 67 | Ok(ShadowWalReader { 68 | position: pos, 69 | file, 70 | left, 71 | }) 72 | } 73 | 74 | pub fn position(&self) -> WalGenerationPos { 75 | self.position.clone() 76 | } 77 | } 78 | 79 | impl Read for ShadowWalReader { 80 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 81 | if self.left == 0 { 82 | // nothing to read 83 | return Ok(0); 84 | } 85 | let n = buf.len() as u64; 86 | if self.left < n { 87 | return Err(std::io::Error::from(io::ErrorKind::Interrupted)); 88 | } 89 | 90 | let ret = self.file.read(buf)?; 91 | self.left -= n; 92 | self.position.offset += n; 93 | 94 | Ok(ret) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /config.md: -------------------------------------------------------------------------------- 1 | 2 | - [Overview](#overview) 3 | - [Log config](#log-config) 4 | - [Database config](#database-config) 5 | - [Replicate Config](#replicate-config) 6 | - [Azure blob Params](#azure-blob-params) 7 | - [File System Params](#file-system-params) 8 | - [Ftp Params](#ftp-params) 9 | - [Gcs Params](#gcs-params) 10 | - [S3 Params](#s3-params) 11 | 12 | 13 | 14 | ## Overview 15 | 16 | replited use `toml` as its config file format, the structure of config is: 17 | 18 | * Log config; 19 | * One or more database configs: 20 | * sqlite database file path; 21 | * one or more database replicate backend. 22 | 23 | See config sample in [sample.toml](./etc/sample.toml) 24 | 25 | ## Log Config 26 | 27 | | item | value | 28 | | :---- | ---- | 29 | | level | Trace/Debug/Info/Warn/Error | 30 | | dir | log files directory | 31 | 32 | ## Database Config 33 | | item | value | 34 | | :---- | ---- | 35 | | db | sqlite database file path | 36 | | replicate | one or more database replicate backend | 37 | 38 | ### Replicate Config 39 | | item | value | 40 | | :---- | ---- | 41 | | name | replicate backend config name, cannot duplicate | 42 | | params | params of backend, see below | 43 | 44 | #### Azure blob Params 45 | | item | value | 46 | | :---- | ---- | 47 | | params.type | "Azb" | 48 | | params.root | root of Azblob service backend. | 49 | | params.container | container name of Azblob service backend. | 50 | | params.endpoint | endpoint of Azblob service backend. | 51 | | params.account_name | account name of Azblob service backend. | 52 | | params.account_key | account key of Azblob service backend. | 53 | 54 | #### File System Params 55 | | item | value | 56 | | :---- | ---- | 57 | | params.type | "Fs" | 58 | | params.root | root directory of file system backend | 59 | 60 | #### Ftp Params 61 | | item | value | 62 | | :---- | ---- | 63 | | params.type | "Ftp" | 64 | | params.endpoint | Endpoint of this ftp, use "ftps://127.0.0.1" by default. | 65 | | params.root | root directory of file system backend, use "/" by default. | 66 | | params.user | user of ftp backend. | 67 | | params.password | password of ftp backend. | 68 | 69 | #### Gcs Params 70 | | item | value | 71 | | :---- | ---- | 72 | | params.type | "Gcs" | 73 | | params.endpoint | Endpoint of this backend, must be full uri, use "https://storage.googleapis.com" by default. | 74 | | params.root | Root URI of gcs operations. | 75 | | params.bucket | Bucket name of this backend. | 76 | | params.credential | Credentials string for GCS service OAuth2 authentication. | 77 | 78 | 79 | #### S3 Params 80 | | item | value | 81 | | :---- | ---- | 82 | | params.type | "S3" | 83 | | params.endpoint | Endpoint of this backend, must be full uri, use "https://s3.amazonaws.com" by default. | 84 | | params.region | Region represent the signing region of this endpoint.If `region` is empty, use env value `AWS_REGION` if it is set, or else use `us-east-1` by default. | 85 | | params.bucket | Bucket name of this backend. | 86 | | params.access_key_id | access_key_id of this backend. | 87 | | params.secret_access_key | secret_access_key of this backend. | 88 | | params.root | root of this backend. | 89 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # replited(Replicate SQLITE Daemon) 2 | 3 | [![GitHub stars](https://img.shields.io/github/stars/lichuang/replited?label=Stars&logo=github)](https://github.com/lichuang/replited) 4 | [![GitHub forks](https://img.shields.io/github/forks/lichuang/replited?label=Forks&logo=github)](https://github.com/lichuang/replited) 5 | 6 | 7 | - [Introduction](#introduction) 8 | - [Why replited](#why-replited) 9 | - [Support Backend](#support-backend) 10 | - [Quick Start](#quick-start) 11 | - [Config](#config) 12 | - [Sub commands](#sub-commands) 13 | - [Replicate](#replicate) 14 | - [Restore](#restore) 15 | 16 | 17 | ## Introduction 18 | 19 | Inspired by [Litestream](https://litestream.io/), with the power of [Rust](https://www.rust-lang.org/) and [OpenDAL](https://opendal.apache.org/), replited target to replicate sqlite to everywhere(file system,s3,ftp,google drive,dropbox,etc). 20 | 21 | ## Why replited 22 | * Using sqlite's [WAL](https://sqlite.org/wal.html) mechanism, instead of backing up full data every time, do incremental backup of data to reduce the amount of synchronised data; 23 | * Support for multiple types of storage backends,such as s3,gcs,ftp,local file system,etc. 24 | 25 | ## Support Backend 26 | 27 | | Type | Services | 28 | | -------------------------- | ------------------------------------------------------------ | 29 | | Standard Storage Protocols | ftp![CI](https://github.com/lichuang/replited/actions/workflows/ftp_integration_test.yml/badge.svg) | 30 | | Object Storage Services | [azblob] [gcs]
[s3]![CI](https://github.com/lichuang/replited/actions/workflows/s3_integration_test.yml/badge.svg) | 31 | | File Storage Services | fs![CI](https://github.com/lichuang/replited/actions/workflows/fs_integration_test.yml/badge.svg) | 32 | 33 | [azblob]: https://azure.microsoft.com/en-us/services/storage/blobs/ 34 | [gcs]: https://cloud.google.com/storage 35 | [s3]: https://aws.amazon.com/s3/ 36 | 37 | 38 | 39 | ## Quick Start 40 | 41 | Start a daemon to replicate sqlite: 42 | 43 | ```shell 44 | replited --config {config file} replicate 45 | ``` 46 | 47 | Restore sqlite from backend: 48 | 49 | ```shell 50 | replited --config {config file} restore --db {db in config file} --output {output sqlite db file path} 51 | ``` 52 | 53 | ## Config 54 | 55 | See [config.md](./config.md) 56 | 57 | 58 | ## Sub commands 59 | ### Replicate 60 | `repicate` sub command will run a background process to replicate db to replicates in config periodically, example: 61 | ``` 62 | replited --config ./etc/sample.toml replicate 63 | ``` 64 | 65 | ### Restore 66 | `restore` sub command will restore db from replicates in config, example: 67 | ``` 68 | replited --config ./etc/sample.toml restore --db /Users/codedump/local/sqlite/test.db --output ./test.db 69 | ``` 70 | 71 | command options: 72 | * `db`: which db will be restore from config 73 | * `output`: which path will restored db saved 74 | 75 | ## Stargazers over time 76 | [![Stargazers over time](https://starchart.cc/lichuang/replited.svg?variant=adaptive)](https://starchart.cc/lichuang/replited) 77 | 78 | ​ 79 | -------------------------------------------------------------------------------- /src/error/error_into.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Debug; 2 | use std::fmt::Display; 3 | use std::fmt::Formatter; 4 | use std::time::SystemTimeError; 5 | 6 | use super::capture; 7 | use crate::database::DbCommand; 8 | use crate::error::Error; 9 | use crate::sync::ReplicateCommand; 10 | 11 | #[derive(thiserror::Error)] 12 | enum OtherErrors { 13 | AnyHow { error: anyhow::Error }, 14 | } 15 | 16 | impl Display for OtherErrors { 17 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 18 | match self { 19 | OtherErrors::AnyHow { error } => write!(f, "{}", error), 20 | } 21 | } 22 | } 23 | 24 | impl Debug for OtherErrors { 25 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 26 | match self { 27 | OtherErrors::AnyHow { error } => write!(f, "{:?}", error), 28 | } 29 | } 30 | } 31 | 32 | impl From for Error { 33 | fn from(error: log::SetLoggerError) -> Self { 34 | Error::InitLoggerError(format!("Set logger error: {}", error)) 35 | } 36 | } 37 | 38 | impl From for Error { 39 | fn from(error: anyhow::Error) -> Self { 40 | Error::create( 41 | Error::INTERNAL, 42 | "anyhow", 43 | format!("{}, source: {:?}", error, error.source()), 44 | String::new(), 45 | Some(Box::new(OtherErrors::AnyHow { error })), 46 | capture(), 47 | ) 48 | } 49 | } 50 | 51 | impl From for Error { 52 | fn from(error: rusqlite::Error) -> Self { 53 | Error::SqliteError(format!("sqlite error: {}", error)) 54 | } 55 | } 56 | 57 | impl From for Error { 58 | fn from(error: std::io::Error) -> Self { 59 | use std::io::ErrorKind; 60 | 61 | let msg = format!("{} ({})", error.kind(), &error); 62 | 63 | match error.kind() { 64 | ErrorKind::NotFound => Error::StorageNotFound(msg), 65 | ErrorKind::PermissionDenied => Error::StoragePermissionDenied(msg), 66 | ErrorKind::UnexpectedEof => Error::UnexpectedEofError(msg), 67 | _ => Error::StorageOther(msg), 68 | } 69 | } 70 | } 71 | 72 | impl From for Error { 73 | fn from(e: std::array::TryFromSliceError) -> Error { 74 | Error::from_std_error(e) 75 | } 76 | } 77 | 78 | impl From for Error { 79 | fn from(e: std::num::ParseIntError) -> Error { 80 | Error::from_std_error(e) 81 | } 82 | } 83 | 84 | impl From for Error { 85 | fn from(e: opendal::Error) -> Error { 86 | Error::OpenDalError(format!("opendal error: {:?}", e.to_string())) 87 | } 88 | } 89 | 90 | impl From for Error { 91 | fn from(e: uuid::Error) -> Error { 92 | Error::UUIDError(format!("uuid error: {:?}", e.to_string())) 93 | } 94 | } 95 | 96 | impl From> for Error { 97 | fn from(e: tokio::sync::mpsc::error::SendError) -> Error { 98 | Error::TokioError(format!( 99 | "tokio send ReplicateCommand error: {:?}", 100 | e.to_string() 101 | )) 102 | } 103 | } 104 | 105 | impl From> for Error { 106 | fn from(e: tokio::sync::mpsc::error::SendError) -> Error { 107 | Error::TokioError(format!("tokio send DbCommand error: {:?}", e.to_string())) 108 | } 109 | } 110 | 111 | impl From for Error { 112 | fn from(e: tokio::sync::broadcast::error::RecvError) -> Error { 113 | Error::TokioError(format!("tokio broadcast recv error: {:?}", e.to_string())) 114 | } 115 | } 116 | 117 | impl From for Error { 118 | fn from(e: SystemTimeError) -> Error { 119 | Error::from_std_error(e) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/sqlite/common.rs: -------------------------------------------------------------------------------- 1 | use std::fs::OpenOptions; 2 | use std::os::unix::fs::FileExt; 3 | use std::os::unix::fs::MetadataExt; 4 | 5 | use crate::error::Error; 6 | use crate::error::Result; 7 | 8 | pub const WAL_FRAME_HEADER_SIZE: u64 = 24; 9 | pub const WAL_HEADER_SIZE: u64 = 32; 10 | 11 | static WAL_HEADER_CHECKSUM_OFFSET: u64 = 24; 12 | static WAL_FRAME_HEADER_CHECKSUM_OFFSET: u64 = 16; 13 | 14 | pub const WAL_HEADER_BIG_ENDIAN_MAGIC: [u8; 4] = [0x37, 0x7f, 0x06, 0x83]; 15 | pub const WAL_HEADER_LITTLE_ENDIAN_MAGIC: [u8; 4] = [0x37, 0x7f, 0x06, 0x82]; 16 | 17 | // SQLite checkpoint modes. 18 | static CHECKPOINT_MODE_PASSIVE: &str = "PASSIVE"; 19 | static CHECKPOINT_MODE_FULL: &str = "FULL"; 20 | static CHECKPOINT_MODE_RESTART: &str = "RESTART"; 21 | static CHECKPOINT_MODE_TRUNCATE: &str = "TRUNCATE"; 22 | 23 | #[derive(Clone)] 24 | pub enum CheckpointMode { 25 | Passive, 26 | Full, 27 | Restart, 28 | Truncate, 29 | } 30 | 31 | impl CheckpointMode { 32 | pub fn as_str(&self) -> &str { 33 | match self { 34 | CheckpointMode::Passive => CHECKPOINT_MODE_PASSIVE, 35 | CheckpointMode::Full => CHECKPOINT_MODE_FULL, 36 | CheckpointMode::Restart => CHECKPOINT_MODE_RESTART, 37 | CheckpointMode::Truncate => CHECKPOINT_MODE_TRUNCATE, 38 | } 39 | } 40 | } 41 | 42 | // implementation of sqlite check algorithm 43 | pub fn checksum(data: &[u8], s1: u32, s2: u32, is_big_endian: bool) -> (u32, u32) { 44 | let mut i = 0; 45 | let mut s1: u32 = s1; 46 | let mut s2: u32 = s2; 47 | while i < data.len() { 48 | let bytes1 = &data[i..i + 4]; 49 | let bytes2 = &data[i + 4..i + 8]; 50 | let (n1, n2) = if is_big_endian { 51 | ( 52 | u32::from_be_bytes(bytes1.try_into().unwrap()), 53 | u32::from_be_bytes(bytes2.try_into().unwrap()), 54 | ) 55 | } else { 56 | ( 57 | u32::from_le_bytes(bytes1.try_into().unwrap()), 58 | u32::from_le_bytes(bytes2.try_into().unwrap()), 59 | ) 60 | }; 61 | // use `wrapping_add` instead of `+` directly, or else will be overflow panic 62 | s1 = s1.wrapping_add(n1).wrapping_add(s2); 63 | s2 = s2.wrapping_add(n2).wrapping_add(s1); 64 | 65 | i += 8; 66 | } 67 | 68 | (s1, s2) 69 | } 70 | 71 | pub fn read_last_checksum(file_name: &str, page_size: u64) -> Result<(u32, u32)> { 72 | let file = OpenOptions::new().read(true).open(file_name)?; 73 | let metadata = file.metadata()?; 74 | let fsize = metadata.size(); 75 | let offset = if fsize > WAL_HEADER_SIZE { 76 | let sz = align_frame(page_size, fsize); 77 | sz - page_size - WAL_FRAME_HEADER_SIZE + WAL_FRAME_HEADER_CHECKSUM_OFFSET 78 | } else { 79 | WAL_HEADER_CHECKSUM_OFFSET 80 | }; 81 | 82 | let mut buf = [0u8; 8]; 83 | let n = file.read_at(&mut buf, offset)?; 84 | if n != buf.len() { 85 | return Err(Error::UnexpectedEofError( 86 | "UnexpectedEOFError when read last checksum".to_string(), 87 | )); 88 | } 89 | 90 | let checksum1 = from_be_bytes_at(&buf, 0)?; 91 | let checksum2 = from_be_bytes_at(&buf, 4)?; 92 | 93 | Ok((checksum1, checksum2)) 94 | } 95 | 96 | // returns a frame-aligned offset. 97 | // Returns zero if offset is less than the WAL header size. 98 | pub fn align_frame(page_size: u64, offset: u64) -> u64 { 99 | if offset < WAL_HEADER_SIZE { 100 | return 0; 101 | } 102 | 103 | let page_size = page_size as i64; 104 | let offset = offset as i64; 105 | let frame_size = WAL_FRAME_HEADER_SIZE as i64 + page_size; 106 | let frame_num = (offset - WAL_HEADER_SIZE as i64) / frame_size; 107 | 108 | (frame_num * frame_size) as u64 + WAL_HEADER_SIZE 109 | } 110 | 111 | pub(crate) fn from_be_bytes_at(data: &[u8], offset: usize) -> Result { 112 | let p = &data[offset..offset + 4]; 113 | Ok(u32::from_be_bytes(p.try_into()?)) 114 | } 115 | 116 | #[cfg(test)] 117 | mod tests { 118 | use super::align_frame; 119 | use crate::error::Result; 120 | 121 | #[test] 122 | fn test_align_frame() -> Result<()> { 123 | assert_eq!(4152, align_frame(4096, 4152)); 124 | 125 | Ok(()) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/error/error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Debug; 2 | use std::fmt::Display; 3 | use std::fmt::Formatter; 4 | use std::sync::Arc; 5 | 6 | use backtrace::Backtrace; 7 | 8 | use super::backtrace::capture; 9 | 10 | #[derive(Clone)] 11 | pub enum ErrorCodeBacktrace { 12 | Serialized(Arc), 13 | Symbols(Arc), 14 | Address(Arc), 15 | } 16 | 17 | #[derive(thiserror::Error)] 18 | pub struct Error { 19 | code: u32, 20 | name: String, 21 | display_text: String, 22 | detail: String, 23 | cause: Option>, 24 | backtrace: Option, 25 | } 26 | 27 | impl Error { 28 | pub fn code(&self) -> u32 { 29 | self.code 30 | } 31 | 32 | pub fn display_text(&self) -> String { 33 | if let Some(cause) = &self.cause { 34 | format!("{}\n{:?}", self.display_text, cause) 35 | } else { 36 | self.display_text.clone() 37 | } 38 | } 39 | 40 | pub fn message(&self) -> String { 41 | let msg = self.display_text(); 42 | if self.detail.is_empty() { 43 | msg 44 | } else { 45 | format!("{}\n{}", msg, self.detail) 46 | } 47 | } 48 | 49 | pub fn backtrace(&self) -> Option { 50 | self.backtrace.clone() 51 | } 52 | 53 | // output a string without backtrace 54 | pub fn simple_string(&self) -> String { 55 | format!( 56 | "{}. Code: {}, Text = {}.", 57 | self.name, 58 | self.code(), 59 | self.message(), 60 | ) 61 | } 62 | } 63 | 64 | pub type Result = std::result::Result; 65 | 66 | impl Debug for Error { 67 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 68 | write!( 69 | f, 70 | "{}. Code: {}, Text = {}.", 71 | self.name, 72 | self.code(), 73 | self.message(), 74 | )?; 75 | 76 | match self.backtrace.as_ref() { 77 | None => write!( 78 | f, 79 | "\n\n " 80 | ), 81 | Some(backtrace) => match backtrace { 82 | ErrorCodeBacktrace::Symbols(backtrace) => write!(f, "\n\n{:?}", backtrace), 83 | ErrorCodeBacktrace::Serialized(backtrace) => write!(f, "\n\n{}", backtrace), 84 | ErrorCodeBacktrace::Address(backtrace) => { 85 | let frames_address = backtrace 86 | .frames() 87 | .iter() 88 | .map(|f| (f.ip() as usize, f.symbol_address() as usize)) 89 | .collect::>(); 90 | write!(f, "\n\n{:?}", frames_address) 91 | } 92 | }, 93 | } 94 | } 95 | } 96 | 97 | impl Display for Error { 98 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 99 | write!( 100 | f, 101 | "{}. Code: {}, Text = {}.", 102 | self.name, 103 | self.code(), 104 | self.message(), 105 | ) 106 | } 107 | } 108 | 109 | impl Error { 110 | /// All std error will be converted to InternalError 111 | pub fn from_std_error(error: T) -> Self { 112 | Error { 113 | code: Error::INTERNAL, 114 | name: String::from("FromStdError"), 115 | display_text: error.to_string(), 116 | detail: String::new(), 117 | cause: None, 118 | backtrace: capture(), 119 | } 120 | } 121 | 122 | pub fn from_error_code(code: u32, display_text: impl ToString) -> Self { 123 | Error { 124 | code, 125 | name: String::new(), 126 | display_text: display_text.to_string(), 127 | detail: String::new(), 128 | cause: None, 129 | backtrace: capture(), 130 | } 131 | } 132 | 133 | pub fn from_string(error: String) -> Self { 134 | Error { 135 | code: Error::INTERNAL, 136 | name: String::from("Internal"), 137 | display_text: error, 138 | detail: String::new(), 139 | cause: None, 140 | backtrace: capture(), 141 | } 142 | } 143 | 144 | pub fn from_string_no_backtrace(error: String) -> Self { 145 | Error { 146 | code: Error::INTERNAL, 147 | name: String::from("Internal"), 148 | display_text: error, 149 | detail: String::new(), 150 | cause: None, 151 | backtrace: capture(), 152 | } 153 | } 154 | 155 | pub fn create( 156 | code: u32, 157 | name: impl ToString, 158 | display_text: String, 159 | detail: String, 160 | cause: Option>, 161 | backtrace: Option, 162 | ) -> Error { 163 | Error { 164 | code, 165 | display_text, 166 | detail, 167 | cause, 168 | name: name.to_string(), 169 | backtrace, 170 | } 171 | } 172 | } 173 | 174 | impl Clone for Error { 175 | fn clone(&self) -> Self { 176 | Error::create( 177 | self.code(), 178 | &self.name, 179 | self.display_text(), 180 | self.detail.clone(), 181 | None, 182 | self.backtrace(), 183 | ) 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/storage/operator.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::sync::Arc; 3 | use std::sync::LazyLock; 4 | use std::time::Duration; 5 | 6 | use log::warn; 7 | use opendal::Builder; 8 | use opendal::Operator; 9 | use opendal::raw::HttpClient; 10 | use opendal::services; 11 | use reqwest_hickory_resolver::HickoryResolver; 12 | 13 | use crate::config::StorageAzblobConfig; 14 | use crate::config::StorageFsConfig; 15 | use crate::config::StorageFtpConfig; 16 | use crate::config::StorageGcsConfig; 17 | use crate::config::StorageParams; 18 | use crate::config::StorageS3Config; 19 | use crate::error::Result; 20 | 21 | /// The global dns resolver for opendal. 22 | static GLOBAL_HICKORY_RESOLVER: LazyLock> = 23 | LazyLock::new(|| Arc::new(HickoryResolver::default())); 24 | 25 | pub fn init_operator(cfg: &StorageParams) -> Result { 26 | let op = match cfg { 27 | StorageParams::Azb(cfg) => build_operator(init_azblob_operator(cfg)?)?, 28 | StorageParams::Fs(cfg) => build_operator(init_fs_operator(cfg)?)?, 29 | StorageParams::Ftp(cfg) => build_operator(init_ftp_operator(cfg)?)?, 30 | StorageParams::Gcs(cfg) => build_operator(init_gcs_operator(cfg)?)?, 31 | StorageParams::S3(cfg) => build_operator(init_s3_operator(cfg)?)?, 32 | }; 33 | 34 | Ok(op) 35 | } 36 | 37 | pub fn build_operator(builder: B) -> Result { 38 | let op = Operator::new(builder)?; 39 | 40 | Ok(op.finish()) 41 | } 42 | 43 | /// init_azblob_operator will init an opendal azblob operator. 44 | pub fn init_azblob_operator(cfg: &StorageAzblobConfig) -> Result { 45 | let builder = services::Azblob::default() 46 | // Endpoint 47 | .endpoint(&cfg.endpoint) 48 | // Container 49 | .container(&cfg.container) 50 | // Root 51 | .root(&cfg.root) 52 | // Credential 53 | .account_name(&cfg.account_name) 54 | .account_key(&cfg.account_key) 55 | .http_client(new_storage_http_client()?); 56 | 57 | Ok(builder) 58 | } 59 | 60 | /// init_gcs_operator will init a opendal gcs operator. 61 | fn init_gcs_operator(cfg: &StorageGcsConfig) -> Result { 62 | let builder = services::Gcs::default() 63 | .endpoint(&cfg.endpoint) 64 | .bucket(&cfg.bucket) 65 | .root(&cfg.root) 66 | .credential(&cfg.credential) 67 | .http_client(new_storage_http_client()?); 68 | 69 | Ok(builder) 70 | } 71 | 72 | /// init_fs_operator will init a opendal fs operator. 73 | fn init_fs_operator(cfg: &StorageFsConfig) -> Result { 74 | let mut builder = services::Fs::default(); 75 | 76 | let mut path = cfg.root.clone(); 77 | if !path.starts_with('/') { 78 | path = env::current_dir().unwrap().join(path).display().to_string(); 79 | } 80 | builder = builder.root(&path); 81 | 82 | Ok(builder) 83 | } 84 | 85 | /// init_ftp_operator will init a opendal ftp operator. 86 | fn init_ftp_operator(cfg: &StorageFtpConfig) -> Result { 87 | let builder = services::Ftp::default() 88 | .endpoint(&cfg.endpoint) 89 | .root(&cfg.root) 90 | .user(&cfg.username) 91 | .password(&cfg.password); 92 | 93 | Ok(builder) 94 | } 95 | 96 | /// Create a new http client for storage. 97 | fn new_storage_http_client() -> Result { 98 | let mut builder = reqwest::ClientBuilder::new(); 99 | 100 | // Disable http2 for better performance. 101 | builder = builder.http1_only(); 102 | 103 | // Set dns resolver. 104 | builder = builder.dns_resolver(GLOBAL_HICKORY_RESOLVER.clone()); 105 | 106 | // Pool max idle per host controls connection pool size. 107 | // Default to no limit, set to `0` for disable it. 108 | let pool_max_idle_per_host = env::var("_LITESYNC_INTERNAL_POOL_MAX_IDLE_PER_HOST") 109 | .ok() 110 | .and_then(|v| v.parse::().ok()) 111 | .unwrap_or(usize::MAX); 112 | builder = builder.pool_max_idle_per_host(pool_max_idle_per_host); 113 | 114 | // Connect timeout default to 30s. 115 | let connect_timeout = env::var("_LITESYNC_INTERNAL_CONNECT_TIMEOUT") 116 | .ok() 117 | .and_then(|v| v.parse::().ok()) 118 | .unwrap_or(30); 119 | builder = builder.connect_timeout(Duration::from_secs(connect_timeout)); 120 | 121 | // Enable TCP keepalive if set. 122 | if let Ok(v) = env::var("_LITESYNC_INTERNAL_TCP_KEEPALIVE") 123 | && let Ok(v) = v.parse::() { 124 | builder = builder.tcp_keepalive(Duration::from_secs(v)); 125 | } 126 | 127 | Ok(HttpClient::build(builder)?) 128 | } 129 | 130 | /// init_s3_operator will init a opendal s3 operator with input s3 config. 131 | fn init_s3_operator(cfg: &StorageS3Config) -> Result { 132 | let mut builder = services::S3::default() 133 | // Endpoint. 134 | .endpoint(&cfg.endpoint) 135 | // Bucket. 136 | .bucket(&cfg.bucket); 137 | 138 | // Region 139 | if !cfg.region.is_empty() { 140 | builder = builder.region(&cfg.region); 141 | } else if let Ok(region) = env::var("AWS_REGION") { 142 | // Try to load region from env if not set. 143 | builder = builder.region(®ion); 144 | } else { 145 | // FIXME: we should return error here but keep those logic for compatibility. 146 | warn!( 147 | "Region is not specified for S3 storage, we will attempt to load it from profiles. If it is still not found, we will use the default region of `us-east-1`." 148 | ); 149 | builder = builder.region("us-east-1"); 150 | } 151 | 152 | // Credential. 153 | builder = builder 154 | .access_key_id(&cfg.access_key_id) 155 | .secret_access_key(&cfg.secret_access_key) 156 | // It's safe to allow anonymous since opendal will perform the check first. 157 | .allow_anonymous() 158 | // Root. 159 | .root(&cfg.root); 160 | 161 | // Disable credential loader 162 | builder = builder.disable_config_load().disable_ec2_metadata(); 163 | 164 | builder = builder.http_client(new_storage_http_client()?); 165 | 166 | Ok(builder) 167 | } 168 | -------------------------------------------------------------------------------- /src/sync/restore.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | use std::fs::OpenOptions; 3 | use std::io::Write; 4 | 5 | use log::debug; 6 | use log::error; 7 | use rusqlite::Connection; 8 | use tempfile::NamedTempFile; 9 | 10 | use crate::base::decompressed_data; 11 | use crate::base::parent_dir; 12 | use crate::config::DbConfig; 13 | use crate::config::RestoreOptions; 14 | use crate::config::StorageConfig; 15 | use crate::error::Error; 16 | use crate::error::Result; 17 | use crate::storage::RestoreInfo; 18 | use crate::storage::RestoreWalSegments; 19 | use crate::storage::SnapshotInfo; 20 | use crate::storage::StorageClient; 21 | use crate::storage::WalSegmentInfo; 22 | 23 | static WAL_CHECKPOINT_TRUNCATE: &str = "PRAGMA wal_checkpoint(TRUNCATE);"; 24 | 25 | struct Restore { 26 | db: String, 27 | config: Vec, 28 | options: RestoreOptions, 29 | } 30 | 31 | impl Restore { 32 | pub fn try_create( 33 | db: String, 34 | config: Vec, 35 | options: RestoreOptions, 36 | ) -> Result { 37 | Ok(Self { 38 | db, 39 | config, 40 | options, 41 | }) 42 | } 43 | 44 | pub async fn decide_restore_info(&self) -> Result> { 45 | let mut latest_restore_info: Option<(RestoreInfo, StorageClient)> = None; 46 | 47 | for config in &self.config { 48 | let client = StorageClient::try_create(self.db.clone(), config.clone())?; 49 | let restore_info = match client.restore_info().await? { 50 | Some(snapshot_into) => snapshot_into, 51 | None => continue, 52 | }; 53 | match &latest_restore_info { 54 | Some(ls) => { 55 | if restore_info.snapshot.generation > ls.0.snapshot.generation { 56 | latest_restore_info = Some((restore_info, client)); 57 | } 58 | } 59 | None => { 60 | latest_restore_info = Some((restore_info, client)); 61 | } 62 | } 63 | } 64 | 65 | Ok(latest_restore_info) 66 | } 67 | 68 | async fn restore_snapshot( 69 | &self, 70 | client: &StorageClient, 71 | snapshot: &SnapshotInfo, 72 | path: &str, 73 | ) -> Result<()> { 74 | let compressed_data = client.read_snapshot(snapshot).await?; 75 | let decompressed_data = decompressed_data(compressed_data)?; 76 | 77 | let mut file = OpenOptions::new() 78 | .write(true) 79 | .create(true) 80 | .truncate(true) 81 | .open(path)?; 82 | 83 | file.write_all(&decompressed_data)?; 84 | 85 | Ok(()) 86 | } 87 | 88 | async fn apply_wal_frames( 89 | &self, 90 | client: &StorageClient, 91 | snapshot: &SnapshotInfo, 92 | wal_segments: &RestoreWalSegments, 93 | db_path: &str, 94 | ) -> Result<()> { 95 | debug!( 96 | "restore db {} apply wal segments: {:?}", 97 | self.db, wal_segments 98 | ); 99 | let wal_file_name = format!("{}-wal", db_path); 100 | 101 | for (index, offsets) in wal_segments { 102 | let mut wal_decompressed_data = Vec::new(); 103 | for offset in offsets { 104 | let wal_segment = WalSegmentInfo { 105 | generation: snapshot.generation.clone(), 106 | index: *index, 107 | offset: *offset, 108 | size: 0, 109 | }; 110 | 111 | let compressed_data = client.read_wal_segment(&wal_segment).await?; 112 | let data = decompressed_data(compressed_data)?; 113 | wal_decompressed_data.extend_from_slice(&data); 114 | } 115 | 116 | // prepare db wal before open db connection 117 | let mut wal_file = OpenOptions::new() 118 | .write(true) 119 | .create(true) 120 | .truncate(true) 121 | .open(&wal_file_name)?; 122 | 123 | wal_file.write_all(&wal_decompressed_data)?; 124 | wal_file.flush()?; 125 | 126 | let connection = Connection::open(db_path)?; 127 | 128 | if let Err(e) = connection.query_row(WAL_CHECKPOINT_TRUNCATE, [], |_row| Ok(())) { 129 | error!( 130 | "truncation checkpoint failed during restore {}:{:?}", 131 | index, offsets 132 | ); 133 | return Err(e.into()); 134 | } 135 | 136 | // connection.close().unwrap(); 137 | } 138 | 139 | Ok(()) 140 | } 141 | 142 | pub async fn run(&self) -> Result<()> { 143 | // Ensure output path does not already exist. 144 | if fs::exists(&self.options.output)? { 145 | println!("db {} already exists but cannot overwrite", self.db); 146 | return Err(Error::OverwriteDbError("cannot overwrite exist db")); 147 | } 148 | 149 | let (latest_restore_info, client) = match self.decide_restore_info().await? { 150 | Some(latest_restore_info) => latest_restore_info, 151 | None => { 152 | debug!("cannot find snapshot"); 153 | return Ok(()); 154 | } 155 | }; 156 | 157 | // create a temp file to write snapshot 158 | let temp_file = NamedTempFile::new()?; 159 | let temp_file_name = temp_file.path().to_str().unwrap().to_string(); 160 | 161 | let dir = parent_dir(&temp_file_name).unwrap(); 162 | fs::create_dir_all(&dir)?; 163 | 164 | // restore snapshot 165 | self.restore_snapshot(&client, &latest_restore_info.snapshot, &temp_file_name) 166 | .await?; 167 | 168 | // apply wal frames 169 | self.apply_wal_frames( 170 | &client, 171 | &latest_restore_info.snapshot, 172 | &latest_restore_info.wal_segments, 173 | &temp_file_name, 174 | ) 175 | .await?; 176 | 177 | // rename the temp file to output file 178 | fs::rename(&temp_file_name, &self.options.output)?; 179 | 180 | println!( 181 | "restore db {} to {} success", 182 | self.options.db, self.options.output 183 | ); 184 | 185 | Ok(()) 186 | } 187 | } 188 | 189 | pub async fn run_restore(config: &DbConfig, options: &RestoreOptions) -> Result<()> { 190 | let restore = 191 | Restore::try_create(config.db.clone(), config.replicate.clone(), options.clone())?; 192 | 193 | restore.run().await?; 194 | 195 | Ok(()) 196 | } 197 | -------------------------------------------------------------------------------- /src/config/storage_params.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Debug; 2 | use std::fmt::Display; 3 | use std::fmt::Formatter; 4 | 5 | use serde::Deserialize; 6 | use serde::Serialize; 7 | 8 | use crate::base::mask_string; 9 | 10 | /// Storage params which contains the detailed storage info. 11 | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] 12 | #[serde(tag = "type")] 13 | pub enum StorageParams { 14 | Azb(Box), 15 | Fs(Box), 16 | Ftp(Box), 17 | Gcs(Box), 18 | S3(Box), 19 | } 20 | 21 | impl StorageParams { 22 | pub fn root(&self) -> String { 23 | match self { 24 | StorageParams::Azb(s) => s.root.clone(), 25 | StorageParams::Fs(s) => s.root.clone(), 26 | StorageParams::Ftp(s) => s.root.clone(), 27 | StorageParams::Gcs(s) => s.root.clone(), 28 | StorageParams::S3(s) => s.root.clone(), 29 | } 30 | } 31 | } 32 | 33 | /// StorageParams will be displayed by `{protocol}://{key1=value1},{key2=value2}` 34 | impl Display for StorageParams { 35 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 36 | match self { 37 | StorageParams::Azb(v) => write!( 38 | f, 39 | "azblob | container={},root={},endpoint={}", 40 | v.container, v.root, v.endpoint 41 | ), 42 | StorageParams::Fs(v) => write!(f, "fs | root={}", v.root), 43 | StorageParams::Ftp(v) => { 44 | write!(f, "ftp | root={},endpoint={}", v.root, v.endpoint) 45 | } 46 | StorageParams::Gcs(v) => write!( 47 | f, 48 | "gcs | bucket={},root={},endpoint={}", 49 | v.bucket, v.root, v.endpoint 50 | ), 51 | StorageParams::S3(v) => { 52 | write!( 53 | f, 54 | "s3 | bucket={},root={},endpoint={}", 55 | v.bucket, v.root, v.endpoint 56 | ) 57 | } 58 | } 59 | } 60 | } 61 | 62 | /// Config for storage backend azblob. 63 | #[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] 64 | pub struct StorageAzblobConfig { 65 | pub endpoint: String, 66 | pub container: String, 67 | pub account_name: String, 68 | pub account_key: String, 69 | pub root: String, 70 | } 71 | 72 | impl Debug for StorageAzblobConfig { 73 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 74 | f.debug_struct("StorageAzblobConfig") 75 | .field("endpoint", &self.endpoint) 76 | .field("container", &self.container) 77 | .field("root", &self.root) 78 | .field("account_name", &self.account_name) 79 | .field("account_key", &mask_string(&self.account_key, 3)) 80 | .finish() 81 | } 82 | } 83 | 84 | /// Config for FTP and FTPS data source 85 | pub const STORAGE_FTP_DEFAULT_ENDPOINT: &str = "ftps://127.0.0.1"; 86 | #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] 87 | pub struct StorageFtpConfig { 88 | pub endpoint: String, 89 | pub root: String, 90 | pub username: String, 91 | pub password: String, 92 | } 93 | 94 | impl Default for StorageFtpConfig { 95 | fn default() -> Self { 96 | Self { 97 | endpoint: STORAGE_FTP_DEFAULT_ENDPOINT.to_string(), 98 | username: "".to_string(), 99 | password: "".to_string(), 100 | root: "/".to_string(), 101 | } 102 | } 103 | } 104 | 105 | impl Debug for StorageFtpConfig { 106 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 107 | f.debug_struct("StorageFtpConfig") 108 | .field("endpoint", &self.endpoint) 109 | .field("root", &self.root) 110 | .field("username", &self.username) 111 | .field("password", &mask_string(self.password.as_str(), 3)) 112 | .finish() 113 | } 114 | } 115 | 116 | /// Config for storage backend fs. 117 | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] 118 | pub struct StorageFsConfig { 119 | pub root: String, 120 | } 121 | 122 | impl Default for StorageFsConfig { 123 | fn default() -> Self { 124 | Self { 125 | root: "_data".to_string(), 126 | } 127 | } 128 | } 129 | 130 | /// Config for storage backend GCS. 131 | pub static STORAGE_GCS_DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com"; 132 | 133 | #[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] 134 | pub struct StorageGcsConfig { 135 | pub endpoint: String, 136 | pub bucket: String, 137 | pub root: String, 138 | pub credential: String, 139 | } 140 | 141 | impl Default for StorageGcsConfig { 142 | fn default() -> Self { 143 | Self { 144 | endpoint: STORAGE_GCS_DEFAULT_ENDPOINT.to_string(), 145 | bucket: String::new(), 146 | root: String::new(), 147 | credential: String::new(), 148 | } 149 | } 150 | } 151 | 152 | impl Debug for StorageGcsConfig { 153 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 154 | f.debug_struct("StorageGcsConfig") 155 | .field("endpoint", &self.endpoint) 156 | .field("bucket", &self.bucket) 157 | .field("root", &self.root) 158 | .field("credential", &mask_string(&self.credential, 3)) 159 | .finish() 160 | } 161 | } 162 | 163 | /// Config for storage backend s3. 164 | pub static STORAGE_S3_DEFAULT_ENDPOINT: &str = "https://s3.amazonaws.com"; 165 | 166 | #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] 167 | pub struct StorageS3Config { 168 | pub endpoint: String, 169 | pub region: String, 170 | pub bucket: String, 171 | pub access_key_id: String, 172 | pub secret_access_key: String, 173 | 174 | pub root: String, 175 | } 176 | 177 | impl Default for StorageS3Config { 178 | fn default() -> Self { 179 | StorageS3Config { 180 | endpoint: STORAGE_S3_DEFAULT_ENDPOINT.to_string(), 181 | region: "".to_string(), 182 | bucket: "".to_string(), 183 | access_key_id: "".to_string(), 184 | secret_access_key: "".to_string(), 185 | root: "".to_string(), 186 | } 187 | } 188 | } 189 | 190 | impl Debug for StorageS3Config { 191 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 192 | f.debug_struct("StorageS3Config") 193 | .field("endpoint", &self.endpoint) 194 | .field("region", &self.region) 195 | .field("bucket", &self.bucket) 196 | .field("root", &self.root) 197 | .field("access_key_id", &mask_string(&self.access_key_id, 3)) 198 | .field( 199 | "secret_access_key", 200 | &mask_string(&self.secret_access_key, 3), 201 | ) 202 | .finish() 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /src/config/config.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::fmt::Debug; 3 | use std::fmt::Display; 4 | use std::fmt::Formatter; 5 | use std::fs; 6 | 7 | use serde::Deserialize; 8 | 9 | use super::StorageParams; 10 | use crate::error::Error; 11 | use crate::error::Result; 12 | 13 | const DEFAULT_MIN_CHECKPOINT_PAGE_NUMBER: u64 = 1000; 14 | const DEFAULT_MAX_CHECKPOINT_PAGE_NUMBER: u64 = 10000; 15 | const DEFAULT_TRUNCATE_PAGE_NUMBER: u64 = 500000; 16 | const DEFAULT_CHECKPOINT_INTERVAL_SECS: u64 = 60; 17 | 18 | #[derive(Clone, PartialEq, Eq, Deserialize)] 19 | pub struct Config { 20 | pub log: LogConfig, 21 | 22 | pub database: Vec, 23 | } 24 | 25 | impl Config { 26 | pub fn load(config_file: &str) -> Result { 27 | let toml_str = match fs::read_to_string(config_file) { 28 | Ok(toml_str) => toml_str, 29 | Err(e) => { 30 | return Err(Error::ReadConfigFail(format!( 31 | "read config file {} fail: {:?}", 32 | config_file, e, 33 | ))); 34 | } 35 | }; 36 | 37 | let config: Config = match toml::from_str(&toml_str) { 38 | Ok(config) => config, 39 | Err(e) => { 40 | return Err(Error::ParseConfigFail(format!( 41 | "parse config file {} fail: {:?}", 42 | config_file, e, 43 | ))); 44 | } 45 | }; 46 | 47 | config.validate()?; 48 | Ok(config) 49 | } 50 | 51 | fn validate(&self) -> Result<()> { 52 | if self.database.is_empty() { 53 | return Err(Error::InvalidConfig( 54 | "config MUST has at least one database config", 55 | )); 56 | } 57 | for db in &self.database { 58 | db.validate()?; 59 | } 60 | Ok(()) 61 | } 62 | } 63 | 64 | /// Config for logging. 65 | #[derive(Clone, Debug, PartialEq, Eq, Deserialize)] 66 | pub struct LogConfig { 67 | pub level: LogLevel, 68 | pub dir: String, 69 | } 70 | 71 | #[derive(Clone, Debug, PartialEq, Eq, Deserialize)] 72 | pub enum LogLevel { 73 | Off, 74 | Error, 75 | Warn, 76 | Info, 77 | Debug, 78 | Trace, 79 | } 80 | 81 | impl Default for LogConfig { 82 | fn default() -> Self { 83 | Self { 84 | level: LogLevel::Info, 85 | dir: "/var/log/replited".to_string(), 86 | } 87 | } 88 | } 89 | 90 | impl Display for LogConfig { 91 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 92 | write!(f, "level={:?}, dir={}", self.level, self.dir) 93 | } 94 | } 95 | 96 | #[derive(Clone, PartialEq, Eq, Deserialize)] 97 | pub struct DbConfig { 98 | // db file full path 99 | pub db: String, 100 | 101 | // replicates of db file config 102 | pub replicate: Vec, 103 | 104 | // Minimum threshold of WAL size, in pages, before a passive checkpoint. 105 | // A passive checkpoint will attempt a checkpoint but fail if there are 106 | // active transactions occurring at the same time. 107 | #[serde(default = "default_min_checkpoint_page_number")] 108 | pub min_checkpoint_page_number: u64, 109 | 110 | // Maximum threshold of WAL size, in pages, before a forced checkpoint. 111 | // A forced checkpoint will block new transactions and wait for existing 112 | // transactions to finish before issuing a checkpoint and resetting the WAL. 113 | // 114 | // If zero, no checkpoints are forced. This can cause the WAL to grow 115 | // unbounded if there are always read transactions occurring. 116 | #[serde(default = "default_max_checkpoint_page_number")] 117 | pub max_checkpoint_page_number: u64, 118 | 119 | // Threshold of WAL size, in pages, before a forced truncation checkpoint. 120 | // A forced truncation checkpoint will block new transactions and wait for 121 | // existing transactions to finish before issuing a checkpoint and 122 | // truncating the WAL. 123 | // 124 | // If zero, no truncates are forced. This can cause the WAL to grow 125 | // unbounded if there's a sudden spike of changes between other 126 | // checkpoints. 127 | #[serde(default = "default_truncate_page_number")] 128 | pub truncate_page_number: u64, 129 | 130 | // Seconds between automatic checkpoints in the WAL. This is done to allow 131 | // more fine-grained WAL files so that restores can be performed with 132 | // better precision. 133 | #[serde(default = "default_checkpoint_interval_secs")] 134 | pub checkpoint_interval_secs: u64, 135 | } 136 | 137 | fn default_min_checkpoint_page_number() -> u64 { 138 | DEFAULT_MIN_CHECKPOINT_PAGE_NUMBER 139 | } 140 | 141 | fn default_max_checkpoint_page_number() -> u64 { 142 | DEFAULT_MAX_CHECKPOINT_PAGE_NUMBER 143 | } 144 | 145 | fn default_truncate_page_number() -> u64 { 146 | DEFAULT_TRUNCATE_PAGE_NUMBER 147 | } 148 | 149 | fn default_checkpoint_interval_secs() -> u64 { 150 | DEFAULT_CHECKPOINT_INTERVAL_SECS 151 | } 152 | 153 | impl Debug for DbConfig { 154 | fn fmt(&self, f: &mut Formatter) -> fmt::Result { 155 | f.debug_struct("ReplicateDbConfig") 156 | .field("db", &self.db) 157 | .field("storage", &self.replicate) 158 | .field( 159 | "min_checkpoint_page_number", 160 | &self.min_checkpoint_page_number, 161 | ) 162 | .field( 163 | "max_checkpoint_page_number", 164 | &self.max_checkpoint_page_number, 165 | ) 166 | .field("truncate_page_number", &self.truncate_page_number) 167 | .field("checkpoint_interval_secs", &self.checkpoint_interval_secs) 168 | .finish() 169 | } 170 | } 171 | 172 | impl DbConfig { 173 | fn validate(&self) -> Result<()> { 174 | if self.replicate.is_empty() { 175 | return Err(Error::InvalidConfig( 176 | "database MUST has at least one replicate config", 177 | )); 178 | } 179 | 180 | if self.min_checkpoint_page_number == 0 { 181 | return Err(Error::InvalidConfig( 182 | "min_checkpoint_page_number cannot be zero", 183 | )); 184 | } 185 | 186 | if self.min_checkpoint_page_number > self.max_checkpoint_page_number { 187 | return Err(Error::InvalidConfig( 188 | "min_checkpoint_page_number cannot bigger than max_checkpoint_page_number", 189 | )); 190 | } 191 | Ok(()) 192 | } 193 | } 194 | 195 | #[derive(Clone, PartialEq, Eq, Deserialize)] 196 | pub struct StorageConfig { 197 | pub name: String, 198 | pub params: StorageParams, 199 | } 200 | 201 | impl Debug for StorageConfig { 202 | fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { 203 | f.debug_struct("StorageS3Config") 204 | .field("name", &self.name) 205 | .field("params", &self.params) 206 | .finish() 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /tests/integration_test.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import os, sys 3 | import random 4 | import string 5 | import subprocess 6 | import time 7 | import shutil 8 | 9 | class Test: 10 | def __init__(self, root, max_records): 11 | self.conn = sqlite3.connect(root + '/test.db') 12 | self.max_records = max_records 13 | self.cursor = self.conn.cursor() 14 | 15 | def create_table(self): 16 | self.conn.execute(''' 17 | CREATE TABLE IF NOT EXISTS random_data ( 18 | id INTEGER PRIMARY KEY AUTOINCREMENT, 19 | name TEXT NOT NULL, 20 | value INTEGER NOT NULL 21 | ) 22 | ''') 23 | 24 | def insert_random_data(self, start): 25 | num_records = random.randint(1, 20) 26 | records = [] 27 | for i in range(num_records): 28 | name = ''.join(random.choices(string.ascii_letters, k=5)) 29 | value = i + start 30 | records.append((name, value)) 31 | self.cursor.executemany('INSERT INTO random_data (name, value) VALUES (?, ?)', records) 32 | self.conn.commit() 33 | 34 | return num_records 35 | 36 | def insert(self): 37 | num_records = 0 38 | sleep_num = 0 39 | while num_records < self.max_records: 40 | num = self.insert_random_data(num_records) 41 | num_records += num 42 | sleep_num += num 43 | if sleep_num > 500: 44 | print("after insert ", sleep_num, " data, total: ", num_records, ", go to sleep(1)") 45 | time.sleep(1) 46 | sleep_num = 0 47 | 48 | print("finish insert test data, total: ", num_records) 49 | 50 | def query_data(self): 51 | cursor = self.conn.cursor() 52 | cursor.execute('SELECT * FROM random_data order by value') 53 | return cursor.fetchall() 54 | 55 | class ConfigGenerator: 56 | def __init__(self): 57 | self.cwd = os.getcwd() 58 | self.root = self.cwd + "/.test" 59 | # clean test dir 60 | try: 61 | shutil.rmtree(self.root) 62 | pass 63 | except: 64 | pass 65 | print("root: ", self.root) 66 | try: 67 | os.makedirs(self.root) 68 | except: 69 | pass 70 | 71 | self.config_file = self.root + "/replited.toml" 72 | 73 | def generate(self): 74 | print("generate config for backend type ", self.type) 75 | self.do_generate() 76 | 77 | class FsConfigGenerator(ConfigGenerator): 78 | def __init__(self): 79 | ConfigGenerator.__init__(self) 80 | self.type = 'Fs' 81 | 82 | def do_generate(self): 83 | # create root dir of fs 84 | try: 85 | os.makedirs(self.root + "/replited") 86 | except: 87 | pass 88 | 89 | # generate config file 90 | file = open(self.cwd + '/tests/config/fs_template.toml') 91 | content = file.read() 92 | content = content.replace('{root}', self.root) 93 | config_file = self.config_file 94 | file = open(config_file, 'w+') 95 | file.write(content) 96 | file.close() 97 | 98 | class S3ConfigGenerator(ConfigGenerator): 99 | def __init__(self): 100 | ConfigGenerator.__init__(self) 101 | self.type = 'S3' 102 | 103 | def do_generate(self): 104 | # create root dir of fs 105 | try: 106 | os.makedirs(self.root + "/replited") 107 | except: 108 | pass 109 | 110 | # generate config file 111 | file = open(self.cwd + '/tests/config/s3_template.toml') 112 | content = file.read() 113 | content = content.replace('{root}', self.root) 114 | config_file = self.config_file 115 | file = open(config_file, 'w+') 116 | file.write(content) 117 | file.close() 118 | 119 | class FtpConfigGenerator(ConfigGenerator): 120 | def __init__(self): 121 | ConfigGenerator.__init__(self) 122 | self.type = 'Ftp' 123 | 124 | def do_generate(self): 125 | # create root dir of fs 126 | try: 127 | os.makedirs(self.root + "/replited") 128 | except: 129 | pass 130 | 131 | # generate config file 132 | file = open(self.cwd + '/tests/config/ftp_template.toml') 133 | content = file.read() 134 | content = content.replace('{root}', self.root) 135 | config_file = self.config_file 136 | file = open(config_file, 'w+') 137 | file.write(content) 138 | file.close() 139 | 140 | def start_replicate(p, config_file): 141 | cmd = p + " --config " + config_file + " replicate &" 142 | print("replicate cmd: ", cmd) 143 | #cmds = [p, "--config", config_file, "replicate", "&"] 144 | #pipe = subprocess.Popen(cmds, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 145 | #ret = os.popen(cmds) 146 | os.system(cmd) 147 | #print("after replicate") 148 | #print("after replicate: ", pipe.stdout.read()) 149 | 150 | def stop_replicate(): 151 | cmd = "killall replited" 152 | os.system(cmd) 153 | #subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 154 | 155 | def test_restore(p, config_file, root, exp_data): 156 | db = root + "/test.db" 157 | output = os.getcwd() + "/test.db" 158 | try: 159 | os.remove(output) 160 | except: 161 | pass 162 | cmd = p + " --config " + config_file + " restore --db " + db + " --output " + output 163 | #cmds = [p, "--config", config_file,"restore", "--db", db, "--output", output] 164 | print("restore: ", cmd) 165 | #pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 166 | #pipe = subprocess.Popen(cmds, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 167 | os.system(cmd) 168 | #print("after restore") 169 | 170 | conn = sqlite3.connect(output) 171 | cursor = conn.cursor() 172 | cursor.execute('SELECT * FROM random_data order by value') 173 | data = cursor.fetchall() 174 | print("data len: ", len(data), ", exp_data len: ", len(exp_data)) 175 | assert data == exp_data 176 | 177 | def decide_config_generator(config_type): 178 | if config_type == "fs": 179 | return FsConfigGenerator() 180 | elif config_type == "s3": 181 | return S3ConfigGenerator() 182 | elif config_type == "ftp": 183 | return FtpConfigGenerator() 184 | else: 185 | print("invalid config type: ", config_type) 186 | sys.exit(-1) 187 | 188 | # python3 tests/integration_test.py [number of data] [config type] [replited bin path] 189 | if __name__ == '__main__': 190 | print("args: ", sys.argv) 191 | number = int(sys.argv[1]) 192 | if number <= 0: 193 | print("invalid number: ", number) 194 | sys.exit(-1) 195 | config_type = sys.argv[2] 196 | bin_path = sys.argv[3] 197 | 198 | stop_replicate() 199 | 200 | config = decide_config_generator(config_type) 201 | config.generate() 202 | 203 | test = Test(config.root, number) 204 | test.create_table() 205 | 206 | start_replicate(bin_path, config.config_file) 207 | 208 | test.insert() 209 | 210 | time.sleep(3) 211 | data = test.query_data() 212 | 213 | stop_replicate() 214 | 215 | test_restore(bin_path, config.config_file, config.root, data) 216 | 217 | -------------------------------------------------------------------------------- /src/base/file.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | use std::path::PathBuf; 3 | use std::sync::LazyLock; 4 | 5 | use regex::Regex; 6 | 7 | use crate::error::Error; 8 | use crate::error::Result; 9 | 10 | static WAL_EXTENDION: &str = ".wal"; 11 | static WAL_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"^([0-9]{10})\.wal$").unwrap()); 12 | static WAL_SEGMENT_EXTENDION: &str = ".wal.lz4"; 13 | static WAL_SEGMENT_REGEX: LazyLock = 14 | LazyLock::new(|| Regex::new(r"^([0-9]{10})(?:_([0-9]{10}))\.wal\.lz4$").unwrap()); 15 | static SNAPSHOT_REGEX: LazyLock = 16 | LazyLock::new(|| Regex::new(r"^([0-9]{10})\.snapshot\.lz4$").unwrap()); 17 | static SNAPSHOT_EXTENDION: &str = ".snapshot.lz4"; 18 | 19 | // return base name of path 20 | pub fn path_base(path: &str) -> Result { 21 | let path_buf = PathBuf::from(path); 22 | path_buf 23 | .file_name() 24 | .map(|name| name.to_string_lossy().to_string()) 25 | .ok_or(Error::InvalidPath(format!("invalid path {}", path))) 26 | } 27 | 28 | pub fn parent_dir(path: &str) -> Option { 29 | let path = Path::new(path); 30 | path.parent() 31 | .map(|parent| parent.to_string_lossy().into_owned()) 32 | } 33 | 34 | // parse wal file path, return wal index 35 | pub fn parse_wal_path(path: &str) -> Result { 36 | let base = path_base(path)?; 37 | let a = WAL_REGEX 38 | .captures(&base) 39 | .ok_or(Error::InvalidPath(format!("invalid wal path {}", path)))?; 40 | let a = a 41 | .get(1) 42 | .ok_or(Error::InvalidPath(format!("invalid wal path {}", path)))? 43 | .as_str(); 44 | 45 | Ok(a.parse::()?) 46 | } 47 | 48 | pub fn format_wal_path(index: u64) -> String { 49 | format!("{:0>10}{}", index, WAL_EXTENDION) 50 | } 51 | 52 | pub fn parse_wal_segment_path(path: &str) -> Result<(u64, u64)> { 53 | let base = path_base(path)?; 54 | let a = WAL_SEGMENT_REGEX 55 | .captures(&base) 56 | .ok_or(Error::InvalidPath(format!( 57 | "invalid wal segment path {}", 58 | path 59 | )))?; 60 | let index = a 61 | .get(1) 62 | .ok_or(Error::InvalidPath(format!( 63 | "invalid wal segment path {}", 64 | path 65 | )))? 66 | .as_str(); 67 | let offset = a 68 | .get(2) 69 | .ok_or(Error::InvalidPath(format!( 70 | "invalid wal segment path {}", 71 | path 72 | )))? 73 | .as_str(); 74 | 75 | let index = index.parse::()?; 76 | let offset = offset.parse::()?; 77 | 78 | Ok((index, offset)) 79 | } 80 | 81 | // parse snapshot file path, return snapshot index 82 | pub fn parse_snapshot_path(path: &str) -> Result { 83 | let base = path_base(path)?; 84 | let a = SNAPSHOT_REGEX 85 | .captures(&base) 86 | .ok_or(Error::InvalidPath(format!( 87 | "invalid snapshot path {}", 88 | path 89 | )))?; 90 | let a = a 91 | .get(1) 92 | .ok_or(Error::InvalidPath(format!( 93 | "invalid snapshot path {}", 94 | path 95 | )))? 96 | .as_str(); 97 | 98 | Ok(a.parse::()?) 99 | } 100 | 101 | pub fn format_snapshot_path(index: u64) -> String { 102 | format!("{:0>10}{}", index, SNAPSHOT_EXTENDION) 103 | } 104 | 105 | pub fn local_generations_dir(meta_dir: &str) -> String { 106 | Path::new(meta_dir) 107 | .join("generations") 108 | .as_path() 109 | .to_str() 110 | .unwrap() 111 | .to_string() 112 | } 113 | 114 | pub fn remote_generations_dir(db_name: &str) -> String { 115 | Path::new(db_name) 116 | .join("generations/") 117 | .as_path() 118 | .to_str() 119 | .unwrap() 120 | .to_string() 121 | } 122 | 123 | // returns the path of a single generation. 124 | pub fn generation_dir(meta_dir: &str, generation: &str) -> String { 125 | Path::new(meta_dir) 126 | .join("generations") 127 | .join(generation) 128 | .as_path() 129 | .to_str() 130 | .unwrap() 131 | .to_string() 132 | } 133 | 134 | pub fn snapshots_dir(db: &str, generation: &str) -> String { 135 | Path::new(&generation_dir(db, generation)) 136 | .join("snapshots/") 137 | .as_path() 138 | .to_str() 139 | .unwrap() 140 | .to_string() 141 | } 142 | 143 | pub fn snapshot_file(db: &str, generation: &str, index: u64) -> String { 144 | Path::new(&generation_dir(db, generation)) 145 | .join("snapshots") 146 | .join(format_snapshot_path(index)) 147 | .as_path() 148 | .to_str() 149 | .unwrap() 150 | .to_string() 151 | } 152 | 153 | pub fn walsegments_dir(db: &str, generation: &str) -> String { 154 | Path::new(&generation_dir(db, generation)) 155 | .join("wal/") 156 | .as_path() 157 | .to_str() 158 | .unwrap() 159 | .to_string() 160 | } 161 | 162 | pub fn walsegment_file(db: &str, generation: &str, index: u64, offset: u64) -> String { 163 | Path::new(&generation_dir(db, generation)) 164 | .join("wal") 165 | .join(format_walsegment_path(index, offset)) 166 | .as_path() 167 | .to_str() 168 | .unwrap() 169 | .to_string() 170 | } 171 | 172 | pub fn format_walsegment_path(index: u64, offset: u64) -> String { 173 | format!("{:0>10}_{:0>10}{}", index, offset, WAL_SEGMENT_EXTENDION) 174 | } 175 | 176 | // returns the path of the name of the current generation. 177 | pub fn generation_file_path(meta_dir: &str) -> String { 178 | Path::new(meta_dir) 179 | .join("generation") 180 | .as_path() 181 | .to_str() 182 | .unwrap() 183 | .to_string() 184 | } 185 | 186 | pub fn shadow_wal_dir(meta_dir: &str, generation: &str) -> String { 187 | Path::new(&generation_dir(meta_dir, generation)) 188 | .join("wal") 189 | .as_path() 190 | .to_str() 191 | .unwrap() 192 | .to_string() 193 | } 194 | 195 | pub fn shadow_wal_file(meta_dir: &str, generation: &str, index: u64) -> String { 196 | Path::new(&shadow_wal_dir(meta_dir, generation)) 197 | .join(format_wal_path(index)) 198 | .as_path() 199 | .to_str() 200 | .unwrap() 201 | .to_string() 202 | } 203 | 204 | #[cfg(test)] 205 | mod tests { 206 | use super::format_snapshot_path; 207 | use super::format_wal_path; 208 | use super::format_walsegment_path; 209 | use super::parent_dir; 210 | use super::parse_snapshot_path; 211 | use super::parse_wal_path; 212 | use super::parse_wal_segment_path; 213 | use super::path_base; 214 | use crate::error::Result; 215 | 216 | #[test] 217 | fn test_path_base() -> Result<()> { 218 | let path = "a/b/c"; 219 | let base = path_base(path)?; 220 | assert_eq!(&base, "c"); 221 | 222 | let path = "c/"; 223 | let base = path_base(path)?; 224 | assert_eq!(&base, "c"); 225 | 226 | let path = "a-b/.."; 227 | let base = path_base(path); 228 | assert!(base.is_err()); 229 | let err = base.unwrap_err(); 230 | assert_eq!(err.code(), 54); 231 | 232 | Ok(()) 233 | } 234 | 235 | #[test] 236 | fn test_parse_wal_path() -> Result<()> { 237 | let path = "a/b/c/0000000019.wal"; 238 | let index = parse_wal_path(path)?; 239 | assert_eq!(index, 19); 240 | 241 | let path = "a/b/c/000000019.wal"; 242 | let index = parse_wal_path(path); 243 | assert!(index.is_err()); 244 | 245 | let path = format!("a/b/{}", format_wal_path(19)); 246 | let index = parse_wal_path(&path)?; 247 | assert_eq!(index, 19); 248 | Ok(()) 249 | } 250 | 251 | #[test] 252 | fn test_parse_snapshot_path() -> Result<()> { 253 | let path = "a/b/c/0000000019.snapshot.lz4"; 254 | let index = parse_snapshot_path(path)?; 255 | assert_eq!(index, 19); 256 | 257 | let path = "a/b/c/000000019.snapshot.lz4"; 258 | let index = parse_snapshot_path(path); 259 | assert!(index.is_err()); 260 | 261 | let path = format!("a/b/{}", format_snapshot_path(19)); 262 | let index = parse_snapshot_path(&path)?; 263 | assert_eq!(index, 19); 264 | Ok(()) 265 | } 266 | 267 | #[test] 268 | fn test_parse_walsegment_path() -> Result<()> { 269 | let path = "a/b/c/0000000019_0000000020.wal.lz4"; 270 | let (index, offset) = parse_wal_segment_path(path)?; 271 | assert_eq!(index, 19); 272 | assert_eq!(offset, 20); 273 | 274 | let path = format!("a/b/{}", format_walsegment_path(19, 20)); 275 | let (index, offset) = parse_wal_segment_path(&path)?; 276 | assert_eq!(index, 19); 277 | assert_eq!(offset, 20); 278 | 279 | Ok(()) 280 | } 281 | 282 | #[test] 283 | fn test_parent_dir() -> Result<()> { 284 | let path = "/b/c/0000000019_0000000020.wal.lz4"; 285 | let dir = parent_dir(path); 286 | assert_eq!(dir, Some("/b/c".to_string())); 287 | 288 | Ok(()) 289 | } 290 | } 291 | -------------------------------------------------------------------------------- /src/sync/replicate.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use log::debug; 4 | use log::error; 5 | use log::info; 6 | use parking_lot::RwLock; 7 | use tokio::select; 8 | use tokio::sync::mpsc::Receiver; 9 | use tokio::sync::mpsc::Sender; 10 | use tokio::task::JoinHandle; 11 | 12 | use super::ShadowWalReader; 13 | use crate::base::Generation; 14 | use crate::base::compress_buffer; 15 | use crate::base::decompressed_data; 16 | use crate::config::StorageConfig; 17 | use crate::database::DatabaseInfo; 18 | use crate::database::DbCommand; 19 | use crate::database::WalGenerationPos; 20 | use crate::error::Error; 21 | use crate::error::Result; 22 | use crate::sqlite::WALFrame; 23 | use crate::sqlite::WALHeader; 24 | use crate::sqlite::align_frame; 25 | use crate::storage::SnapshotInfo; 26 | use crate::storage::StorageClient; 27 | use crate::storage::WalSegmentInfo; 28 | 29 | #[derive(Clone, Debug)] 30 | pub enum ReplicateCommand { 31 | DbChanged(WalGenerationPos), 32 | Snapshot((WalGenerationPos, Vec)), 33 | } 34 | 35 | #[derive(Debug, Clone, PartialEq)] 36 | enum ReplicateState { 37 | WaitDbChanged, 38 | WaitSnapshot, 39 | } 40 | 41 | #[derive(Debug, Clone)] 42 | pub struct Replicate { 43 | db: String, 44 | index: usize, 45 | client: StorageClient, 46 | db_notifier: Sender, 47 | position: Arc>, 48 | state: ReplicateState, 49 | info: DatabaseInfo, 50 | config: StorageConfig, 51 | } 52 | 53 | impl Replicate { 54 | pub fn new( 55 | config: StorageConfig, 56 | db: String, 57 | index: usize, 58 | db_notifier: Sender, 59 | info: DatabaseInfo, 60 | ) -> Result { 61 | Ok(Self { 62 | db: db.clone(), 63 | index, 64 | position: Arc::new(RwLock::new(WalGenerationPos::default())), 65 | db_notifier, 66 | client: StorageClient::try_create(db, config.clone())?, 67 | config, 68 | state: ReplicateState::WaitDbChanged, 69 | info, 70 | }) 71 | } 72 | 73 | pub fn start(s: Replicate, rx: Receiver) -> Result> { 74 | info!("start replicate {:?} of db {}", s.config, s.db); 75 | let s = s.clone(); 76 | let handle = tokio::spawn(async move { 77 | let _ = Replicate::main(s, rx).await; 78 | }); 79 | 80 | Ok(handle) 81 | } 82 | 83 | pub async fn main(s: Replicate, rx: Receiver) -> Result<()> { 84 | let mut rx = rx; 85 | let mut s = s; 86 | loop { 87 | select! { 88 | cmd = rx.recv() => if let Some(cmd) = cmd { 89 | s.command(cmd).await? 90 | } 91 | } 92 | } 93 | } 94 | 95 | // returns the last snapshot in a generation. 96 | async fn max_snapshot(&self, generation: &str) -> Result { 97 | let snapshots = self.client.snapshots(generation).await?; 98 | if snapshots.is_empty() { 99 | return Err(Error::NoSnapshotError(generation)); 100 | } 101 | let mut max_index = 0; 102 | let mut max_snapshot_index = 0; 103 | for (i, snapshot) in snapshots.iter().enumerate() { 104 | if snapshot.index > max_snapshot_index { 105 | max_snapshot_index = snapshot.index; 106 | max_index = i; 107 | } 108 | } 109 | 110 | Ok(snapshots[max_index].clone()) 111 | } 112 | 113 | // returns the highest WAL segment in a generation. 114 | async fn max_wal_segment(&self, generation: &str) -> Result { 115 | let wal_segments = self.client.wal_segments(generation).await?; 116 | if wal_segments.is_empty() { 117 | return Err(Error::NoWalsegmentError(generation)); 118 | } 119 | let mut max_index = 0; 120 | let mut max_wg_index = 0; 121 | for (i, wg) in wal_segments.iter().enumerate() { 122 | if wg.index > max_wg_index { 123 | max_wg_index = wg.index; 124 | max_index = i; 125 | } 126 | } 127 | 128 | Ok(wal_segments[max_index].clone()) 129 | } 130 | 131 | async fn calculate_generation_position(&self, generation: &str) -> Result { 132 | // Fetch last snapshot. Return error if no snapshots exist. 133 | let snapshot = self.max_snapshot(generation).await?; 134 | let generation = Generation::try_create(generation)?; 135 | 136 | // Determine last WAL segment available. 137 | let segment = self.max_wal_segment(generation.as_str()).await; 138 | let segment = match segment { 139 | Err(e) => { 140 | if e.code() == Error::NO_WALSEGMENT_ERROR { 141 | // Use snapshot if none exist. 142 | return Ok(WalGenerationPos { 143 | generation, 144 | index: snapshot.index, 145 | offset: 0, 146 | }); 147 | } else { 148 | return Err(e); 149 | } 150 | } 151 | Ok(segment) => segment, 152 | }; 153 | 154 | let compressed_data = self.client.read_wal_segment(&segment).await?; 155 | let decompressed_data = decompressed_data(compressed_data)?; 156 | 157 | Ok(WalGenerationPos { 158 | generation: segment.generation.clone(), 159 | index: segment.index, 160 | offset: segment.offset + decompressed_data.len() as u64, 161 | }) 162 | } 163 | 164 | async fn sync_wal(&mut self) -> Result<()> { 165 | let mut reader = ShadowWalReader::try_create(self.position(), &self.info)?; 166 | 167 | // Obtain initial position from shadow reader. 168 | // It may have moved to the next index if previous position was at the end. 169 | let init_pos = reader.position(); 170 | let mut data = Vec::new(); 171 | 172 | debug!("db {} write wal segment position {:?}", self.db, init_pos,); 173 | 174 | // Copy header if at offset zero. 175 | let mut salt1 = 0; 176 | let mut salt2 = 0; 177 | if init_pos.offset == 0 { 178 | let wal_header = WALHeader::read_from(&mut reader)?; 179 | salt1 = wal_header.salt1; 180 | salt2 = wal_header.salt2; 181 | data.extend_from_slice(&wal_header.data); 182 | } 183 | 184 | // Copy frames. 185 | loop { 186 | if reader.left == 0 { 187 | break; 188 | } 189 | 190 | let pos = reader.position(); 191 | debug_assert_eq!(pos.offset, align_frame(self.info.page_size, pos.offset)); 192 | 193 | let wal_frame = WALFrame::read(&mut reader, self.info.page_size)?; 194 | 195 | if (salt1 != 0 && salt1 != wal_frame.salt1) || (salt2 != 0 && salt2 != wal_frame.salt2) 196 | { 197 | return Err(Error::SqliteInvalidWalFrameError(format!( 198 | "db {} Invalid WAL frame at offset {}", 199 | self.db, pos.offset 200 | ))); 201 | } 202 | salt1 = wal_frame.salt1; 203 | salt2 = wal_frame.salt2; 204 | 205 | data.extend_from_slice(&wal_frame.data); 206 | } 207 | let compressed_data = compress_buffer(&data)?; 208 | 209 | self.client 210 | .write_wal_segment(&init_pos, compressed_data) 211 | .await?; 212 | 213 | // update position 214 | let mut position = self.position.write(); 215 | *position = reader.position(); 216 | Ok(()) 217 | } 218 | 219 | pub fn position(&self) -> WalGenerationPos { 220 | let position = self.position.read(); 221 | position.clone() 222 | } 223 | 224 | fn reset_position(&self) { 225 | let mut position = self.position.write(); 226 | *position = WalGenerationPos::default(); 227 | } 228 | 229 | async fn command(&mut self, cmd: ReplicateCommand) -> Result<()> { 230 | match cmd { 231 | ReplicateCommand::DbChanged(pos) => { 232 | if let Err(e) = self.sync(pos).await { 233 | error!("sync db error: {:?}", e); 234 | // Clear last position if if an error occurs during sync. 235 | self.reset_position(); 236 | } 237 | } 238 | ReplicateCommand::Snapshot((pos, compressed_data)) => { 239 | if let Err(e) = self.sync_snapshot(pos, compressed_data).await { 240 | error!("sync db snapshot error: {:?}", e); 241 | } 242 | } 243 | } 244 | Ok(()) 245 | } 246 | 247 | async fn sync_snapshot( 248 | &mut self, 249 | pos: WalGenerationPos, 250 | compressed_data: Vec, 251 | ) -> Result<()> { 252 | info!("db {} sync snapshot {:?}", self.db, pos); 253 | debug_assert_eq!(self.state, ReplicateState::WaitSnapshot); 254 | if pos.offset == 0 { 255 | return Ok(()); 256 | } 257 | 258 | let _ = self.client.write_snapshot(&pos, compressed_data).await?; 259 | 260 | // change state from WaitSnapshot to WaitDbChanged 261 | self.state = ReplicateState::WaitDbChanged; 262 | self.sync(pos).await 263 | } 264 | 265 | async fn sync(&mut self, pos: WalGenerationPos) -> Result<()> { 266 | info!( 267 | "db {} replicate {} replica sync pos: {:?}\n", 268 | self.db, self.config.name, pos 269 | ); 270 | 271 | if self.state == ReplicateState::WaitSnapshot { 272 | return Ok(()); 273 | } 274 | 275 | if pos.offset == 0 { 276 | return Ok(()); 277 | } 278 | 279 | // Create a new snapshot and update the current replica position if 280 | // the generation on the database has changed. 281 | let generation = pos.generation.clone(); 282 | let position = { 283 | let position = self.position.read(); 284 | position.clone() 285 | }; 286 | debug!( 287 | "db {} replicate {} replicate position: {:?}", 288 | self.db, self.config.name, position 289 | ); 290 | if generation != position.generation { 291 | let snapshots = self.client.snapshots(generation.as_str()).await?; 292 | debug!( 293 | "db {} replicate {} snapshot {} num: {}", 294 | self.db, 295 | self.config.name, 296 | generation.as_str(), 297 | snapshots.len() 298 | ); 299 | if snapshots.is_empty() { 300 | // Create snapshot if no snapshots exist for generation. 301 | self.db_notifier 302 | .send(DbCommand::Snapshot(self.index)) 303 | .await?; 304 | self.state = ReplicateState::WaitSnapshot; 305 | return Ok(()); 306 | } 307 | 308 | let pos = self 309 | .calculate_generation_position(generation.as_str()) 310 | .await?; 311 | debug!( 312 | "db {} replicate {} calc position: {:?}", 313 | self.db, self.config.name, pos 314 | ); 315 | *self.position.write() = pos; 316 | } 317 | 318 | // Read all WAL files since the last position. 319 | loop { 320 | if let Err(e) = self.sync_wal().await 321 | && e.code() == Error::UNEXPECTED_EOF_ERROR { 322 | break; 323 | } 324 | } 325 | Ok(()) 326 | } 327 | } 328 | -------------------------------------------------------------------------------- /src/storage/storage_client.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | 3 | use chrono::DateTime; 4 | use chrono::Utc; 5 | use log::debug; 6 | use log::error; 7 | use opendal::Metakey; 8 | use opendal::Operator; 9 | 10 | use super::init_operator; 11 | use crate::base::Generation; 12 | use crate::base::parent_dir; 13 | use crate::base::parse_snapshot_path; 14 | use crate::base::parse_wal_segment_path; 15 | use crate::base::path_base; 16 | use crate::base::remote_generations_dir; 17 | use crate::base::snapshot_file; 18 | use crate::base::snapshots_dir; 19 | use crate::base::walsegment_file; 20 | use crate::base::walsegments_dir; 21 | use crate::config::StorageConfig; 22 | use crate::database::WalGenerationPos; 23 | use crate::error::Error; 24 | use crate::error::Result; 25 | 26 | #[derive(Debug, Clone)] 27 | pub struct StorageClient { 28 | operator: Operator, 29 | root: String, 30 | db_path: String, 31 | db_name: String, 32 | } 33 | 34 | #[derive(Debug, Clone, Default)] 35 | pub struct SnapshotInfo { 36 | pub generation: Generation, 37 | pub index: u64, 38 | pub size: u64, 39 | pub created_at: DateTime, 40 | } 41 | 42 | #[derive(Debug, Clone)] 43 | pub struct WalSegmentInfo { 44 | pub generation: Generation, 45 | pub index: u64, 46 | pub offset: u64, 47 | pub size: u64, 48 | } 49 | 50 | // restore wal_segments formats: vector> 51 | pub type RestoreWalSegments = Vec<(u64, Vec)>; 52 | 53 | #[derive(Debug)] 54 | pub struct RestoreInfo { 55 | pub snapshot: SnapshotInfo, 56 | 57 | pub wal_segments: RestoreWalSegments, 58 | } 59 | 60 | impl StorageClient { 61 | pub fn try_create(db_path: String, config: StorageConfig) -> Result { 62 | Ok(Self { 63 | root: config.params.root(), 64 | operator: init_operator(&config.params)?, 65 | db_name: path_base(&db_path)?, 66 | db_path, 67 | }) 68 | } 69 | 70 | async fn ensure_parent_exist(&self, path: &str) -> Result<()> { 71 | let base = format!("{}/", parent_dir(path).unwrap()); 72 | 73 | let mut exist = false; 74 | match self.operator.is_exist(&base).await { 75 | Err(e) => { 76 | debug!("check path {} parent_dir error: {}", path, e.kind()) 77 | } 78 | Ok(r) => { 79 | exist = r; 80 | } 81 | } 82 | 83 | if !exist { 84 | debug!("create dir {}", base); 85 | self.operator.create_dir(&base).await?; 86 | } 87 | 88 | Ok(()) 89 | } 90 | 91 | pub async fn write_wal_segment( 92 | &self, 93 | pos: &WalGenerationPos, 94 | compressed_data: Vec, 95 | ) -> Result<()> { 96 | let file = walsegment_file( 97 | &self.db_path, 98 | pos.generation.as_str(), 99 | pos.index, 100 | pos.offset, 101 | ); 102 | 103 | self.ensure_parent_exist(&file).await?; 104 | 105 | self.operator.write(&file, compressed_data).await?; 106 | 107 | Ok(()) 108 | } 109 | 110 | pub async fn write_snapshot( 111 | &self, 112 | pos: &WalGenerationPos, 113 | compressed_data: Vec, 114 | ) -> Result { 115 | let snapshot_file = snapshot_file(&self.db_name, pos.generation.as_str(), pos.index); 116 | let snapshot_info = SnapshotInfo { 117 | generation: pos.generation.clone(), 118 | index: pos.index, 119 | size: compressed_data.len() as u64, 120 | created_at: Utc::now(), 121 | }; 122 | 123 | self.ensure_parent_exist(&snapshot_file).await?; 124 | 125 | self.operator.write(&snapshot_file, compressed_data).await?; 126 | 127 | Ok(snapshot_info) 128 | } 129 | 130 | pub async fn read_snapshot(&self, info: &SnapshotInfo) -> Result> { 131 | let snapshot_file = snapshot_file(&self.db_name, info.generation.as_str(), info.index); 132 | 133 | let data = self.operator.read(&snapshot_file).await?; 134 | 135 | Ok(data.to_vec()) 136 | } 137 | 138 | pub async fn snapshots(&self, generation: &str) -> Result> { 139 | let generation = Generation::try_create(generation)?; 140 | let snapshots_dir = snapshots_dir(&self.db_name, generation.as_str()); 141 | let entries = match self 142 | .operator 143 | .list_with(&snapshots_dir) 144 | .metakey(Metakey::ContentLength) 145 | .metakey(Metakey::LastModified) 146 | .await 147 | { 148 | Ok(entries) => entries, 149 | Err(e) => { 150 | debug!( 151 | "list snapshots {:?} error kind: {:?}", 152 | generation, 153 | e.to_string() 154 | ); 155 | if e.kind() == opendal::ErrorKind::NotADirectory 156 | || e.kind() == opendal::ErrorKind::Unexpected 157 | { 158 | return Ok(vec![]); 159 | } else { 160 | return Err(e.into()); 161 | } 162 | } 163 | }; 164 | 165 | let mut snapshots = vec![]; 166 | for entry in entries { 167 | let metadata = entry.metadata(); 168 | if !metadata.is_file() { 169 | continue; 170 | } 171 | let index = parse_snapshot_path(entry.name())?; 172 | snapshots.push(SnapshotInfo { 173 | generation: generation.clone(), 174 | index, 175 | size: metadata.content_length(), 176 | created_at: metadata.last_modified().unwrap(), 177 | }) 178 | } 179 | 180 | Ok(snapshots) 181 | } 182 | 183 | async fn max_snapshot(&self, generation: &str) -> Result> { 184 | let generation = Generation::try_create(generation)?; 185 | let snapshots_dir = snapshots_dir(&self.db_name, generation.as_str()); 186 | let entries = self 187 | .operator 188 | .list_with(&snapshots_dir) 189 | .metakey(Metakey::ContentLength) 190 | .metakey(Metakey::LastModified) 191 | .await?; 192 | 193 | let mut snapshot = None; 194 | let mut max_index = None; 195 | for entry in entries { 196 | let metadata = entry.metadata(); 197 | if !metadata.is_file() { 198 | continue; 199 | } 200 | let index = parse_snapshot_path(entry.name())?; 201 | let mut update = false; 202 | match max_index { 203 | Some(mi) => { 204 | if index > mi { 205 | update = true; 206 | } 207 | } 208 | None => { 209 | update = true; 210 | } 211 | } 212 | 213 | if !update { 214 | continue; 215 | } 216 | 217 | max_index = Some(index); 218 | snapshot = Some(SnapshotInfo { 219 | generation: generation.clone(), 220 | index, 221 | size: metadata.content_length(), 222 | created_at: metadata.last_modified().unwrap(), 223 | }); 224 | } 225 | 226 | Ok(snapshot) 227 | } 228 | 229 | pub async fn wal_segments(&self, generation: &str) -> Result> { 230 | let generation = Generation::try_create(generation)?; 231 | let walsegments_dir = walsegments_dir(&self.db_name, generation.as_str()); 232 | let entries = self 233 | .operator 234 | .list_with(&walsegments_dir) 235 | .metakey(Metakey::ContentLength) 236 | .await?; 237 | 238 | let mut wal_segments = vec![]; 239 | for entry in entries { 240 | let metadata = entry.metadata(); 241 | if !metadata.is_file() { 242 | continue; 243 | } 244 | let (index, offset) = parse_wal_segment_path(entry.name())?; 245 | wal_segments.push(WalSegmentInfo { 246 | generation: generation.clone(), 247 | index, 248 | offset, 249 | size: entry.metadata().content_length(), 250 | }) 251 | } 252 | 253 | Ok(wal_segments) 254 | } 255 | 256 | pub async fn read_wal_segment(&self, info: &WalSegmentInfo) -> Result> { 257 | let generation = &info.generation; 258 | let index = info.index; 259 | let offset = info.offset; 260 | 261 | let wal_segment_file = walsegment_file(&self.db_name, generation.as_str(), index, offset); 262 | let bytes = self.operator.read(&wal_segment_file).await?.to_vec(); 263 | Ok(bytes) 264 | } 265 | 266 | async fn restore_wal_segments_of(&self, snapshot: &SnapshotInfo) -> Result { 267 | let mut wal_segments = self.wal_segments(snapshot.generation.as_str()).await?; 268 | 269 | // sort wal segments first by index, then offset 270 | wal_segments.sort_by(|a, b| { 271 | let ordering = a.index.partial_cmp(&b.index).unwrap(); 272 | if ordering.is_eq() { 273 | a.offset.partial_cmp(&b.offset).unwrap() 274 | } else { 275 | ordering 276 | } 277 | }); 278 | 279 | let mut restore_wal_segments: BTreeMap> = BTreeMap::new(); 280 | 281 | for wal_segment in wal_segments { 282 | if wal_segment.index < snapshot.index { 283 | continue; 284 | } 285 | 286 | match restore_wal_segments.get_mut(&wal_segment.index) { 287 | Some(offsets) => { 288 | if *offsets.last().unwrap() >= wal_segment.offset { 289 | let msg = format!( 290 | "wal segment out of order, generation: {:?}, index: {}, offset: {}", 291 | snapshot.generation.as_str(), 292 | wal_segment.index, 293 | wal_segment.offset 294 | ); 295 | error!("{}", msg); 296 | return Err(Error::InvalidWalSegmentError(msg)); 297 | } 298 | offsets.push(wal_segment.offset); 299 | } 300 | None => { 301 | if wal_segment.offset != 0 { 302 | let msg = format!( 303 | "missing initial wal segment, generation: {:?}, index: {}, offset: {}", 304 | snapshot.generation.as_str(), 305 | wal_segment.index, 306 | wal_segment.offset 307 | ); 308 | error!("{}", msg); 309 | return Err(Error::InvalidWalSegmentError(msg)); 310 | } 311 | restore_wal_segments.insert(wal_segment.index, vec![wal_segment.offset]); 312 | } 313 | } 314 | } 315 | 316 | Ok(restore_wal_segments.into_iter().collect()) 317 | } 318 | 319 | pub async fn restore_info(&self) -> Result> { 320 | let dir = remote_generations_dir(&self.db_name); 321 | let entries = self.operator.list(&dir).await?; 322 | 323 | let mut entry_with_generation = Vec::with_capacity(entries.len()); 324 | for entry in entries { 325 | let metadata = entry.metadata(); 326 | if !metadata.is_dir() { 327 | continue; 328 | } 329 | let generation = path_base(entry.name())?; 330 | 331 | let generation = match Generation::try_create(&generation) { 332 | Ok(generation) => generation, 333 | Err(_e) => { 334 | error!("dir {} is not valid generation dir", generation,); 335 | continue; 336 | } 337 | }; 338 | 339 | entry_with_generation.push((entry, generation)); 340 | } 341 | 342 | // sort the entries in reverse order 343 | entry_with_generation.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); 344 | 345 | for (_entry, generation) in entry_with_generation { 346 | let snapshot = match self.max_snapshot(generation.as_str()).await? { 347 | Some(snapshot) => snapshot, 348 | // if generation has no snapshot, ignore and skip to the next generation 349 | None => { 350 | error!("dir {:?} has no snapshots", generation); 351 | continue; 352 | } 353 | }; 354 | 355 | // return only if wal segments in this snapshot is valid. 356 | if let Ok(wal_segments) = self.restore_wal_segments_of(&snapshot).await { 357 | return Ok(Some(RestoreInfo { 358 | snapshot, 359 | wal_segments, 360 | })); 361 | } 362 | } 363 | 364 | Ok(None) 365 | } 366 | } 367 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 2.1, February 1999 3 | 4 | Copyright (C) 1991, 1999 Free Software Foundation, Inc. 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | [This is the first released version of the Lesser GPL. It also counts 10 | as the successor of the GNU Library Public License, version 2, hence 11 | the version number 2.1.] 12 | 13 | Preamble 14 | 15 | The licenses for most software are designed to take away your 16 | freedom to share and change it. By contrast, the GNU General Public 17 | Licenses are intended to guarantee your freedom to share and change 18 | free software--to make sure the software is free for all its users. 19 | 20 | This license, the Lesser General Public License, applies to some 21 | specially designated software packages--typically libraries--of the 22 | Free Software Foundation and other authors who decide to use it. You 23 | can use it too, but we suggest you first think carefully about whether 24 | this license or the ordinary General Public License is the better 25 | strategy to use in any particular case, based on the explanations below. 26 | 27 | When we speak of free software, we are referring to freedom of use, 28 | not price. Our General Public Licenses are designed to make sure that 29 | you have the freedom to distribute copies of free software (and charge 30 | for this service if you wish); that you receive source code or can get 31 | it if you want it; that you can change the software and use pieces of 32 | it in new free programs; and that you are informed that you can do 33 | these things. 34 | 35 | To protect your rights, we need to make restrictions that forbid 36 | distributors to deny you these rights or to ask you to surrender these 37 | rights. These restrictions translate to certain responsibilities for 38 | you if you distribute copies of the library or if you modify it. 39 | 40 | For example, if you distribute copies of the library, whether gratis 41 | or for a fee, you must give the recipients all the rights that we gave 42 | you. You must make sure that they, too, receive or can get the source 43 | code. If you link other code with the library, you must provide 44 | complete object files to the recipients, so that they can relink them 45 | with the library after making changes to the library and recompiling 46 | it. And you must show them these terms so they know their rights. 47 | 48 | We protect your rights with a two-step method: (1) we copyright the 49 | library, and (2) we offer you this license, which gives you legal 50 | permission to copy, distribute and/or modify the library. 51 | 52 | To protect each distributor, we want to make it very clear that 53 | there is no warranty for the free library. Also, if the library is 54 | modified by someone else and passed on, the recipients should know 55 | that what they have is not the original version, so that the original 56 | author's reputation will not be affected by problems that might be 57 | introduced by others. 58 | 59 | Finally, software patents pose a constant threat to the existence of 60 | any free program. We wish to make sure that a company cannot 61 | effectively restrict the users of a free program by obtaining a 62 | restrictive license from a patent holder. Therefore, we insist that 63 | any patent license obtained for a version of the library must be 64 | consistent with the full freedom of use specified in this license. 65 | 66 | Most GNU software, including some libraries, is covered by the 67 | ordinary GNU General Public License. This license, the GNU Lesser 68 | General Public License, applies to certain designated libraries, and 69 | is quite different from the ordinary General Public License. We use 70 | this license for certain libraries in order to permit linking those 71 | libraries into non-free programs. 72 | 73 | When a program is linked with a library, whether statically or using 74 | a shared library, the combination of the two is legally speaking a 75 | combined work, a derivative of the original library. The ordinary 76 | General Public License therefore permits such linking only if the 77 | entire combination fits its criteria of freedom. The Lesser General 78 | Public License permits more lax criteria for linking other code with 79 | the library. 80 | 81 | We call this license the "Lesser" General Public License because it 82 | does Less to protect the user's freedom than the ordinary General 83 | Public License. It also provides other free software developers Less 84 | of an advantage over competing non-free programs. These disadvantages 85 | are the reason we use the ordinary General Public License for many 86 | libraries. However, the Lesser license provides advantages in certain 87 | special circumstances. 88 | 89 | For example, on rare occasions, there may be a special need to 90 | encourage the widest possible use of a certain library, so that it becomes 91 | a de-facto standard. To achieve this, non-free programs must be 92 | allowed to use the library. A more frequent case is that a free 93 | library does the same job as widely used non-free libraries. In this 94 | case, there is little to gain by limiting the free library to free 95 | software only, so we use the Lesser General Public License. 96 | 97 | In other cases, permission to use a particular library in non-free 98 | programs enables a greater number of people to use a large body of 99 | free software. For example, permission to use the GNU C Library in 100 | non-free programs enables many more people to use the whole GNU 101 | operating system, as well as its variant, the GNU/Linux operating 102 | system. 103 | 104 | Although the Lesser General Public License is Less protective of the 105 | users' freedom, it does ensure that the user of a program that is 106 | linked with the Library has the freedom and the wherewithal to run 107 | that program using a modified version of the Library. 108 | 109 | The precise terms and conditions for copying, distribution and 110 | modification follow. Pay close attention to the difference between a 111 | "work based on the library" and a "work that uses the library". The 112 | former contains code derived from the library, whereas the latter must 113 | be combined with the library in order to run. 114 | 115 | GNU LESSER GENERAL PUBLIC LICENSE 116 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 117 | 118 | 0. This License Agreement applies to any software library or other 119 | program which contains a notice placed by the copyright holder or 120 | other authorized party saying it may be distributed under the terms of 121 | this Lesser General Public License (also called "this License"). 122 | Each licensee is addressed as "you". 123 | 124 | A "library" means a collection of software functions and/or data 125 | prepared so as to be conveniently linked with application programs 126 | (which use some of those functions and data) to form executables. 127 | 128 | The "Library", below, refers to any such software library or work 129 | which has been distributed under these terms. A "work based on the 130 | Library" means either the Library or any derivative work under 131 | copyright law: that is to say, a work containing the Library or a 132 | portion of it, either verbatim or with modifications and/or translated 133 | straightforwardly into another language. (Hereinafter, translation is 134 | included without limitation in the term "modification".) 135 | 136 | "Source code" for a work means the preferred form of the work for 137 | making modifications to it. For a library, complete source code means 138 | all the source code for all modules it contains, plus any associated 139 | interface definition files, plus the scripts used to control compilation 140 | and installation of the library. 141 | 142 | Activities other than copying, distribution and modification are not 143 | covered by this License; they are outside its scope. The act of 144 | running a program using the Library is not restricted, and output from 145 | such a program is covered only if its contents constitute a work based 146 | on the Library (independent of the use of the Library in a tool for 147 | writing it). Whether that is true depends on what the Library does 148 | and what the program that uses the Library does. 149 | 150 | 1. You may copy and distribute verbatim copies of the Library's 151 | complete source code as you receive it, in any medium, provided that 152 | you conspicuously and appropriately publish on each copy an 153 | appropriate copyright notice and disclaimer of warranty; keep intact 154 | all the notices that refer to this License and to the absence of any 155 | warranty; and distribute a copy of this License along with the 156 | Library. 157 | 158 | You may charge a fee for the physical act of transferring a copy, 159 | and you may at your option offer warranty protection in exchange for a 160 | fee. 161 | 162 | 2. You may modify your copy or copies of the Library or any portion 163 | of it, thus forming a work based on the Library, and copy and 164 | distribute such modifications or work under the terms of Section 1 165 | above, provided that you also meet all of these conditions: 166 | 167 | a) The modified work must itself be a software library. 168 | 169 | b) You must cause the files modified to carry prominent notices 170 | stating that you changed the files and the date of any change. 171 | 172 | c) You must cause the whole of the work to be licensed at no 173 | charge to all third parties under the terms of this License. 174 | 175 | d) If a facility in the modified Library refers to a function or a 176 | table of data to be supplied by an application program that uses 177 | the facility, other than as an argument passed when the facility 178 | is invoked, then you must make a good faith effort to ensure that, 179 | in the event an application does not supply such function or 180 | table, the facility still operates, and performs whatever part of 181 | its purpose remains meaningful. 182 | 183 | (For example, a function in a library to compute square roots has 184 | a purpose that is entirely well-defined independent of the 185 | application. Therefore, Subsection 2d requires that any 186 | application-supplied function or table used by this function must 187 | be optional: if the application does not supply it, the square 188 | root function must still compute square roots.) 189 | 190 | These requirements apply to the modified work as a whole. If 191 | identifiable sections of that work are not derived from the Library, 192 | and can be reasonably considered independent and separate works in 193 | themselves, then this License, and its terms, do not apply to those 194 | sections when you distribute them as separate works. But when you 195 | distribute the same sections as part of a whole which is a work based 196 | on the Library, the distribution of the whole must be on the terms of 197 | this License, whose permissions for other licensees extend to the 198 | entire whole, and thus to each and every part regardless of who wrote 199 | it. 200 | 201 | Thus, it is not the intent of this section to claim rights or contest 202 | your rights to work written entirely by you; rather, the intent is to 203 | exercise the right to control the distribution of derivative or 204 | collective works based on the Library. 205 | 206 | In addition, mere aggregation of another work not based on the Library 207 | with the Library (or with a work based on the Library) on a volume of 208 | a storage or distribution medium does not bring the other work under 209 | the scope of this License. 210 | 211 | 3. You may opt to apply the terms of the ordinary GNU General Public 212 | License instead of this License to a given copy of the Library. To do 213 | this, you must alter all the notices that refer to this License, so 214 | that they refer to the ordinary GNU General Public License, version 2, 215 | instead of to this License. (If a newer version than version 2 of the 216 | ordinary GNU General Public License has appeared, then you can specify 217 | that version instead if you wish.) Do not make any other change in 218 | these notices. 219 | 220 | Once this change is made in a given copy, it is irreversible for 221 | that copy, so the ordinary GNU General Public License applies to all 222 | subsequent copies and derivative works made from that copy. 223 | 224 | This option is useful when you wish to copy part of the code of 225 | the Library into a program that is not a library. 226 | 227 | 4. You may copy and distribute the Library (or a portion or 228 | derivative of it, under Section 2) in object code or executable form 229 | under the terms of Sections 1 and 2 above provided that you accompany 230 | it with the complete corresponding machine-readable source code, which 231 | must be distributed under the terms of Sections 1 and 2 above on a 232 | medium customarily used for software interchange. 233 | 234 | If distribution of object code is made by offering access to copy 235 | from a designated place, then offering equivalent access to copy the 236 | source code from the same place satisfies the requirement to 237 | distribute the source code, even though third parties are not 238 | compelled to copy the source along with the object code. 239 | 240 | 5. A program that contains no derivative of any portion of the 241 | Library, but is designed to work with the Library by being compiled or 242 | linked with it, is called a "work that uses the Library". Such a 243 | work, in isolation, is not a derivative work of the Library, and 244 | therefore falls outside the scope of this License. 245 | 246 | However, linking a "work that uses the Library" with the Library 247 | creates an executable that is a derivative of the Library (because it 248 | contains portions of the Library), rather than a "work that uses the 249 | library". The executable is therefore covered by this License. 250 | Section 6 states terms for distribution of such executables. 251 | 252 | When a "work that uses the Library" uses material from a header file 253 | that is part of the Library, the object code for the work may be a 254 | derivative work of the Library even though the source code is not. 255 | Whether this is true is especially significant if the work can be 256 | linked without the Library, or if the work is itself a library. The 257 | threshold for this to be true is not precisely defined by law. 258 | 259 | If such an object file uses only numerical parameters, data 260 | structure layouts and accessors, and small macros and small inline 261 | functions (ten lines or less in length), then the use of the object 262 | file is unrestricted, regardless of whether it is legally a derivative 263 | work. (Executables containing this object code plus portions of the 264 | Library will still fall under Section 6.) 265 | 266 | Otherwise, if the work is a derivative of the Library, you may 267 | distribute the object code for the work under the terms of Section 6. 268 | Any executables containing that work also fall under Section 6, 269 | whether or not they are linked directly with the Library itself. 270 | 271 | 6. As an exception to the Sections above, you may also combine or 272 | link a "work that uses the Library" with the Library to produce a 273 | work containing portions of the Library, and distribute that work 274 | under terms of your choice, provided that the terms permit 275 | modification of the work for the customer's own use and reverse 276 | engineering for debugging such modifications. 277 | 278 | You must give prominent notice with each copy of the work that the 279 | Library is used in it and that the Library and its use are covered by 280 | this License. You must supply a copy of this License. If the work 281 | during execution displays copyright notices, you must include the 282 | copyright notice for the Library among them, as well as a reference 283 | directing the user to the copy of this License. Also, you must do one 284 | of these things: 285 | 286 | a) Accompany the work with the complete corresponding 287 | machine-readable source code for the Library including whatever 288 | changes were used in the work (which must be distributed under 289 | Sections 1 and 2 above); and, if the work is an executable linked 290 | with the Library, with the complete machine-readable "work that 291 | uses the Library", as object code and/or source code, so that the 292 | user can modify the Library and then relink to produce a modified 293 | executable containing the modified Library. (It is understood 294 | that the user who changes the contents of definitions files in the 295 | Library will not necessarily be able to recompile the application 296 | to use the modified definitions.) 297 | 298 | b) Use a suitable shared library mechanism for linking with the 299 | Library. A suitable mechanism is one that (1) uses at run time a 300 | copy of the library already present on the user's computer system, 301 | rather than copying library functions into the executable, and (2) 302 | will operate properly with a modified version of the library, if 303 | the user installs one, as long as the modified version is 304 | interface-compatible with the version that the work was made with. 305 | 306 | c) Accompany the work with a written offer, valid for at 307 | least three years, to give the same user the materials 308 | specified in Subsection 6a, above, for a charge no more 309 | than the cost of performing this distribution. 310 | 311 | d) If distribution of the work is made by offering access to copy 312 | from a designated place, offer equivalent access to copy the above 313 | specified materials from the same place. 314 | 315 | e) Verify that the user has already received a copy of these 316 | materials or that you have already sent this user a copy. 317 | 318 | For an executable, the required form of the "work that uses the 319 | Library" must include any data and utility programs needed for 320 | reproducing the executable from it. However, as a special exception, 321 | the materials to be distributed need not include anything that is 322 | normally distributed (in either source or binary form) with the major 323 | components (compiler, kernel, and so on) of the operating system on 324 | which the executable runs, unless that component itself accompanies 325 | the executable. 326 | 327 | It may happen that this requirement contradicts the license 328 | restrictions of other proprietary libraries that do not normally 329 | accompany the operating system. Such a contradiction means you cannot 330 | use both them and the Library together in an executable that you 331 | distribute. 332 | 333 | 7. You may place library facilities that are a work based on the 334 | Library side-by-side in a single library together with other library 335 | facilities not covered by this License, and distribute such a combined 336 | library, provided that the separate distribution of the work based on 337 | the Library and of the other library facilities is otherwise 338 | permitted, and provided that you do these two things: 339 | 340 | a) Accompany the combined library with a copy of the same work 341 | based on the Library, uncombined with any other library 342 | facilities. This must be distributed under the terms of the 343 | Sections above. 344 | 345 | b) Give prominent notice with the combined library of the fact 346 | that part of it is a work based on the Library, and explaining 347 | where to find the accompanying uncombined form of the same work. 348 | 349 | 8. You may not copy, modify, sublicense, link with, or distribute 350 | the Library except as expressly provided under this License. Any 351 | attempt otherwise to copy, modify, sublicense, link with, or 352 | distribute the Library is void, and will automatically terminate your 353 | rights under this License. However, parties who have received copies, 354 | or rights, from you under this License will not have their licenses 355 | terminated so long as such parties remain in full compliance. 356 | 357 | 9. You are not required to accept this License, since you have not 358 | signed it. However, nothing else grants you permission to modify or 359 | distribute the Library or its derivative works. These actions are 360 | prohibited by law if you do not accept this License. Therefore, by 361 | modifying or distributing the Library (or any work based on the 362 | Library), you indicate your acceptance of this License to do so, and 363 | all its terms and conditions for copying, distributing or modifying 364 | the Library or works based on it. 365 | 366 | 10. Each time you redistribute the Library (or any work based on the 367 | Library), the recipient automatically receives a license from the 368 | original licensor to copy, distribute, link with or modify the Library 369 | subject to these terms and conditions. You may not impose any further 370 | restrictions on the recipients' exercise of the rights granted herein. 371 | You are not responsible for enforcing compliance by third parties with 372 | this License. 373 | 374 | 11. If, as a consequence of a court judgment or allegation of patent 375 | infringement or for any other reason (not limited to patent issues), 376 | conditions are imposed on you (whether by court order, agreement or 377 | otherwise) that contradict the conditions of this License, they do not 378 | excuse you from the conditions of this License. If you cannot 379 | distribute so as to satisfy simultaneously your obligations under this 380 | License and any other pertinent obligations, then as a consequence you 381 | may not distribute the Library at all. For example, if a patent 382 | license would not permit royalty-free redistribution of the Library by 383 | all those who receive copies directly or indirectly through you, then 384 | the only way you could satisfy both it and this License would be to 385 | refrain entirely from distribution of the Library. 386 | 387 | If any portion of this section is held invalid or unenforceable under any 388 | particular circumstance, the balance of the section is intended to apply, 389 | and the section as a whole is intended to apply in other circumstances. 390 | 391 | It is not the purpose of this section to induce you to infringe any 392 | patents or other property right claims or to contest validity of any 393 | such claims; this section has the sole purpose of protecting the 394 | integrity of the free software distribution system which is 395 | implemented by public license practices. Many people have made 396 | generous contributions to the wide range of software distributed 397 | through that system in reliance on consistent application of that 398 | system; it is up to the author/donor to decide if he or she is willing 399 | to distribute software through any other system and a licensee cannot 400 | impose that choice. 401 | 402 | This section is intended to make thoroughly clear what is believed to 403 | be a consequence of the rest of this License. 404 | 405 | 12. If the distribution and/or use of the Library is restricted in 406 | certain countries either by patents or by copyrighted interfaces, the 407 | original copyright holder who places the Library under this License may add 408 | an explicit geographical distribution limitation excluding those countries, 409 | so that distribution is permitted only in or among countries not thus 410 | excluded. In such case, this License incorporates the limitation as if 411 | written in the body of this License. 412 | 413 | 13. The Free Software Foundation may publish revised and/or new 414 | versions of the Lesser General Public License from time to time. 415 | Such new versions will be similar in spirit to the present version, 416 | but may differ in detail to address new problems or concerns. 417 | 418 | Each version is given a distinguishing version number. If the Library 419 | specifies a version number of this License which applies to it and 420 | "any later version", you have the option of following the terms and 421 | conditions either of that version or of any later version published by 422 | the Free Software Foundation. If the Library does not specify a 423 | license version number, you may choose any version ever published by 424 | the Free Software Foundation. 425 | 426 | 14. If you wish to incorporate parts of the Library into other free 427 | programs whose distribution conditions are incompatible with these, 428 | write to the author to ask for permission. For software which is 429 | copyrighted by the Free Software Foundation, write to the Free 430 | Software Foundation; we sometimes make exceptions for this. Our 431 | decision will be guided by the two goals of preserving the free status 432 | of all derivatives of our free software and of promoting the sharing 433 | and reuse of software generally. 434 | 435 | NO WARRANTY 436 | 437 | 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO 438 | WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. 439 | EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR 440 | OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY 441 | KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE 442 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 443 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE 444 | LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME 445 | THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 446 | 447 | 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN 448 | WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY 449 | AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU 450 | FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR 451 | CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE 452 | LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING 453 | RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A 454 | FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF 455 | SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH 456 | DAMAGES. 457 | 458 | END OF TERMS AND CONDITIONS 459 | 460 | How to Apply These Terms to Your New Libraries 461 | 462 | If you develop a new library, and you want it to be of the greatest 463 | possible use to the public, we recommend making it free software that 464 | everyone can redistribute and change. You can do so by permitting 465 | redistribution under these terms (or, alternatively, under the terms of the 466 | ordinary General Public License). 467 | 468 | To apply these terms, attach the following notices to the library. It is 469 | safest to attach them to the start of each source file to most effectively 470 | convey the exclusion of warranty; and each file should have at least the 471 | "copyright" line and a pointer to where the full notice is found. 472 | 473 | 474 | Copyright (C) 475 | 476 | This library is free software; you can redistribute it and/or 477 | modify it under the terms of the GNU Lesser General Public 478 | License as published by the Free Software Foundation; either 479 | version 2.1 of the License, or (at your option) any later version. 480 | 481 | This library is distributed in the hope that it will be useful, 482 | but WITHOUT ANY WARRANTY; without even the implied warranty of 483 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 484 | Lesser General Public License for more details. 485 | 486 | You should have received a copy of the GNU Lesser General Public 487 | License along with this library; if not, write to the Free Software 488 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 489 | USA 490 | 491 | Also add information on how to contact you by electronic and paper mail. 492 | 493 | You should also get your employer (if you work as a programmer) or your 494 | school, if any, to sign a "copyright disclaimer" for the library, if 495 | necessary. Here is a sample; alter the names: 496 | 497 | Yoyodyne, Inc., hereby disclaims all copyright interest in the 498 | library `Frob' (a library for tweaking knobs) written by James Random 499 | Hacker. 500 | 501 | , 1 April 1990 502 | Ty Coon, President of Vice 503 | 504 | That's all there is to it! 505 | -------------------------------------------------------------------------------- /src/database/database.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | use std::fs::File; 3 | use std::fs::OpenOptions; 4 | use std::io::Read; 5 | use std::io::Seek; 6 | use std::io::SeekFrom; 7 | use std::io::Write; 8 | use std::os::unix::fs::MetadataExt; 9 | use std::os::unix::fs::PermissionsExt; 10 | use std::path::Path; 11 | use std::path::PathBuf; 12 | use std::time::Duration; 13 | use std::time::SystemTime; 14 | 15 | use log::debug; 16 | use log::error; 17 | use log::info; 18 | use rusqlite::Connection; 19 | use rusqlite::DropBehavior; 20 | use tempfile::NamedTempFile; 21 | use tempfile::tempfile; 22 | use tokio::select; 23 | use tokio::sync::mpsc; 24 | use tokio::sync::mpsc::Receiver; 25 | use tokio::sync::mpsc::Sender; 26 | use tokio::task::JoinHandle; 27 | use tokio::time::sleep; 28 | 29 | use crate::base::Generation; 30 | use crate::base::compress_file; 31 | use crate::base::generation_dir; 32 | use crate::base::generation_file_path; 33 | use crate::base::local_generations_dir; 34 | use crate::base::parent_dir; 35 | use crate::base::parse_wal_path; 36 | use crate::base::path_base; 37 | use crate::base::shadow_wal_dir; 38 | use crate::base::shadow_wal_file; 39 | use crate::config::DbConfig; 40 | use crate::error::Error; 41 | use crate::error::Result; 42 | use crate::sqlite::CheckpointMode; 43 | use crate::sqlite::WAL_FRAME_HEADER_SIZE; 44 | use crate::sqlite::WAL_HEADER_SIZE; 45 | use crate::sqlite::WALFrame; 46 | use crate::sqlite::WALHeader; 47 | use crate::sqlite::align_frame; 48 | use crate::sqlite::checksum; 49 | use crate::sqlite::read_last_checksum; 50 | use crate::sync::Replicate; 51 | use crate::sync::ReplicateCommand; 52 | 53 | const GENERATION_LEN: usize = 32; 54 | 55 | // MaxIndex is the maximum possible WAL index. 56 | // If this index is reached then a new generation will be started. 57 | const MAX_WAL_INDEX: u64 = 0x7FFFFFFF; 58 | 59 | // Default DB settings. 60 | const DEFAULT_MONITOR_INTERVAL: Duration = Duration::from_secs(1); 61 | 62 | #[derive(Clone, Debug)] 63 | pub enum DbCommand { 64 | Snapshot(usize), 65 | } 66 | 67 | #[derive(Debug, Clone)] 68 | pub struct DatabaseInfo { 69 | // Path to the database metadata. 70 | pub meta_dir: String, 71 | 72 | pub page_size: u64, 73 | } 74 | 75 | pub struct Database { 76 | config: DbConfig, 77 | 78 | // Path to the database metadata. 79 | meta_dir: String, 80 | 81 | // full wal file name of db file 82 | wal_file: String, 83 | page_size: u64, 84 | 85 | connection: Connection, 86 | 87 | // database connection for transaction, None if there if no tranction 88 | tx_connection: Option, 89 | 90 | // for sync 91 | sync_notifiers: Vec>, 92 | sync_handle: Vec>, 93 | syncs: Vec, 94 | } 95 | 96 | // position info of wal for a generation 97 | #[derive(Debug, Clone, Default)] 98 | pub struct WalGenerationPos { 99 | // generation name 100 | pub generation: Generation, 101 | 102 | // wal file index 103 | pub index: u64, 104 | 105 | // offset within wal file 106 | pub offset: u64, 107 | } 108 | 109 | impl WalGenerationPos { 110 | pub fn is_empty(&self) -> bool { 111 | self.generation.is_empty() && self.index == 0 && self.offset == 0 112 | } 113 | } 114 | 115 | #[derive(Debug, Default)] 116 | struct SyncInfo { 117 | pub generation: Generation, 118 | pub db_mod_time: Option, 119 | pub index: u64, 120 | pub wal_size: u64, 121 | pub shadow_wal_file: String, 122 | pub shadow_wal_size: u64, 123 | pub reason: Option, 124 | pub restart: bool, 125 | } 126 | 127 | impl Database { 128 | fn init_params(db: &str, connection: &Connection) -> Result<()> { 129 | let max_try_num = 10; 130 | // busy timeout 131 | connection.busy_timeout(Duration::from_secs(1))?; 132 | 133 | let mut try_num = 0; 134 | while try_num < max_try_num { 135 | try_num += 1; 136 | // PRAGMA journal_mode = wal; 137 | if let Err(e) = 138 | connection.pragma_update_and_check(None, "journal_mode", "WAL", |_param| { 139 | // println!("journal_mode param: {:?}\n", param); 140 | Ok(()) 141 | }) 142 | { 143 | error!("set journal_mode=wal error: {:?}", e); 144 | continue; 145 | } 146 | try_num = 0; 147 | break; 148 | } 149 | if try_num >= max_try_num { 150 | error!("try set journal_mode=wal failed"); 151 | return Err(Error::SqliteError(format!( 152 | "set journal_mode=wal for db {} failed", 153 | db, 154 | ))); 155 | } 156 | 157 | let mut try_num = 0; 158 | while try_num < max_try_num { 159 | try_num += 1; 160 | // PRAGMA wal_autocheckpoint = 0; 161 | if let Err(e) = 162 | connection.pragma_update_and_check(None, "wal_autocheckpoint", "0", |_param| { 163 | // println!("wal_autocheckpoint param: {:?}\n", param); 164 | Ok(()) 165 | }) 166 | { 167 | error!("set wal_autocheckpoint=0 error: {:?}", e); 168 | continue; 169 | } 170 | try_num = 0; 171 | break; 172 | } 173 | if try_num >= max_try_num { 174 | error!("try set wal_autocheckpoint=0 failed"); 175 | return Err(Error::SqliteError(format!( 176 | "set wal_autocheckpoint=0 for db {} failed", 177 | db, 178 | ))); 179 | } 180 | 181 | Ok(()) 182 | } 183 | 184 | fn create_internal_tables(connection: &Connection) -> Result<()> { 185 | connection.execute( 186 | "CREATE TABLE IF NOT EXISTS _replited_seq (id INTEGER PRIMARY KEY, seq INTEGER);", 187 | (), 188 | )?; 189 | 190 | connection.execute( 191 | "CREATE TABLE IF NOT EXISTS _replited_lock (id INTEGER);", 192 | (), 193 | )?; 194 | Ok(()) 195 | } 196 | 197 | // acquire_read_lock begins a read transaction on the database to prevent checkpointing. 198 | fn acquire_read_lock(&mut self) -> Result<()> { 199 | if self.tx_connection.is_none() { 200 | let tx_connection = Connection::open(&self.config.db)?; 201 | // Execute read query to obtain read lock. 202 | tx_connection.execute_batch("BEGIN;SELECT COUNT(1) FROM _replited_seq;")?; 203 | self.tx_connection = Some(tx_connection); 204 | } 205 | 206 | Ok(()) 207 | } 208 | 209 | fn release_read_lock(&mut self) -> Result<()> { 210 | // Rollback & clear read transaction. 211 | if let Some(tx) = &self.tx_connection { 212 | tx.execute_batch("ROLLBACK;")?; 213 | self.tx_connection = None; 214 | } 215 | 216 | Ok(()) 217 | } 218 | 219 | // init replited directory 220 | fn init_directory(config: &DbConfig) -> Result { 221 | let file_path = PathBuf::from(&config.db); 222 | let db_name = file_path.file_name().unwrap().to_str().unwrap(); 223 | let dir_path = file_path.parent().unwrap_or_else(|| Path::new(".")); 224 | let meta_dir = format!("{}/.{}-replited/", dir_path.to_str().unwrap(), db_name,); 225 | fs::create_dir_all(&meta_dir)?; 226 | 227 | Ok(meta_dir) 228 | } 229 | 230 | fn try_create(config: DbConfig) -> Result<(Self, Receiver)> { 231 | info!("start database with config: {:?}\n", config); 232 | let connection = Connection::open(&config.db)?; 233 | 234 | Database::init_params(&config.db, &connection)?; 235 | 236 | Database::create_internal_tables(&connection)?; 237 | 238 | let page_size = connection.pragma_query_value(None, "page_size", |row| row.get(0))?; 239 | let wal_file = format!("{}-wal", config.db); 240 | 241 | // init path 242 | let meta_dir = Database::init_directory(&config)?; 243 | 244 | // init replicate 245 | let (db_notifier, db_receiver) = mpsc::channel(16); 246 | let mut sync_handle = Vec::with_capacity(config.replicate.len()); 247 | let mut sync_notifiers = Vec::with_capacity(config.replicate.len()); 248 | let mut syncs = Vec::with_capacity(config.replicate.len()); 249 | let info = DatabaseInfo { 250 | meta_dir: meta_dir.clone(), 251 | page_size, 252 | }; 253 | let db = Path::new(&config.db) 254 | .file_name() 255 | .unwrap() 256 | .to_str() 257 | .unwrap() 258 | .to_string(); 259 | for (index, replicate) in config.replicate.iter().enumerate() { 260 | let (sync_notifier, sync_receiver) = mpsc::channel(16); 261 | let s = Replicate::new( 262 | replicate.clone(), 263 | db.clone(), 264 | index, 265 | db_notifier.clone(), 266 | info.clone(), 267 | )?; 268 | syncs.push(s.clone()); 269 | let h = Replicate::start(s, sync_receiver)?; 270 | sync_handle.push(h); 271 | sync_notifiers.push(sync_notifier); 272 | } 273 | 274 | let mut db = Self { 275 | config: config.clone(), 276 | connection, 277 | meta_dir, 278 | wal_file, 279 | page_size, 280 | tx_connection: None, 281 | sync_notifiers, 282 | sync_handle, 283 | syncs, 284 | }; 285 | 286 | db.acquire_read_lock()?; 287 | 288 | // If we have an existing shadow WAL, ensure the headers match. 289 | if let Err(err) = db.verify_header_match() { 290 | debug!( 291 | "db {} cannot determine last wal position, error: {:?}, clearing generation", 292 | db.config.db, err 293 | ); 294 | 295 | if let Err(e) = fs::remove_file(generation_file_path(&db.meta_dir)) { 296 | error!("db {} remove generation file error: {:?}", db.config.db, e); 297 | } 298 | } 299 | 300 | // Clean up previous generations. 301 | if let Err(e) = db.clean() { 302 | error!( 303 | "db {} clean previous generations error {:?} when startup", 304 | db.config.db, e 305 | ); 306 | return Err(e); 307 | } 308 | 309 | Ok((db, db_receiver)) 310 | } 311 | 312 | // verify if primary wal and last shadow wal header match 313 | fn verify_header_match(&self) -> Result<()> { 314 | let generation = self.current_generation()?; 315 | if generation.is_empty() { 316 | return Ok(()); 317 | } 318 | 319 | let shadow_wal_file = self.current_shadow_wal_file(&generation)?; 320 | 321 | let wal_header = WALHeader::read(&self.wal_file)?; 322 | let shadow_wal_header = WALHeader::read(&shadow_wal_file)?; 323 | 324 | if wal_header != shadow_wal_header { 325 | return Err(Error::MismatchWalHeaderError(format!( 326 | "db {} wal header mismatched", 327 | self.config.db 328 | ))); 329 | } 330 | 331 | Ok(()) 332 | } 333 | 334 | fn calc_wal_size(&self, n: u64) -> u64 { 335 | WAL_HEADER_SIZE + (WAL_FRAME_HEADER_SIZE + self.page_size) * n 336 | } 337 | 338 | fn decide_checkpoint_mode( 339 | &self, 340 | orig_wal_size: u64, 341 | new_wal_size: u64, 342 | info: &SyncInfo, 343 | ) -> Option { 344 | // If WAL size is great than max threshold, force checkpoint. 345 | // If WAL size is greater than min threshold, attempt checkpoint. 346 | if self.config.truncate_page_number > 0 347 | && orig_wal_size >= self.calc_wal_size(self.config.truncate_page_number) 348 | { 349 | debug!( 350 | "checkpoint by orig_wal_size({}) > truncate_page_number({})", 351 | orig_wal_size, self.config.truncate_page_number 352 | ); 353 | return Some(CheckpointMode::Truncate); 354 | } else if self.config.max_checkpoint_page_number > 0 355 | && new_wal_size >= self.calc_wal_size(self.config.max_checkpoint_page_number) 356 | { 357 | debug!( 358 | "checkpoint by new_wal_size({}) > max_checkpoint_page_number({})", 359 | new_wal_size, self.config.max_checkpoint_page_number 360 | ); 361 | return Some(CheckpointMode::Restart); 362 | } else if new_wal_size >= self.calc_wal_size(self.config.min_checkpoint_page_number) { 363 | debug!( 364 | "checkpoint by new_wal_size({}) > min_checkpoint_page_number({})", 365 | new_wal_size, self.config.min_checkpoint_page_number 366 | ); 367 | return Some(CheckpointMode::Passive); 368 | } else if self.config.checkpoint_interval_secs > 0 369 | && let Some(db_mod_time) = &info.db_mod_time { 370 | let now = SystemTime::now(); 371 | 372 | if let Ok(duration) = now.duration_since(*db_mod_time) 373 | && duration.as_secs() > self.config.checkpoint_interval_secs 374 | && new_wal_size > self.calc_wal_size(1) 375 | { 376 | debug!( 377 | "checkpoint by db_mod_time > checkpoint_interval_secs({})", 378 | self.config.checkpoint_interval_secs 379 | ); 380 | return Some(CheckpointMode::Passive); 381 | } 382 | } 383 | 384 | None 385 | } 386 | 387 | // copy pending data from wal to shadow wal 388 | async fn sync(&mut self) -> Result<()> { 389 | debug!("sync database: {}", self.config.db); 390 | 391 | // make sure wal file has at least one frame in it 392 | self.ensure_wal_exists()?; 393 | 394 | // Verify our last sync matches the current state of the WAL. 395 | // This ensures that we have an existing generation & that the last sync 396 | // position of the real WAL hasn't been overwritten by another process. 397 | let mut info = self.verify()?; 398 | debug!("db {} sync info: {:?}", self.config.db, info); 399 | 400 | // Track if anything in the shadow WAL changes and then notify at the end. 401 | let mut changed = 402 | info.wal_size != info.shadow_wal_size || info.restart || info.reason.is_some(); 403 | 404 | if let Some(reason) = &info.reason { 405 | // Start new generation & notify user via log message. 406 | info.generation = self.create_generation().await?; 407 | info!( 408 | "db {} sync new generation: {}, reason: {}", 409 | self.config.db, 410 | info.generation.as_str(), 411 | reason 412 | ); 413 | 414 | // Clear shadow wal info. 415 | info.shadow_wal_file = shadow_wal_file(&self.meta_dir, info.generation.as_str(), 0); 416 | info.shadow_wal_size = WAL_HEADER_SIZE; 417 | info.restart = false; 418 | info.reason = None; 419 | } 420 | 421 | // Synchronize real WAL with current shadow WAL. 422 | let (orig_wal_size, new_wal_size) = self.sync_wal(&info)?; 423 | debug!( 424 | "db {} sync_wal {}:{}", 425 | self.config.db, orig_wal_size, new_wal_size 426 | ); 427 | 428 | // decide if need to do checkpoint 429 | let checkmode = self.decide_checkpoint_mode(orig_wal_size, new_wal_size, &info); 430 | if let Some(checkmode) = checkmode { 431 | changed = true; 432 | 433 | self.do_checkpoint(info.generation.as_str(), checkmode.as_str())?; 434 | } 435 | 436 | // Clean up any old files. 437 | self.clean()?; 438 | 439 | // notify the database has been changed 440 | if changed { 441 | let generation_pos = self.wal_generation_position()?; 442 | self.sync_notifiers[0] 443 | .send(ReplicateCommand::DbChanged(generation_pos)) 444 | .await?; 445 | } 446 | 447 | debug!("sync db {} ok", self.config.db); 448 | Ok(()) 449 | } 450 | 451 | pub fn wal_generation_position(&self) -> Result { 452 | let generation = Generation::try_create(&self.current_generation()?)?; 453 | 454 | let (index, _total_size) = self.current_shadow_index(generation.as_str())?; 455 | 456 | let shadow_wal_file = self.shadow_wal_file(generation.as_str(), index); 457 | 458 | match fs::exists(&shadow_wal_file) { 459 | Err(e) => { 460 | error!( 461 | "check db {} shadow_wal file {} err: {}", 462 | self.config.db, shadow_wal_file, e, 463 | ); 464 | return Err(e.into()); 465 | } 466 | Ok(exist) => { 467 | // shadow wal file not exists, return pos with offset = 0 468 | if !exist { 469 | return Ok(WalGenerationPos { 470 | generation, 471 | index, 472 | offset: 0, 473 | }); 474 | } 475 | } 476 | } 477 | let file_metadata = fs::metadata(&shadow_wal_file)?; 478 | 479 | Ok(WalGenerationPos { 480 | generation, 481 | index, 482 | offset: align_frame(self.page_size, file_metadata.size()), 483 | }) 484 | } 485 | 486 | // CurrentShadowWALPath returns the path to the last shadow WAL in a generation. 487 | fn current_shadow_wal_file(&self, generation: &str) -> Result { 488 | let (index, _total_size) = self.current_shadow_index(generation)?; 489 | 490 | Ok(self.shadow_wal_file(generation, index)) 491 | } 492 | 493 | // returns the path of a single shadow WAL file. 494 | fn shadow_wal_file(&self, generation: &str, index: u64) -> String { 495 | shadow_wal_file(&self.meta_dir, generation, index) 496 | } 497 | 498 | // create_generation initiates a new generation by establishing the generation 499 | // directory, capturing snapshots for each replica, and refreshing the current 500 | // generation name. 501 | async fn create_generation(&mut self) -> Result { 502 | let generation = Generation::new(); 503 | 504 | // create a temp file to write new generation 505 | let temp_file = NamedTempFile::new()?; 506 | let temp_file_name = temp_file.path().to_str().unwrap().to_string(); 507 | 508 | fs::write(&temp_file_name, generation.as_str())?; 509 | 510 | // create new directory. 511 | let dir = generation_dir(&self.meta_dir, generation.as_str()); 512 | fs::create_dir_all(&dir)?; 513 | 514 | // init first(index 0) shadow wal file 515 | self.init_shadow_wal_file(&self.shadow_wal_file(generation.as_str(), 0))?; 516 | 517 | // rename the temp file to generation file 518 | let generation_file = generation_file_path(&self.meta_dir); 519 | fs::rename(&temp_file_name, &generation_file)?; 520 | 521 | // Remove old generations. 522 | self.clean()?; 523 | 524 | Ok(generation) 525 | } 526 | 527 | // copies pending bytes from the real WAL to the shadow WAL. 528 | fn sync_wal(&self, info: &SyncInfo) -> Result<(u64, u64)> { 529 | let (orig_size, new_size) = self.copy_to_shadow_wal(&info.shadow_wal_file)?; 530 | debug!( 531 | "db {} sync_wal copy_to_shadow_wal: {}, {}", 532 | self.config.db, orig_size, new_size 533 | ); 534 | 535 | if !info.restart { 536 | return Ok((orig_size, new_size)); 537 | } 538 | 539 | // Parse index of current shadow WAL file. 540 | let index = parse_wal_path(&info.shadow_wal_file)?; 541 | 542 | // Start a new shadow WAL file with next index. 543 | let new_shadow_wal_file = 544 | shadow_wal_file(&self.meta_dir, info.generation.as_str(), index + 1); 545 | let new_size = self.init_shadow_wal_file(&new_shadow_wal_file)?; 546 | 547 | Ok((orig_size, new_size)) 548 | } 549 | 550 | // remove old generations files and wal files 551 | fn clean(&mut self) -> Result<()> { 552 | self.clean_generations()?; 553 | 554 | self.clean_wal()?; 555 | 556 | Ok(()) 557 | } 558 | 559 | fn clean_generations(&self) -> Result<()> { 560 | let generation = self.current_generation()?; 561 | let genetations_dir = local_generations_dir(&self.meta_dir); 562 | 563 | if !fs::exists(&genetations_dir)? { 564 | return Ok(()); 565 | } 566 | for entry in fs::read_dir(&genetations_dir)? { 567 | let entry = entry?; 568 | let file_name = entry.file_name().as_os_str().to_str().unwrap().to_string(); 569 | let base = path_base(&file_name)?; 570 | // skip the current generation 571 | if base == generation { 572 | continue; 573 | } 574 | let path = Path::new(&genetations_dir) 575 | .join(&file_name) 576 | .as_path() 577 | .to_str() 578 | .unwrap() 579 | .to_string(); 580 | fs::remove_dir_all(&path)?; 581 | } 582 | Ok(()) 583 | } 584 | 585 | // removes WAL files that have been replicated. 586 | fn clean_wal(&self) -> Result<()> { 587 | let generation = self.current_generation()?; 588 | 589 | let mut min = None; 590 | for sync in &self.syncs { 591 | // let sync = sync.read().await; 592 | let mut position = sync.position(); 593 | if position.generation.as_str() != generation { 594 | position = WalGenerationPos::default(); 595 | } 596 | match min { 597 | None => min = Some(position.index), 598 | Some(m) => { 599 | if position.index < m { 600 | min = Some(position.index); 601 | } 602 | } 603 | } 604 | } 605 | 606 | // Skip if our lowest index is too small. 607 | let mut min = match min { 608 | None => return Ok(()), 609 | Some(min) => min, 610 | }; 611 | 612 | if min == 0 { 613 | return Ok(()); 614 | } 615 | // Keep an extra WAL file. 616 | min -= 1; 617 | 618 | // Remove all WAL files for the generation before the lowest index. 619 | let dir = shadow_wal_dir(&self.meta_dir, generation.as_str()); 620 | if !fs::exists(&dir)? { 621 | return Ok(()); 622 | } 623 | for entry in fs::read_dir(&dir)?.flatten() { 624 | let file_name = entry.file_name().as_os_str().to_str().unwrap().to_string(); 625 | let index = parse_wal_path(&file_name)?; 626 | if index >= min { 627 | continue; 628 | } 629 | let path = Path::new(&dir) 630 | .join(&file_name) 631 | .as_path() 632 | .to_str() 633 | .unwrap() 634 | .to_string(); 635 | fs::remove_file(&path)?; 636 | } 637 | 638 | Ok(()) 639 | } 640 | 641 | fn init_shadow_wal_file(&self, shadow_wal: &String) -> Result { 642 | debug!("init_shadow_wal_file {}", shadow_wal); 643 | 644 | // read wal file header 645 | let wal_header = WALHeader::read(&self.wal_file)?; 646 | if wal_header.page_size != self.page_size { 647 | return Err(Error::SqliteInvalidWalHeaderError("Invalid page size")); 648 | } 649 | 650 | // create new shadow wal file 651 | let db_file_metadata = fs::metadata(&self.config.db)?; 652 | let mode = db_file_metadata.mode(); 653 | let dir = parent_dir(shadow_wal); 654 | if let Some(dir) = dir { 655 | fs::create_dir_all(&dir)?; 656 | } else { 657 | debug!("db {} cannot find parent dir of shadow wal", self.config.db); 658 | } 659 | let mut shadow_wal_file = fs::File::create(shadow_wal)?; 660 | let mut permissions = shadow_wal_file.metadata()?.permissions(); 661 | permissions.set_mode(mode); 662 | shadow_wal_file.set_permissions(permissions)?; 663 | std::os::unix::fs::chown( 664 | shadow_wal, 665 | Some(db_file_metadata.uid()), 666 | Some(db_file_metadata.gid()), 667 | )?; 668 | // write wal file header into shadow wal file 669 | shadow_wal_file.write_all(&wal_header.data)?; 670 | shadow_wal_file.flush()?; 671 | debug!("create shadow wal file {}", shadow_wal); 672 | 673 | // copy wal file frame into shadow wal file 674 | let (orig_size, new_size) = self.copy_to_shadow_wal(shadow_wal)?; 675 | debug!( 676 | "db {} init_shadow_wal_file copy_to_shadow_wal: {}, {}", 677 | self.config.db, orig_size, new_size 678 | ); 679 | 680 | Ok(new_size) 681 | } 682 | 683 | // return original wal file size and new wal size 684 | fn copy_to_shadow_wal(&self, shadow_wal: &String) -> Result<(u64, u64)> { 685 | let wal_file_name = &self.wal_file; 686 | let wal_file_metadata = fs::metadata(wal_file_name)?; 687 | let orig_wal_size = align_frame(self.page_size, wal_file_metadata.size()); 688 | 689 | let shadow_wal_file_metadata = fs::metadata(shadow_wal)?; 690 | let orig_shadow_wal_size = align_frame(self.page_size, shadow_wal_file_metadata.size()); 691 | debug!( 692 | "copy_to_shadow_wal orig_wal_size: {}, orig_shadow_wal_size: {}", 693 | orig_wal_size, orig_shadow_wal_size 694 | ); 695 | 696 | // read shadow wal header 697 | let wal_header = WALHeader::read(shadow_wal)?; 698 | 699 | // create a temp file to copy wal frames 700 | let mut temp_file = tempfile()?; 701 | 702 | // seek on real db wal 703 | let mut wal_file = File::open(wal_file_name)?; 704 | wal_file.seek(SeekFrom::Start(orig_shadow_wal_size))?; 705 | // read last checksum of shadow wal file 706 | let (mut ck1, mut ck2) = read_last_checksum(shadow_wal, self.page_size)?; 707 | let mut offset = orig_shadow_wal_size; 708 | let mut last_commit_size = orig_shadow_wal_size; 709 | // Read through WAL from last position to find the page of the last 710 | // committed transaction. 711 | loop { 712 | let wal_frame = WALFrame::read(&mut wal_file, self.page_size); 713 | let wal_frame = match wal_frame { 714 | Ok(wal_frame) => wal_frame, 715 | Err(e) => { 716 | if e.code() == Error::UNEXPECTED_EOF_ERROR { 717 | debug!("copy_to_shadow_wal EOF at offset: {}", offset); 718 | break; 719 | } else { 720 | error!("copy_to_shadow_wal error {} at offset {}", e, offset); 721 | return Err(e); 722 | } 723 | } 724 | }; 725 | 726 | // compare wal frame salts with wal header salts, break if mismatch 727 | if wal_frame.salt1 != wal_header.salt1 || wal_frame.salt2 != wal_header.salt2 { 728 | debug!( 729 | "db {} copy shadow wal frame salt mismatch at offset {}", 730 | self.config.db, offset 731 | ); 732 | break; 733 | } 734 | 735 | // frame header 736 | (ck1, ck2) = checksum(&wal_frame.data[0..8], ck1, ck2, wal_header.is_big_endian); 737 | // frame data 738 | (ck1, ck2) = checksum(&wal_frame.data[24..], ck1, ck2, wal_header.is_big_endian); 739 | if ck1 != wal_frame.checksum1 || ck2 != wal_frame.checksum2 { 740 | debug!( 741 | "db {} copy shadow wal checksum mismatch at offset {}, check: ({},{}),({},{})", 742 | self.config.db, offset, ck1, wal_frame.checksum1, ck2, wal_frame.checksum2, 743 | ); 744 | break; 745 | } 746 | 747 | // Write page to temporary WAL file. 748 | temp_file.write_all(&wal_frame.data)?; 749 | 750 | offset += wal_frame.data.len() as u64; 751 | if wal_frame.db_size != 0 { 752 | last_commit_size = offset; 753 | } 754 | } 755 | 756 | // If no WAL writes found, exit. 757 | if last_commit_size == orig_shadow_wal_size { 758 | return Ok((orig_wal_size, last_commit_size)); 759 | } 760 | 761 | // copy frames from temp file to shadow wal file 762 | temp_file.flush()?; 763 | temp_file.seek(SeekFrom::Start(0))?; 764 | 765 | let mut buffer = Vec::new(); 766 | temp_file.read_to_end(&mut buffer)?; 767 | 768 | // append wal frames to end of shadow wal file 769 | let mut shadow_wal_file = OpenOptions::new().append(true).open(shadow_wal)?; 770 | shadow_wal_file.write_all(&buffer)?; 771 | shadow_wal_file.flush()?; 772 | 773 | // in debug mode, assert last frame match 774 | #[cfg(debug_assertions)] 775 | { 776 | let shadow_wal_file = fs::metadata(shadow_wal)?; 777 | let shadow_wal_size = align_frame(self.page_size, shadow_wal_file.len()); 778 | 779 | let offset = shadow_wal_size - self.page_size - WAL_FRAME_HEADER_SIZE; 780 | 781 | let mut shadow_wal_file = OpenOptions::new().read(true).open(shadow_wal)?; 782 | shadow_wal_file.seek(SeekFrom::Start(offset))?; 783 | let shadow_wal_last_frame = WALFrame::read(&mut shadow_wal_file, self.page_size)?; 784 | 785 | let mut wal_file = OpenOptions::new().read(true).open(wal_file_name)?; 786 | wal_file.seek(SeekFrom::Start(offset))?; 787 | let wal_last_frame = WALFrame::read(&mut wal_file, self.page_size)?; 788 | 789 | assert_eq!(shadow_wal_last_frame, wal_last_frame); 790 | } 791 | Ok((orig_wal_size, last_commit_size)) 792 | } 793 | 794 | // make sure wal file has at least one frame in it 795 | fn ensure_wal_exists(&self) -> Result<()> { 796 | if fs::exists(&self.wal_file)? { 797 | let stat = fs::metadata(&self.wal_file)?; 798 | if !stat.is_file() { 799 | return Err(Error::SqliteWalError(format!( 800 | "wal {} is not a file", 801 | self.wal_file, 802 | ))); 803 | } 804 | 805 | if stat.len() >= WAL_HEADER_SIZE { 806 | return Ok(()); 807 | } 808 | } 809 | 810 | // create transaction that updates the internal table. 811 | self.connection.execute( 812 | "INSERT INTO _replited_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1;", 813 | (), 814 | )?; 815 | 816 | Ok(()) 817 | } 818 | 819 | // current_shadow_index returns the current WAL index & total size. 820 | fn current_shadow_index(&self, generation: &str) -> Result<(u64, u64)> { 821 | let wal_dir = shadow_wal_dir(&self.meta_dir, generation); 822 | if !fs::exists(&wal_dir)? { 823 | return Ok((0, 0)); 824 | } 825 | 826 | let entries = fs::read_dir(&wal_dir)?; 827 | let mut total_size = 0; 828 | let mut index = 0; 829 | for entry in entries.flatten() { 830 | let file_type = entry.file_type()?; 831 | let file_name = entry.file_name().into_string().unwrap(); 832 | let path = Path::new(&wal_dir).join(&file_name); 833 | if !fs::exists(&path)? { 834 | // file was deleted after os.ReadDir returned 835 | debug!( 836 | "db {} shadow wal {:?} deleted after read_dir return", 837 | self.config.db, path 838 | ); 839 | continue; 840 | } 841 | if !file_type.is_file() { 842 | continue; 843 | } 844 | let metadata = entry.metadata()?; 845 | total_size += metadata.size(); 846 | match parse_wal_path(&file_name) { 847 | Err(e) => { 848 | debug!("invalid wal file {:?}, err:{:?}", file_name, e); 849 | continue; 850 | } 851 | Ok(i) => { 852 | if i > index { 853 | index = i; 854 | } 855 | } 856 | } 857 | } 858 | 859 | Ok((index, total_size)) 860 | } 861 | 862 | // current_generation returns the name of the generation saved to the "generation" 863 | // file in the meta data directory. 864 | // Returns empty string if none exists. 865 | fn current_generation(&self) -> Result { 866 | let generation_file = generation_file_path(&self.meta_dir); 867 | if !fs::exists(&generation_file)? { 868 | return Ok("".to_string()); 869 | } 870 | let generation = fs::read_to_string(&generation_file)?; 871 | if generation.len() != GENERATION_LEN { 872 | return Ok("".to_string()); 873 | } 874 | 875 | Ok(generation) 876 | } 877 | 878 | // verify ensures the current shadow WAL state matches where it left off from 879 | // the real WAL. Returns generation & WAL sync information. If info.reason is 880 | // not blank, verification failed and a new generation should be started. 881 | fn verify(&mut self) -> Result { 882 | let mut info = SyncInfo::default(); 883 | 884 | // get existing generation 885 | let generation = self.current_generation()?; 886 | if generation.is_empty() { 887 | info.reason = Some("no generation exists".to_string()); 888 | return Ok(info); 889 | } 890 | info.generation = Generation::try_create(&generation)?; 891 | 892 | let db_file = fs::metadata(&self.config.db)?; 893 | if let Ok(db_mod_time) = db_file.modified() { 894 | info.db_mod_time = Some(db_mod_time); 895 | } else { 896 | info.db_mod_time = None; 897 | } 898 | 899 | // total bytes of real WAL. 900 | let wal_file = fs::metadata(&self.wal_file)?; 901 | let wal_size = align_frame(self.page_size, wal_file.len()); 902 | info.wal_size = wal_size; 903 | 904 | // get current shadow wal index 905 | let (index, _total_size) = self.current_shadow_index(&generation)?; 906 | if index > MAX_WAL_INDEX { 907 | info.reason = Some("max index exceeded".to_string()); 908 | return Ok(info); 909 | } 910 | info.shadow_wal_file = self.shadow_wal_file(&generation, index); 911 | 912 | // Determine shadow WAL current size. 913 | if !fs::exists(&info.shadow_wal_file)? { 914 | info.reason = Some("no shadow wal".to_string()); 915 | return Ok(info); 916 | } 917 | let shadow_wal_file = fs::metadata(&info.shadow_wal_file)?; 918 | info.shadow_wal_size = align_frame(self.page_size, shadow_wal_file.len()); 919 | if info.shadow_wal_size < WAL_HEADER_SIZE { 920 | info.reason = Some("short shadow wal".to_string()); 921 | return Ok(info); 922 | } 923 | 924 | if info.shadow_wal_size > info.wal_size { 925 | info.reason = Some("wal truncated by another process".to_string()); 926 | return Ok(info); 927 | } 928 | 929 | // Compare WAL headers. Start a new shadow WAL if they are mismatched. 930 | let wal_header = WALHeader::read(&self.wal_file)?; 931 | let shadow_wal_header = WALHeader::read(&info.shadow_wal_file)?; 932 | if wal_header != shadow_wal_header { 933 | info.restart = true; 934 | } 935 | 936 | if info.shadow_wal_size == WAL_HEADER_SIZE && info.restart { 937 | info.reason = Some("wal header only, mismatched".to_string()); 938 | return Ok(info); 939 | } 940 | 941 | // Verify last page synced still matches. 942 | if info.shadow_wal_size > WAL_HEADER_SIZE { 943 | let offset = info.shadow_wal_size - self.page_size - WAL_FRAME_HEADER_SIZE; 944 | 945 | let mut wal_file = OpenOptions::new().read(true).open(&self.wal_file)?; 946 | wal_file.seek(SeekFrom::Start(offset))?; 947 | let wal_last_frame = WALFrame::read(&mut wal_file, self.page_size)?; 948 | 949 | let mut shadow_wal_file = OpenOptions::new().read(true).open(&info.shadow_wal_file)?; 950 | shadow_wal_file.seek(SeekFrom::Start(offset))?; 951 | let shadow_wal_last_frame = WALFrame::read(&mut shadow_wal_file, self.page_size)?; 952 | if wal_last_frame != shadow_wal_last_frame { 953 | debug!( 954 | "db {} verify offset: {}, shadow_wal_file: {}, shadow salt1: {}, wal file: {}, wal salt1: {} last frame mismatched", 955 | self.config.db, 956 | offset, 957 | info.shadow_wal_file, 958 | shadow_wal_last_frame.salt1, 959 | self.wal_file, 960 | wal_last_frame.salt1 961 | ); 962 | info.reason = Some("wal overwritten by another process".to_string()); 963 | return Ok(info); 964 | } 965 | } 966 | 967 | Ok(info) 968 | } 969 | 970 | fn checkpoint(&mut self, mode: CheckpointMode) -> Result<()> { 971 | let generation = self.current_generation()?; 972 | 973 | self.do_checkpoint(&generation, mode.as_str()) 974 | } 975 | 976 | // checkpoint performs a checkpoint on the WAL file and initializes a 977 | // new shadow WAL file. 978 | fn do_checkpoint(&mut self, generation: &str, mode: &str) -> Result<()> { 979 | // Try getting a checkpoint lock, will fail during snapshots. 980 | 981 | let shadow_wal_file = self.current_shadow_wal_file(generation)?; 982 | 983 | // Read WAL header before checkpoint to check if it has been restarted. 984 | let wal_header1 = WALHeader::read(&self.wal_file)?; 985 | 986 | // Copy shadow WAL before checkpoint to copy as much as possible. 987 | let (orig_size, new_size) = self.copy_to_shadow_wal(&shadow_wal_file)?; 988 | debug!( 989 | "db {} do_checkpoint copy_to_shadow_wal: {}, {}", 990 | self.config.db, orig_size, new_size 991 | ); 992 | 993 | // Execute checkpoint and immediately issue a write to the WAL to ensure 994 | // a new page is written. 995 | self.exec_checkpoint(mode)?; 996 | 997 | self.connection.execute( 998 | "INSERT INTO _replited_seq (id, seq) VALUES (1, 1) ON CONFLICT (id) DO UPDATE SET seq = seq + 1;", 999 | (), 1000 | )?; 1001 | 1002 | // If WAL hasn't been restarted, exit. 1003 | let wal_header2 = WALHeader::read(&self.wal_file)?; 1004 | if wal_header1 == wal_header2 { 1005 | return Ok(()); 1006 | } 1007 | 1008 | // Start a transaction. This will be promoted immediately after. 1009 | let mut connection = Connection::open(&self.config.db)?; 1010 | let mut tx = connection.transaction()?; 1011 | tx.set_drop_behavior(DropBehavior::Rollback); 1012 | 1013 | // Insert into the lock table to promote to a write tx. The lock table 1014 | // insert will never actually occur because our tx will be rolled back, 1015 | // however, it will ensure our tx grabs the write lock. Unfortunately, 1016 | // we can't call "BEGIN IMMEDIATE" as we are already in a transaction. 1017 | tx.execute("INSERT INTO _replited_lock (id) VALUES (1);", ())?; 1018 | 1019 | // Copy the end of the previous WAL before starting a new shadow WAL. 1020 | let (orig_size, new_size) = self.copy_to_shadow_wal(&shadow_wal_file)?; 1021 | debug!( 1022 | "db {} do_checkpoint after checkpoint copy_to_shadow_wal: {}, {}", 1023 | self.config.db, orig_size, new_size 1024 | ); 1025 | 1026 | // Parse index of current shadow WAL file. 1027 | let index = parse_wal_path(&shadow_wal_file)?; 1028 | 1029 | // Start a new shadow WAL file with next index. 1030 | let new_shadow_wal_file = self.shadow_wal_file(generation, index + 1); 1031 | self.init_shadow_wal_file(&new_shadow_wal_file)?; 1032 | 1033 | Ok(()) 1034 | } 1035 | 1036 | fn exec_checkpoint(&mut self, mode: &str) -> Result<()> { 1037 | // Ensure the read lock has been removed before issuing a checkpoint. 1038 | // We defer the re-acquire to ensure it occurs even on an early return. 1039 | self.release_read_lock()?; 1040 | 1041 | // A non-forced checkpoint is issued as "PASSIVE". This will only checkpoint 1042 | // if there are not pending transactions. A forced checkpoint ("RESTART") 1043 | // will wait for pending transactions to end & block new transactions before 1044 | // forcing the checkpoint and restarting the WAL. 1045 | // 1046 | // See: https://www.sqlite.org/pragma.html#pragma_wal_checkpoint 1047 | let sql = format!("PRAGMA wal_checkpoint({})", mode); 1048 | 1049 | let ret = self.connection.execute_batch(&sql); 1050 | 1051 | // Reacquire the read lock immediately after the checkpoint. 1052 | self.acquire_read_lock()?; 1053 | 1054 | ret?; 1055 | 1056 | Ok(()) 1057 | } 1058 | 1059 | pub async fn handle_db_command(&mut self, cmd: DbCommand) -> Result<()> { 1060 | match cmd { 1061 | DbCommand::Snapshot(i) => self.handle_db_snapshot_command(i).await?, 1062 | } 1063 | Ok(()) 1064 | } 1065 | 1066 | fn snapshot(&mut self) -> Result<(Vec, WalGenerationPos)> { 1067 | // Issue a passive checkpoint to flush any pages to disk before snapshotting. 1068 | self.checkpoint(CheckpointMode::Passive)?; 1069 | 1070 | // Prevent internal checkpoints during snapshot. 1071 | 1072 | // Acquire a read lock on the database during snapshot to prevent external checkpoints. 1073 | self.acquire_read_lock()?; 1074 | 1075 | // Obtain current position. 1076 | let pos = self.wal_generation_position()?; 1077 | if pos.is_empty() { 1078 | return Err(Error::NoGenerationError("no generation")); 1079 | } 1080 | 1081 | // compress db file 1082 | let compressed_data = compress_file(&self.config.db)?; 1083 | 1084 | Ok((compressed_data.to_owned(), pos)) 1085 | } 1086 | 1087 | async fn handle_db_snapshot_command(&mut self, index: usize) -> Result<()> { 1088 | let (compressed_data, generation_pos) = self.snapshot()?; 1089 | debug!( 1090 | "db {} snapshot {} data of pos {:?}", 1091 | self.config.db, 1092 | compressed_data.len(), 1093 | generation_pos 1094 | ); 1095 | self.sync_notifiers[index] 1096 | .send(ReplicateCommand::Snapshot(( 1097 | generation_pos, 1098 | compressed_data, 1099 | ))) 1100 | .await?; 1101 | Ok(()) 1102 | } 1103 | } 1104 | 1105 | impl Drop for Database { 1106 | fn drop(&mut self) { 1107 | let _ = self.release_read_lock(); 1108 | } 1109 | } 1110 | 1111 | pub async fn run_database(config: DbConfig) -> Result<()> { 1112 | let (mut database, mut db_receiver) = match Database::try_create(config.clone()) { 1113 | Ok((db, receiver)) => (db, receiver), 1114 | Err(e) => { 1115 | error!("run_database for {:?} error: {:?}", config, e); 1116 | return Err(e); 1117 | } 1118 | }; 1119 | loop { 1120 | select! { 1121 | cmd = db_receiver.recv() => { 1122 | if let Some(cmd) = cmd && let Err(e) = database.handle_db_command(cmd).await { 1123 | error!("handle_db_command of db {} error: {:?}", database.config.db, e); 1124 | } 1125 | } 1126 | _ = sleep(DEFAULT_MONITOR_INTERVAL) => { 1127 | if let Err(e) = database.sync().await { 1128 | error!("sync db {} error: {:?}", database.config.db, e); 1129 | } 1130 | } 1131 | } 1132 | } 1133 | } 1134 | --------------------------------------------------------------------------------