├── .github └── workflows │ ├── pages.yml │ └── release.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── build-linux.sh ├── cfg └── config.go ├── config.toml ├── core └── events.go ├── db ├── change_log.go ├── change_log_event.go ├── cleanup.go ├── global_change_log_script.tmpl ├── sqlite.go ├── table_change_log_script.tmpl └── utils.go ├── docs ├── .eslintrc.json ├── .gitignore ├── .prettierignore ├── next.config.js ├── package.json ├── postcss.config.js ├── public │ ├── logo.png │ ├── marmot-logo-transparent.svg │ └── sillouhette-smooth.svg ├── src │ ├── components │ │ ├── Hero.tsx │ │ └── Icons │ │ │ ├── MarmotLogo.tsx │ │ │ └── MarmotLogoTransparent.tsx │ ├── pages │ │ ├── _app.tsx │ │ ├── _meta.json │ │ ├── demo.mdx │ │ ├── index.mdx │ │ ├── internals.mdx │ │ └── intro.mdx │ └── styles │ │ └── globals.css ├── tailwind.config.js ├── theme.config.js ├── tsconfig.json └── yarn.lock ├── examples ├── node-1-config.toml ├── node-2-config.toml ├── node-3-config.toml └── run-cluster.sh ├── go.mod ├── go.sum ├── logstream ├── replication_event.go ├── replication_state.go ├── replicator.go └── replicator_meta_store.go ├── marmot.go ├── pool ├── connection_pool.go └── sqlite_driver_connector.go ├── snapshot ├── db_snapshot.go ├── nats_snapshot.go ├── nats_storage.go ├── s3_storage.go ├── sftp_storage.go └── webdav_storage.go ├── stream ├── embedded_nats.go ├── nats.go ├── nats_logger.go └── routes_discover.go ├── telemetry └── telemetry.go └── utils ├── deep.go ├── state_context.go ├── stop_watch.go └── timeout.go /.github/workflows/pages.yml: -------------------------------------------------------------------------------- 1 | name: 📕 Deploy docs site to Pages 2 | 3 | on: 4 | push: 5 | branches: ["master"] 6 | 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | pages: write 12 | id-token: write 13 | 14 | concurrency: 15 | group: "pages" 16 | cancel-in-progress: true 17 | 18 | jobs: 19 | build: 20 | defaults: 21 | run: 22 | working-directory: docs 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v3 27 | - name: Detect package manager 28 | id: detect-package-manager 29 | run: | 30 | if [ -f "${{ github.workspace }}/docs/yarn.lock" ]; then 31 | echo "::set-output name=manager::yarn" 32 | echo "::set-output name=command::install" 33 | echo "::set-output name=runner::yarn" 34 | exit 0 35 | elif [ -f "${{ github.workspace }}/docs/package.json" ]; then 36 | echo "::set-output name=manager::npm" 37 | echo "::set-output name=command::ci" 38 | echo "::set-output name=runner::npx --no-install" 39 | exit 0 40 | else 41 | echo "Unable to determine packager manager" 42 | exit 1 43 | fi 44 | - name: Setup Node 45 | uses: actions/setup-node@v3 46 | with: 47 | node-version: "16" 48 | cache: ${{ steps.detect-package-manager.outputs.manager }} 49 | cache-dependency-path: docs/yarn.lock 50 | - name: Setup Pages 51 | uses: actions/configure-pages@v2 52 | with: 53 | static_site_generator: next 54 | - name: Restore cache 55 | uses: actions/cache@v3 56 | with: 57 | path: | 58 | docs/.next/cache 59 | key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}-${{ hashFiles('**.[jt]s', '**.[jt]sx') }} 60 | restore-keys: | 61 | ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}- 62 | - name: Install dependencies 63 | run: ${{ steps.detect-package-manager.outputs.manager }} ${{ steps.detect-package-manager.outputs.command }} 64 | - name: Build with Next.js 65 | run: ${{ steps.detect-package-manager.outputs.runner }} next build 66 | - name: Static HTML export with Next.js 67 | run: ${{ steps.detect-package-manager.outputs.runner }} next export 68 | - name: Upload artifact 69 | uses: actions/upload-pages-artifact@v1 70 | with: 71 | path: docs/out 72 | 73 | deploy: 74 | environment: 75 | name: github-pages 76 | url: ${{ steps.deployment.outputs.page_url }} 77 | runs-on: ubuntu-latest 78 | needs: build 79 | steps: 80 | - name: Deploy to GitHub Pages 81 | id: deployment 82 | uses: actions/deploy-pages@v1 83 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # .github/workflows/release.yaml 2 | name: Release 3 | 4 | on: 5 | release: 6 | types: [created] 7 | push: 8 | branches: [master] 9 | pull_request: 10 | types: 11 | - opened 12 | - synchronize 13 | - reopened 14 | 15 | jobs: 16 | linux-build: 17 | runs-on: ubuntu-latest 18 | strategy: 19 | matrix: 20 | include: 21 | - arch: amd64 22 | cc: gcc 23 | 24 | - arch: amd64 25 | cc: gcc 26 | static: true 27 | 28 | - arch: arm64 29 | cc: aarch64-linux-gnu-gcc 30 | 31 | - arch: arm64 32 | cc: aarch64-linux-gnu-gcc 33 | static: true 34 | - arch: arm 35 | arm: 6 36 | cc: arm-linux-gnueabi-gcc 37 | 38 | - arch: arm 39 | arm: 6 40 | cc: arm-linux-gnueabi-gcc 41 | static: true 42 | 43 | - arch: arm 44 | arm: 7 45 | cc: arm-linux-gnueabihf-gcc 46 | 47 | - arch: arm 48 | arm: 7 49 | cc: arm-linux-gnueabihf-gcc 50 | static: true 51 | env: 52 | GOOS: linux 53 | GOARCH: ${{ matrix.arch }} 54 | GOARM: ${{ matrix.arm }} 55 | CC: ${{ matrix.cc }} 56 | LDFLAGS: ${{ matrix.static && '-extldflags "-static"' || '' }} 57 | SUFFIX: "${{ matrix.static && '-static' || ''}}" 58 | VERSION: "${{ github.event_name == 'release' && github.event.release.name || github.sha }}" 59 | steps: 60 | - uses: actions/checkout@v3 61 | 62 | - uses: actions/setup-go@v4 63 | with: 64 | go-version: '^1.21.3' 65 | 66 | - name: Install cross-compilers 67 | run: | 68 | sudo apt-get update 69 | sudo apt-get install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf gcc-arm-linux-gnueabi 70 | 71 | - name: Build marmot 72 | run: | 73 | CGO_ENABLED=1 go build -ldflags "-s -w ${{ env.LDFLAGS }}" -o marmot . 74 | tar -czvf marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz marmot config.toml LICENSE README.md examples/* 75 | 76 | - name: Upload binary artifact 77 | uses: actions/upload-artifact@v2 78 | with: 79 | name: marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz 80 | path: marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz 81 | if-no-files-found: error 82 | 83 | - name: Get release 84 | id: release 85 | uses: bruceadams/get-release@v1.2.3 86 | if: github.event_name == 'release' 87 | env: 88 | GITHUB_TOKEN: ${{ github.token }} 89 | 90 | - name: Upload release tarball 91 | uses: actions/upload-release-asset@v1.0.2 92 | if: github.event_name == 'release' 93 | env: 94 | GITHUB_TOKEN: ${{ github.token }} 95 | with: 96 | upload_url: ${{ steps.release.outputs.upload_url }} 97 | asset_path: marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz 98 | asset_name: marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.GOARM }}${{ env.SUFFIX }}.tar.gz 99 | asset_content_type: application/gzip 100 | mac-build: 101 | runs-on: macos-latest 102 | strategy: 103 | matrix: 104 | include: 105 | - arch: arm64 106 | cc: gcc 107 | 108 | - arch: amd64 109 | cc: gcc 110 | env: 111 | GOOS: darwin 112 | GOARCH: ${{ matrix.arch }} 113 | CC: ${{ matrix.cc }} 114 | LDFLAGS: ${{ matrix.static && '-extldflags "-static"' || '' }} 115 | SUFFIX: "${{ matrix.static && '-static' || ''}}" 116 | VERSION: "${{ github.event_name == 'release' && github.event.release.name || github.sha }}" 117 | steps: 118 | - uses: actions/checkout@v3 119 | 120 | - uses: actions/setup-go@v4 121 | with: 122 | go-version: '^1.21.3' 123 | 124 | - name: Build marmot 125 | run: | 126 | CGO_ENABLED=1 go build -o marmot . 127 | tar -czvf marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.SUFFIX }}.tar.gz marmot config.toml LICENSE README.md examples/* 128 | 129 | - name: Upload binary artifact 130 | uses: actions/upload-artifact@v2 131 | with: 132 | name: marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.SUFFIX }}.tar.gz 133 | path: marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.SUFFIX }}.tar.gz 134 | if-no-files-found: error 135 | 136 | - name: Get release 137 | id: release 138 | uses: bruceadams/get-release@v1.2.3 139 | if: github.event_name == 'release' 140 | env: 141 | GITHUB_TOKEN: ${{ github.token }} 142 | 143 | - name: Upload release tarball 144 | uses: actions/upload-release-asset@v1.0.2 145 | if: github.event_name == 'release' 146 | env: 147 | GITHUB_TOKEN: ${{ github.token }} 148 | with: 149 | upload_url: ${{ steps.release.outputs.upload_url }} 150 | asset_path: marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.SUFFIX }}.tar.gz 151 | asset_name: marmot-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}${{ env.SUFFIX }}.tar.gz 152 | asset_content_type: application/gzip 153 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | build/ 17 | .idea 18 | .vscode 19 | .DS_Store 20 | *.cert 21 | *.crt 22 | *.key 23 | *.pem 24 | marmot 25 | dist/ 26 | 27 | # NPM Stuff 28 | node_modules/ 29 | npm-debug.log 30 | yarn-error.log 31 | yarn-debug.log 32 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | marmot-coc@googlegroups.com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Zohaib Sibte Hassan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Marmot 2 | 3 | [![Go Report Card](https://goreportcard.com/badge/github.com/maxpert/marmot)](https://goreportcard.com/report/github.com/maxpert/marmot) 4 | [![Discord](https://badgen.net/badge/icon/discord?icon=discord&label=Marmot)](https://discord.gg/AWUwY66XsE) 5 | ![GitHub](https://img.shields.io/github/license/maxpert/marmot) 6 | 7 | ## What & Why? 8 | 9 | Marmot is a distributed SQLite replicator with leaderless, and eventual consistency. It allows you to build a robust replication 10 | between your nodes by building on top of fault-tolerant [NATS JetStream](https://nats.io/). 11 | 12 | So if you are running a read heavy website based on SQLite, you should be easily able to scale it out by adding more SQLite replicated nodes. 13 | SQLite is probably the most ubiquitous DB that exists almost everywhere, Marmot aims to make it even more ubiquitous for server 14 | side applications by building a replication layer on top. 15 | 16 | ## Quick Start 17 | 18 | Download [latest](https://github.com/maxpert/marmot/releases/latest) Marmot and extract package using: 19 | 20 | ``` 21 | tar vxzf marmot-v*.tar.gz 22 | ``` 23 | 24 | From extracted directory run `examples/run-cluster.sh`. Make a change in `/tmp/marmot-1.db` using: 25 | 26 | ``` 27 | bash > sqlite3 /tmp/marmot-1.db 28 | sqlite3 > INSERT INTO Books (title, author, publication_year) VALUES ('Pride and Prejudice', 'Jane Austen', 1813); 29 | ``` 30 | 31 | Now observe changes getting propagated to other database `/tmp/marmot-2.db`: 32 | 33 | ``` 34 | bash > sqlite3 /tmp/marmot-2.db 35 | sqlite3 > SELECT * FROM Books; 36 | ``` 37 | 38 | You should be able to make changes interchangeably and see the changes getting propagated. 39 | 40 | ## Out in wild 41 | 42 | Here are some official, and community demos/usages showing Marmot out in wild: 43 | - [2-node HA for edge Kubernetes - Using Marmot](https://www.youtube.com/watch?v=HycGtLjlikI) 44 | - [Scaling Isso with Marmot on Fly.io](https://maxpert.github.io/marmot/demo) 45 | - [Scaling PocketBase with Marmot on Fly.io](https://github.com/maxpert/marmot-pocketbase-flyio) 46 | - [Scaling PocketBase with Marmot 0.4.x](https://www.youtube.com/watch?v=QqZl61bJ9BA) 47 | - [Scaling Keystone 6 with Marmot 0.4.x](https://youtu.be/GQ5x8pc9vuI) 48 | 49 | ## What is the difference from others? 50 | 51 | Marmot is essentially a CDC (Change Data Capture) and replication pipeline running top of NATS. It can automatically configure appropriate 52 | JetStreams making sure those streams evenly distribute load over those shards, so scaling simply boils down to adding more nodes, and 53 | re-balancing those JetStreams (auto rebalancing not implemented yet). 54 | 55 | There are a few solutions like [rqlite](https://github.com/rqlite/rqlite), [dqlite](https://dqlite.io/), and 56 | [LiteFS](https://github.com/superfly/litefs) etc. All of them either are layers on top of SQLite (e.g. 57 | rqlite, dqlite) that requires them to sit in the middle with network layer in order to provide 58 | replication; or intercept physical page level writes to stream them off to replicas. In both 59 | cases they require a single primary node where all the writes have to go, and then these 60 | changes are applied to multiple readonly replicas. 61 | 62 | Marmot on the other hand is born different. It's born to act as a side-car to your existing processes: 63 | - Instead of requiring single primary, there is **no primary**! Which means **any node can make changes to its local DB**. 64 | Marmot will use triggers to capture your changes, and then stream them off to NATS. 65 | - Instead of being strongly consistent, Marmot is **eventually consistent**. Which means no locking, or blocking of nodes. 66 | - It does not require any changes to your existing SQLite application logic for reading/writing. 67 | 68 | Making these choices has multiple benefits: 69 | 70 | - You can read, and write to your SQLite database like you normally do. No extension, or VFS changes. 71 | - You can write on any node! You don't have to go to single primary for writing your data. 72 | - As long as you start with same copy of database, all the mutations will eventually converge 73 | (hence eventually consistent). 74 | 75 | ## What happens when there is a race condition? 76 | 77 | In Marmot every row is uniquely mapped to a JetStream. This guarantees that for any node to publish changes for a row it has to go through 78 | same JetStream as everyone else. If two nodes perform a change to same row in parallel, both of the nodes will compete to publish their 79 | change to JetStream cluster. Due to [RAFT quorum](https://docs.nats.io/running-a-nats-service/configuration/clustering/jetstream_clustering#raft) 80 | constraint only one of the writer will be able to get its changes published first. Now as these changes are applied (even the publisher applies 81 | its own changes to database) the **last writer** will always win. This means there is NO serializability guarantee of a transaction 82 | spanning multiple tables. This is a design choice, in order to avoid any sort of global locking, and performance. 83 | 84 | ## Stargazers over time 85 | [![Stargazers over time](https://starchart.cc/maxpert/marmot.svg?variant=adaptive)](https://starchart.cc/maxpert/marmot) 86 | 87 | ## Limitations 88 | Right now there are a few limitations on current solution: 89 | - Marmot does not support schema changes propagation, so any tables you create or columns you change won't be reflected. 90 | This feature is being [debated](https://github.com/maxpert/marmot/discussions/59) and will be available in future 91 | versions of Marmot. 92 | - You can't watch tables selectively on a DB. This is due to various limitations around snapshot and restore mechanism. 93 | - WAL mode required - since your DB is going to be processed by multiple processes the only way to have multi-process 94 | changes reliably is via WAL. 95 | - Marmot is eventually consistent - This simply means rows can get synced out of order, and `SERIALIZABLE` assumptions 96 | on transactions might not hold true anymore. However your application can choose to redirect writes to single node 97 | so that your changes are always replayed in order. 98 | 99 | ## Features 100 | 101 | ![Eventually Consistent](https://img.shields.io/badge/Eventually%20Consistent-✔️-green) 102 | ![Leaderless Replication](https://img.shields.io/badge/Leaderless%20Replication-✔️-green) 103 | ![Fault Tolerant](https://img.shields.io/badge/Fault%20Tolerant-✔️-green) 104 | ![Built on NATS](https://img.shields.io/badge/Built%20on%20NATS-✔️-green) 105 | 106 | - Leaderless replication never requiring a single node to handle all write load. 107 | - Ability to snapshot and fully recover from those snapshots. Multiple storage options for snapshot: 108 | - ![NATS Blob Storage](https://img.shields.io/badge/NATS%20Blob-%E2%9C%94%EF%B8%8F-green) 109 | - ![WebDAV](https://img.shields.io/badge/WebDAV-%E2%9C%94%EF%B8%8F-green) 110 | - ![SFTP](https://img.shields.io/badge/SFTP-%E2%9C%94%EF%B8%8F-green) 111 | - S3 Compatible: 112 | - ![AWS S3](https://img.shields.io/badge/AWS%20S3-%E2%9C%94%EF%B8%8F-green) 113 | - ![Minio](https://img.shields.io/badge/Minio-%E2%9C%94%EF%B8%8F-green) 114 | - ![Blackblaze](https://img.shields.io/badge/Blackblaze-%E2%9C%94%EF%B8%8F-green) 115 | - ![SeaweedFS](https://img.shields.io/badge/SeaweedFS-%E2%9C%94%EF%B8%8F-green) 116 | 117 | - Built with NATS, abstracting stream distribution and replication. 118 | - Support for log entry compression, handling content heavy CMS needs. 119 | - Sleep timeout support for serverless scenarios. 120 | 121 | 122 | ## Dependencies 123 | Starting 0.8+ Marmot comes with embedded [nats-server](https://nats.io/download/) with JetStream support. This not only reduces 124 | the dependencies/processes that one might have to spin up, but also provides with out-of-box tooling like 125 | [nat-cli](https://github.com/nats-io/natscli). You can also use existing libraries to build additional 126 | tooling and scripts due to standard library support. Here is one example using Deno: 127 | 128 | ``` 129 | deno run --allow-net https://gist.githubusercontent.com/maxpert/d50a49dfb2f307b30b7cae841c9607e1/raw/6d30803c140b0ba602545c1c0878d3394be548c3/watch-marmot-change-logs.ts -u -p -s 130 | ``` 131 | 132 | The output will look something like this: 133 | ![image](https://user-images.githubusercontent.com/22441/196061378-21f885b3-7958-4a7e-994b-09d4e86df721.png) 134 | 135 | ## Production status 136 | 137 | - `v0.8.x` introduced support for embedded NATS. This is recommended version for production. 138 | - `v0.7.x` moves to file based configuration rather than CLI flags, and S3 compatible snapshot storage. 139 | - `v0.6.x` introduces snapshot save/restore. It's in pre-production state. 140 | - `v0.5.x` introduces change log compression with zstd. 141 | - `v0.4.x` introduces NATS based change log streaming, and continuous multi-directional sync. 142 | - `v0.3.x` is deprecated, and unstable. DO NOT USE IT IN PRODUCTION. 143 | 144 | ## CLI Documentation 145 | 146 | Marmot picks simplicity, and lesser knobs to configure by choice. Here are command line options you can use to 147 | configure marmot: 148 | 149 | - `config` - Path to a TOML configuration file. Check out `config.toml` comments for detailed documentation 150 | on various configurable options. 151 | - `cleanup` (default: `false`) - Just cleanup and exit marmot. Useful for scenarios where you are 152 | performing a cleanup of hooks and change logs. 153 | - `save-snapshot` (default: `false` `Since 0.6.x`) - Just snapshot the local database, and upload snapshot 154 | to NATS/S3 server 155 | - `cluster-addr` (default: none `Since 0.8.x`) - Sets the binding address for cluster, when specifying 156 | this flag at-least two nodes will be required (or `replication_log.replicas`). It's a simple 157 | `:` pair that can be used to bind cluster listening server. 158 | - Since `v0.8.4` Marmot will automatically expose a leaf server on `:`. This is 159 | intended to reduce the number for flags. So if you expose cluster on port `4222` the port `4223` will 160 | be automatically a leaf server listener. 161 | - `cluster-peers` (default: none `Since 0.8.x`) - Comma separated list of `nats://:/` peers of 162 | NATS cluster. You can also use (Since version `v0.8.4` ) `dns://:/` to A/AAAA record lookups. 163 | Marmot will automatically resolve the DNS IPs at boot time to expand the routes with value of 164 | `nats://:/` value, where `` is replaced with all the DNS entries queried. There 165 | are two additional query parameters you can use: 166 | - `min` - forcing Marmot to wait for minimum number of entries (e.g. `dns://foo:4222/?min=3` will require 167 | 3 DNS entries to be present before embedded NATs server is started) 168 | - `interval_ms` - delay between DNS queries, which will prevent Marmot from flooding DNS server. 169 | - `leaf-server` (default: none `Since v0.8.4` )- Comma separated list of `nats://:/` 170 | or `dns://:/` just like `cluster-peers` can be used to connect to a cluster 171 | as a leaf node. 172 | 173 | For more details and internal workings of marmot [go to these docs](https://maxpert.github.io/marmot/). 174 | 175 | ## FAQs & Community 176 | 177 | - For FAQs visit [this page](https://maxpert.github.io/marmot/intro#faq) 178 | - For community visit our [discord](https://discord.gg/AWUwY66XsE) or discussions on GitHub 179 | 180 | ## Our sponsor 181 | 182 | Last but not least we would like to thank our sponsors who have been supporting development of this project. 183 | 184 | [GoLand logo. 185 | JetBrains Logo (Main) logo.](https://www.jetbrains.com/?utm_medium=opensource&utm_source=marmot) 186 | -------------------------------------------------------------------------------- /build-linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # docker run --rm -v "$PWD":/usr/src/myapp -w /usr/src/myapp -e CGO_ENABLED=1 -e GOARCH=amd64 golang:1.18 go build -v -o build/marmot-linux-amd64 marmot.go 4 | 5 | CC=x86_64-linux-musl-gcc \ 6 | CXX=x86_64-linux-musl-g++ \ 7 | GOARCH=amd64 GOOS=linux CGO_ENABLED=1 \ 8 | go build -ldflags "-linkmode external -extldflags -static" -o dist/linux/amd64/marmot 9 | 10 | -------------------------------------------------------------------------------- /cfg/config.go: -------------------------------------------------------------------------------- 1 | package cfg 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "hash/fnv" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | 11 | "github.com/BurntSushi/toml" 12 | "github.com/denisbrodbeck/machineid" 13 | "github.com/google/uuid" 14 | "github.com/rs/zerolog/log" 15 | ) 16 | 17 | type SnapshotStoreType string 18 | 19 | const NodeNamePrefix = "marmot-node" 20 | const EmbeddedClusterName = "e-marmot" 21 | const ( 22 | Nats SnapshotStoreType = "nats" 23 | S3 SnapshotStoreType = "s3" 24 | WebDAV SnapshotStoreType = "webdav" 25 | SFTP SnapshotStoreType = "sftp" 26 | ) 27 | 28 | type ReplicationLogConfiguration struct { 29 | Shards uint64 `toml:"shards"` 30 | MaxEntries int64 `toml:"max_entries"` 31 | Replicas int `toml:"replicas"` 32 | Compress bool `toml:"compress"` 33 | UpdateExisting bool `toml:"update_existing"` 34 | } 35 | 36 | type WebDAVConfiguration struct { 37 | Url string `toml:"url"` 38 | } 39 | 40 | type SFTPConfiguration struct { 41 | Url string `toml:"url"` 42 | } 43 | 44 | type S3Configuration struct { 45 | DirPath string `toml:"path"` 46 | Endpoint string `toml:"endpoint"` 47 | AccessKey string `toml:"access_key"` 48 | SecretKey string `toml:"secret"` 49 | SessionToken string `toml:"session_token"` 50 | Bucket string `toml:"bucket"` 51 | UseSSL bool `toml:"use_ssl"` 52 | } 53 | 54 | type ObjectStoreConfiguration struct { 55 | Replicas int `toml:"replicas"` 56 | BucketName string `toml:"bucket"` 57 | } 58 | 59 | type SnapshotConfiguration struct { 60 | Enable bool `toml:"enabled"` 61 | Interval uint32 `toml:"interval"` 62 | StoreType SnapshotStoreType `toml:"store"` 63 | Nats ObjectStoreConfiguration `toml:"nats"` 64 | S3 S3Configuration `toml:"s3"` 65 | WebDAV WebDAVConfiguration `toml:"webdav"` 66 | SFTP SFTPConfiguration `toml:"sftp"` 67 | } 68 | 69 | type NATSConfiguration struct { 70 | URLs []string `toml:"urls"` 71 | SubjectPrefix string `toml:"subject_prefix"` 72 | StreamPrefix string `toml:"stream_prefix"` 73 | ServerConfigFile string `toml:"server_config"` 74 | SeedFile string `toml:"seed_file"` 75 | CredsUser string `toml:"user_name"` 76 | CredsPassword string `toml:"user_password"` 77 | CAFile string `toml:"ca_file"` 78 | CertFile string `toml:"cert_file"` 79 | KeyFile string `toml:"key_file"` 80 | BindAddress string `toml:"bind_address"` 81 | ConnectRetries int `toml:"connect_retries"` 82 | ReconnectWaitSeconds int `toml:"reconnect_wait_seconds"` 83 | } 84 | 85 | type LoggingConfiguration struct { 86 | Verbose bool `toml:"verbose"` 87 | Format string `toml:"format"` 88 | } 89 | 90 | type PrometheusConfiguration struct { 91 | Bind string `toml:"bind"` 92 | Enable bool `toml:"enable"` 93 | Namespace string `toml:"namespace"` 94 | Subsystem string `toml:"subsystem"` 95 | } 96 | 97 | type Configuration struct { 98 | SeqMapPath string `toml:"seq_map_path"` 99 | DBPath string `toml:"db_path"` 100 | NodeID uint64 `toml:"node_id"` 101 | Publish bool `toml:"publish"` 102 | Replicate bool `toml:"replicate"` 103 | ScanMaxChanges uint32 `toml:"scan_max_changes"` 104 | CleanupInterval uint32 `toml:"cleanup_interval"` 105 | SleepTimeout uint32 `toml:"sleep_timeout"` 106 | PollingInterval uint32 `toml:"polling_interval"` 107 | 108 | Snapshot SnapshotConfiguration `toml:"snapshot"` 109 | ReplicationLog ReplicationLogConfiguration `toml:"replication_log"` 110 | NATS NATSConfiguration `toml:"nats"` 111 | Logging LoggingConfiguration `toml:"logging"` 112 | Prometheus PrometheusConfiguration `toml:"prometheus"` 113 | } 114 | 115 | var ConfigPathFlag = flag.String("config", "", "Path to configuration file") 116 | var CleanupFlag = flag.Bool("cleanup", false, "Only cleanup marmot triggers and changelogs") 117 | var SaveSnapshotFlag = flag.Bool("save-snapshot", false, "Only take snapshot and upload") 118 | var ClusterAddrFlag = flag.String("cluster-addr", "", "Cluster listening address") 119 | var ClusterPeersFlag = flag.String("cluster-peers", "", "Comma separated list of clusters") 120 | var LeafServerFlag = flag.String("leaf-servers", "", "Comma separated list of leaf servers") 121 | var ProfServer = flag.String("pprof", "", "PProf listening address") 122 | 123 | var DataRootDir = os.TempDir() 124 | var Config = &Configuration{ 125 | SeqMapPath: path.Join(DataRootDir, "seq-map.cbor"), 126 | DBPath: path.Join(DataRootDir, "marmot.db"), 127 | NodeID: 0, 128 | Publish: true, 129 | Replicate: true, 130 | ScanMaxChanges: 512, 131 | CleanupInterval: 5000, 132 | SleepTimeout: 0, 133 | PollingInterval: 0, 134 | 135 | Snapshot: SnapshotConfiguration{ 136 | Enable: true, 137 | Interval: 0, 138 | StoreType: Nats, 139 | Nats: ObjectStoreConfiguration{ 140 | Replicas: 1, 141 | }, 142 | S3: S3Configuration{}, 143 | WebDAV: WebDAVConfiguration{}, 144 | SFTP: SFTPConfiguration{}, 145 | }, 146 | 147 | ReplicationLog: ReplicationLogConfiguration{ 148 | Shards: 1, 149 | MaxEntries: 1024, 150 | Replicas: 1, 151 | Compress: true, 152 | UpdateExisting: false, 153 | }, 154 | 155 | NATS: NATSConfiguration{ 156 | URLs: []string{}, 157 | SubjectPrefix: "marmot-change-log", 158 | StreamPrefix: "marmot-changes", 159 | ServerConfigFile: "", 160 | SeedFile: "", 161 | CredsPassword: "", 162 | CredsUser: "", 163 | BindAddress: ":-1", 164 | ConnectRetries: 5, 165 | ReconnectWaitSeconds: 2, 166 | }, 167 | 168 | Logging: LoggingConfiguration{ 169 | Verbose: false, 170 | Format: "console", 171 | }, 172 | 173 | Prometheus: PrometheusConfiguration{ 174 | Bind: ":3010", 175 | Enable: false, 176 | Namespace: "marmot", 177 | Subsystem: "", 178 | }, 179 | } 180 | 181 | func init() { 182 | id, err := machineid.ID() 183 | if err != nil { 184 | log.Warn().Err(err).Msg("⚠️⚠️⚠️ Unable to read machine ID from OS, generating random ID ⚠️⚠️⚠️") 185 | id = uuid.NewString() 186 | } 187 | 188 | hasher := fnv.New64() 189 | _, err = hasher.Write([]byte(id)) 190 | if err != nil { 191 | panic(err) 192 | } 193 | 194 | Config.NodeID = hasher.Sum64() 195 | } 196 | 197 | func Load(filePath string) error { 198 | _, err := toml.DecodeFile(filePath, Config) 199 | if os.IsNotExist(err) { 200 | return nil 201 | } 202 | 203 | if err != nil { 204 | return err 205 | } 206 | 207 | DataRootDir, err = filepath.Abs(path.Dir(Config.DBPath)) 208 | if err != nil { 209 | return err 210 | } 211 | 212 | if Config.SeqMapPath == "" { 213 | Config.SeqMapPath = path.Join(DataRootDir, "seq-map.cbor") 214 | } 215 | 216 | return nil 217 | } 218 | 219 | func (c *Configuration) SnapshotStorageType() SnapshotStoreType { 220 | return c.Snapshot.StoreType 221 | } 222 | 223 | func (c *Configuration) NodeName() string { 224 | return fmt.Sprintf("%s-%d", NodeNamePrefix, c.NodeID) 225 | } 226 | -------------------------------------------------------------------------------- /config.toml: -------------------------------------------------------------------------------- 1 | # Path to target SQLite database 2 | db_path="/tmp/marmot.db" 3 | 4 | # ID to uniquely identify your nodes in your cluster 5 | # It's recommended to always configure this 6 | # node_id=1 7 | 8 | # Path to persist the saved sequence map on disk for warm reboot 9 | # If this file is missing Marmot has to download snapshot 10 | # and replay all logs in order to restore database 11 | # seq_map_path="/tmp/seq-map.cbor" 12 | 13 | # Replication enabled/disabled (default: true) 14 | # This will allow process to consume incoming changes from NATS 15 | # replicate = true 16 | 17 | # Publishing enabled/disabled (default: true) 18 | # This will allow process to control publishing of local DB changes to NATS 19 | # publish = true 20 | 21 | # Number of maximum rows to process per change allows configuring the maximum number of rows Marmot 22 | # will process (scan/load in memory) before publishing to NATS (default: 512) 23 | # scan_max_changes = 512 24 | 25 | # Cleanup interval in milliseconds used to clean up published rows. This is done in order to reduce write 26 | # load on the system (default: 5000) 27 | # cleanup_interval = 5000 28 | 29 | # Sleep timeout in milliseconds, useful for serverless scenarios. If there is no activity within given timelimit, 30 | # a snapshot will be performed, and process will exit. Value of 0 means it's disabled (default: 0). 31 | # sleep_timeout = 15000 32 | 33 | # Polling interval in milliseconds, that will explicitly check DB for change logs. This should not be required, 34 | # it's only useful for broken or buggy file system watchers. Value of 0 means it's disabled (default: 0) 35 | # polling_interval = 0 36 | 37 | # Snapshots are used to limit log size and have a database snapshot backedup on your 38 | # configured blob storage (NATS for now). This helps speedier recovery or cold boot 39 | # nodes to come up. A Snapshot is taken every log entries are close to max_entries 40 | # configured in replication_log section. It's recommended to use a large value 41 | # for maximum entries in replication log, because SQLite can do 1000s of TPS 42 | # replaying a couple thousands of entries should be really quick. 43 | [snapshot] 44 | # Disabling snapshot disables both restore and save 45 | enabled=true 46 | # Storage for snapshot can be "nats" | "webdav" | "s3" (default "nats") 47 | store="nats" 48 | # Interval sets perodic interval in milliseconds after which an automatic snapshot should be saved 49 | # If there was a snapshot saved within interval range due to other log threshold triggers, then 50 | # new snapshot won't be saved (since it's within time range), a value of 0 means it's disabled. 51 | interval=0 52 | 53 | # When setting snapshot.store to "nats" [snapshot.nats] will be used to configure snapshotting details 54 | # NATS connection settings (urls etc.) will be loaded from global [nats] configurations 55 | [snapshot.nats] 56 | # Number of NATS replicas of snapshot object store (max 5). Recommended values: 2-3 57 | replicas=1 58 | # Bucket name for object store to save snapshot on. 59 | #bucket="custom-bucket-name" 60 | 61 | # When setting snapshot.store to "s3" [snapshot.s3] will be used to configure snapshotting details 62 | [snapshot.s3] 63 | # For S3 this will be `s3.region-code.amazonaws.com` (check your AWS Console for details). 64 | # For Minio this will point to the host where Minio lives 65 | endpoint="127.0.0.1:9000" 66 | 67 | # Directory path within bucket where snapshot is saved and restore from 68 | path="snapshots/marmot" 69 | 70 | # By default false but should be set to true depending upong Minio configuration, for S3 it should be 71 | # always true. This essentially lets you select between https and http for your hosting. 72 | use_ssl=false 73 | 74 | # Access key ID or Minio user name 75 | #access_key="marmot" 76 | 77 | # Secret Key or Minio password 78 | #secret="ChangeMe" 79 | 80 | # Bucket name where snapshots live 81 | bucket="marmot" 82 | 83 | [snapshot.webdav] 84 | # URL of the WebDAV server root 85 | url="https:///?dir=/snapshots/path/for/marmot&login=&secret=" 86 | 87 | [snapshot.sftp] 88 | # URL of the SFTP server with path 89 | url="sftp://:@:/path/to/save/snapshot" 90 | 91 | # Change log that is published and persisted in JetStreams by Marmot. 92 | # Marmot auto-configures missing JetStreams when booting up for you. 93 | [replication_log] 94 | # Number of replicas per log to configure (user > 1 for failover and redundancy). 95 | replicas=1 96 | # Number of shards to divide the logs over, each JetStream and subject will be prefixed 97 | # by the configured `subject_prefix` and `stream_prefix` under nats 98 | shards=1 99 | # Max log entries JetStream should persist, JetStream is configured to drop older entries 100 | # Each JetStream is configured to persist on file. 101 | max_entries=1024 102 | # Enable log compression, uses zstd to compress logs as they are streamd to NATS 103 | # This is useful for DB storing large blobs that can be compressed. 104 | compress=true 105 | # Update existing stream if the configurations of JetStream don't match up with configurations 106 | # generated due to parameters above. Use this option carefully because changing shards, 107 | # or max_etries etc. might have undesired side-effects on existing running cluster 108 | update_existing=false 109 | 110 | 111 | # NATS server configurations 112 | [nats] 113 | # List of NATS server to use as boot server. Reference NATS documentation on how to pass 114 | # authentication credentials as part of URL. Leaving out this list empty will result 115 | # in embedded NATS server being started with node named `marmot-node-{node_id}`. 116 | # NATS configuration can provided via `server_config` variable 117 | urls=[ 118 | # "nats://localhost:4222" 119 | # "nats://:@:" 120 | ] 121 | # Embedded server bind address 122 | bind_address="0.0.0.0:4222" 123 | # Embedded server config file (will only be used if URLs array is empty) 124 | server_config="" 125 | # Subject prefix used when publishing log entries, it's usually suffixed by shard number 126 | # to get the full subject name 127 | subject_prefix="marmot-change-log" 128 | # JetStream name prefix used for publishing log entries, it's usually suffixed by shard number 129 | # to get the full JetStream name 130 | stream_prefix="marmot-changes" 131 | # Seed file used for client nkey authentication 132 | # nk -gen user > user.seed 133 | # nk -inkey user.seed -pubout > user.pub 134 | # Set to user.seed 135 | # Reference https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt#what-are-nkeys 136 | seed_file="" 137 | # User credentials used for plain user password authentication 138 | user_name="" 139 | user_password="" 140 | # Number of retries when establishing the NATS server connection (will only be used if URLs array is not empty) 141 | connect_retries=5 142 | # Wait time between NATS reconnect attempts (will only be used if URLs array is not empty) 143 | reconnect_wait_seconds=2 144 | 145 | [prometheus] 146 | # Enable/Disable prometheus telemetry collection 147 | enable=false 148 | # HTTP endpoint to expose for prometheus matrix collection 149 | # bind=":3010" 150 | # Namespace for prometheus (default: `marmot`), applies to all counters, gaugues, histograms 151 | # namespace="" 152 | # Subsystem for prometheus (default: empty), applies to all counters, gauges, histograms 153 | # subsystem="" 154 | 155 | # Console STDOUT configurations 156 | [logging] 157 | # Configure console logging 158 | verbose=true 159 | # "console" | "json" 160 | format="console" 161 | -------------------------------------------------------------------------------- /core/events.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "github.com/fxamacker/cbor/v2" 4 | 5 | var CBORTags = cbor.NewTagSet() 6 | 7 | type ReplicableEvent[T any] interface { 8 | Wrap() (T, error) 9 | Unwrap() (T, error) 10 | } 11 | -------------------------------------------------------------------------------- /db/change_log.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "database/sql" 7 | "errors" 8 | "fmt" 9 | "regexp" 10 | "strings" 11 | "text/template" 12 | "time" 13 | 14 | "github.com/maxpert/marmot/cfg" 15 | "github.com/maxpert/marmot/utils" 16 | 17 | _ "embed" 18 | 19 | "github.com/doug-martin/goqu/v9" 20 | "github.com/fsnotify/fsnotify" 21 | "github.com/rs/zerolog/log" 22 | "github.com/samber/lo" 23 | ) 24 | 25 | var ErrNoTableMapping = errors.New("no table mapping found") 26 | var ErrLogNotReadyToPublish = errors.New("not ready to publish changes") 27 | var ErrEndOfWatch = errors.New("watching event finished") 28 | 29 | //go:embed table_change_log_script.tmpl 30 | var tableChangeLogScriptTemplate string 31 | 32 | //go:embed global_change_log_script.tmpl 33 | var globalChangeLogScriptTemplate string 34 | var tableChangeLogTpl *template.Template 35 | var globalChangeLogTpl *template.Template 36 | 37 | var spaceStripper = regexp.MustCompile(`\n\s+`) 38 | 39 | type ChangeLogState = int16 40 | 41 | const ( 42 | Pending ChangeLogState = 0 43 | Published ChangeLogState = 1 44 | Failed ChangeLogState = -1 45 | ) 46 | const changeLogName = "change_log" 47 | const upsertQuery = `INSERT OR REPLACE INTO %s(%s) VALUES (%s)` 48 | 49 | type globalChangeLogTemplateData struct { 50 | Prefix string 51 | } 52 | 53 | type triggerTemplateData struct { 54 | Prefix string 55 | TableName string 56 | Columns []*ColumnInfo 57 | Triggers map[string]string 58 | } 59 | 60 | type globalChangeLogEntry struct { 61 | Id int64 `db:"id"` 62 | ChangeTableId int64 `db:"change_table_id"` 63 | TableName string `db:"table_name"` 64 | } 65 | 66 | type changeLogEntry struct { 67 | Id int64 `db:"id"` 68 | Type string `db:"type"` 69 | State string `db:"state"` 70 | } 71 | 72 | func init() { 73 | tableChangeLogTpl = template.Must( 74 | template.New("tableChangeLogScriptTemplate").Parse(tableChangeLogScriptTemplate), 75 | ) 76 | 77 | globalChangeLogTpl = template.Must( 78 | template.New("globalChangeLogScriptTemplate").Parse(globalChangeLogScriptTemplate), 79 | ) 80 | } 81 | 82 | func (conn *SqliteStreamDB) Replicate(event *ChangeLogEvent) error { 83 | if err := conn.consumeReplicationEvent(event); err != nil { 84 | return err 85 | } 86 | return nil 87 | } 88 | 89 | func (conn *SqliteStreamDB) CleanupChangeLogs(beforeTime time.Time) (int64, error) { 90 | sqlConn, err := conn.pool.Borrow() 91 | if err != nil { 92 | return 0, err 93 | } 94 | defer sqlConn.Return() 95 | 96 | total := int64(0) 97 | for name := range conn.watchTablesSchema { 98 | metaTableName := conn.metaTable(name, changeLogName) 99 | rs, err := sqlConn.DB().Delete(metaTableName). 100 | Where( 101 | goqu.C("state").Eq(Published), 102 | goqu.C("created_at").Lte(beforeTime.UnixMilli()), 103 | ). 104 | Prepared(true). 105 | Executor(). 106 | Exec() 107 | 108 | if err != nil { 109 | return 0, err 110 | } 111 | 112 | count, err := rs.RowsAffected() 113 | if err != nil { 114 | return 0, err 115 | } 116 | 117 | total += count 118 | } 119 | 120 | return total, nil 121 | } 122 | 123 | func (conn *SqliteStreamDB) metaTable(tableName string, name string) string { 124 | return conn.prefix + tableName + "_" + name 125 | } 126 | 127 | func (conn *SqliteStreamDB) globalMetaTable() string { 128 | return conn.prefix + "_change_log_global" 129 | } 130 | 131 | func (conn *SqliteStreamDB) globalCDCScript() (string, error) { 132 | buf := new(bytes.Buffer) 133 | err := globalChangeLogTpl.Execute(buf, &globalChangeLogTemplateData{ 134 | Prefix: conn.prefix, 135 | }) 136 | 137 | if err != nil { 138 | return "", err 139 | } 140 | 141 | return spaceStripper.ReplaceAllString(buf.String(), "\n "), nil 142 | } 143 | 144 | func (conn *SqliteStreamDB) tableCDCScriptFor(tableName string) (string, error) { 145 | columns, ok := conn.watchTablesSchema[tableName] 146 | if !ok { 147 | return "", errors.New("table info not found") 148 | } 149 | 150 | buf := new(bytes.Buffer) 151 | err := tableChangeLogTpl.Execute(buf, &triggerTemplateData{ 152 | Prefix: conn.prefix, 153 | Triggers: map[string]string{"insert": "NEW", "update": "NEW", "delete": "OLD"}, 154 | Columns: columns, 155 | TableName: tableName, 156 | }) 157 | 158 | if err != nil { 159 | return "", err 160 | } 161 | 162 | return spaceStripper.ReplaceAllString(buf.String(), "\n "), nil 163 | } 164 | 165 | func (conn *SqliteStreamDB) consumeReplicationEvent(event *ChangeLogEvent) error { 166 | sqlConn, err := conn.pool.Borrow() 167 | if err != nil { 168 | return err 169 | } 170 | defer sqlConn.Return() 171 | 172 | return sqlConn.DB().WithTx(func(tnx *goqu.TxDatabase) error { 173 | primaryKeyMap := conn.getPrimaryKeyMap(event) 174 | if primaryKeyMap == nil { 175 | return ErrNoTableMapping 176 | } 177 | 178 | logEv := log.Debug(). 179 | Int64("event_id", event.Id). 180 | Str("type", event.Type) 181 | 182 | for k, v := range primaryKeyMap { 183 | logEv = logEv.Str(event.TableName+"."+k, fmt.Sprintf("%v", v)) 184 | } 185 | 186 | logEv.Send() 187 | 188 | return replicateRow(tnx, event, primaryKeyMap) 189 | }) 190 | } 191 | 192 | func (conn *SqliteStreamDB) getPrimaryKeyMap(event *ChangeLogEvent) map[string]any { 193 | ret := make(map[string]any) 194 | tableColsSchema, ok := conn.watchTablesSchema[event.TableName] 195 | if !ok { 196 | return nil 197 | } 198 | 199 | for _, col := range tableColsSchema { 200 | if col.IsPrimaryKey { 201 | ret[col.Name] = event.Row[col.Name] 202 | } 203 | } 204 | 205 | return ret 206 | } 207 | 208 | func (conn *SqliteStreamDB) initGlobalChangeLog() error { 209 | sqlConn, err := conn.pool.Borrow() 210 | if err != nil { 211 | return err 212 | } 213 | defer sqlConn.Return() 214 | 215 | script, err := conn.globalCDCScript() 216 | if err != nil { 217 | return err 218 | } 219 | 220 | log.Info().Msg("Creating global change log table") 221 | _, err = sqlConn.DB().Exec(script) 222 | if err != nil { 223 | return err 224 | } 225 | 226 | return nil 227 | } 228 | 229 | func (conn *SqliteStreamDB) initTriggers(tableName string) error { 230 | sqlConn, err := conn.pool.Borrow() 231 | if err != nil { 232 | return err 233 | } 234 | defer sqlConn.Return() 235 | 236 | name := strings.TrimSpace(tableName) 237 | if strings.HasPrefix(name, "sqlite_") || strings.HasPrefix(name, conn.prefix) { 238 | return fmt.Errorf("invalid table to watch %s", tableName) 239 | } 240 | 241 | script, err := conn.tableCDCScriptFor(name) 242 | if err != nil { 243 | log.Error().Err(err).Msg("Failed to prepare CDC statement") 244 | return err 245 | } 246 | 247 | log.Info().Msg(fmt.Sprintf("Creating trigger for %v", name)) 248 | _, err = sqlConn.DB().Exec(script) 249 | if err != nil { 250 | return err 251 | } 252 | 253 | return nil 254 | } 255 | 256 | func (conn *SqliteStreamDB) filterChangesTo(changed chan fsnotify.Event, watcher *fsnotify.Watcher) { 257 | for { 258 | select { 259 | case ev, ok := <-watcher.Events: 260 | if !ok { 261 | close(changed) 262 | return 263 | } 264 | 265 | if ev.Op == fsnotify.Chmod { 266 | continue 267 | } 268 | 269 | changed <- ev 270 | } 271 | } 272 | } 273 | 274 | func (conn *SqliteStreamDB) watchChanges(watcher *fsnotify.Watcher, path string) { 275 | shmPath := path + "-shm" 276 | walPath := path + "-wal" 277 | 278 | errDB := watcher.Add(path) 279 | errShm := watcher.Add(shmPath) 280 | errWal := watcher.Add(walPath) 281 | dbChanged := make(chan fsnotify.Event) 282 | 283 | tickerDur := time.Duration(cfg.Config.PollingInterval) * time.Millisecond 284 | changeLogTicker := utils.NewTimeoutPublisher(tickerDur) 285 | 286 | // Publish change logs for any residual change logs before starting watcher 287 | conn.publishChangeLog() 288 | go conn.filterChangesTo(dbChanged, watcher) 289 | 290 | for { 291 | changeLogTicker.Reset() 292 | 293 | err := conn.WithReadTx(func(_tx *sql.Tx) error { 294 | select { 295 | case ev, ok := <-dbChanged: 296 | if !ok { 297 | return ErrEndOfWatch 298 | } 299 | 300 | log.Debug().Int("change", int(ev.Op)).Msg("Change detected") 301 | conn.publishChangeLog() 302 | case <-changeLogTicker.Channel(): 303 | log.Debug().Dur("timeout", tickerDur).Msg("Change polling timeout") 304 | conn.publishChangeLog() 305 | } 306 | 307 | return nil 308 | }) 309 | 310 | if err != nil { 311 | log.Warn().Err(err).Msg("Error watching changes; trying to resubscribe...") 312 | errDB = watcher.Add(path) 313 | errShm = watcher.Add(shmPath) 314 | errWal = watcher.Add(walPath) 315 | } 316 | 317 | if errDB != nil { 318 | errDB = watcher.Add(path) 319 | } 320 | 321 | if errShm != nil { 322 | errShm = watcher.Add(shmPath) 323 | } 324 | 325 | if errWal != nil { 326 | errWal = watcher.Add(walPath) 327 | } 328 | } 329 | } 330 | 331 | func (conn *SqliteStreamDB) getGlobalChanges(limit uint32) ([]globalChangeLogEntry, error) { 332 | sw := utils.NewStopWatch("scan_changes") 333 | defer sw.Log(log.Debug(), conn.stats.scanChanges) 334 | 335 | sqlConn, err := conn.pool.Borrow() 336 | if err != nil { 337 | return nil, err 338 | } 339 | defer sqlConn.Return() 340 | 341 | var entries []globalChangeLogEntry 342 | err = sqlConn.DB(). 343 | From(conn.globalMetaTable()). 344 | Order(goqu.I("id").Asc()). 345 | Limit(uint(limit)). 346 | ScanStructs(&entries) 347 | 348 | if err != nil { 349 | return nil, err 350 | } 351 | return entries, nil 352 | } 353 | 354 | func (conn *SqliteStreamDB) countChanges() (int64, error) { 355 | sw := utils.NewStopWatch("count_changes") 356 | defer sw.Log(log.Debug(), conn.stats.countChanges) 357 | 358 | sqlConn, err := conn.pool.Borrow() 359 | if err != nil { 360 | return -1, err 361 | } 362 | defer sqlConn.Return() 363 | 364 | return sqlConn.DB(). 365 | From(conn.globalMetaTable()). 366 | Count() 367 | } 368 | 369 | func (conn *SqliteStreamDB) publishChangeLog() { 370 | if !conn.publishLock.TryLock() { 371 | log.Warn().Msg("Publish in progress skipping...") 372 | return 373 | } 374 | defer conn.publishLock.Unlock() 375 | 376 | cnt, err := conn.countChanges() 377 | if err != nil { 378 | log.Error().Err(err).Msg("Unable to count global changes") 379 | return 380 | } 381 | 382 | conn.stats.pendingPublish.Set(float64(cnt)) 383 | if cnt <= 0 { 384 | log.Debug().Msg("no new rows") 385 | return 386 | } 387 | 388 | changes, err := conn.getGlobalChanges(cfg.Config.ScanMaxChanges) 389 | if err != nil { 390 | log.Error().Err(err).Msg("Unable to scan global changes") 391 | return 392 | } 393 | 394 | if len(changes) < 0 { 395 | return 396 | } 397 | 398 | for _, change := range changes { 399 | logEntry := changeLogEntry{} 400 | found := false 401 | found, err = conn.getChangeEntry(&logEntry, change) 402 | 403 | if err != nil { 404 | log.Error().Err(err).Msg("Error scanning last row ID") 405 | return 406 | } 407 | 408 | if !found { 409 | log.Panic(). 410 | Str("table", change.TableName). 411 | Int64("id", change.ChangeTableId). 412 | Msg("Global change log row not found in corresponding table") 413 | return 414 | } 415 | 416 | err = conn.consumeChangeLogs(change.TableName, []*changeLogEntry{&logEntry}) 417 | if err != nil { 418 | if errors.Is(err, ErrLogNotReadyToPublish) || errors.Is(err, context.Canceled) { 419 | break 420 | } 421 | 422 | log.Error().Err(err).Msg("Unable to consume changes") 423 | } 424 | 425 | err = conn.markChangePublished(change) 426 | if err != nil { 427 | log.Error().Err(err).Msg("Unable to cleanup change log") 428 | } 429 | 430 | conn.stats.published.Inc() 431 | } 432 | } 433 | 434 | func (conn *SqliteStreamDB) markChangePublished(change globalChangeLogEntry) error { 435 | sqlConn, err := conn.pool.Borrow() 436 | if err != nil { 437 | return err 438 | } 439 | defer sqlConn.Return() 440 | 441 | return sqlConn.DB().WithTx(func(tx *goqu.TxDatabase) error { 442 | _, err = tx.Update(conn.metaTable(change.TableName, changeLogName)). 443 | Set(goqu.Record{"state": Published}). 444 | Where(goqu.Ex{"id": change.ChangeTableId}). 445 | Prepared(true). 446 | Executor(). 447 | Exec() 448 | 449 | if err != nil { 450 | return err 451 | } 452 | 453 | _, err = tx.Delete(conn.globalMetaTable()). 454 | Where(goqu.C("id").Eq(change.Id)). 455 | Prepared(true). 456 | Executor(). 457 | Exec() 458 | 459 | if err != nil { 460 | return err 461 | } 462 | 463 | return nil 464 | }) 465 | } 466 | 467 | func (conn *SqliteStreamDB) getChangeEntry(entry *changeLogEntry, change globalChangeLogEntry) (bool, error) { 468 | sqlConn, err := conn.pool.Borrow() 469 | if err != nil { 470 | return false, err 471 | } 472 | defer sqlConn.Return() 473 | 474 | return sqlConn.DB().Select("id", "type", "state"). 475 | From(conn.metaTable(change.TableName, changeLogName)). 476 | Where( 477 | goqu.C("state").Eq(Pending), 478 | goqu.C("id").Eq(change.ChangeTableId), 479 | ). 480 | Prepared(true). 481 | ScanStruct(entry) 482 | } 483 | 484 | func (conn *SqliteStreamDB) consumeChangeLogs(tableName string, changes []*changeLogEntry) error { 485 | rowIds := lo.Map(changes, func(e *changeLogEntry, i int) int64 { 486 | return e.Id 487 | }) 488 | 489 | changeMap := lo.Associate( 490 | changes, 491 | func(l *changeLogEntry) (int64, *changeLogEntry) { 492 | return l.Id, l 493 | }, 494 | ) 495 | 496 | idColumnName := conn.prefix + "change_log_id" 497 | rawRows, err := conn.fetchChangeRows(tableName, idColumnName, rowIds) 498 | if err != nil { 499 | return err 500 | } 501 | 502 | rows := &EnhancedRows{rawRows} 503 | defer rows.Finalize() 504 | 505 | for rows.Next() { 506 | row, err := rows.fetchRow() 507 | if err != nil { 508 | return err 509 | } 510 | 511 | changeRowID := row[idColumnName].(int64) 512 | changeRow := changeMap[changeRowID] 513 | delete(row, idColumnName) 514 | 515 | logger := log.With(). 516 | Int64("rowid", changeRowID). 517 | Str("table", tableName). 518 | Str("type", changeRow.Type). 519 | Logger() 520 | 521 | if conn.OnChange != nil { 522 | err = conn.OnChange(&ChangeLogEvent{ 523 | Id: changeRowID, 524 | Type: changeRow.Type, 525 | TableName: tableName, 526 | Row: row, 527 | tableInfo: conn.watchTablesSchema[tableName], 528 | }) 529 | 530 | if err != nil { 531 | if err == ErrLogNotReadyToPublish || err == context.Canceled { 532 | return err 533 | } 534 | 535 | logger.Error().Err(err).Msg("Failed to publish for table " + tableName) 536 | return err 537 | } 538 | } 539 | } 540 | 541 | return nil 542 | } 543 | 544 | func (conn *SqliteStreamDB) fetchChangeRows( 545 | tableName string, 546 | idColumnName string, 547 | rowIds []int64, 548 | ) (*sql.Rows, error) { 549 | sqlConn, err := conn.pool.Borrow() 550 | if err != nil { 551 | return nil, err 552 | } 553 | defer sqlConn.Return() 554 | 555 | columnNames := make([]any, 0) 556 | tableCols := conn.watchTablesSchema[tableName] 557 | columnNames = append(columnNames, goqu.C("id").As(idColumnName)) 558 | for _, col := range tableCols { 559 | columnNames = append(columnNames, goqu.C("val_"+col.Name).As(col.Name)) 560 | } 561 | 562 | query, params, err := sqlConn.DB().From(conn.metaTable(tableName, changeLogName)). 563 | Select(columnNames...). 564 | Where(goqu.C("id").In(rowIds)). 565 | Prepared(true). 566 | ToSQL() 567 | if err != nil { 568 | return nil, err 569 | } 570 | 571 | rawRows, err := sqlConn.DB().Query(query, params...) 572 | if err != nil { 573 | return nil, err 574 | } 575 | 576 | return rawRows, nil 577 | } 578 | 579 | func replicateRow(tx *goqu.TxDatabase, event *ChangeLogEvent, pkMap map[string]any) error { 580 | if event.Type == "insert" || event.Type == "update" { 581 | return replicateUpsert(tx, event, pkMap) 582 | } 583 | 584 | if event.Type == "delete" { 585 | return replicateDelete(tx, event, pkMap) 586 | } 587 | 588 | return fmt.Errorf("invalid operation type %s", event.Type) 589 | } 590 | 591 | func replicateUpsert(tx *goqu.TxDatabase, event *ChangeLogEvent, _ map[string]any) error { 592 | columnNames := make([]string, 0, len(event.Row)) 593 | columnValues := make([]any, 0, len(event.Row)) 594 | for k, v := range event.Row { 595 | columnNames = append(columnNames, k) 596 | columnValues = append(columnValues, v) 597 | } 598 | 599 | query := fmt.Sprintf( 600 | upsertQuery, 601 | event.TableName, 602 | strings.Join(columnNames, ", "), 603 | strings.Join(strings.Split(strings.Repeat("?", len(columnNames)), ""), ", "), 604 | ) 605 | 606 | stmt, err := tx.Prepare(query) 607 | if err != nil { 608 | return err 609 | } 610 | 611 | _, err = stmt.Exec(columnValues...) 612 | return err 613 | } 614 | 615 | func replicateDelete(tx *goqu.TxDatabase, event *ChangeLogEvent, pkMap map[string]any) error { 616 | _, err := tx.Delete(event.TableName). 617 | Where(goqu.Ex(pkMap)). 618 | Prepared(true). 619 | Executor(). 620 | Exec() 621 | 622 | return err 623 | } 624 | -------------------------------------------------------------------------------- /db/change_log_event.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "hash/fnv" 5 | "reflect" 6 | "sort" 7 | "sync" 8 | "time" 9 | 10 | "github.com/fxamacker/cbor/v2" 11 | "github.com/maxpert/marmot/core" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | var tablePKColumnsCache = make(map[string][]string) 16 | var tablePKColumnsLock = sync.RWMutex{} 17 | 18 | type sensitiveTypeWrapper struct { 19 | Time *time.Time `cbor:"1,keyasint,omitempty"` 20 | } 21 | 22 | type ChangeLogEvent struct { 23 | Id int64 24 | Type string 25 | TableName string 26 | Row map[string]any 27 | tableInfo []*ColumnInfo `cbor:"-"` 28 | } 29 | 30 | func init() { 31 | err := core.CBORTags.Add( 32 | cbor.TagOptions{ 33 | DecTag: cbor.DecTagRequired, 34 | EncTag: cbor.EncTagRequired, 35 | }, 36 | reflect.TypeOf(sensitiveTypeWrapper{}), 37 | 32, 38 | ) 39 | 40 | log.Panic().Err(err) 41 | } 42 | 43 | func (s sensitiveTypeWrapper) GetValue() any { 44 | // Right now only sensitive value is Time 45 | return s.Time 46 | } 47 | 48 | func (e ChangeLogEvent) Wrap() (ChangeLogEvent, error) { 49 | return e.prepare(), nil 50 | } 51 | 52 | func (e ChangeLogEvent) Unwrap() (ChangeLogEvent, error) { 53 | ret := ChangeLogEvent{ 54 | Id: e.Id, 55 | TableName: e.TableName, 56 | Type: e.Type, 57 | Row: map[string]any{}, 58 | tableInfo: e.tableInfo, 59 | } 60 | 61 | for k, v := range e.Row { 62 | if st, ok := v.(sensitiveTypeWrapper); ok { 63 | ret.Row[k] = st.GetValue() 64 | continue 65 | } 66 | 67 | ret.Row[k] = v 68 | } 69 | 70 | return ret, nil 71 | } 72 | 73 | func (e ChangeLogEvent) Hash() (uint64, error) { 74 | hasher := fnv.New64() 75 | enc := cbor.NewEncoder(hasher) 76 | err := enc.StartIndefiniteArray() 77 | if err != nil { 78 | return 0, err 79 | } 80 | 81 | err = enc.Encode(e.TableName) 82 | if err != nil { 83 | return 0, err 84 | } 85 | 86 | pkColumns := e.getSortedPKColumns() 87 | for _, pk := range pkColumns { 88 | err = enc.Encode([]any{pk, e.Row[pk]}) 89 | if err != nil { 90 | return 0, err 91 | } 92 | } 93 | 94 | err = enc.EndIndefinite() 95 | if err != nil { 96 | return 0, err 97 | } 98 | 99 | return hasher.Sum64(), nil 100 | } 101 | 102 | func (e ChangeLogEvent) getSortedPKColumns() []string { 103 | tablePKColumnsLock.RLock() 104 | 105 | if values, found := tablePKColumnsCache[e.TableName]; found { 106 | tablePKColumnsLock.RUnlock() 107 | return values 108 | } 109 | tablePKColumnsLock.RUnlock() 110 | 111 | pkColumns := make([]string, 0, len(e.tableInfo)) 112 | for _, itm := range e.tableInfo { 113 | if itm.IsPrimaryKey { 114 | pkColumns = append(pkColumns, itm.Name) 115 | } 116 | } 117 | sort.Strings(pkColumns) 118 | 119 | tablePKColumnsLock.Lock() 120 | defer tablePKColumnsLock.Unlock() 121 | 122 | tablePKColumnsCache[e.TableName] = pkColumns 123 | return pkColumns 124 | } 125 | 126 | func (e ChangeLogEvent) prepare() ChangeLogEvent { 127 | needsTransform := false 128 | preparedRow := map[string]any{} 129 | for k, v := range e.Row { 130 | if t, ok := v.(time.Time); ok { 131 | preparedRow[k] = sensitiveTypeWrapper{Time: &t} 132 | needsTransform = true 133 | } else { 134 | preparedRow[k] = v 135 | } 136 | } 137 | 138 | if !needsTransform { 139 | return e 140 | } 141 | 142 | return ChangeLogEvent{ 143 | Id: e.Id, 144 | Type: e.Type, 145 | TableName: e.TableName, 146 | Row: preparedRow, 147 | tableInfo: e.tableInfo, 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /db/cleanup.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/doug-martin/goqu/v9" 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | const deleteTriggerQuery = `DROP TRIGGER IF EXISTS %s` 11 | const deleteMarmotTables = `DROP TABLE IF EXISTS %s;` 12 | 13 | func removeMarmotTriggers(conn *goqu.Database, prefix string) error { 14 | triggers := make([]string, 0) 15 | err := conn. 16 | Select("name"). 17 | From("sqlite_master"). 18 | Where(goqu.C("type").Eq("trigger"), goqu.C("name").Like(prefix+"%")). 19 | Prepared(true). 20 | ScanVals(&triggers) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | for _, name := range triggers { 26 | query := fmt.Sprintf(deleteTriggerQuery, name) 27 | _, err = conn.Exec(query) 28 | if err != nil { 29 | log.Error().Err(err).Str("name", name).Msg("Unable to delete trigger") 30 | return err 31 | } 32 | } 33 | 34 | return nil 35 | } 36 | 37 | func removeMarmotTables(conn *goqu.Database, prefix string) error { 38 | tables := make([]string, 0) 39 | err := conn. 40 | Select("name"). 41 | From("sqlite_master"). 42 | Where(goqu.C("type").Eq("table"), goqu.C("name").Like(prefix+"%")). 43 | Prepared(true). 44 | ScanVals(&tables) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | for _, name := range tables { 50 | query := fmt.Sprintf(deleteMarmotTables, name) 51 | _, err = conn.Exec(query) 52 | if err != nil { 53 | log.Error().Err(err).Msg("Unable to delete marmot tables") 54 | return err 55 | } 56 | } 57 | 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /db/global_change_log_script.tmpl: -------------------------------------------------------------------------------- 1 | {{$GlobalChangeLogTableName := (printf "%s_change_log_global" .Prefix)}} 2 | 3 | CREATE TABLE IF NOT EXISTS {{$GlobalChangeLogTableName}} ( 4 | id INTEGER PRIMARY KEY AUTOINCREMENT, 5 | change_table_id INTEGER, 6 | table_name TEXT 7 | ); 8 | -------------------------------------------------------------------------------- /db/sqlite.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "database/sql/driver" 7 | "fmt" 8 | "io" 9 | "os" 10 | "sync" 11 | "time" 12 | 13 | "github.com/doug-martin/goqu/v9" 14 | "github.com/fsnotify/fsnotify" 15 | "github.com/mattn/go-sqlite3" 16 | "github.com/maxpert/marmot/pool" 17 | "github.com/maxpert/marmot/telemetry" 18 | "github.com/rs/zerolog/log" 19 | ) 20 | 21 | const snapshotTransactionMode = "exclusive" 22 | 23 | var PoolSize = 4 24 | var MarmotPrefix = "__marmot__" 25 | 26 | type statsSqliteStreamDB struct { 27 | published telemetry.Counter 28 | pendingPublish telemetry.Gauge 29 | countChanges telemetry.Histogram 30 | scanChanges telemetry.Histogram 31 | } 32 | 33 | type SqliteStreamDB struct { 34 | OnChange func(event *ChangeLogEvent) error 35 | pool *pool.SQLitePool 36 | rawConnection *sqlite3.SQLiteConn 37 | publishLock *sync.Mutex 38 | 39 | dbPath string 40 | prefix string 41 | watchTablesSchema map[string][]*ColumnInfo 42 | stats *statsSqliteStreamDB 43 | } 44 | 45 | type ColumnInfo struct { 46 | Name string `db:"name"` 47 | Type string `db:"type"` 48 | NotNull bool `db:"notnull"` 49 | DefaultValue any `db:"dflt_value"` 50 | PrimaryKeyIndex int `db:"pk"` 51 | IsPrimaryKey bool 52 | } 53 | 54 | func RestoreFrom(destPath, bkFilePath string) error { 55 | dnsTpl := "%s?_journal_mode=WAL&_foreign_keys=false&_busy_timeout=30000&_sync=FULL&_txlock=%s" 56 | dns := fmt.Sprintf(dnsTpl, destPath, snapshotTransactionMode) 57 | destDB, dest, err := pool.OpenRaw(dns) 58 | if err != nil { 59 | return err 60 | } 61 | defer dest.Close() 62 | 63 | dns = fmt.Sprintf(dnsTpl, bkFilePath, snapshotTransactionMode) 64 | srcDB, src, err := pool.OpenRaw(dns) 65 | if err != nil { 66 | return err 67 | } 68 | defer src.Close() 69 | 70 | dgSQL := goqu.New("sqlite", destDB) 71 | sgSQL := goqu.New("sqlite", srcDB) 72 | 73 | // Source locking is required so that any lock related metadata is mirrored in destination 74 | // Transacting on both src and dest in immediate mode makes sure nobody 75 | // else is modifying or interacting with DB 76 | err = sgSQL.WithTx(func(dtx *goqu.TxDatabase) error { 77 | return dgSQL.WithTx(func(_ *goqu.TxDatabase) error { 78 | err = copyFile(destPath, bkFilePath) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | err = copyFile(destPath+"-shm", bkFilePath+"-shm") 84 | if err != nil { 85 | return err 86 | } 87 | 88 | err = copyFile(destPath+"-wal", bkFilePath+"-wal") 89 | if err != nil { 90 | return err 91 | } 92 | 93 | return nil 94 | }) 95 | }) 96 | 97 | if err != nil { 98 | return err 99 | } 100 | 101 | err = performCheckpoint(dgSQL) 102 | if err != nil { 103 | return err 104 | } 105 | 106 | return nil 107 | } 108 | 109 | func GetAllDBTables(path string) ([]string, error) { 110 | connectionStr := fmt.Sprintf("%s?_journal_mode=WAL", path) 111 | conn, rawConn, err := pool.OpenRaw(connectionStr) 112 | if err != nil { 113 | return nil, err 114 | } 115 | defer rawConn.Close() 116 | defer conn.Close() 117 | 118 | gSQL := goqu.New("sqlite", conn) 119 | names := make([]string, 0) 120 | err = gSQL.WithTx(func(tx *goqu.TxDatabase) error { 121 | return listDBTables(&names, tx) 122 | }) 123 | 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | return names, nil 129 | } 130 | 131 | func OpenStreamDB(path string) (*SqliteStreamDB, error) { 132 | dbPool, err := pool.NewSQLitePool(fmt.Sprintf("%s?_journal_mode=WAL", path), PoolSize, true) 133 | if err != nil { 134 | return nil, err 135 | } 136 | 137 | conn, err := dbPool.Borrow() 138 | if err != nil { 139 | return nil, err 140 | } 141 | defer conn.Return() 142 | 143 | err = performCheckpoint(conn.DB()) 144 | if err != nil { 145 | return nil, err 146 | } 147 | 148 | ret := &SqliteStreamDB{ 149 | pool: dbPool, 150 | dbPath: path, 151 | prefix: MarmotPrefix, 152 | publishLock: &sync.Mutex{}, 153 | watchTablesSchema: map[string][]*ColumnInfo{}, 154 | stats: &statsSqliteStreamDB{ 155 | published: telemetry.NewCounter("published", "number of rows published"), 156 | pendingPublish: telemetry.NewGauge("pending_publish", "rows pending publishing"), 157 | countChanges: telemetry.NewHistogram("count_changes", "latency counting changes in microseconds"), 158 | scanChanges: telemetry.NewHistogram("scan_changes", "latency scanning change rows in DB"), 159 | }, 160 | } 161 | 162 | return ret, nil 163 | } 164 | 165 | func (conn *SqliteStreamDB) InstallCDC(tables []string) error { 166 | sqlConn, err := conn.pool.Borrow() 167 | if err != nil { 168 | return err 169 | } 170 | defer sqlConn.Return() 171 | 172 | err = sqlConn.DB().WithTx(func(tx *goqu.TxDatabase) error { 173 | for _, n := range tables { 174 | colInfo, err := getTableInfo(tx, n) 175 | if err != nil { 176 | return err 177 | } 178 | 179 | conn.watchTablesSchema[n] = colInfo 180 | } 181 | 182 | return nil 183 | }) 184 | if err != nil { 185 | return err 186 | } 187 | 188 | err = conn.installChangeLogTriggers() 189 | if err != nil { 190 | return err 191 | } 192 | 193 | watcher, err := fsnotify.NewWatcher() 194 | if err != nil { 195 | return err 196 | } 197 | 198 | go conn.watchChanges(watcher, conn.dbPath) 199 | return nil 200 | } 201 | 202 | func (conn *SqliteStreamDB) RemoveCDC(tables bool) error { 203 | sqlConn, err := conn.pool.Borrow() 204 | if err != nil { 205 | return err 206 | } 207 | defer sqlConn.Return() 208 | 209 | log.Info().Msg("Uninstalling all CDC triggers...") 210 | err = removeMarmotTriggers(sqlConn.DB(), conn.prefix) 211 | if err != nil { 212 | return err 213 | } 214 | 215 | if tables { 216 | return removeMarmotTables(sqlConn.DB(), conn.prefix) 217 | } 218 | 219 | return nil 220 | } 221 | 222 | func (conn *SqliteStreamDB) installChangeLogTriggers() error { 223 | if err := conn.initGlobalChangeLog(); err != nil { 224 | return err 225 | } 226 | 227 | for tableName := range conn.watchTablesSchema { 228 | err := conn.initTriggers(tableName) 229 | if err != nil { 230 | return err 231 | } 232 | } 233 | return nil 234 | } 235 | 236 | func getTableInfo(tx *goqu.TxDatabase, table string) ([]*ColumnInfo, error) { 237 | query := "SELECT name, type, `notnull`, dflt_value, pk FROM pragma_table_info(?)" 238 | stmt, err := tx.Prepare(query) 239 | if err != nil { 240 | return nil, err 241 | } 242 | 243 | rows, err := stmt.Query(table) 244 | if err != nil { 245 | return nil, err 246 | } 247 | 248 | tableInfo := make([]*ColumnInfo, 0) 249 | hasPrimaryKey := false 250 | for rows.Next() { 251 | if rows.Err() != nil { 252 | return nil, rows.Err() 253 | } 254 | 255 | c := ColumnInfo{} 256 | err = rows.Scan(&c.Name, &c.Type, &c.NotNull, &c.DefaultValue, &c.PrimaryKeyIndex) 257 | if err != nil { 258 | return nil, err 259 | } 260 | 261 | c.IsPrimaryKey = c.PrimaryKeyIndex > 0 262 | 263 | if c.IsPrimaryKey { 264 | hasPrimaryKey = true 265 | } 266 | 267 | tableInfo = append(tableInfo, &c) 268 | } 269 | 270 | if !hasPrimaryKey { 271 | tableInfo = append(tableInfo, &ColumnInfo{ 272 | Name: "rowid", 273 | IsPrimaryKey: true, 274 | Type: "INT", 275 | NotNull: true, 276 | DefaultValue: nil, 277 | }) 278 | } 279 | 280 | return tableInfo, nil 281 | } 282 | 283 | func (conn *SqliteStreamDB) BackupTo(bkFilePath string) error { 284 | sqlDB, rawDB, err := pool.OpenRaw(fmt.Sprintf("%s?mode=ro&_foreign_keys=false&_journal_mode=WAL", conn.dbPath)) 285 | if err != nil { 286 | return err 287 | } 288 | defer sqlDB.Close() 289 | defer rawDB.Close() 290 | 291 | _, err = rawDB.Exec("VACUUM main INTO ?;", []driver.Value{bkFilePath}) 292 | if err != nil { 293 | return err 294 | } 295 | 296 | err = rawDB.Close() 297 | if err != nil { 298 | return err 299 | } 300 | 301 | err = sqlDB.Close() 302 | if err != nil { 303 | return err 304 | } 305 | 306 | // Now since we have separate copy of DB we don't need to deal with WAL journals or foreign keys 307 | // We need to remove all the marmot specific tables, triggers, and vacuum out the junk. 308 | sqlDB, rawDB, err = pool.OpenRaw(fmt.Sprintf("%s?_foreign_keys=false&_journal_mode=TRUNCATE", bkFilePath)) 309 | if err != nil { 310 | return err 311 | } 312 | 313 | gSQL := goqu.New("sqlite", sqlDB) 314 | err = removeMarmotTriggers(gSQL, conn.prefix) 315 | if err != nil { 316 | return err 317 | } 318 | 319 | err = removeMarmotTables(gSQL, conn.prefix) 320 | if err != nil { 321 | return err 322 | } 323 | 324 | _, err = gSQL.Exec("VACUUM;") 325 | if err != nil { 326 | return err 327 | } 328 | 329 | return nil 330 | } 331 | 332 | func (conn *SqliteStreamDB) GetRawConnection() *sqlite3.SQLiteConn { 333 | return conn.rawConnection 334 | } 335 | 336 | func (conn *SqliteStreamDB) GetPath() string { 337 | return conn.dbPath 338 | } 339 | 340 | func (conn *SqliteStreamDB) WithReadTx(cb func(tx *sql.Tx) error) error { 341 | var tx *sql.Tx = nil 342 | db, _, err := pool.OpenRaw(fmt.Sprintf("%s?_journal_mode=WAL", conn.dbPath)) 343 | if err != nil { 344 | return err 345 | } 346 | 347 | ctx, cancel := context.WithCancel(context.Background()) 348 | defer func() { 349 | if r := recover(); r != nil { 350 | log.Error().Any("recover", r).Msg("Recovered read transaction") 351 | } 352 | 353 | if tx != nil { 354 | err = tx.Rollback() 355 | if err != nil { 356 | log.Error().Err(err).Msg("Error performing read transaction") 357 | } 358 | } 359 | 360 | db.Close() 361 | cancel() 362 | }() 363 | 364 | tx, err = db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) 365 | return cb(tx) 366 | } 367 | 368 | func copyFile(toPath, fromPath string) error { 369 | fi, err := os.OpenFile(fromPath, os.O_RDWR, 0) 370 | if err != nil { 371 | return err 372 | } 373 | defer fi.Close() 374 | 375 | fo, err := os.OpenFile(toPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_SYNC, 0) 376 | if err != nil { 377 | return err 378 | } 379 | defer fo.Close() 380 | 381 | bytesWritten, err := io.Copy(fo, fi) 382 | log.Debug(). 383 | Int64("bytes", bytesWritten). 384 | Str("from", fromPath). 385 | Str("to", toPath). 386 | Msg("File copied...") 387 | return err 388 | } 389 | 390 | func listDBTables(names *[]string, gSQL *goqu.TxDatabase) error { 391 | err := gSQL.Select("name").From("sqlite_schema").Where( 392 | goqu.C("type").Eq("table"), 393 | goqu.C("name").NotLike("sqlite_%"), 394 | goqu.C("name").NotLike(MarmotPrefix+"%"), 395 | ).ScanVals(names) 396 | 397 | if err != nil { 398 | return err 399 | } 400 | 401 | return nil 402 | } 403 | 404 | func performCheckpoint(gSQL *goqu.Database) error { 405 | rBusy, rLog, rCheckpoint := int64(1), int64(0), int64(0) 406 | log.Debug().Msg("Forcing WAL checkpoint") 407 | 408 | for rBusy != 0 { 409 | row := gSQL.QueryRow("PRAGMA wal_checkpoint(truncate);") 410 | err := row.Scan(&rBusy, &rLog, &rCheckpoint) 411 | if err != nil { 412 | return err 413 | } 414 | 415 | if rBusy != 0 { 416 | log.Debug(). 417 | Int64("busy", rBusy). 418 | Int64("log", rLog). 419 | Int64("checkpoint", rCheckpoint). 420 | Msg("Waiting checkpoint...") 421 | 422 | time.Sleep(100 * time.Millisecond) 423 | } 424 | } 425 | 426 | return nil 427 | } 428 | -------------------------------------------------------------------------------- /db/table_change_log_script.tmpl: -------------------------------------------------------------------------------- 1 | {{$ChangeLogTableName := (printf "%s%s_change_log" .Prefix .TableName)}} 2 | {{$GlobalChangeLogTableName := (printf "%s_change_log_global" .Prefix)}} 3 | 4 | CREATE TABLE IF NOT EXISTS {{$ChangeLogTableName}} ( 5 | id INTEGER PRIMARY KEY AUTOINCREMENT, 6 | {{range $index, $col := .Columns}} 7 | val_{{$col.Name}} {{$col.Type}}, 8 | {{end}} 9 | type TEXT, 10 | created_at INTEGER, 11 | state INTEGER 12 | ); 13 | 14 | CREATE INDEX IF NOT EXISTS {{$ChangeLogTableName}}_state_index ON {{$ChangeLogTableName}} (state); 15 | 16 | {{range $trigger, $read_target := .Triggers}} 17 | DROP TRIGGER IF EXISTS {{$ChangeLogTableName}}_on_{{$trigger}}; 18 | CREATE TRIGGER IF NOT EXISTS {{$ChangeLogTableName}}_on_{{$trigger}} 19 | AFTER {{$trigger}} ON {{$.TableName}} 20 | WHEN (SELECT COUNT(*) FROM pragma_function_list WHERE name='marmot_version') < 1 21 | BEGIN 22 | 23 | INSERT INTO {{$ChangeLogTableName}}( 24 | {{range $col := $.Columns}} 25 | val_{{$col.Name}}, 26 | {{end}} 27 | type, 28 | created_at, 29 | state 30 | ) VALUES( 31 | {{range $col := $.Columns}} 32 | {{$read_target}}.{{$col.Name}}, 33 | {{end}} 34 | '{{$trigger}}', 35 | CAST((strftime('%s','now') || substr(strftime('%f','now'),4)) as INT), 36 | 0 -- Pending 37 | ); 38 | 39 | INSERT INTO {{$GlobalChangeLogTableName}} (change_table_id, table_name) 40 | VALUES ( 41 | last_insert_rowid(), 42 | '{{$.TableName}}' 43 | ); 44 | 45 | END; 46 | {{end}} -------------------------------------------------------------------------------- /db/utils.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/rs/zerolog/log" 7 | ) 8 | 9 | type EnhancedStatement struct { 10 | *sql.Stmt 11 | } 12 | 13 | func (stmt *EnhancedStatement) Finalize() { 14 | err := stmt.Close() 15 | if err != nil { 16 | log.Error().Err(err).Msg("Unable to close statement") 17 | } 18 | } 19 | 20 | type EnhancedRows struct { 21 | *sql.Rows 22 | } 23 | 24 | func (rs *EnhancedRows) fetchRow() (map[string]any, error) { 25 | columns, err := rs.Columns() 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | scanRow := make([]any, len(columns)) 31 | rowPointers := make([]any, len(columns)) 32 | for i := range scanRow { 33 | rowPointers[i] = &scanRow[i] 34 | } 35 | 36 | if err := rs.Scan(rowPointers...); err != nil { 37 | return nil, err 38 | } 39 | 40 | row := make(map[string]any) 41 | for i, column := range columns { 42 | row[column] = scanRow[i] 43 | } 44 | 45 | return row, nil 46 | } 47 | 48 | func (rs *EnhancedRows) Finalize() { 49 | err := rs.Close() 50 | if err != nil { 51 | log.Error().Err(err).Msg("Unable to close result set") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /docs/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true, 5 | "node": true 6 | }, 7 | "extends": [ 8 | "eslint:recommended", 9 | "plugin:react/recommended", 10 | "plugin:@typescript-eslint/recommended", 11 | "plugin:@next/next/recommended" 12 | ], 13 | "overrides": [], 14 | "parser": "@typescript-eslint/parser", 15 | "parserOptions": { 16 | "ecmaVersion": "latest", 17 | "sourceType": "module" 18 | }, 19 | "plugins": ["react", "@typescript-eslint"], 20 | "rules": { 21 | "react/react-in-jsx-scope": "off", 22 | "react/prop-types": "off" 23 | }, 24 | "settings": { 25 | "react": { 26 | "version": "detect" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | .pnpm-debug.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts -------------------------------------------------------------------------------- /docs/.prettierignore: -------------------------------------------------------------------------------- 1 | .next 2 | node_modules 3 | -------------------------------------------------------------------------------- /docs/next.config.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-var-requires */ 2 | /** 3 | * @type {import('next').NextConfig} 4 | */ 5 | const nextConfig = { 6 | reactStrictMode: true, 7 | basePath: "/marmot", 8 | }; 9 | 10 | const withNextra = require("nextra")({ 11 | theme: "nextra-theme-docs", 12 | themeConfig: "./theme.config.js", 13 | // unstable_staticImage: true, 14 | }); 15 | module.exports = withNextra(nextConfig); 16 | -------------------------------------------------------------------------------- /docs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "docs", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "private": true, 6 | "license": "MIT", 7 | "scripts": { 8 | "start": "next start", 9 | "build": "next build", 10 | "dev": "next" 11 | }, 12 | "dependencies": { 13 | "next": "^12.3.1", 14 | "nextra": "^2.0.0-beta.41", 15 | "nextra-theme-docs": "^2.0.0-beta.41", 16 | "octokit": "^2.0.10", 17 | "react": "^18.2.0", 18 | "react-dom": "^18.2.0" 19 | }, 20 | "devDependencies": { 21 | "@next/eslint-plugin-next": "^12.3.1", 22 | "@types/node": "18.11.9", 23 | "@types/react": "18.0.25", 24 | "@typescript-eslint/eslint-plugin": "^5.42.1", 25 | "@typescript-eslint/parser": "^5.42.1", 26 | "autoprefixer": "^10.4.13", 27 | "eslint": "^8.27.0", 28 | "eslint-plugin-react": "^7.31.10", 29 | "postcss": "^8.4.18", 30 | "tailwindcss": "^3.2.2", 31 | "typescript": "4.8.4" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /docs/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /docs/public/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxpert/marmot/fe252af4a57e472e21455acdbbaec0dcfd12221d/docs/public/logo.png -------------------------------------------------------------------------------- /docs/public/marmot-logo-transparent.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | MARMOTA distributed SQLite replicator 82 | -------------------------------------------------------------------------------- /docs/public/sillouhette-smooth.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 18 | 37 | 41 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /docs/src/components/Hero.tsx: -------------------------------------------------------------------------------- 1 | import Link from "next/link"; 2 | import MemoMarmotLogo from "./Icons/MarmotLogo"; 3 | 4 | export const Hero = () => { 5 | return ( 6 |
7 |
8 | 9 |
10 | {/*

Some text above

*/} 11 |

12 | Marmot 13 |

14 |

15 | A distributed SQLite replicator 16 |

17 |
18 | 19 | 38 |
39 |
40 | ); 41 | }; 42 | -------------------------------------------------------------------------------- /docs/src/components/Icons/MarmotLogo.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | type Props = React.SVGProps & { 4 | width?: number | string; 5 | height?: number | string; 6 | }; 7 | 8 | function MarmotLogo(props: Props) { 9 | const { width = 200, height = 200 } = props; 10 | return ( 11 | 12 | 16 | 17 | ); 18 | } 19 | 20 | const MemoMarmotLogo = React.memo(MarmotLogo); 21 | export default MemoMarmotLogo; 22 | -------------------------------------------------------------------------------- /docs/src/components/Icons/MarmotLogoTransparent.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | type Props = React.SVGProps & { 4 | width?: number | string; 5 | height?: number | string; 6 | }; 7 | 8 | function MarmotLogoTransparent(props: Props) { 9 | const { width = 250, height = 250 } = props; 10 | return ( 11 | 12 | 13 | 14 | 15 | 25 | 26 | 31 | {"MARMOT"} 32 | 33 | 34 | 35 | 48 | 49 | {"A distributed SQLite replicator"} 50 | 51 | 52 | 56 | 57 | ); 58 | } 59 | 60 | const MemoMarmotLogoTransparent = React.memo(MarmotLogoTransparent); 61 | export default MemoMarmotLogoTransparent; 62 | -------------------------------------------------------------------------------- /docs/src/pages/_app.tsx: -------------------------------------------------------------------------------- 1 | import "../styles/globals.css"; 2 | 3 | export default function Nextra({ Component, pageProps }) { 4 | const getLayout = Component.getLayout || ((page) => page); 5 | return getLayout(); 6 | } 7 | -------------------------------------------------------------------------------- /docs/src/pages/_meta.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": "Welcome", 3 | "demo": "Demo", 4 | "intro": "Overview", 5 | "internals": "Internals" 6 | } 7 | -------------------------------------------------------------------------------- /docs/src/pages/demo.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Marmot + Isso Demo 3 | --- 4 | 5 | ## What is Marmot? 6 | 7 | [Marmot](https://github.com/maxpert/marmot) is a distributed SQLite replicator that runs as a side-car to you service, and replicates 8 | data across cluster using embedded NATS. Marmot relies on [JetStream](https://docs.nats.io/nats-concepts/jetstream) 9 | based CDC (Change Data Capture) to replicate changes. JetStream under the hood uses RAFT for consensus 10 | and allows for eventually consistent SQLite replicas (multi-primary replication). 11 | 12 | ## What is Isso? 13 | 14 | [Isso](https://isso-comments.de/) is a commenting system like Disqus. Users can edit or delete own 15 | comments (within 15 minutes by default). Best part about Isso is that it uses SQLite! 16 | 17 | ## What is Fly.io? 18 | 19 | Fly is a platform for running full stack apps and databases close to your users. Compute jobs at Fly.io are 20 | virtualized using Firecracker, the virtualization engine developed at AWS as the 21 | engine for Lambda and Fargate. 22 | 23 | ## Why should I care? 24 | 25 | This demo effectively shows how Isso can be scaled out and pushed closer to the edge. Many out of box SQLite tools can be scaled 26 | similarly. Your fly.io nodes scale up or down based on traffic, and write from everywhere. This allows horizontal scalability 27 | of your Isso close to the user. With NATS embedded into Marmot, a sharded RAFT is used to capture changes, and replay them 28 | across the fly nodes. So if you are running a single node SQLite site you can now add redundancy to that with Marmot. 29 | 30 | ## How do I configure one for myself? 31 | 32 | You can access code and follow the instructions [here](https://gitlab.com/maxpert/marmot-denokv). 33 | 34 | ## Demo 35 | 36 |
37 | 38 |
39 | 40 | 41 | -------------------------------------------------------------------------------- /docs/src/pages/index.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Marmot - A distributed SQLite replicator 3 | --- 4 | 5 | import { Hero } from "../components/Hero.tsx"; 6 | 7 | 8 | -------------------------------------------------------------------------------- /docs/src/pages/internals.mdx: -------------------------------------------------------------------------------- 1 | # How does it work? 2 | 3 | There are multiple components in Marmot that make it work. Let's look at them one by one: 4 | 5 | 6 | ## Triggers and data capture 7 | 8 | Marmot works by using a pretty basic trick so that each process that's access database can capture changes, 9 | and then Marmot can publish them to rest of the nodes. This is how it works internally: 10 | 11 | - Global table `__marmot___global_change_log` that captures exact sequence of operations committed to DB. 12 | Ideally we should be able to keep the changed as JSON blob inline, but due to limitation of SQLites 13 | JSON not able to serialize BLOBs as JSON property we for now keep a different table with values as 14 | columns in the table. So each table gets a `__marmot___change_log` that will record 15 | the value changed. Each column name in table change log is prefixed by `val_` prefix (i.e. `id` 16 | becomes `val_id`). These triggers are compiled via Go's builtin templating system, and installed 17 | to database at boot time. 18 | 19 | - Each `insert`, `update`, `delete` triggers for every table `AFTER` the changes have been 20 | committed to the table. These triggers record `OLD` (ON DELETE) or `NEW` (ON INSERT OR 21 | UPDATE) values into the table. 22 | 23 | ## Replication 24 | 25 | When you are running Marmot process, it's watching for changes on DB file and WAL file. Everytime there is a change 26 | Marmot: 27 | 28 | - Gathers all change records, and for each record calculate a consistent hash based on table name + primary keys. 29 | - Using the hash decide JetStream and subject the change belongs to. And publish the change into that specific JetStream. 30 | - Once JetStream has replicated the change log, mark the change published. 31 | - As soon as change is published to JetStream rest of the nodes replay that log, and row changes are applied via state machine 32 | to local tables of the node. This means every row in database due to RAFT consensus at stream level will have only one 33 | deterministic order of changes getting in cluster in case of race-conditions. 34 | - Once the order is determined for a change it's applied in an upsert or delete manner to the table. So it's quite 35 | possible that a row committed locally is overwritten or replaced later because it was not the last one 36 | in order of cluster wide commit order. 37 | 38 | ## Changelog format 39 | 40 | Changelog is a CBOR serialized (and compressed if configured) payload that has following interface definition: 41 | 42 | ```typescript 43 | interface MarmotPublishedRow { 44 | FromNodeId: number; 45 | Payload: { 46 | Id: number; 47 | Type: "insert" | "update" | "delete"; 48 | TableName: string; 49 | Row: {[ColumnName: string]: any} 50 | }; 51 | } 52 | ``` 53 | 54 | `FromNodeId` points to node ID who sent the changelog (configured when launching). 55 | `Payload.TableName` points to the table that changed with mutation type of 56 | `Payload.Type`. Then `Payload.Row` contains flat map of column 57 | name to value. 58 | 59 | > There is alot of optimization that can be done to this payload overall, in future using 60 | > ProtoBuf or more optimized serialization format is absolutely an open option. 61 | 62 | ## Snapshotting 63 | 64 | In a normal distributed system it's pretty typical for nodes to go down for really long time. In that particular 65 | case a node coming back up can be lagging so far behind that `max-log-entries` might not be sufficient to 66 | fully play all logs and restore the state of database. In that case it required that a node restores a 67 | snapshot and apply current log entries to be fully up-to-date. 68 | 69 | As of `v0.6.0+` Marmot supports taking snapshot and being able to fully restore it via 70 | [NATS Object Storage](https://docs.nats.io/using-nats/developer/develop_jetstream/object). 71 | 72 | > In future Marmot plans to support more storage mechanisms like S3 (and compatible APIs including BlackBlaze 73 | > and Minio), Azure Blob, and SFTP. 74 | 75 | ### Saving Snapshot 76 | Everytime a node publishes a new change log to NATs, it saves sequence number of the entry in JetStream. The saved 77 | sequence number will be used later once the node tries to boot up. Let's first look at how snapshot is saved. 78 | 79 | In order to keep enough headroom it calculates max snapshot entries by dividing `max-log-entries` by total number 80 | of `shards` e.g. `max-log-entries` of 1024 and total 8 shards will result in 128 max snapshot entries. Now anytime 81 | sequence number of shard 1 is multiple of 128 a snapshot will be taken and uploaded to NATS Object. The snapshot 82 | will be saved in `OBJ_-snapshot-store`. Statistically due to even distribution of hashes among shards 83 | everytime shard 1 hit max snapshot entries, rest of the shards will have almost same number of new entries. 84 | 85 | Once it has been decided that system wants to save snapshot, a temporary path is created where we used awesome feature 86 | of SQLite called `VACUUM INTO `. Where SQLite optimizes the DB, and gives us a compacted snapshot of database. 87 | After which Marmot removes all the hooks, and triggers from the snapshot, and re-VACUUM packs the database for upload. 88 | Once done this snapshot is uploaded to `OBJ_-snapshot-store` (`-snapshot-store` in 89 | client API). 90 | 91 | One important thing to keep in mind is everytime Marmot sees a sequence number in a shard higher than what it 92 | has seen before it will record it against that shard, and this whole mapping is saved on specified sequence 93 | map file path `seq-map-path`. 94 | 95 | 96 | ### Restoring Snapshot 97 | Whenever a node boots up it first verifies the DB file integrity and performs any WAL checkpoints. Then it loads the 98 | shard mapping from `seq-map-path` and compare it to corresponding JetStream's starting sequence number. If the 99 | sequence number that node has is less than start sequence number that simply means node can't reconstruct the 100 | exact state of database by replaying all logs it's missing. In this case node downloads the snapshot from 101 | NATs server as specified in section above. This snapshot is downloaded in a temporary path. Marmot then 102 | uses `exclusive` transaction lock to prevent any writers from getting into DB while it copies over 103 | the files. 104 | 105 | Once snapshot has been copied over, Marmot installs same triggers, and change log tables again in DB, and starts 106 | processing logs applying them all on the snapshot. This means while Marmot is restoring the DB it's quite 107 | possible that you might have an outdated copy of database until the logs are fully applied. 108 | -------------------------------------------------------------------------------- /docs/src/pages/intro.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Marmot - Introduction 3 | --- 4 | 5 | # What & Why? 6 | 7 | Marmot is a distributed SQLite replicator with leaderless, and eventual consistency. It allows you to build a robust replication 8 | between your nodes by building on top of fault-tolerant [NATS JetStream](https://nats.io/). 9 | 10 | So if you are running a read heavy website based on SQLite, you should be easily able to scale it out by adding more SQLite replicated nodes. 11 | SQLite is probably the most ubiquitous DB that exists almost everywhere, Marmot aims to make it even more ubiquitous for server 12 | side applications by building a replication layer on top. 13 | 14 | ## Why? 15 | 16 | SQLite is a probably the most ubiquitous DB that exists almost everywhere, this project aims to make it even more ubiquitous for server 17 | side applications by building a master-less replication layer on top. This means if you are running a read heavy website based on SQLite 18 | you should be easily able to scale it out by adding more nodes of your app with SQLite replicated nodes. 19 | 20 | ## Quick Start 21 | 22 | Download [latest](https://github.com/maxpert/marmot/releases/latest) Marmot and extract package using: 23 | 24 | ``` 25 | tar vxzf marmot-v*.tar.gz 26 | ``` 27 | 28 | From extracted directory run `examples/run-cluster.sh`. Make a change in `/tmp/marmot-1.db` using: 29 | 30 | ``` 31 | bash > sqlite3 /tmp/marmot-1.db 32 | sqlite3 > INSERT INTO Books (title, author, publication_year) VALUES ('Pride and Prejudice', 'Jane Austen', 1813); 33 | ``` 34 | 35 | Now observe changes getting propagated to other database `/tmp/marmot-2.db`: 36 | 37 | ``` 38 | bash > sqlite3 /tmp/marmot-2.db 39 | sqlite3 > SELECT * FROM Books; 40 | ``` 41 | 42 | You should be able to make changes interchangeably and see the changes getting propagated. 43 | 44 | For more complicated demos, checkout following (older versions): 45 | 46 |
47 | 48 |
49 | 50 |
51 | 52 | ## What is the difference from others? 53 | 54 | Marmot is essentially a CDC (Change Data Capture) and replication pipeline running top of NATS. It can automatically configure appropriate 55 | JetStreams making sure those streams evenly distribute load over those shards, so scaling simply boils down to adding more nodes, and 56 | re-balancing those JetStreams (auto re-balancing not implemented yet). 57 | 58 | There are a few solutions like [rqlite](https://github.com/rqlite/rqlite), [dqlite](https://dqlite.io/), and 59 | [LiteFS](https://github.com/superfly/litefs) etc. All of them either are layers on top of SQLite (e.g. 60 | rqlite, dqlite) that requires them to sit in the middle with network layer in order to provide 61 | replication; or intercept physical page level writes to stream them off to replicas. In both 62 | cases they require a single primary node where all the writes have to go, and then these 63 | changes are applied to multiple readonly replicas. 64 | 65 | Marmot on the other hand is born different. It's born to act as a side-car to your existing processes: 66 | - Instead of requiring single primary, there is no primary! Which means any node can make changes to its local DB. 67 | Marmot will use triggers to capture your changes (hence atomic records), and then stream them off to NATS. 68 | - Instead of being strongly consistent, it's eventually consistent. Which means no locking, or blocking of nodes. 69 | - It does not require any changes to your application logic for reading/writing. 70 | 71 | Making these choices has multiple benefits: 72 | 73 | - You can read and write to your SQLite database like you normally do. No extension, or VFS changes. 74 | - You can write on any node! You don't have to go to single primary for writing your data. 75 | - As long as you start with same copy of database, all the mutations will eventually converge 76 | (hence eventually consistent). 77 | 78 | ## FAQ 79 | 80 | ### What happens when there is a race condition? 81 | 82 | In Marmot every row is uniquely mapped to a JetStream. This guarantees that for any node to publish changes for a row it 83 | has to go through same JetStream as everyone else. If two nodes perform a change to same row in parallel, both of the 84 | nodes will compete to publish their change to JetStream cluster. Due to 85 | [RAFT quorum](https://docs.nats.io/running-a-nats-service/configuration/clustering/jetstream_clustering#raft) 86 | constraint only one of the writer will be able to get its changes published first. Now as these changes are applied 87 | (even the publisher applies its own changes to database) the **last writer** will always win. This means there is 88 | NO serializability guarantee of a transaction spanning multiple tables. This is a design choice, in order to avoid 89 | any sort of global locking, and performance. 90 | 91 | ### Won't capturing changes with triggers use more disk space? 92 | 93 | Yes it will require additional storage to old/new values from triggers. But right now that is the only way 94 | sqlite can and should allow one to capture changes. However, in a typical setting these captured 95 | changes will be picked up pretty quickly, and cleaned up as soon as they have been pushed to NATS. 96 | Disk space is usually the cheapest part of modern cloud, so it should not be a huge problem. 97 | 98 | ### How do I cleanup my database? 99 | 100 | Ask marmot to remove hooks and log tables by: 101 | `marmot -config /path/to/config.toml -cleanup` 102 | 103 | ### How many shards should I have? 104 | 105 | Mostly you won't need more than 1. But it depends on your use-case, and what problem you are solving for. While read 106 | scaling won't be a problem, your write throughput will depend on your network and disk speeds (Network being 107 | the biggest culprit). Shards are there to alleviate the problem when you are writing fast enough to cause 108 | bottleneck by NATS JetStream (very unlikely with a commodity SQLite node). 109 | 110 | ### Can I use Marmot as single primary and multiple replicas? 111 | 112 | Yes you can. There are two flags in configuration that will allow you to do that. First flag `publish` 113 | enables/disables publishing local changes to NATS, you should disable `publish` (set it to `false`) 114 | on replicas. Second flag is `replicate` that enables/disables replicating changes from NATS on to 115 | local node. You should disable `replicate` (set it to `false`) on primary. 116 | -------------------------------------------------------------------------------- /docs/src/styles/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | --color-bg-dark: #111; 7 | --color-border-dark: #eee; 8 | --color-text-dark: #ccc; 9 | 10 | --color-bg-light: #fff; 11 | --color-border-light: #ddd; 12 | --color-text-light: #999; 13 | } 14 | 15 | @media (prefers-color-scheme: dark) { 16 | .isso-postbox input, .isso-postbox textarea { 17 | background-color: var(--color-bg-dark) !important; 18 | border: var(--color-border-dark) 1px solid !important; 19 | color: var(--color-text-dark) !important; 20 | } 21 | } 22 | 23 | @media (prefers-color-scheme: light) { 24 | .isso-postbox input, .isso-postbox textarea { 25 | background-color: var(--color-bg-light) !important; 26 | border: var(--color-border-light) 1px solid !important; 27 | color: var(--color-text-light) !important; 28 | } 29 | } 30 | 31 | .isso-postbox input, .isso-postbox textarea { 32 | border-radius: 5px; 33 | } -------------------------------------------------------------------------------- /docs/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: ["./src/pages/**/*.{js,ts,jsx,tsx,mdx}", "./src/components/**/*.{js,ts,jsx,tsx}"], 4 | theme: { 5 | extend: { 6 | colors: { 7 | "marmot-blue": { 8 | 100: "#698EA6", 9 | 200: "#5384A3", 10 | 300: "#437A9D", 11 | 400: "#327199", 12 | 500: "#226997", 13 | 600: "#126397", 14 | 700: "#015D98", 15 | 800: "#0E517B", 16 | 900: "#174765", 17 | DEFAULT: "#126397", 18 | }, 19 | }, 20 | }, 21 | }, 22 | plugins: [], 23 | }; 24 | -------------------------------------------------------------------------------- /docs/theme.config.js: -------------------------------------------------------------------------------- 1 | const basePath = "/marmot"; 2 | 3 | /** 4 | * @type {import("nextra-theme-docs").DocsThemeConfig} 5 | */ 6 | export default { 7 | project: { link: "https://github.com/maxpert/marmot" }, 8 | docsRepositoryBase: "https://github.com/maxpert/marmot/tree/master/docs", 9 | titleSuffix: " - Marmot", 10 | logo: ( 11 | <> 12 | 13 | 14 | A distributed SQLite replicator built on top of NATS 15 | 16 | 17 | ), 18 | head: ( 19 | <> 20 | 21 | 22 | 23 | 24 | 28 | 32 | 33 | 34 | 35 | 36 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | ), 50 | navigation: true, 51 | footer: { text: <>MIT {new Date().getFullYear()} © Marmot. }, 52 | editLink: { text: "Edit this page on GitHub" }, 53 | unstable_faviconGlyph: "👋", 54 | }; 55 | -------------------------------------------------------------------------------- /docs/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "strict": false, 12 | "forceConsistentCasingInFileNames": true, 13 | "noEmit": true, 14 | "incremental": true, 15 | "esModuleInterop": true, 16 | "module": "esnext", 17 | "moduleResolution": "node", 18 | "resolveJsonModule": true, 19 | "isolatedModules": true, 20 | "jsx": "preserve" 21 | }, 22 | "include": [ 23 | "next-env.d.ts", 24 | "**/*.ts", 25 | "**/*.tsx" 26 | ], 27 | "exclude": [ 28 | "node_modules" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /examples/node-1-config.toml: -------------------------------------------------------------------------------- 1 | seq_map_path="/tmp/marmot-1-sm.cbor" 2 | db_path="/tmp/marmot-1.db" 3 | node_id=1 -------------------------------------------------------------------------------- /examples/node-2-config.toml: -------------------------------------------------------------------------------- 1 | seq_map_path="/tmp/marmot-2-sm.cbor" 2 | db_path="/tmp/marmot-2.db" 3 | node_id=2 -------------------------------------------------------------------------------- /examples/node-3-config.toml: -------------------------------------------------------------------------------- 1 | seq_map_path="/tmp/marmot-3-sm.cbor" 2 | db_path="/tmp/marmot-3.db" 3 | node_id=3 -------------------------------------------------------------------------------- /examples/run-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | create_db() { 3 | local db_file="$1" 4 | cat <>1) + 1 344 | } 345 | 346 | if replicas > 5 { 347 | replicas = 5 348 | } 349 | 350 | return &nats.StreamConfig{ 351 | Name: streamName, 352 | Subjects: []string{subjectName(shardID)}, 353 | Discard: nats.DiscardOld, 354 | MaxMsgs: cfg.Config.ReplicationLog.MaxEntries, 355 | Storage: nats.FileStorage, 356 | Retention: nats.LimitsPolicy, 357 | AllowDirect: true, 358 | MaxConsumers: -1, 359 | MaxMsgsPerSubject: -1, 360 | Duplicates: 0, 361 | DenyDelete: true, 362 | Replicas: replicas, 363 | } 364 | } 365 | 366 | func eqShardStreamConfig(a *nats.StreamConfig, b *nats.StreamConfig) bool { 367 | return a.Name == b.Name && 368 | len(a.Subjects) == 1 && 369 | len(b.Subjects) == 1 && 370 | a.Subjects[0] == b.Subjects[0] && 371 | a.Discard == b.Discard && 372 | a.MaxMsgs == b.MaxMsgs && 373 | a.Storage == b.Storage && 374 | a.Retention == b.Retention && 375 | a.AllowDirect == b.AllowDirect && 376 | a.MaxConsumers == b.MaxConsumers && 377 | a.MaxMsgsPerSubject == b.MaxMsgsPerSubject && 378 | a.Duplicates == b.Duplicates && 379 | a.DenyDelete == b.DenyDelete && 380 | a.Replicas == b.Replicas 381 | } 382 | 383 | func streamName(shardID uint64, compressed bool) string { 384 | compPostfix := "" 385 | if compressed { 386 | compPostfix = "-c" 387 | } 388 | 389 | return fmt.Sprintf("%s%s-%d", cfg.Config.NATS.StreamPrefix, compPostfix, shardID) 390 | } 391 | 392 | func subjectName(shardID uint64) string { 393 | return fmt.Sprintf("%s-%d", cfg.Config.NATS.SubjectPrefix, shardID) 394 | } 395 | 396 | func payloadCompress(payload []byte) ([]byte, error) { 397 | enc, err := zstd.NewWriter(nil) 398 | if err != nil { 399 | return nil, err 400 | } 401 | 402 | return enc.EncodeAll(payload, nil), nil 403 | } 404 | 405 | func payloadDecompress(payload []byte) ([]byte, error) { 406 | dec, err := zstd.NewReader(nil) 407 | if err != nil { 408 | return nil, err 409 | } 410 | 411 | return dec.DecodeAll(payload, nil) 412 | } 413 | -------------------------------------------------------------------------------- /logstream/replicator_meta_store.go: -------------------------------------------------------------------------------- 1 | package logstream 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/fxamacker/cbor/v2" 8 | "github.com/maxpert/marmot/cfg" 9 | "github.com/nats-io/nats.go" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | type replicatorMetaStore struct { 14 | nats.KeyValue 15 | } 16 | 17 | type replicatorLockInfo struct { 18 | NodeID uint64 19 | Timestamp int64 20 | } 21 | 22 | func newReplicatorMetaStore(name string, nc *nats.Conn) (*replicatorMetaStore, error) { 23 | jsx, err := nc.JetStream() 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | kv, err := jsx.KeyValue(name) 29 | if err == nats.ErrBucketNotFound { 30 | kv, err = jsx.CreateKeyValue(&nats.KeyValueConfig{ 31 | Storage: nats.FileStorage, 32 | Bucket: name, 33 | Replicas: cfg.Config.ReplicationLog.Replicas, 34 | }) 35 | } 36 | 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | return &replicatorMetaStore{KeyValue: kv}, nil 42 | } 43 | 44 | func (m *replicatorMetaStore) AcquireLease(name string, duration time.Duration) (bool, error) { 45 | now := time.Now().UnixMilli() 46 | info := &replicatorLockInfo{ 47 | NodeID: cfg.Config.NodeID, 48 | Timestamp: now, 49 | } 50 | payload, err := info.Serialize() 51 | if err != nil { 52 | return false, err 53 | } 54 | 55 | entry, err := m.Get(name) 56 | if err == nats.ErrKeyNotFound { 57 | rev := uint64(0) 58 | rev, err = m.Create(name, payload) 59 | if rev != 0 && err == nil { 60 | return true, nil 61 | } 62 | } 63 | 64 | if err != nil { 65 | return false, err 66 | } 67 | 68 | err = info.DeserializeFrom(entry.Value()) 69 | if err != nil { 70 | return false, err 71 | } 72 | 73 | if info.NodeID != cfg.Config.NodeID && info.Timestamp+duration.Milliseconds() > now { 74 | return false, err 75 | } 76 | 77 | _, err = m.Update(name, payload, entry.Revision()) 78 | if err != nil { 79 | return false, err 80 | } 81 | 82 | return true, nil 83 | } 84 | 85 | func (m *replicatorMetaStore) ContextRefreshingLease( 86 | name string, 87 | duration time.Duration, 88 | ctx context.Context, 89 | ) (bool, error) { 90 | locked, err := m.AcquireLease(name, duration) 91 | go func(locked bool, err error) { 92 | if !locked || err != nil { 93 | return 94 | } 95 | 96 | refresh := time.NewTicker(duration / 2) 97 | for { 98 | locked, err = m.AcquireLease(name, duration) 99 | if err != nil { 100 | log.Warn().Err(err).Str("name", name).Msg("Error acquiring lease") 101 | return 102 | } else if !locked { 103 | log.Warn().Str("name", name).Msg("Unable to acquire lease") 104 | continue 105 | } 106 | 107 | refresh.Reset(duration / 2) 108 | select { 109 | case <-refresh.C: 110 | continue 111 | case <-ctx.Done(): 112 | return 113 | } 114 | } 115 | }(locked, err) 116 | 117 | return locked, err 118 | } 119 | 120 | func (r *replicatorLockInfo) Serialize() ([]byte, error) { 121 | return cbor.Marshal(r) 122 | } 123 | 124 | func (r *replicatorLockInfo) DeserializeFrom(data []byte) error { 125 | return cbor.Unmarshal(data, r) 126 | } 127 | -------------------------------------------------------------------------------- /marmot.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "io" 7 | "net/http" 8 | "net/http/pprof" 9 | _ "net/http/pprof" 10 | "os" 11 | "time" 12 | 13 | "github.com/maxpert/marmot/telemetry" 14 | "github.com/maxpert/marmot/utils" 15 | 16 | "github.com/maxpert/marmot/cfg" 17 | "github.com/maxpert/marmot/db" 18 | "github.com/maxpert/marmot/logstream" 19 | "github.com/maxpert/marmot/snapshot" 20 | 21 | "github.com/asaskevich/EventBus" 22 | "github.com/rs/zerolog" 23 | "github.com/rs/zerolog/log" 24 | ) 25 | 26 | func main() { 27 | flag.Parse() 28 | err := cfg.Load(*cfg.ConfigPathFlag) 29 | if err != nil { 30 | panic(err) 31 | } 32 | 33 | var writer io.Writer = zerolog.NewConsoleWriter() 34 | if cfg.Config.Logging.Format == "json" { 35 | writer = os.Stdout 36 | } 37 | gLog := zerolog.New(writer). 38 | With(). 39 | Timestamp(). 40 | Uint64("node_id", cfg.Config.NodeID). 41 | Logger() 42 | 43 | if cfg.Config.Logging.Verbose { 44 | log.Logger = gLog.Level(zerolog.DebugLevel) 45 | } else { 46 | log.Logger = gLog.Level(zerolog.InfoLevel) 47 | } 48 | 49 | if *cfg.ProfServer != "" { 50 | go func() { 51 | mux := http.NewServeMux() 52 | mux.HandleFunc("/debug/pprof/", pprof.Index) 53 | mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) 54 | mux.HandleFunc("/debug/pprof/profile", pprof.Profile) 55 | mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 56 | mux.HandleFunc("/debug/pprof/trace", pprof.Trace) 57 | 58 | err := http.ListenAndServe(*cfg.ProfServer, mux) 59 | if err != nil { 60 | log.Error().Err(err).Msg("unable to bind profiler server") 61 | } 62 | }() 63 | } 64 | 65 | log.Debug().Msg("Initializing telemetry") 66 | telemetry.InitializeTelemetry() 67 | 68 | log.Debug().Str("path", cfg.Config.DBPath).Msg("Opening database") 69 | streamDB, err := db.OpenStreamDB(cfg.Config.DBPath) 70 | if err != nil { 71 | log.Error().Err(err).Msg("Unable to open database") 72 | return 73 | } 74 | 75 | if *cfg.CleanupFlag { 76 | err = streamDB.RemoveCDC(true) 77 | if err != nil { 78 | log.Panic().Err(err).Msg("Unable to clean up...") 79 | } else { 80 | log.Info().Msg("Cleanup complete...") 81 | } 82 | 83 | return 84 | } 85 | 86 | snpStore, err := snapshot.NewSnapshotStorage() 87 | if err != nil { 88 | log.Panic().Err(err).Msg("Unable to initialize snapshot storage") 89 | } 90 | 91 | replicator, err := logstream.NewReplicator(snapshot.NewNatsDBSnapshot(streamDB, snpStore)) 92 | if err != nil { 93 | log.Panic().Err(err).Msg("Unable to initialize replicators") 94 | } 95 | 96 | if *cfg.SaveSnapshotFlag { 97 | replicator.ForceSaveSnapshot() 98 | return 99 | } 100 | 101 | if cfg.Config.Snapshot.Enable && cfg.Config.Replicate { 102 | err = replicator.RestoreSnapshot() 103 | if err != nil { 104 | log.Panic().Err(err).Msg("Unable to restore snapshot") 105 | } 106 | } 107 | 108 | log.Info().Msg("Listing tables to watch...") 109 | tableNames, err := db.GetAllDBTables(cfg.Config.DBPath) 110 | if err != nil { 111 | log.Error().Err(err).Msg("Unable to list all tables") 112 | return 113 | } 114 | 115 | eventBus := EventBus.New() 116 | ctxSt := utils.NewStateContext() 117 | 118 | streamDB.OnChange = onTableChanged(replicator, ctxSt, eventBus, cfg.Config.NodeID) 119 | log.Info().Msg("Starting change data capture pipeline...") 120 | if err := streamDB.InstallCDC(tableNames); err != nil { 121 | log.Error().Err(err).Msg("Unable to install change data capture pipeline") 122 | return 123 | } 124 | 125 | errChan := make(chan error) 126 | for i := uint64(0); i < cfg.Config.ReplicationLog.Shards; i++ { 127 | go changeListener(streamDB, replicator, ctxSt, eventBus, i+1, errChan) 128 | } 129 | 130 | sleepTimeout := utils.AutoResetEventTimer( 131 | eventBus, 132 | "pulse", 133 | time.Duration(cfg.Config.SleepTimeout)*time.Millisecond, 134 | ) 135 | cleanupInterval := time.Duration(cfg.Config.CleanupInterval) * time.Millisecond 136 | cleanupTicker := time.NewTicker(cleanupInterval) 137 | defer cleanupTicker.Stop() 138 | 139 | snapshotInterval := time.Duration(cfg.Config.Snapshot.Interval) * time.Millisecond 140 | snapshotTicker := utils.NewTimeoutPublisher(snapshotInterval) 141 | defer snapshotTicker.Stop() 142 | 143 | for { 144 | select { 145 | case err = <-errChan: 146 | if err != nil { 147 | log.Panic().Err(err).Msg("Terminated listener") 148 | } 149 | case t := <-cleanupTicker.C: 150 | cnt, err := streamDB.CleanupChangeLogs(t.Add(-cleanupInterval)) 151 | if err != nil { 152 | log.Warn().Err(err).Msg("Unable to cleanup change logs") 153 | } else if cnt > 0 { 154 | log.Debug().Int64("count", cnt).Msg("Cleaned up DB change logs") 155 | } 156 | case <-snapshotTicker.Channel(): 157 | if cfg.Config.Snapshot.Enable && cfg.Config.Publish { 158 | lastSnapshotTime := replicator.LastSaveSnapshotTime() 159 | now := time.Now() 160 | if now.Sub(lastSnapshotTime) >= snapshotInterval { 161 | log.Info(). 162 | Time("last_snapshot", lastSnapshotTime). 163 | Dur("duration", now.Sub(lastSnapshotTime)). 164 | Msg("Triggering timer based snapshot save") 165 | replicator.SaveSnapshot() 166 | } 167 | } 168 | case <-sleepTimeout.Channel(): 169 | log.Info().Msg("No more events to process, initiating shutdown") 170 | ctxSt.Cancel() 171 | if cfg.Config.Snapshot.Enable && cfg.Config.Publish { 172 | log.Info().Msg("Saving snapshot before going to sleep") 173 | replicator.ForceSaveSnapshot() 174 | } 175 | 176 | os.Exit(0) 177 | } 178 | } 179 | } 180 | 181 | func changeListener( 182 | streamDB *db.SqliteStreamDB, 183 | rep *logstream.Replicator, 184 | ctxSt *utils.StateContext, 185 | events EventBus.BusPublisher, 186 | shard uint64, 187 | errChan chan error, 188 | ) { 189 | log.Debug().Uint64("shard", shard).Msg("Listening stream") 190 | err := rep.Listen(shard, onChangeEvent(streamDB, ctxSt, events)) 191 | if err != nil { 192 | errChan <- err 193 | } 194 | } 195 | 196 | func onChangeEvent(streamDB *db.SqliteStreamDB, ctxSt *utils.StateContext, events EventBus.BusPublisher) func(data []byte) error { 197 | return func(data []byte) error { 198 | events.Publish("pulse") 199 | if ctxSt.IsCanceled() { 200 | return context.Canceled 201 | } 202 | 203 | if !cfg.Config.Replicate { 204 | return nil 205 | } 206 | 207 | ev := &logstream.ReplicationEvent[db.ChangeLogEvent]{} 208 | err := ev.Unmarshal(data) 209 | if err != nil { 210 | log.Error().Err(err).Send() 211 | return err 212 | } 213 | 214 | return streamDB.Replicate(&ev.Payload) 215 | } 216 | } 217 | 218 | func onTableChanged(r *logstream.Replicator, ctxSt *utils.StateContext, events EventBus.BusPublisher, nodeID uint64) func(event *db.ChangeLogEvent) error { 219 | return func(event *db.ChangeLogEvent) error { 220 | events.Publish("pulse") 221 | if ctxSt.IsCanceled() { 222 | return context.Canceled 223 | } 224 | 225 | if !cfg.Config.Publish { 226 | return nil 227 | } 228 | 229 | ev := &logstream.ReplicationEvent[db.ChangeLogEvent]{ 230 | FromNodeId: nodeID, 231 | Payload: *event, 232 | } 233 | 234 | data, err := ev.Marshal() 235 | if err != nil { 236 | return err 237 | } 238 | 239 | hash, err := event.Hash() 240 | if err != nil { 241 | return err 242 | } 243 | 244 | err = r.Publish(hash, data) 245 | if err != nil { 246 | return err 247 | } 248 | 249 | return nil 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /pool/connection_pool.go: -------------------------------------------------------------------------------- 1 | package pool 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "sync/atomic" 7 | "time" 8 | 9 | "github.com/doug-martin/goqu/v9" 10 | "github.com/mattn/go-sqlite3" 11 | ) 12 | 13 | var ErrWrongPool = errors.New("returning object to wrong pool") 14 | 15 | type ConnectionDisposer interface { 16 | Dispose(obj *SQLiteConnection) error 17 | } 18 | 19 | type SQLiteConnection struct { 20 | db *sql.DB 21 | raw *sqlite3.SQLiteConn 22 | gSQL *goqu.Database 23 | disposer ConnectionDisposer 24 | state int32 25 | } 26 | 27 | type SQLitePool struct { 28 | connections chan *SQLiteConnection 29 | dns string 30 | } 31 | 32 | func (q *SQLiteConnection) SQL() *sql.DB { 33 | return q.db 34 | } 35 | 36 | func (q *SQLiteConnection) Raw() *sqlite3.SQLiteConn { 37 | return q.raw 38 | } 39 | 40 | func (q *SQLiteConnection) DB() *goqu.Database { 41 | return q.gSQL 42 | } 43 | 44 | func (q *SQLiteConnection) Return() error { 45 | return q.disposer.Dispose(q) 46 | } 47 | 48 | func (q *SQLiteConnection) init(dns string, disposer ConnectionDisposer) error { 49 | if !atomic.CompareAndSwapInt32(&q.state, 0, 1) { 50 | return nil 51 | } 52 | 53 | dbC, rawC, err := OpenRaw(dns) 54 | if err != nil { 55 | atomic.SwapInt32(&q.state, 0) 56 | return err 57 | } 58 | 59 | q.raw = rawC 60 | q.db = dbC 61 | q.gSQL = goqu.New("sqlite", dbC) 62 | q.disposer = disposer 63 | return nil 64 | } 65 | 66 | func (q *SQLiteConnection) reset() { 67 | if !atomic.CompareAndSwapInt32(&q.state, 1, 0) { 68 | return 69 | } 70 | 71 | q.db.Close() 72 | q.raw.Close() 73 | 74 | q.db = nil 75 | q.raw = nil 76 | q.gSQL = nil 77 | q.disposer = nil 78 | } 79 | 80 | func NewSQLitePool(dns string, poolSize int, lazy bool) (*SQLitePool, error) { 81 | ret := &SQLitePool{ 82 | connections: make(chan *SQLiteConnection, poolSize), 83 | dns: dns, 84 | } 85 | 86 | for i := 0; i < poolSize; i++ { 87 | con := &SQLiteConnection{} 88 | if !lazy { 89 | err := con.init(dns, ret) 90 | if err != nil { 91 | return nil, err 92 | } 93 | } 94 | ret.connections <- con 95 | } 96 | 97 | return ret, nil 98 | } 99 | 100 | func (q *SQLitePool) Borrow() (*SQLiteConnection, error) { 101 | c := <-q.connections 102 | err := c.init(q.dns, q) 103 | 104 | if err != nil { 105 | q.connections <- &SQLiteConnection{} 106 | return nil, err 107 | } 108 | 109 | return c, nil 110 | } 111 | 112 | func (q *SQLitePool) Dispose(obj *SQLiteConnection) error { 113 | if obj.disposer != q { 114 | return ErrWrongPool 115 | } 116 | 117 | q.connections <- obj 118 | return nil 119 | } 120 | 121 | func OpenRaw(dns string) (*sql.DB, *sqlite3.SQLiteConn, error) { 122 | var rawConn *sqlite3.SQLiteConn 123 | d := &sqlite3.SQLiteDriver{ 124 | ConnectHook: func(conn *sqlite3.SQLiteConn) error { 125 | rawConn = conn 126 | return conn.RegisterFunc("marmot_version", func() string { 127 | return "0.1" 128 | }, true) 129 | }, 130 | } 131 | 132 | conn := sql.OpenDB(SqliteDriverConnector{driver: d, dns: dns}) 133 | conn.SetConnMaxLifetime(0) 134 | conn.SetConnMaxIdleTime(10 * time.Second) 135 | 136 | err := conn.Ping() 137 | if err != nil { 138 | return nil, nil, err 139 | } 140 | 141 | return conn, rawConn, nil 142 | } 143 | -------------------------------------------------------------------------------- /pool/sqlite_driver_connector.go: -------------------------------------------------------------------------------- 1 | package pool 2 | 3 | import ( 4 | "context" 5 | "database/sql/driver" 6 | ) 7 | 8 | type SqliteDriverConnector struct { 9 | driver driver.Driver 10 | dns string 11 | } 12 | 13 | func (t SqliteDriverConnector) Connect(_ context.Context) (driver.Conn, error) { 14 | return t.driver.Open(t.dns) 15 | } 16 | 17 | func (t SqliteDriverConnector) Driver() driver.Driver { 18 | return t.driver 19 | } 20 | -------------------------------------------------------------------------------- /snapshot/db_snapshot.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "hash/fnv" 7 | "io" 8 | "os" 9 | "path" 10 | "sync" 11 | "time" 12 | 13 | "github.com/maxpert/marmot/db" 14 | "github.com/rs/zerolog/log" 15 | ) 16 | 17 | var ErrPendingSnapshot = errors.New("system busy capturing snapshot") 18 | 19 | const snapshotFileName = "snapshot.db" 20 | const tempDirPattern = "marmot-snapshot-*" 21 | 22 | type NatsDBSnapshot struct { 23 | mutex *sync.Mutex 24 | db *db.SqliteStreamDB 25 | storage Storage 26 | } 27 | 28 | func NewNatsDBSnapshot(d *db.SqliteStreamDB, snapshotStorage Storage) *NatsDBSnapshot { 29 | return &NatsDBSnapshot{ 30 | mutex: &sync.Mutex{}, 31 | db: d, 32 | storage: snapshotStorage, 33 | } 34 | } 35 | 36 | func (n *NatsDBSnapshot) SaveSnapshot() error { 37 | locked := n.mutex.TryLock() 38 | if !locked { 39 | return ErrPendingSnapshot 40 | } 41 | 42 | defer n.mutex.Unlock() 43 | tmpSnapshot, err := os.MkdirTemp(os.TempDir(), tempDirPattern) 44 | if err != nil { 45 | return err 46 | } 47 | defer cleanupDir(tmpSnapshot) 48 | 49 | bkFilePath := path.Join(tmpSnapshot, snapshotFileName) 50 | err = n.db.BackupTo(bkFilePath) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | return n.storage.Upload(snapshotFileName, bkFilePath) 56 | } 57 | 58 | func (n *NatsDBSnapshot) RestoreSnapshot() error { 59 | n.mutex.Lock() 60 | defer n.mutex.Unlock() 61 | 62 | tmpSnapshotPath, err := os.MkdirTemp(os.TempDir(), tempDirPattern) 63 | if err != nil { 64 | return err 65 | } 66 | defer cleanupDir(tmpSnapshotPath) 67 | 68 | bkFilePath := path.Join(tmpSnapshotPath, snapshotFileName) 69 | err = n.storage.Download(bkFilePath, snapshotFileName) 70 | if err == ErrNoSnapshotFound { 71 | log.Warn().Err(err).Msg("System will now continue without restoring snapshot") 72 | return nil 73 | } 74 | 75 | if err != nil { 76 | return err 77 | } 78 | 79 | log.Info().Str("path", bkFilePath).Msg("Downloaded snapshot, restoring...") 80 | err = db.RestoreFrom(n.db.GetPath(), bkFilePath) 81 | if err != nil { 82 | return err 83 | } 84 | 85 | log.Info().Str("path", bkFilePath).Msg("Restore complete...") 86 | return nil 87 | } 88 | 89 | func cleanupDir(p string) { 90 | for i := 0; i < 5; i++ { 91 | err := os.RemoveAll(p) 92 | if err == nil { 93 | return 94 | } 95 | 96 | log.Warn().Err(err).Str("path", p).Msg("Unable to cleanup directory path") 97 | time.Sleep(1 * time.Second) 98 | } 99 | 100 | log.Error().Str("path", p).Msg("Unable to cleanup temp path, this might cause disk wastage") 101 | } 102 | 103 | func fileHash(p string) (string, error) { 104 | f, err := os.Open(p) 105 | if err != nil { 106 | return "", err 107 | } 108 | defer f.Close() 109 | 110 | h := fnv.New64() 111 | if _, err := io.Copy(h, f); err != nil { 112 | return "", err 113 | } 114 | 115 | return fmt.Sprintf("%x", h.Sum64()), nil 116 | } 117 | -------------------------------------------------------------------------------- /snapshot/nats_snapshot.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/maxpert/marmot/cfg" 7 | ) 8 | 9 | var ErrInvalidStorageType = errors.New("invalid snapshot storage type") 10 | var ErrNoSnapshotFound = errors.New("no snapshot found") 11 | var ErrRequiredParameterMissing = errors.New("required parameter missing") 12 | 13 | type NatsSnapshot interface { 14 | SaveSnapshot() error 15 | RestoreSnapshot() error 16 | } 17 | 18 | type Storage interface { 19 | Upload(name, filePath string) error 20 | Download(filePath, name string) error 21 | } 22 | 23 | func NewSnapshotStorage() (Storage, error) { 24 | c := cfg.Config 25 | 26 | switch c.SnapshotStorageType() { 27 | case cfg.SFTP: 28 | return newSFTPStorage() 29 | case cfg.WebDAV: 30 | return newWebDAVStorage() 31 | case cfg.Nats: 32 | return newNatsStorage() 33 | case cfg.S3: 34 | return newS3Storage() 35 | } 36 | 37 | return nil, ErrInvalidStorageType 38 | } 39 | -------------------------------------------------------------------------------- /snapshot/nats_storage.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | "github.com/maxpert/marmot/cfg" 8 | "github.com/maxpert/marmot/stream" 9 | "github.com/nats-io/nats.go" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | const hashHeaderKey = "marmot-snapshot-tag" 14 | 15 | type natsStorage struct { 16 | nc *nats.Conn 17 | } 18 | 19 | func (n *natsStorage) Upload(name, filePath string) error { 20 | blb, err := getBlobStore(n.nc) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | err = blb.Delete(name) 26 | if err != nil && err != nats.ErrObjectNotFound { 27 | return err 28 | } 29 | 30 | hash, err := fileHash(filePath) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | rfl, err := os.Open(filePath) 36 | if err != nil { 37 | return err 38 | } 39 | defer rfl.Close() 40 | 41 | info, err := blb.Put(&nats.ObjectMeta{ 42 | Name: name, 43 | Headers: map[string][]string{ 44 | hashHeaderKey: {hash}, 45 | }, 46 | }, rfl) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | log.Info(). 52 | Str("hash", hash). 53 | Str("file_name", name). 54 | Str("file_path", filePath). 55 | Uint64("size", info.Size). 56 | Uint32("chunks", info.Chunks). 57 | Msg("Snapshot saved to NATS") 58 | 59 | return nil 60 | } 61 | 62 | func (n *natsStorage) Download(filePath, name string) error { 63 | blb, err := getBlobStore(n.nc) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | for { 69 | err = blb.GetFile(name, filePath) 70 | if err == nil { 71 | return nil 72 | } 73 | 74 | if err == nats.ErrObjectNotFound { 75 | return ErrNoSnapshotFound 76 | } 77 | 78 | if jsmErr, ok := err.(nats.JetStreamError); ok { 79 | log.Warn(). 80 | Err(err). 81 | Int("Status", jsmErr.APIError().Code). 82 | Msg("Error downloading snapshot") 83 | 84 | if jsmErr.APIError().Code == 503 { 85 | time.Sleep(time.Second) 86 | continue 87 | } 88 | } 89 | 90 | return err 91 | } 92 | } 93 | 94 | func getBlobStore(conn *nats.Conn) (nats.ObjectStore, error) { 95 | js, err := conn.JetStream(nats.MaxWait(30 * time.Second)) 96 | if err != nil { 97 | return nil, err 98 | } 99 | 100 | blb, err := js.ObjectStore(blobBucketName()) 101 | if err == nats.ErrStreamNotFound { 102 | blb, err = js.CreateObjectStore(&nats.ObjectStoreConfig{ 103 | Bucket: blobBucketName(), 104 | Replicas: cfg.Config.Snapshot.Nats.Replicas, 105 | Storage: nats.FileStorage, 106 | Description: "Bucket to store snapshot", 107 | }) 108 | } 109 | 110 | return blb, err 111 | } 112 | 113 | func newNatsStorage() (*natsStorage, error) { 114 | nc, err := stream.Connect() 115 | if err != nil { 116 | return nil, err 117 | } 118 | 119 | return &natsStorage{nc: nc}, nil 120 | } 121 | 122 | func blobBucketName() string { 123 | if cfg.Config.Snapshot.Nats.BucketName == "" { 124 | return cfg.Config.NATS.StreamPrefix + "-snapshot-store" 125 | } 126 | 127 | return cfg.Config.Snapshot.Nats.BucketName 128 | } 129 | -------------------------------------------------------------------------------- /snapshot/s3_storage.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/maxpert/marmot/cfg" 10 | "github.com/minio/minio-go/v7" 11 | "github.com/minio/minio-go/v7/pkg/credentials" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | type s3Storage struct { 16 | mc *minio.Client 17 | } 18 | 19 | func (s s3Storage) Upload(name, filePath string) error { 20 | ctx := context.Background() 21 | cS3 := cfg.Config.Snapshot.S3 22 | bucketPath := fmt.Sprintf("%s/%s", cS3.DirPath, name) 23 | info, err := s.mc.FPutObject(ctx, cS3.Bucket, bucketPath, filePath, minio.PutObjectOptions{}) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | log.Info(). 29 | Str("file_name", name). 30 | Int64("size", info.Size). 31 | Str("file_path", filePath). 32 | Str("bucket", info.Bucket). 33 | Msg("Snapshot saved to S3") 34 | 35 | return nil 36 | } 37 | 38 | func (s s3Storage) Download(filePath, name string) error { 39 | ctx := context.Background() 40 | cS3 := cfg.Config.Snapshot.S3 41 | bucketPath := fmt.Sprintf("%s/%s", cS3.DirPath, name) 42 | err := s.mc.FGetObject(ctx, cS3.Bucket, bucketPath, filePath, minio.GetObjectOptions{}) 43 | if mErr, ok := err.(minio.ErrorResponse); ok { 44 | if mErr.StatusCode == http.StatusNotFound { 45 | return ErrNoSnapshotFound 46 | } 47 | } 48 | 49 | return err 50 | } 51 | 52 | func newS3Storage() (*s3Storage, error) { 53 | c := cfg.Config 54 | cS3 := c.Snapshot.S3 55 | v := credentials.Value{ 56 | AccessKeyID: cS3.AccessKey, 57 | SecretAccessKey: cS3.SecretKey, 58 | SessionToken: cS3.SessionToken, 59 | } 60 | 61 | if cS3.AccessKey == "" && cS3.SecretKey == "" { 62 | v.SignerType = credentials.SignatureAnonymous 63 | } else { 64 | v.SignerType = credentials.SignatureV4 65 | } 66 | 67 | chain := []credentials.Provider{ 68 | &credentials.EnvAWS{}, 69 | &credentials.EnvMinio{}, 70 | &credentials.Static{ 71 | Value: v, 72 | }, 73 | } 74 | 75 | creds := credentials.NewChainCredentials(chain) 76 | mc, err := minio.New(cS3.Endpoint, &minio.Options{ 77 | Creds: creds, 78 | Secure: cS3.UseSSL, 79 | }) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 85 | defer cancel() 86 | 87 | exists, err := mc.BucketExists(ctx, cS3.Bucket) 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | if !exists { 93 | err = mc.MakeBucket(ctx, cS3.Bucket, minio.MakeBucketOptions{}) 94 | if err != nil { 95 | return nil, err 96 | } 97 | } 98 | 99 | return &s3Storage{ 100 | mc: mc, 101 | }, nil 102 | } 103 | -------------------------------------------------------------------------------- /snapshot/sftp_storage.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "net" 5 | "net/url" 6 | "os" 7 | "path" 8 | 9 | "github.com/maxpert/marmot/cfg" 10 | "github.com/pkg/sftp" 11 | "github.com/rs/zerolog/log" 12 | "golang.org/x/crypto/ssh" 13 | ) 14 | 15 | type sftpStorage struct { 16 | client *sftp.Client 17 | uploadPath string 18 | } 19 | 20 | func (s *sftpStorage) Upload(name, filePath string) error { 21 | err := s.client.MkdirAll(s.uploadPath) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | srcFile, err := os.Open(filePath) 27 | if err != nil { 28 | return err 29 | } 30 | defer srcFile.Close() 31 | 32 | uploadPath := path.Join(s.uploadPath, name) 33 | dstFile, err := s.client.OpenFile(uploadPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC) 34 | if err != nil { 35 | return err 36 | } 37 | defer dstFile.Close() 38 | 39 | bytes, err := dstFile.ReadFrom(srcFile) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | log.Info(). 45 | Str("file_name", name). 46 | Str("file_path", filePath). 47 | Str("sftp_path", uploadPath). 48 | Int64("bytes", bytes). 49 | Msg("Snapshot uploaded to SFTP server") 50 | return nil 51 | } 52 | 53 | func (s *sftpStorage) Download(filePath, name string) error { 54 | remotePath := path.Join(s.uploadPath, name) 55 | srcFile, err := s.client.Open(remotePath) 56 | if err != nil { 57 | if err.Error() == "file does not exist" { 58 | return ErrNoSnapshotFound 59 | } 60 | return err 61 | } 62 | defer srcFile.Close() 63 | 64 | dstFile, err := os.Create(filePath) 65 | if err != nil { 66 | return err 67 | } 68 | defer dstFile.Close() 69 | 70 | bytes, err := dstFile.ReadFrom(srcFile) 71 | log.Info(). 72 | Str("file_name", name). 73 | Str("file_path", filePath). 74 | Str("sftp_path", remotePath). 75 | Int64("bytes", bytes). 76 | Msg("Snapshot downloaded from SFTP server") 77 | return err 78 | } 79 | 80 | func newSFTPStorage() (*sftpStorage, error) { 81 | // Get the SFTP URL from the environment 82 | sftpURL := cfg.Config.Snapshot.SFTP.Url 83 | u, err := url.Parse(sftpURL) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | password, hasPassword := u.User.Password() 89 | authMethod := make([]ssh.AuthMethod, 0) 90 | if hasPassword { 91 | authMethod = append(authMethod, ssh.Password(password)) 92 | } 93 | 94 | // Set up the SSH config 95 | config := &ssh.ClientConfig{ 96 | User: u.User.Username(), 97 | Auth: authMethod, 98 | HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { 99 | log.Info(). 100 | Str("hostname", hostname). 101 | Str("remote", remote.String()). 102 | Str("public_key_type", key.Type()). 103 | Msg("Host connected for SFTP storage") 104 | return nil 105 | }, 106 | BannerCallback: func(message string) error { 107 | log.Info().Str("message", message).Msgf("Server message...") 108 | return nil 109 | }, 110 | } 111 | 112 | // Connect to the SSH server 113 | conn, err := ssh.Dial("tcp", u.Host, config) 114 | if err != nil { 115 | return nil, err 116 | } 117 | 118 | // Open the SFTP client 119 | client, err := sftp.NewClient(conn) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | return &sftpStorage{client: client, uploadPath: u.Path}, nil 125 | } 126 | -------------------------------------------------------------------------------- /snapshot/webdav_storage.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/fs" 7 | "net/url" 8 | "os" 9 | "path" 10 | "time" 11 | 12 | "github.com/maxpert/marmot/cfg" 13 | "github.com/rs/zerolog/log" 14 | "github.com/studio-b12/gowebdav" 15 | ) 16 | 17 | const queryParamTargetDir = "dir" 18 | const queryParamLogin = "login" 19 | const queryParamSecret = "secret" 20 | 21 | type webDAVStorage struct { 22 | client *gowebdav.Client 23 | path string 24 | } 25 | 26 | func (w *webDAVStorage) Upload(name, filePath string) error { 27 | rfl, err := os.Open(filePath) 28 | if err != nil { 29 | return err 30 | } 31 | defer rfl.Close() 32 | 33 | err = w.makeStoragePath() 34 | if err != nil { 35 | return err 36 | } 37 | 38 | nodePath := fmt.Sprintf("%s-%d-temp-%s", cfg.Config.NodeName(), time.Now().UnixMilli(), name) 39 | err = w.client.WriteStream(nodePath, rfl, 0644) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | completedPath := path.Join("/", w.path, name) 45 | err = w.client.Rename(nodePath, completedPath, true) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | log.Info(). 51 | Str("file_name", name). 52 | Str("file_path", filePath). 53 | Str("webdav_path", completedPath). 54 | Msg("Snapshot saved to WebDAV") 55 | return nil 56 | } 57 | 58 | func (w *webDAVStorage) Download(filePath, name string) error { 59 | completedPath := path.Join(w.path, name) 60 | rst, err := w.client.ReadStream(completedPath) 61 | if err != nil { 62 | if fsErr, ok := err.(*fs.PathError); ok { 63 | if wdErr, ok := fsErr.Err.(gowebdav.StatusError); ok && wdErr.Status == 404 { 64 | return ErrNoSnapshotFound 65 | } 66 | } 67 | return err 68 | } 69 | defer rst.Close() 70 | 71 | wst, err := os.Create(filePath) 72 | if err != nil { 73 | return err 74 | } 75 | defer wst.Close() 76 | 77 | if _, err = io.Copy(wst, rst); err != nil { 78 | return err 79 | } 80 | 81 | log.Info(). 82 | Str("file_name", name). 83 | Str("file_path", filePath). 84 | Str("webdav_path", completedPath). 85 | Msg("Snapshot downloaded from WebDAV") 86 | return nil 87 | } 88 | 89 | func (w *webDAVStorage) makeStoragePath() error { 90 | err := w.client.MkdirAll(w.path, 0740) 91 | if err == nil { 92 | return nil 93 | } 94 | 95 | if fsError, ok := err.(*os.PathError); ok { 96 | if wdErr, ok := fsError.Err.(gowebdav.StatusError); ok { 97 | if wdErr.Status == 409 { // Conflict means directory already exists! 98 | return nil 99 | } 100 | } 101 | } 102 | 103 | return err 104 | } 105 | 106 | func newWebDAVStorage() (*webDAVStorage, error) { 107 | webDAVCfg := cfg.Config.Snapshot.WebDAV 108 | u, err := url.Parse(webDAVCfg.Url) 109 | if err != nil { 110 | return nil, err 111 | } 112 | 113 | qp := u.Query() 114 | targetDir := qp.Get(queryParamTargetDir) 115 | if targetDir == "" { 116 | targetDir = "/" 117 | } 118 | 119 | login := qp.Get(queryParamLogin) 120 | secret := qp.Get(queryParamSecret) 121 | if login == "" || secret == "" { 122 | return nil, ErrRequiredParameterMissing 123 | } 124 | 125 | // Remove webdav parameters from query params 126 | qp.Del(queryParamTargetDir) 127 | qp.Del(queryParamLogin) 128 | qp.Del(queryParamSecret) 129 | 130 | // Set query params without parameters 131 | u.RawQuery = qp.Encode() 132 | cl := gowebdav.NewAuthClient(u.String(), gowebdav.NewAutoAuth(login, secret)) 133 | ret := &webDAVStorage{client: cl, path: targetDir} 134 | 135 | err = cl.Connect() 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | return ret, nil 141 | } 142 | -------------------------------------------------------------------------------- /stream/embedded_nats.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "net" 5 | "path" 6 | "strconv" 7 | "sync" 8 | "time" 9 | 10 | "github.com/maxpert/marmot/cfg" 11 | "github.com/nats-io/nats-server/v2/server" 12 | "github.com/nats-io/nats.go" 13 | "github.com/rs/zerolog/log" 14 | ) 15 | 16 | type embeddedNats struct { 17 | server *server.Server 18 | lock *sync.Mutex 19 | } 20 | 21 | var embeddedIns = &embeddedNats{ 22 | server: nil, 23 | lock: &sync.Mutex{}, 24 | } 25 | 26 | func parseHostAndPort(adr string) (string, int, error) { 27 | host, portStr, err := net.SplitHostPort(adr) 28 | if err != nil { 29 | return "", 0, err 30 | } 31 | 32 | port, err := strconv.Atoi(portStr) 33 | if err != nil { 34 | return "", 0, err 35 | } 36 | 37 | return host, port, nil 38 | } 39 | 40 | func startEmbeddedServer(nodeName string) (*embeddedNats, error) { 41 | embeddedIns.lock.Lock() 42 | defer embeddedIns.lock.Unlock() 43 | 44 | if embeddedIns.server != nil { 45 | return embeddedIns, nil 46 | } 47 | 48 | host, port, err := parseHostAndPort(cfg.Config.NATS.BindAddress) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | opts := &server.Options{ 54 | ServerName: nodeName, 55 | Host: host, 56 | Port: port, 57 | NoSigs: true, 58 | JetStream: true, 59 | JetStreamMaxMemory: -1, 60 | JetStreamMaxStore: -1, 61 | Cluster: server.ClusterOpts{ 62 | Name: cfg.EmbeddedClusterName, 63 | }, 64 | LeafNode: server.LeafNodeOpts{}, 65 | } 66 | 67 | if *cfg.ClusterPeersFlag != "" { 68 | opts.Routes = server.RoutesFromStr(*cfg.ClusterPeersFlag) 69 | } 70 | 71 | if *cfg.ClusterAddrFlag != "" { 72 | host, port, err := parseHostAndPort(*cfg.ClusterAddrFlag) 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | opts.Cluster.ListenStr = *cfg.ClusterAddrFlag 78 | opts.Cluster.Host = host 79 | opts.Cluster.Port = port 80 | } 81 | 82 | if *cfg.LeafServerFlag != "" { 83 | opts.LeafNode.Remotes = parseRemoteLeafOpts() 84 | } 85 | 86 | if cfg.Config.NATS.ServerConfigFile != "" { 87 | err := opts.ProcessConfigFile(cfg.Config.NATS.ServerConfigFile) 88 | if err != nil { 89 | return nil, err 90 | } 91 | } 92 | 93 | originalRoutes := opts.Routes 94 | if len(opts.Routes) != 0 { 95 | opts.Routes = flattenRoutes(originalRoutes, true) 96 | } 97 | 98 | if opts.StoreDir == "" { 99 | opts.StoreDir = path.Join(cfg.DataRootDir, "nats", nodeName) 100 | } 101 | 102 | s, err := server.NewServer(opts) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | s.SetLogger( 108 | &natsLogger{log.With().Str("from", "nats").Logger()}, 109 | opts.Debug, 110 | opts.Trace, 111 | ) 112 | s.Start() 113 | 114 | embeddedIns.server = s 115 | return embeddedIns, nil 116 | } 117 | 118 | func (e *embeddedNats) prepareConnection(opts ...nats.Option) (*nats.Conn, error) { 119 | e.lock.Lock() 120 | s := e.server 121 | e.lock.Unlock() 122 | 123 | for !s.ReadyForConnections(1 * time.Second) { 124 | continue 125 | } 126 | 127 | opts = append(opts, nats.InProcessServer(s)) 128 | for { 129 | c, err := nats.Connect("", opts...) 130 | if err != nil { 131 | log.Warn().Err(err).Msg("NATS server not accepting connections...") 132 | continue 133 | } 134 | 135 | j, err := c.JetStream() 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | st, err := j.StreamInfo("marmot-r", nats.MaxWait(1*time.Second)) 141 | if err == nats.ErrStreamNotFound || st != nil { 142 | log.Info().Msg("Streaming ready...") 143 | return c, nil 144 | } 145 | 146 | c.Close() 147 | log.Debug().Err(err).Msg("Streams not ready, waiting for NATS streams to come up...") 148 | time.Sleep(1 * time.Second) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /stream/nats.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/maxpert/marmot/cfg" 8 | "github.com/nats-io/nats.go" 9 | "github.com/rs/zerolog/log" 10 | ) 11 | 12 | func Connect() (*nats.Conn, error) { 13 | opts := setupConnOptions() 14 | 15 | creds, err := getNatsAuthFromConfig() 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | tls, err := getNatsTLSFromConfig() 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | opts = append(opts, creds...) 26 | opts = append(opts, tls...) 27 | if len(cfg.Config.NATS.URLs) == 0 { 28 | embedded, err := startEmbeddedServer(cfg.Config.NodeName()) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | return embedded.prepareConnection(opts...) 34 | } 35 | 36 | url := strings.Join(cfg.Config.NATS.URLs, ", ") 37 | 38 | var conn *nats.Conn 39 | for i := 0; i < cfg.Config.NATS.ConnectRetries; i++ { 40 | conn, err = nats.Connect(url, opts...) 41 | if err == nil && conn.Status() == nats.CONNECTED { 42 | break 43 | } 44 | 45 | log.Warn(). 46 | Err(err). 47 | Int("attempt", i+1). 48 | Int("attempt_limit", cfg.Config.NATS.ConnectRetries). 49 | Str("status", conn.Status().String()). 50 | Msg("NATS connection failed") 51 | } 52 | 53 | return conn, err 54 | } 55 | 56 | func getNatsAuthFromConfig() ([]nats.Option, error) { 57 | opts := make([]nats.Option, 0) 58 | 59 | if cfg.Config.NATS.CredsUser != "" { 60 | opt := nats.UserInfo(cfg.Config.NATS.CredsUser, cfg.Config.NATS.CredsPassword) 61 | opts = append(opts, opt) 62 | } 63 | 64 | if cfg.Config.NATS.SeedFile != "" { 65 | opt, err := nats.NkeyOptionFromSeed(cfg.Config.NATS.SeedFile) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | opts = append(opts, opt) 71 | } 72 | 73 | return opts, nil 74 | } 75 | 76 | func getNatsTLSFromConfig() ([]nats.Option, error) { 77 | opts := make([]nats.Option, 0) 78 | 79 | if cfg.Config.NATS.CAFile != "" { 80 | opt := nats.RootCAs(cfg.Config.NATS.CAFile) 81 | opts = append(opts, opt) 82 | } 83 | 84 | if cfg.Config.NATS.CertFile != "" && cfg.Config.NATS.KeyFile != "" { 85 | opt := nats.ClientCert(cfg.Config.NATS.CertFile, cfg.Config.NATS.KeyFile) 86 | opts = append(opts, opt) 87 | } 88 | 89 | return opts, nil 90 | } 91 | 92 | func setupConnOptions() []nats.Option { 93 | return []nats.Option{ 94 | nats.Name(cfg.Config.NodeName()), 95 | nats.RetryOnFailedConnect(true), 96 | nats.ReconnectWait(time.Duration(cfg.Config.NATS.ReconnectWaitSeconds) * time.Second), 97 | nats.MaxReconnects(cfg.Config.NATS.ConnectRetries), 98 | nats.ClosedHandler(func(nc *nats.Conn) { 99 | log.Error(). 100 | Err(nc.LastError()). 101 | Msg("NATS client exiting") 102 | }), 103 | nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { 104 | log.Error(). 105 | Err(err). 106 | Msg("NATS client disconnected") 107 | }), 108 | nats.ReconnectHandler(func(nc *nats.Conn) { 109 | log.Info(). 110 | Str("url", nc.ConnectedUrl()). 111 | Msg("NATS client reconnected") 112 | }), 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /stream/nats_logger.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import "github.com/rs/zerolog" 4 | 5 | type natsLogger struct { 6 | zerolog.Logger 7 | } 8 | 9 | func (n *natsLogger) Noticef(format string, v ...interface{}) { 10 | n.Info().Msgf(format, v...) 11 | } 12 | 13 | func (n *natsLogger) Warnf(format string, v ...interface{}) { 14 | n.Warn().Msgf(format, v...) 15 | } 16 | 17 | func (n *natsLogger) Fatalf(format string, v ...interface{}) { 18 | n.Fatal().Msgf(format, v...) 19 | } 20 | 21 | func (n *natsLogger) Errorf(format string, v ...interface{}) { 22 | n.Error().Msgf(format, v...) 23 | } 24 | 25 | func (n *natsLogger) Debugf(format string, v ...interface{}) { 26 | n.Debug().Msgf(format, v...) 27 | } 28 | 29 | func (n *natsLogger) Tracef(format string, v ...interface{}) { 30 | n.Trace().Msgf(format, v...) 31 | } 32 | -------------------------------------------------------------------------------- /stream/routes_discover.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/url" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "github.com/maxpert/marmot/cfg" 12 | "github.com/nats-io/nats-server/v2/server" 13 | "github.com/rs/zerolog/log" 14 | "github.com/samber/lo" 15 | ) 16 | 17 | func parseRemoteLeafOpts() []*server.RemoteLeafOpts { 18 | leafServers := server.RoutesFromStr(*cfg.LeafServerFlag) 19 | if len(leafServers) != 0 { 20 | leafServers = flattenRoutes(leafServers, true) 21 | } 22 | 23 | return lo.Map[*url.URL, *server.RemoteLeafOpts]( 24 | leafServers, 25 | func(u *url.URL, _ int) *server.RemoteLeafOpts { 26 | hub := u.Query().Get("hub") == "true" 27 | r := &server.RemoteLeafOpts{ 28 | URLs: []*url.URL{u}, 29 | Hub: hub, 30 | } 31 | 32 | return r 33 | }) 34 | } 35 | 36 | func flattenRoutes(urls []*url.URL, waitDNSEntries bool) []*url.URL { 37 | ret := make([]*url.URL, 0) 38 | for _, u := range urls { 39 | if u.Scheme == "dns" { 40 | ret = append(ret, queryDNSRoutes(u, waitDNSEntries)...) 41 | continue 42 | } 43 | 44 | ret = append(ret, u) 45 | } 46 | 47 | return ret 48 | } 49 | 50 | func queryDNSRoutes(u *url.URL, waitDNSEntries bool) []*url.URL { 51 | minPeerStr := u.Query().Get("min") 52 | intervalStr := u.Query().Get("interval_ms") 53 | 54 | minPeers, err := strconv.Atoi(minPeerStr) 55 | if err != nil { 56 | minPeers = 2 57 | } 58 | 59 | interval, err := strconv.Atoi(intervalStr) 60 | if err != nil { 61 | interval = 1000 62 | } 63 | 64 | log.Info(). 65 | Str("url", u.String()). 66 | Int("min_peers", minPeers). 67 | Int("interval", interval). 68 | Bool("wait_dns_entries", waitDNSEntries). 69 | Msg("Starting DNS A/AAAA peer discovery") 70 | 71 | if waitDNSEntries { 72 | minPeers = 0 73 | } 74 | 75 | for { 76 | peers, err := getDirectNATSAddresses(u) 77 | if err != nil { 78 | log.Error(). 79 | Err(err). 80 | Str("url", u.String()). 81 | Msg("Unable to discover peer URLs") 82 | time.Sleep(time.Duration(interval) * time.Millisecond) 83 | continue 84 | } 85 | 86 | urls := strings.Join( 87 | lo.Map[*url.URL, string](peers, func(i *url.URL, _ int) string { return i.String() }), 88 | ", ", 89 | ) 90 | log.Info().Str("urls", urls).Msg("Peers discovered") 91 | 92 | if len(peers) >= minPeers { 93 | return peers 94 | } else { 95 | time.Sleep(time.Duration(interval) * time.Millisecond) 96 | } 97 | } 98 | } 99 | 100 | func getDirectNATSAddresses(u *url.URL) ([]*url.URL, error) { 101 | v4, v6, err := queryDNS(u.Hostname()) 102 | if err != nil { 103 | return nil, err 104 | } 105 | var ret []*url.URL 106 | for _, ip := range v4 { 107 | peerUrl := fmt.Sprintf("nats://%s:%s/", ip, u.Port()) 108 | peer, err := url.Parse(peerUrl) 109 | if err != nil { 110 | log.Warn(). 111 | Str("peer_url", peerUrl). 112 | Msg("Unable to parse URL, might be due to bad DNS entry") 113 | continue 114 | } 115 | ret = append(ret, peer) 116 | } 117 | 118 | for _, ip := range v6 { 119 | peerUrl := fmt.Sprintf("nats://[%s]:%s/", ip, u.Port()) 120 | peer, err := url.Parse(peerUrl) 121 | if err != nil { 122 | log.Warn(). 123 | Str("peer_url", peerUrl). 124 | Msg("Unable to parse URL, might be due to bad DNS entry") 125 | continue 126 | } 127 | ret = append(ret, peer) 128 | } 129 | 130 | return ret, nil 131 | } 132 | 133 | func queryDNS(domain string) ([]string, []string, error) { 134 | ips, err := net.LookupIP(domain) 135 | if err != nil { 136 | return nil, nil, err 137 | } 138 | 139 | var ipv4 []string 140 | var ipv6 []string 141 | for _, ip := range ips { 142 | if v4 := ip.To4(); v4 != nil { 143 | ipv4 = append(ipv4, v4.String()) // A record 144 | } else if v6 := ip.To16(); v6 != nil { 145 | ipv6 = append(ipv6, v6.String()) // AAAA record 146 | } 147 | } 148 | 149 | return ipv4, ipv6, nil 150 | } 151 | -------------------------------------------------------------------------------- /telemetry/telemetry.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | "github.com/maxpert/marmot/cfg" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/promhttp" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | var registry *prometheus.Registry 14 | 15 | type Histogram interface { 16 | Observe(float64) 17 | } 18 | 19 | type Counter interface { 20 | Inc() 21 | Add(float64) 22 | } 23 | 24 | type Gauge interface { 25 | Set(float64) 26 | Inc() 27 | Dec() 28 | Add(float64) 29 | Sub(float64) 30 | SetToCurrentTime() 31 | } 32 | 33 | type NoopStat struct{} 34 | 35 | func (n NoopStat) Observe(float64) { 36 | } 37 | 38 | func (n NoopStat) Set(float64) { 39 | } 40 | 41 | func (n NoopStat) Dec() { 42 | } 43 | 44 | func (n NoopStat) Sub(float64) { 45 | } 46 | 47 | func (n NoopStat) SetToCurrentTime() { 48 | } 49 | 50 | func (n NoopStat) Inc() { 51 | } 52 | 53 | func (n NoopStat) Add(float64) { 54 | } 55 | 56 | func NewCounter(name string, help string) Counter { 57 | if registry == nil { 58 | return NoopStat{} 59 | } 60 | 61 | ret := prometheus.NewCounter(prometheus.CounterOpts{ 62 | Namespace: cfg.Config.Prometheus.Namespace, 63 | Subsystem: cfg.Config.Prometheus.Subsystem, 64 | Name: name, 65 | Help: help, 66 | ConstLabels: map[string]string{ 67 | "node_id": strconv.FormatUint(cfg.Config.NodeID, 10), 68 | }, 69 | }) 70 | 71 | registry.MustRegister(ret) 72 | return ret 73 | } 74 | 75 | func NewGauge(name string, help string) Gauge { 76 | if registry == nil { 77 | return NoopStat{} 78 | } 79 | 80 | ret := prometheus.NewGauge(prometheus.GaugeOpts{ 81 | Namespace: cfg.Config.Prometheus.Namespace, 82 | Subsystem: cfg.Config.Prometheus.Subsystem, 83 | Name: name, 84 | Help: help, 85 | ConstLabels: map[string]string{ 86 | "node_id": strconv.FormatUint(cfg.Config.NodeID, 10), 87 | }, 88 | }) 89 | 90 | registry.MustRegister(ret) 91 | return ret 92 | } 93 | 94 | func NewHistogram(name string, help string) Histogram { 95 | if registry == nil { 96 | return NoopStat{} 97 | } 98 | 99 | ret := prometheus.NewHistogram(prometheus.HistogramOpts{ 100 | Namespace: cfg.Config.Prometheus.Namespace, 101 | Subsystem: cfg.Config.Prometheus.Subsystem, 102 | Name: name, 103 | Help: help, 104 | ConstLabels: map[string]string{ 105 | "node_id": strconv.FormatUint(cfg.Config.NodeID, 10), 106 | }, 107 | }) 108 | 109 | registry.MustRegister(ret) 110 | return ret 111 | } 112 | 113 | func InitializeTelemetry() { 114 | if !cfg.Config.Prometheus.Enable { 115 | return 116 | } 117 | 118 | registry = prometheus.NewRegistry() 119 | server := http.Server{ 120 | Addr: cfg.Config.Prometheus.Bind, 121 | Handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{Registry: registry}), 122 | } 123 | 124 | go func() { 125 | if err := server.ListenAndServe(); err != nil { 126 | log.Error().Err(err).Msg("Unable to start controller listener") 127 | } 128 | }() 129 | } 130 | -------------------------------------------------------------------------------- /utils/deep.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | 7 | "github.com/fxamacker/cbor/v2" 8 | ) 9 | 10 | func DeepCopy(dst, src any) error { 11 | var buf bytes.Buffer 12 | if err := cbor.NewEncoder(&buf).Encode(src); err != nil { 13 | return err 14 | } 15 | return cbor.NewDecoder(&buf).Decode(dst) 16 | } 17 | 18 | func DeepEqualArray[T any](a, b []T) bool { 19 | if len(a) != len(b) { 20 | return false 21 | } 22 | 23 | for i := 0; i < len(a); i++ { 24 | if !reflect.DeepEqual(a[i], b[i]) { 25 | return false 26 | } 27 | } 28 | 29 | return true 30 | } 31 | -------------------------------------------------------------------------------- /utils/state_context.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | type StateContext struct { 9 | ctx context.Context 10 | cancel context.CancelFunc 11 | } 12 | 13 | func NewStateContext() *StateContext { 14 | ctx, cancel := context.WithCancel(context.Background()) 15 | return &StateContext{ 16 | ctx: ctx, 17 | cancel: cancel, 18 | } 19 | } 20 | 21 | func (s *StateContext) Cancel() { 22 | s.cancel() 23 | } 24 | 25 | func (s *StateContext) IsCanceled() bool { 26 | select { 27 | case <-s.ctx.Done(): 28 | return s.ctx.Err() == context.Canceled 29 | case <-time.After(0): 30 | return false 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /utils/stop_watch.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/maxpert/marmot/telemetry" 7 | "github.com/rs/zerolog" 8 | ) 9 | 10 | type StopWatch struct { 11 | startTime time.Time 12 | name string 13 | } 14 | 15 | func NewStopWatch(name string) *StopWatch { 16 | return &StopWatch{ 17 | startTime: time.Now(), 18 | name: name, 19 | } 20 | } 21 | 22 | func (t *StopWatch) Stop() time.Duration { 23 | return time.Since(t.startTime) 24 | } 25 | 26 | func (t *StopWatch) Log(e *zerolog.Event, hist telemetry.Histogram) { 27 | dur := t.Stop() 28 | if hist != nil { 29 | hist.Observe(float64(dur.Microseconds())) 30 | } 31 | 32 | e.Dur("duration", dur).Str("name", t.name).Send() 33 | } 34 | -------------------------------------------------------------------------------- /utils/timeout.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/asaskevich/EventBus" 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | type TimeoutPublisher struct { 11 | duration time.Duration 12 | ticker *time.Ticker 13 | publisher chan time.Time 14 | } 15 | 16 | func AutoResetEventTimer(bus EventBus.Bus, eventName string, duration time.Duration) *TimeoutPublisher { 17 | t := NewTimeoutPublisher(duration) 18 | err := bus.Subscribe(eventName, func(args ...any) { 19 | t.Reset() 20 | }) 21 | 22 | if err != nil { 23 | log.Panic().Err(err).Msg("Unable to subscribe timeout event bus") 24 | } 25 | 26 | return t 27 | } 28 | 29 | func NewTimeoutPublisher(duration time.Duration) *TimeoutPublisher { 30 | if duration == 0 { 31 | return &TimeoutPublisher{ 32 | duration: duration, 33 | ticker: nil, 34 | publisher: make(chan time.Time), 35 | } 36 | } 37 | 38 | ticker := time.NewTicker(duration) 39 | return &TimeoutPublisher{duration: duration, ticker: ticker, publisher: nil} 40 | } 41 | 42 | func (t *TimeoutPublisher) Reset() { 43 | if t.ticker == nil { 44 | return 45 | } 46 | 47 | t.ticker.Reset(t.duration) 48 | } 49 | 50 | func (t *TimeoutPublisher) Stop() { 51 | if t.ticker == nil { 52 | return 53 | } 54 | 55 | t.ticker.Stop() 56 | } 57 | 58 | func (t *TimeoutPublisher) Channel() <-chan time.Time { 59 | if t.ticker == nil { 60 | return t.publisher 61 | } 62 | 63 | return t.ticker.C 64 | } 65 | --------------------------------------------------------------------------------